Harsh200415 commited on
Commit
b2eef7c
·
0 Parent(s):

MVP working - audio analysis pipeline with Whisper, MFCC diarization, Groq insights

Browse files
.gitignore ADDED
@@ -0,0 +1,21 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Python
2
+ __pycache__/
3
+ *.pyc
4
+ *.pyo
5
+ .env
6
+
7
+ # Database
8
+ *.db
9
+
10
+ # Node
11
+ node_modules/
12
+ dist/
13
+
14
+ # Conda/pip
15
+ venv/
16
+
17
+ # Mac
18
+ .DS_Store
19
+
20
+ # Temporary audio files
21
+ /tmp/behavioral_mirror/
backend/db/database.py ADDED
@@ -0,0 +1,24 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from sqlalchemy import create_engine, Column, String, Text, DateTime
2
+ from sqlalchemy.ext.declarative import declarative_base
3
+ from sqlalchemy.orm import sessionmaker
4
+ from datetime import datetime
5
+
6
+ SQLALCHEMY_DATABASE_URL = "sqlite:///./behavioral_mirror.db"
7
+
8
+ engine = create_engine(
9
+ SQLALCHEMY_DATABASE_URL,
10
+ connect_args={"check_same_thread": False}
11
+ )
12
+
13
+ SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine)
14
+ Base = declarative_base()
15
+
16
+ class Session(Base):
17
+ __tablename__ = "sessions"
18
+
19
+ id = Column(String, primary_key=True)
20
+ user_id = Column(String, index=True)
21
+ context = Column(String)
22
+ signals_json = Column(Text)
23
+ insights_json = Column(Text)
24
+ created_at = Column(DateTime, default=datetime.utcnow)
backend/environment.yml ADDED
@@ -0,0 +1,263 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ name: behavioral-mirror
2
+ channels:
3
+ - conda-forge
4
+ - defaults
5
+ - https://repo.anaconda.com/pkgs/main
6
+ - https://repo.anaconda.com/pkgs/r
7
+ dependencies:
8
+ - aom=3.9.1=h7bae524_0
9
+ - bzip2=1.0.8=h80987f9_6
10
+ - ca-certificates=2026.4.22=hbd8a1cb_0
11
+ - cairo=1.18.4=he0f2337_1
12
+ - dav1d=1.2.1=hb547adb_0
13
+ - dbus=1.16.2=h3ff7a7c_1
14
+ - ffmpeg=8.1.1=gpl_h246f3d5_101
15
+ - font-ttf-dejavu-sans-mono=2.37=hab24e00_0
16
+ - font-ttf-inconsolata=3.000=h77eed37_0
17
+ - font-ttf-source-code-pro=2.038=h77eed37_0
18
+ - font-ttf-ubuntu=0.83=h77eed37_3
19
+ - fontconfig=2.17.1=h2b252f5_0
20
+ - fonts-conda-ecosystem=1=0
21
+ - fonts-conda-forge=1=hc364b38_1
22
+ - fribidi=1.0.16=hc919400_0
23
+ - gdk-pixbuf=2.44.6=h4e57454_0
24
+ - glslang=16.3.0=h7cb4797_0
25
+ - gmp=6.3.0=h7bae524_2
26
+ - graphite2=1.3.14=hec049ff_2
27
+ - harfbuzz=14.2.0=h3103d1b_0
28
+ - icu=78.3=hef89b57_0
29
+ - lame=3.100=h1a8c8d9_1003
30
+ - lcms2=2.19.1=hdfa7624_0
31
+ - lerc=4.1.0=h1eee2c3_0
32
+ - libabseil=20260107.1=cxx17_h2062a1b_0
33
+ - libass=0.17.4=hcbd7ca7_0
34
+ - libbrotlicommon=1.2.0=hc919400_1
35
+ - libbrotlidec=1.2.0=hc919400_1
36
+ - libbrotlienc=1.2.0=hc919400_1
37
+ - libcxx=22.1.2=hf894667_0
38
+ - libdeflate=1.25=hc11a715_0
39
+ - libdovi=3.3.2=h78f8ca3_4
40
+ - libexpat=2.8.0=h50f4ffc_0
41
+ - libffi=3.5.2=hcf2aa1b_0
42
+ - libfreetype=2.14.3=hce30654_0
43
+ - libfreetype6=2.14.3=hdfa99f5_0
44
+ - libglib=2.88.1=ha08bb59_2
45
+ - libhwloc=2.13.0=default_ha97f43a_1000
46
+ - libhwy=1.4.0=ha332bbd_0
47
+ - libiconv=1.18=h23cfdf5_2
48
+ - libintl=0.25.1=h493aca8_0
49
+ - libjpeg-turbo=3.1.4.1=h84a0fba_0
50
+ - libjxl=0.11.2=h934fa54_1
51
+ - liblzma=5.8.3=h8088a28_0
52
+ - liblzma-devel=5.8.3=h8088a28_0
53
+ - libogg=1.3.5=h48c0fde_1
54
+ - libopenvino=2026.0.0=h3e6d54f_1
55
+ - libopenvino-arm-cpu-plugin=2026.0.0=h3e6d54f_1
56
+ - libopenvino-auto-batch-plugin=2026.0.0=h2406d2e_1
57
+ - libopenvino-auto-plugin=2026.0.0=h2406d2e_1
58
+ - libopenvino-hetero-plugin=2026.0.0=h85cbfa6_1
59
+ - libopenvino-ir-frontend=2026.0.0=h85cbfa6_1
60
+ - libopenvino-onnx-frontend=2026.0.0=h41365f2_1
61
+ - libopenvino-paddle-frontend=2026.0.0=h41365f2_1
62
+ - libopenvino-pytorch-frontend=2026.0.0=hf6b4638_1
63
+ - libopenvino-tensorflow-frontend=2026.0.0=hc295da0_1
64
+ - libopenvino-tensorflow-lite-frontend=2026.0.0=hf6b4638_1
65
+ - libopus=1.6.1=h1a92334_0
66
+ - libplacebo=7.360.1=h176d363_0
67
+ - libpng=1.6.58=h132b30e_0
68
+ - libprotobuf=6.33.5=h4a5acfd_0
69
+ - librsvg=2.62.1=he8aa2a2_0
70
+ - libsqlite=3.53.1=h1b79a29_0
71
+ - libtiff=4.7.1=h4030677_1
72
+ - libusb=1.0.29=hbc156a2_0
73
+ - libvorbis=1.3.7=h81086ad_2
74
+ - libvpx=1.15.2=ha759d40_0
75
+ - libvulkan-loader=1.4.341.0=h3feff0a_0
76
+ - libwebp-base=1.6.0=h07db88b_0
77
+ - libxml2=2.15.3=h5654f7c_0
78
+ - libxml2-16=2.15.3=h5ef1a60_0
79
+ - libzlib=1.3.2=h8088a28_2
80
+ - ncurses=6.5=hee39554_0
81
+ - openh264=2.6.0=hb5b2745_0
82
+ - openssl=3.6.2=hd24854e_0
83
+ - pango=1.56.4=hf80efc4_1
84
+ - pcre2=10.47=h30297fc_0
85
+ - pip=26.0.1=pyhc872135_1
86
+ - pixman=0.46.4=h81086ad_1
87
+ - pugixml=1.15=hd3d436d_0
88
+ - python=3.10.20=h1b19095_0_cpython
89
+ - readline=8.3=h0b18652_0
90
+ - sdl2=2.32.56=h248ca61_0
91
+ - sdl3=3.4.8=h6fa9c73_0
92
+ - shaderc=2026.2=hf31e910_0
93
+ - snappy=1.2.2=hada39a4_1
94
+ - spirv-tools=2026.1=h4ddebb9_0
95
+ - sqlite=3.51.2=h67002bf_0
96
+ - svt-av1=4.0.1=h0cb729a_0
97
+ - tbb=2023.0.0=he0260a5_2
98
+ - tk=8.6.15=hcd8a7d5_0
99
+ - wheel=0.46.3=py310hca03da5_0
100
+ - x264=1!164.3095=h57fd34a_2
101
+ - x265=3.5=hbc6ce65_3
102
+ - xz=5.8.3=hd0f0c4f_0
103
+ - xz-gpl-tools=5.8.3=hd0f0c4f_0
104
+ - xz-tools=5.8.3=h8088a28_0
105
+ - zlib=1.3.2=h8088a28_2
106
+ - zstd=1.5.7=hbf9d68e_6
107
+ - pip:
108
+ - aiohappyeyeballs==2.6.1
109
+ - aiohttp==3.13.5
110
+ - aiosignal==1.4.0
111
+ - alembic==1.18.4
112
+ - annotated-doc==0.0.4
113
+ - annotated-types==0.7.0
114
+ - anthropic==0.102.0
115
+ - antlr4-python3-runtime==4.9.3
116
+ - anyio==4.13.0
117
+ - asteroid-filterbanks==0.4.0
118
+ - async-timeout==5.0.1
119
+ - attrs==26.1.0
120
+ - audioread==3.1.0
121
+ - av==17.0.1
122
+ - certifi==2026.4.22
123
+ - cffi==2.0.0
124
+ - charset-normalizer==3.4.7
125
+ - click==8.3.3
126
+ - coloredlogs==15.0.1
127
+ - colorlog==6.10.1
128
+ - contourpy==1.3.2
129
+ - ctranslate2==4.7.1
130
+ - cycler==0.12.1
131
+ - decorator==5.2.1
132
+ - distro==1.9.0
133
+ - docstring-parser==0.18.0
134
+ - einops==0.8.2
135
+ - exceptiongroup==1.3.1
136
+ - fastapi==0.136.1
137
+ - faster-whisper==1.2.1
138
+ - filelock==3.29.0
139
+ - flatbuffers==25.12.19
140
+ - fonttools==4.63.0
141
+ - frozenlist==1.8.0
142
+ - fsspec==2026.4.0
143
+ - googleapis-common-protos==1.75.0
144
+ - groq==1.2.0
145
+ - grpcio==1.80.0
146
+ - h11==0.16.0
147
+ - hf-xet==1.5.0
148
+ - httpcore==1.0.9
149
+ - httpx==0.28.1
150
+ - huggingface-hub==0.36.2
151
+ - humanfriendly==10.0
152
+ - hyperpyyaml==1.2.3
153
+ - idna==3.15
154
+ - importlib-metadata==8.7.1
155
+ - jinja2==3.1.6
156
+ - jiter==0.14.0
157
+ - joblib==1.5.3
158
+ - julius==0.2.7
159
+ - kiwisolver==1.5.0
160
+ - lazy-loader==0.5
161
+ - librosa==0.11.0
162
+ - lightning==2.6.1
163
+ - lightning-utilities==0.15.3
164
+ - llvmlite==0.47.0
165
+ - mako==1.3.12
166
+ - markdown-it-py==4.2.0
167
+ - markupsafe==3.0.3
168
+ - matplotlib==3.10.9
169
+ - mdurl==0.1.2
170
+ - more-itertools==11.0.2
171
+ - mpmath==1.3.0
172
+ - msgpack==1.1.2
173
+ - multidict==6.7.1
174
+ - networkx==3.4.2
175
+ - nltk==3.9.4
176
+ - numba==0.65.1
177
+ - numpy==1.26.4
178
+ - omegaconf==2.3.0
179
+ - onnxruntime==1.23.2
180
+ - openai-whisper==20250625
181
+ - opentelemetry-api==1.41.1
182
+ - opentelemetry-exporter-otlp==1.41.1
183
+ - opentelemetry-exporter-otlp-proto-common==1.41.1
184
+ - opentelemetry-exporter-otlp-proto-grpc==1.41.1
185
+ - opentelemetry-exporter-otlp-proto-http==1.41.1
186
+ - opentelemetry-proto==1.41.1
187
+ - opentelemetry-sdk==1.41.1
188
+ - opentelemetry-semantic-conventions==0.62b1
189
+ - optuna==4.8.0
190
+ - packaging==26.2
191
+ - pandas==2.3.3
192
+ - pillow==12.2.0
193
+ - platformdirs==4.9.6
194
+ - pooch==1.9.0
195
+ - primepy==1.3
196
+ - propcache==0.5.2
197
+ - protobuf==6.33.6
198
+ - pyannote-audio==3.1.1
199
+ - pyannote-core==6.0.1
200
+ - pyannote-database==6.1.1
201
+ - pyannote-metrics==4.1
202
+ - pyannote-pipeline==4.0.0
203
+ - pyannoteai-sdk==0.4.0
204
+ - pycparser==3.0
205
+ - pydantic==2.13.4
206
+ - pydantic-core==2.46.4
207
+ - pydub==0.25.1
208
+ - pygments==2.20.0
209
+ - pyparsing==3.3.2
210
+ - python-dateutil==2.9.0.post0
211
+ - python-dotenv==1.2.2
212
+ - python-multipart==0.0.28
213
+ - pytorch-lightning==2.6.1
214
+ - pytorch-metric-learning==2.9.0
215
+ - pytz==2026.2
216
+ - pyyaml==6.0.3
217
+ - regex==2026.5.9
218
+ - requests==2.34.2
219
+ - resemblyzer==0.1.4
220
+ - rich==15.0.0
221
+ - ruamel-yaml==0.18.17
222
+ - ruamel-yaml-clib==0.2.15
223
+ - safetensors==0.7.0
224
+ - scikit-learn==1.7.2
225
+ - scipy==1.15.3
226
+ - semver==3.0.4
227
+ - sentencepiece==0.2.1
228
+ - setuptools==82.0.1
229
+ - shellingham==1.5.4
230
+ - six==1.17.0
231
+ - sniffio==1.3.1
232
+ - sortedcontainers==2.4.0
233
+ - soundfile==0.13.1
234
+ - soxr==1.1.0
235
+ - speechbrain==1.1.0
236
+ - sqlalchemy==2.0.49
237
+ - starlette==1.0.0
238
+ - sympy==1.13.1
239
+ - tensorboardx==2.6.5
240
+ - threadpoolctl==3.6.0
241
+ - tiktoken==0.13.0
242
+ - tokenizers==0.22.2
243
+ - tomli==2.4.1
244
+ - torch==2.5.1
245
+ - torch-audiomentations==0.12.0
246
+ - torch-pitch-shift==1.2.5
247
+ - torchaudio==2.8.0
248
+ - torchcodec==0.7.0
249
+ - torchmetrics==1.9.0
250
+ - torchvision==0.23.0
251
+ - tqdm==4.67.3
252
+ - transformers==4.57.6
253
+ - typer==0.25.1
254
+ - typing==3.7.4.3
255
+ - typing-extensions==4.15.0
256
+ - typing-inspection==0.4.2
257
+ - tzdata==2026.2
258
+ - urllib3==2.7.0
259
+ - uvicorn==0.47.0
260
+ - webrtcvad==2.0.10
261
+ - yarl==1.23.0
262
+ - zipp==3.23.1
263
+ prefix: /opt/anaconda3/envs/behavioral-mirror
backend/main.py ADDED
@@ -0,0 +1,161 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from fastapi import FastAPI, UploadFile, File, Form, HTTPException
2
+ from fastapi.middleware.cors import CORSMiddleware
3
+ import os, shutil, uuid, json
4
+ from dotenv import load_dotenv
5
+ import numpy as np
6
+
7
+ load_dotenv()
8
+
9
+ from pipeline.transcriber import Transcriber
10
+ from pipeline.diarizer import Diarizer
11
+ from pipeline.signal_extractor import SignalExtractor
12
+ from pipeline.insight_generator import InsightGenerator
13
+ from db.database import SessionLocal, Session, Base, engine
14
+
15
+ Base.metadata.create_all(bind=engine)
16
+
17
+ app = FastAPI()
18
+
19
+ app.add_middleware(
20
+ CORSMiddleware,
21
+ allow_origins=["http://localhost:5173"],
22
+ allow_methods=["*"],
23
+ allow_headers=["*"]
24
+ )
25
+
26
+ UPLOAD_DIR = "/tmp/behavioral_mirror"
27
+ os.makedirs(UPLOAD_DIR, exist_ok=True)
28
+
29
+ print("Loading models... this may take a minute on first run.")
30
+ transcriber = Transcriber(model_size="base")
31
+ diarizer = Diarizer(hf_token=os.getenv("HF_TOKEN"))
32
+ insight_gen = InsightGenerator(api_key=os.getenv("GROQ_API_KEY"))
33
+ print("All models loaded. Server ready.")
34
+
35
+
36
+ @app.get("/")
37
+ def root():
38
+ return {"status": "Behavioral Mirror API is running"}
39
+
40
+
41
+ @app.post("/api/analyze")
42
+ async def analyze_session(
43
+ audio: UploadFile = File(...),
44
+ context: str = Form("casual"),
45
+ user_speaker: str = Form("SPEAKER_00"),
46
+ num_speakers: int = Form(2),
47
+ user_id: str = Form("default_user")
48
+ ):
49
+ session_id = str(uuid.uuid4())
50
+ audio_path = f"{UPLOAD_DIR}/{session_id}.wav"
51
+
52
+ # Save uploaded file temporarily
53
+ temp_path = f"{UPLOAD_DIR}/{session_id}_temp"
54
+ with open(temp_path, "wb") as f:
55
+ shutil.copyfileobj(audio.file, f)
56
+
57
+ # Convert to proper WAV format using ffmpeg
58
+ import subprocess
59
+ result_conv = subprocess.run([
60
+ "ffmpeg", "-i", temp_path,
61
+ "-ar", "16000",
62
+ "-ac", "1",
63
+ "-y", audio_path
64
+ ], capture_output=True)
65
+ os.remove(temp_path)
66
+
67
+ try:
68
+ # Step 1: Transcribe
69
+ print(f"[{session_id}] Transcribing...")
70
+ transcript = transcriber.transcribe(audio_path)
71
+
72
+ # Step 2: Diarize
73
+ print(f"[{session_id}] Diarizing...")
74
+ diarization = diarizer.diarize(audio_path, num_speakers=num_speakers)
75
+
76
+ # Step 3: Merge
77
+ print(f"[{session_id}] Merging transcript and speakers...")
78
+ merged = diarizer.merge_transcript_with_speakers(
79
+ transcript["segments"],
80
+ diarization
81
+ )
82
+
83
+ # Step 4: Extract signals
84
+ print(f"[{session_id}] Extracting signals...")
85
+ extractor = SignalExtractor(audio_path, merged, user_speaker)
86
+ signals = extractor.extract_all()
87
+
88
+ # Step 5: Get baseline
89
+ db = SessionLocal()
90
+ baseline = get_user_baseline(db, user_id)
91
+
92
+ # Step 6: Generate insights
93
+ print(f"[{session_id}] Generating insights...")
94
+ insights = insight_gen.generate(signals, context, baseline)
95
+
96
+ # Step 7: Save session
97
+ save_session(db, session_id, user_id, signals, insights, context)
98
+ db.close()
99
+
100
+ # Delete audio immediately after processing
101
+ os.remove(audio_path)
102
+ print(f"[{session_id}] Done.")
103
+
104
+ return {
105
+ "session_id": session_id,
106
+ "signals": signals,
107
+ "insights": insights
108
+ }
109
+
110
+ except Exception as e:
111
+ if os.path.exists(audio_path):
112
+ os.remove(audio_path)
113
+ print(f"Error: {e}")
114
+ raise HTTPException(status_code=500, detail=str(e))
115
+
116
+
117
+ @app.get("/api/sessions/{user_id}")
118
+ def get_sessions(user_id: str):
119
+ db = SessionLocal()
120
+ sessions = db.query(Session).filter(
121
+ Session.user_id == user_id
122
+ ).order_by(Session.created_at.desc()).all()
123
+ db.close()
124
+
125
+ return [
126
+ {
127
+ "session_id": s.id,
128
+ "context": s.context,
129
+ "created_at": s.created_at.isoformat(),
130
+ "insights": json.loads(s.insights_json),
131
+ "signals": json.loads(s.signals_json)
132
+ }
133
+ for s in sessions
134
+ ]
135
+
136
+
137
+ def get_user_baseline(db, user_id: str):
138
+ sessions = db.query(Session).filter(Session.user_id == user_id).all()
139
+ if len(sessions) < 3:
140
+ return None
141
+
142
+ all_signals = [json.loads(s.signals_json) for s in sessions[-10:]]
143
+
144
+ return {
145
+ "avg_speech_rate_wpm": float(np.mean([s["speech_rate"]["overall_wpm"] for s in all_signals])),
146
+ "avg_talk_ratio": float(np.mean([s["talk_ratio"]["user_ratio"] for s in all_signals])),
147
+ "avg_filler_rate": float(np.mean([s["filler_words"]["rate_per_100_words"] for s in all_signals])),
148
+ "avg_response_latency_s": float(np.mean([s["pauses"]["response_latency"]["mean_s"] for s in all_signals]))
149
+ }
150
+
151
+
152
+ def save_session(db, session_id, user_id, signals, insights, context):
153
+ session = Session(
154
+ id=session_id,
155
+ user_id=user_id,
156
+ context=context,
157
+ signals_json=json.dumps(signals),
158
+ insights_json=json.dumps(insights)
159
+ )
160
+ db.add(session)
161
+ db.commit()
backend/models/schemas.py ADDED
File without changes
backend/pipeline/diarizer.py ADDED
@@ -0,0 +1,77 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import numpy as np
2
+ import librosa
3
+ from sklearn.cluster import KMeans
4
+
5
+ class Diarizer:
6
+ def __init__(self, hf_token: str = None):
7
+ print("Diarizer ready (librosa-based).")
8
+
9
+ def diarize(self, audio_path: str, num_speakers: int = 2) -> list:
10
+ print(f"Diarizing: {audio_path}")
11
+
12
+ audio, sr = librosa.load(audio_path, sr=16000, mono=True)
13
+
14
+ # Split into 1-second windows with 0.5s hop
15
+ window = sr # 1 second
16
+ hop = sr // 2 # 0.5 second
17
+
18
+ embeddings = []
19
+ timestamps = []
20
+
21
+ for start in range(0, len(audio) - window, hop):
22
+ chunk = audio[start:start + window]
23
+
24
+ # Extract simple features: MFCCs
25
+ mfcc = librosa.feature.mfcc(y=chunk, sr=sr, n_mfcc=13)
26
+ embedding = np.concatenate([mfcc.mean(axis=1), mfcc.std(axis=1)])
27
+ embeddings.append(embedding)
28
+ timestamps.append({
29
+ "start": round(start / sr, 3),
30
+ "end": round((start + window) / sr, 3)
31
+ })
32
+
33
+ if len(embeddings) < num_speakers:
34
+ # Not enough audio — assign everything to speaker 0
35
+ return [{"speaker": "SPEAKER_00", "start": 0.0,
36
+ "end": round(len(audio) / sr, 3)}]
37
+
38
+ # Cluster into num_speakers groups
39
+ kmeans = KMeans(n_clusters=num_speakers, random_state=0, n_init=10)
40
+ labels = kmeans.fit_predict(np.array(embeddings))
41
+
42
+ # Build segment list
43
+ raw_segments = []
44
+ for ts, label in zip(timestamps, labels):
45
+ raw_segments.append({
46
+ "speaker": f"SPEAKER_0{label}",
47
+ "start": ts["start"],
48
+ "end": ts["end"]
49
+ })
50
+
51
+ # Merge consecutive same-speaker segments
52
+ merged = []
53
+ for seg in raw_segments:
54
+ if (merged and
55
+ merged[-1]["speaker"] == seg["speaker"] and
56
+ seg["start"] - merged[-1]["end"] < 1.0):
57
+ merged[-1]["end"] = seg["end"]
58
+ else:
59
+ merged.append(dict(seg))
60
+
61
+ return merged
62
+
63
+ def merge_transcript_with_speakers(
64
+ self,
65
+ transcript_segments: list,
66
+ diarization_segments: list
67
+ ) -> list:
68
+ merged = []
69
+ for t_seg in transcript_segments:
70
+ t_mid = (t_seg["start"] + t_seg["end"]) / 2
71
+ assigned_speaker = "SPEAKER_00"
72
+ for d_seg in diarization_segments:
73
+ if d_seg["start"] <= t_mid <= d_seg["end"]:
74
+ assigned_speaker = d_seg["speaker"]
75
+ break
76
+ merged.append({**t_seg, "speaker": assigned_speaker})
77
+ return merged
backend/pipeline/insight_generator.py ADDED
@@ -0,0 +1,117 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import json
2
+ from groq import Groq
3
+
4
+ class InsightGenerator:
5
+ def __init__(self, api_key: str):
6
+ self.client = Groq(api_key=api_key)
7
+
8
+ def generate(self, signals: dict, context: str, user_baseline: dict = None) -> dict:
9
+ structured_input = self._prepare_input(signals, context, user_baseline)
10
+ prompt = self._build_prompt(structured_input)
11
+
12
+ response = self.client.chat.completions.create(
13
+ model="llama-3.3-70b-versatile",
14
+ messages=[{"role": "user", "content": prompt}],
15
+ temperature=0.3,
16
+ max_tokens=1500
17
+ )
18
+
19
+ raw_output = response.choices[0].message.content
20
+ return self._parse_output(raw_output)
21
+
22
+ def _prepare_input(self, signals: dict, context: str, baseline: dict) -> dict:
23
+ prepared = {
24
+ "context": context,
25
+ "session_duration_minutes": round(signals["session_duration_s"] / 60, 1),
26
+ "talk_ratio_user": signals["talk_ratio"]["user_ratio"],
27
+ "speech_rate_wpm": signals["speech_rate"]["overall_wpm"],
28
+ "speech_rate_variability": signals["speech_rate"]["variability"],
29
+ "avg_response_latency_s": signals["pauses"]["response_latency"]["mean_s"],
30
+ "within_turn_pause_count": signals["pauses"]["within_turn_pauses"]["count"],
31
+ "user_interrupted_other": signals["interruptions"]["user_interrupted_other"],
32
+ "user_was_interrupted": signals["interruptions"]["user_was_interrupted"],
33
+ "filler_rate_per_100_words": signals["filler_words"]["rate_per_100_words"],
34
+ "top_filler_words": list(signals["filler_words"]["breakdown"].keys())[:3],
35
+ "pitch_std_hz": signals["pitch_features"].get("std_hz"),
36
+ "other_engagement_trend": signals["engagement_proxy"].get("other_speaker_turn_length_trend")
37
+ }
38
+
39
+ if baseline:
40
+ def pct_delta(current, base):
41
+ if not base or base == 0:
42
+ return None
43
+ return round((current - base) / base * 100, 1)
44
+
45
+ prepared["baseline_comparison"] = {
46
+ "speech_rate_delta_pct": pct_delta(
47
+ prepared["speech_rate_wpm"],
48
+ baseline.get("avg_speech_rate_wpm")
49
+ ),
50
+ "filler_rate_delta_pct": pct_delta(
51
+ prepared["filler_rate_per_100_words"],
52
+ baseline.get("avg_filler_rate")
53
+ ),
54
+ "response_latency_delta_pct": pct_delta(
55
+ prepared["avg_response_latency_s"],
56
+ baseline.get("avg_response_latency_s")
57
+ ),
58
+ "talk_ratio_delta": round(
59
+ prepared["talk_ratio_user"] - baseline.get("avg_talk_ratio", 0.5), 3
60
+ )
61
+ }
62
+
63
+ return prepared
64
+
65
+ def _build_prompt(self, data: dict) -> str:
66
+ return f"""You are a behavioral communication analyst. Generate reflective, non-judgmental observations about conversational behavior patterns.
67
+
68
+ STRICT RULES:
69
+ 1. Never diagnose or make personality claims
70
+ 2. Always use probabilistic language: "may suggest", "could indicate", "appeared to", "was observed"
71
+ 3. Never say things like "you are nervous", "you are dominant", "you lack confidence"
72
+ 4. Observations must be grounded in the specific signals provided
73
+ 5. Generate exactly 3 observations
74
+ 6. Output must be valid JSON only — no extra text, no markdown, no code fences
75
+
76
+ SESSION DATA:
77
+ {json.dumps(data, indent=2)}
78
+
79
+ Output this exact JSON structure:
80
+ {{
81
+ "summary_sentence": "One sentence describing the overall conversational pattern.",
82
+ "observations": [
83
+ {{
84
+ "signal": "talk_ratio",
85
+ "observation": "Behavioral observation in non-judgmental language.",
86
+ "resonance_prompt": "A gentle reflective question for the user."
87
+ }},
88
+ {{
89
+ "signal": "speech_rate",
90
+ "observation": "Behavioral observation in non-judgmental language.",
91
+ "resonance_prompt": "A gentle reflective question for the user."
92
+ }},
93
+ {{
94
+ "signal": "pauses",
95
+ "observation": "Behavioral observation in non-judgmental language.",
96
+ "resonance_prompt": "A gentle reflective question for the user."
97
+ }}
98
+ ],
99
+ "notable_pattern": "One sentence about the most notable pattern, or null.",
100
+ "data_confidence": "high"
101
+ }}"""
102
+
103
+ def _parse_output(self, raw: str) -> dict:
104
+ try:
105
+ clean = raw.strip()
106
+ if "```" in clean:
107
+ clean = clean.split("```")[1]
108
+ if clean.startswith("json"):
109
+ clean = clean[4:]
110
+ return json.loads(clean.strip())
111
+ except Exception:
112
+ return {
113
+ "summary_sentence": "Session analyzed successfully.",
114
+ "observations": [],
115
+ "notable_pattern": None,
116
+ "data_confidence": "low"
117
+ }
backend/pipeline/signal_extractor.py ADDED
@@ -0,0 +1,244 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import librosa
2
+ import numpy as np
3
+ import re
4
+
5
+ class SignalExtractor:
6
+
7
+ FILLER_WORDS = ["um", "uh", "like", "you know", "basically",
8
+ "literally", "actually", "so", "right", "i mean",
9
+ "matlab", "yaani", "bas", "toh", "na", "aur",
10
+ "waise", "dekh", "yaar", "arre", "haan"]
11
+
12
+ def __init__(self, audio_path: str, merged_segments: list, user_speaker_id: str):
13
+ self.audio_path = audio_path
14
+ self.segments = merged_segments
15
+ self.user_speaker = user_speaker_id
16
+ self.other_speaker = None
17
+
18
+ speakers = set(s["speaker"] for s in merged_segments if s["speaker"] != "UNKNOWN")
19
+ others = speakers - {user_speaker_id}
20
+ if others:
21
+ self.other_speaker = list(others)[0]
22
+
23
+ self.audio, self.sr = librosa.load(audio_path, sr=16000)
24
+
25
+ def extract_all(self) -> dict:
26
+ user_segs = [s for s in self.segments if s["speaker"] == self.user_speaker]
27
+ other_segs = [s for s in self.segments if s["speaker"] == self.other_speaker] if self.other_speaker else []
28
+
29
+ return {
30
+ "session_duration_s": self._get_session_duration(),
31
+ "talk_ratio": self._compute_talk_ratio(user_segs, other_segs),
32
+ "speech_rate": self._compute_speech_rate(user_segs),
33
+ "pauses": self._compute_pauses(user_segs, other_segs),
34
+ "interruptions": self._compute_interruptions(user_segs, other_segs),
35
+ "filler_words": self._compute_filler_words(user_segs),
36
+ "turn_dynamics": self._compute_turn_dynamics(user_segs, other_segs),
37
+ "pitch_features": self._compute_pitch_features(user_segs),
38
+ "engagement_proxy": self._compute_engagement_proxy(other_segs),
39
+ "timeline": self._build_timeline(user_segs)
40
+ }
41
+
42
+ def _get_session_duration(self) -> float:
43
+ if not self.segments:
44
+ return 0.0
45
+ return round(self.segments[-1]["end"], 2)
46
+
47
+ def _compute_talk_ratio(self, user_segs, other_segs) -> dict:
48
+ user_time = sum(s["end"] - s["start"] for s in user_segs)
49
+ other_time = sum(s["end"] - s["start"] for s in other_segs)
50
+ total = user_time + other_time
51
+
52
+ return {
53
+ "user_speaking_time_s": round(user_time, 2),
54
+ "other_speaking_time_s": round(other_time, 2),
55
+ "user_ratio": round(user_time / total, 3) if total > 0 else 0.5
56
+ }
57
+
58
+ def _compute_speech_rate(self, user_segs) -> dict:
59
+ if not user_segs:
60
+ return {"overall_wpm": 0, "segments": [], "variability": 0}
61
+
62
+ segment_rates = []
63
+ for seg in user_segs:
64
+ duration_min = (seg["end"] - seg["start"]) / 60
65
+ if duration_min < 0.05:
66
+ continue
67
+ words = seg.get("words", [])
68
+ wpm = round(len(words) / duration_min, 1) if duration_min > 0 else 0
69
+ segment_rates.append({
70
+ "start": seg["start"],
71
+ "end": seg["end"],
72
+ "wpm": wpm
73
+ })
74
+
75
+ all_words = sum(len(s.get("words", [])) for s in user_segs)
76
+ total_time_min = sum(s["end"] - s["start"] for s in user_segs) / 60
77
+
78
+ return {
79
+ "overall_wpm": round(all_words / total_time_min, 1) if total_time_min > 0 else 0,
80
+ "segments": segment_rates,
81
+ "variability": round(float(np.std([s["wpm"] for s in segment_rates])), 1) if segment_rates else 0
82
+ }
83
+
84
+ def _compute_pauses(self, user_segs, other_segs) -> dict:
85
+ within_turn_pauses = []
86
+ response_latencies = []
87
+
88
+ for seg in user_segs:
89
+ words = seg.get("words", [])
90
+ for i in range(1, len(words)):
91
+ gap = words[i].get("start", 0) - words[i-1].get("end", 0)
92
+ if 0.15 < gap < 3.0:
93
+ within_turn_pauses.append(gap)
94
+
95
+ other_ends = [s["end"] for s in other_segs]
96
+ user_starts = [s["start"] for s in user_segs]
97
+
98
+ for u_start in user_starts:
99
+ preceding = [t for t in other_ends if t < u_start and u_start - t < 5.0]
100
+ if preceding:
101
+ latency = u_start - max(preceding)
102
+ if latency > 0.1:
103
+ response_latencies.append(latency)
104
+
105
+ return {
106
+ "within_turn_pauses": {
107
+ "count": len(within_turn_pauses),
108
+ "mean_duration_s": round(float(np.mean(within_turn_pauses)), 3) if within_turn_pauses else 0,
109
+ "total_pause_time_s": round(sum(within_turn_pauses), 2)
110
+ },
111
+ "response_latency": {
112
+ "mean_s": round(float(np.mean(response_latencies)), 3) if response_latencies else 0,
113
+ "max_s": round(float(max(response_latencies)), 3) if response_latencies else 0
114
+ }
115
+ }
116
+
117
+ def _compute_interruptions(self, user_segs, other_segs) -> dict:
118
+ user_interrupts_other = 0
119
+ other_interrupts_user = 0
120
+
121
+ for u in user_segs:
122
+ for o in other_segs:
123
+ if o["start"] < u["start"] < o["end"]:
124
+ if o["end"] - u["start"] > 0.3:
125
+ user_interrupts_other += 1
126
+
127
+ for o in other_segs:
128
+ for u in user_segs:
129
+ if u["start"] < o["start"] < u["end"]:
130
+ if u["end"] - o["start"] > 0.3:
131
+ other_interrupts_user += 1
132
+
133
+ return {
134
+ "user_interrupted_other": user_interrupts_other,
135
+ "user_was_interrupted": other_interrupts_user
136
+ }
137
+
138
+ def _compute_filler_words(self, user_segs) -> dict:
139
+ all_text = " ".join(s.get("text", "").lower() for s in user_segs)
140
+ all_words_count = sum(len(s.get("words", [])) for s in user_segs)
141
+
142
+ filler_counts = {}
143
+ total_fillers = 0
144
+ for filler in self.FILLER_WORDS:
145
+ count = len(re.findall(r'\b' + filler + r'\b', all_text))
146
+ if count > 0:
147
+ filler_counts[filler] = count
148
+ total_fillers += count
149
+
150
+ return {
151
+ "total_count": total_fillers,
152
+ "rate_per_100_words": round((total_fillers / all_words_count * 100), 2) if all_words_count > 0 else 0,
153
+ "breakdown": filler_counts
154
+ }
155
+
156
+ def _compute_turn_dynamics(self, user_segs, other_segs) -> dict:
157
+ user_turn_lengths = [s["end"] - s["start"] for s in user_segs]
158
+ other_turn_lengths = [s["end"] - s["start"] for s in other_segs]
159
+
160
+ return {
161
+ "total_turns": len(user_segs) + len(other_segs),
162
+ "user_turns": len(user_segs),
163
+ "other_turns": len(other_segs),
164
+ "avg_user_turn_length_s": round(float(np.mean(user_turn_lengths)), 2) if user_turn_lengths else 0,
165
+ "avg_other_turn_length_s": round(float(np.mean(other_turn_lengths)), 2) if other_turn_lengths else 0
166
+ }
167
+
168
+ def _compute_pitch_features(self, user_segs) -> dict:
169
+ pitches = []
170
+
171
+ for seg in user_segs:
172
+ start_sample = int(seg["start"] * self.sr)
173
+ end_sample = int(seg["end"] * self.sr)
174
+ segment_audio = self.audio[start_sample:end_sample]
175
+
176
+ if len(segment_audio) < self.sr * 0.5:
177
+ continue
178
+
179
+ f0, voiced_flag, _ = librosa.pyin(
180
+ segment_audio,
181
+ fmin=librosa.note_to_hz('C2'),
182
+ fmax=librosa.note_to_hz('C7'),
183
+ sr=self.sr
184
+ )
185
+
186
+ voiced_f0 = f0[voiced_flag & ~np.isnan(f0)]
187
+ if len(voiced_f0) > 0:
188
+ pitches.extend(voiced_f0.tolist())
189
+
190
+ if not pitches:
191
+ return {"mean_hz": None, "std_hz": None, "range_hz": None}
192
+
193
+ return {
194
+ "mean_hz": round(float(np.mean(pitches)), 2),
195
+ "std_hz": round(float(np.std(pitches)), 2),
196
+ "range_hz": round(float(np.max(pitches) - np.min(pitches)), 2)
197
+ }
198
+
199
+ def _compute_engagement_proxy(self, other_segs) -> dict:
200
+ if len(other_segs) < 4:
201
+ return {"trend": "insufficient_data", "interpretation_note": "probabilistic proxy only"}
202
+
203
+ first_half = other_segs[:len(other_segs)//2]
204
+ second_half = other_segs[len(other_segs)//2:]
205
+
206
+ first_avg = float(np.mean([s["end"] - s["start"] for s in first_half]))
207
+ second_avg = float(np.mean([s["end"] - s["start"] for s in second_half]))
208
+
209
+ delta = (second_avg - first_avg) / first_avg if first_avg > 0 else 0
210
+
211
+ return {
212
+ "other_speaker_turn_length_trend": round(delta, 3),
213
+ "interpretation_note": "probabilistic proxy only"
214
+ }
215
+
216
+ def _build_timeline(self, user_segs) -> list:
217
+ if not self.segments:
218
+ return []
219
+
220
+ duration = self.segments[-1]["end"]
221
+ window_size = 60
222
+ windows = []
223
+
224
+ for window_start in range(0, int(duration), window_size):
225
+ window_end = window_start + window_size
226
+ window_segs = [s for s in user_segs
227
+ if s["start"] >= window_start and s["end"] <= window_end]
228
+
229
+ if not window_segs:
230
+ continue
231
+
232
+ words_in_window = sum(len(s.get("words", [])) for s in window_segs)
233
+ speaking_time = sum(s["end"] - s["start"] for s in window_segs)
234
+ wpm = round(words_in_window / (speaking_time / 60), 1) if speaking_time > 0 else 0
235
+
236
+ windows.append({
237
+ "window_start_s": window_start,
238
+ "window_end_s": min(window_end, duration),
239
+ "speech_rate_wpm": wpm,
240
+ "speaking_time_s": round(speaking_time, 1),
241
+ "turn_count": len(window_segs)
242
+ })
243
+
244
+ return windows
backend/pipeline/transcriber.py ADDED
@@ -0,0 +1,109 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import whisper
2
+ import subprocess
3
+ import os
4
+
5
+ class Transcriber:
6
+ def __init__(self, model_size="base"):
7
+ print(f"Loading Whisper model: {model_size}")
8
+ self.model = whisper.load_model(model_size)
9
+ print("Whisper model loaded.")
10
+
11
+ def convert_to_wav(self, audio_path: str) -> str:
12
+ """Convert any audio format to WAV using ffmpeg."""
13
+ wav_path = audio_path.replace(".m4a", ".wav").replace(".mp3", ".wav").replace(".mp4", ".wav")
14
+ if wav_path == audio_path:
15
+ wav_path = audio_path + ".wav"
16
+
17
+ subprocess.run([
18
+ "ffmpeg", "-i", audio_path,
19
+ "-ar", "16000",
20
+ "-ac", "1",
21
+ "-y", wav_path
22
+ ], capture_output=True)
23
+
24
+ return wav_path
25
+
26
+ def transcribe(self, audio_path: str) -> dict:
27
+ print(f"Transcribing: {audio_path}")
28
+
29
+ # Convert to WAV if needed
30
+ if not audio_path.endswith(".wav"):
31
+ print("Converting to WAV...")
32
+ wav_path = self.convert_to_wav(audio_path)
33
+ else:
34
+ wav_path = audio_path
35
+
36
+ result = self.model.transcribe(
37
+ wav_path,
38
+ word_timestamps=True,
39
+ verbose=False
40
+ )
41
+
42
+ # Clean up converted file
43
+ if wav_path != audio_path and os.path.exists(wav_path):
44
+ os.remove(wav_path)
45
+
46
+ segments = []
47
+ for seg in result["segments"]:
48
+ words = []
49
+ for w in seg.get("words", []):
50
+ words.append({
51
+ "word": w["word"].strip(),
52
+ "start": round(w["start"], 3),
53
+ "end": round(w["end"], 3),
54
+ "score": round(w.get("probability", 0), 3)
55
+ })
56
+
57
+ segments.append({
58
+ "start": round(seg["start"], 3),
59
+ "end": round(seg["end"], 3),
60
+ "text": seg["text"].strip(),
61
+ "words": words
62
+ })
63
+
64
+ return {
65
+ "segments": segments,
66
+ "language": result.get("language", "en")
67
+ }
68
+
69
+
70
+
71
+ # import whisper
72
+ # import json
73
+
74
+ # class Transcriber:
75
+ # def __init__(self, model_size="base"):
76
+ # print(f"Loading Whisper model: {model_size}")
77
+ # self.model = whisper.load_model(model_size)
78
+ # print("Whisper model loaded.")
79
+
80
+ # def transcribe(self, audio_path: str) -> dict:
81
+ # print(f"Transcribing: {audio_path}")
82
+ # result = self.model.transcribe(
83
+ # audio_path,
84
+ # word_timestamps=True,
85
+ # verbose=False
86
+ # )
87
+
88
+ # segments = []
89
+ # for seg in result["segments"]:
90
+ # words = []
91
+ # for w in seg.get("words", []):
92
+ # words.append({
93
+ # "word": w["word"].strip(),
94
+ # "start": round(w["start"], 3),
95
+ # "end": round(w["end"], 3),
96
+ # "score": round(w.get("probability", 0), 3)
97
+ # })
98
+
99
+ # segments.append({
100
+ # "start": round(seg["start"], 3),
101
+ # "end": round(seg["end"], 3),
102
+ # "text": seg["text"].strip(),
103
+ # "words": words
104
+ # })
105
+
106
+ # return {
107
+ # "segments": segments,
108
+ # "language": result.get("language", "en")
109
+ # }
backend/requirements.txt ADDED
@@ -0,0 +1,14 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ fastapi
2
+ uvicorn
3
+ python-multipart
4
+ anthropic
5
+ sqlalchemy
6
+ pydantic
7
+ python-dotenv
8
+ pydub
9
+ librosa
10
+ numpy
11
+ scipy
12
+ openai-whisper
13
+ pyannote.audio
14
+ whisperx
backend/requirements_frozen.txt ADDED
@@ -0,0 +1,154 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ aiohappyeyeballs==2.6.1
2
+ aiohttp==3.13.5
3
+ aiosignal==1.4.0
4
+ alembic==1.18.4
5
+ annotated-doc==0.0.4
6
+ annotated-types==0.7.0
7
+ anthropic==0.102.0
8
+ antlr4-python3-runtime==4.9.3
9
+ anyio==4.13.0
10
+ asteroid-filterbanks==0.4.0
11
+ async-timeout==5.0.1
12
+ attrs==26.1.0
13
+ audioread==3.1.0
14
+ av==17.0.1
15
+ certifi==2026.4.22
16
+ cffi==2.0.0
17
+ charset-normalizer==3.4.7
18
+ click==8.3.3
19
+ coloredlogs==15.0.1
20
+ colorlog==6.10.1
21
+ contourpy==1.3.2
22
+ ctranslate2==4.7.1
23
+ cycler==0.12.1
24
+ decorator==5.2.1
25
+ distro==1.9.0
26
+ docstring_parser==0.18.0
27
+ einops==0.8.2
28
+ exceptiongroup==1.3.1
29
+ fastapi==0.136.1
30
+ faster-whisper==1.2.1
31
+ filelock==3.29.0
32
+ flatbuffers==25.12.19
33
+ fonttools==4.63.0
34
+ frozenlist==1.8.0
35
+ fsspec==2026.4.0
36
+ googleapis-common-protos==1.75.0
37
+ groq==1.2.0
38
+ grpcio==1.80.0
39
+ h11==0.16.0
40
+ hf-xet==1.5.0
41
+ httpcore==1.0.9
42
+ httpx==0.28.1
43
+ huggingface_hub==0.36.2
44
+ humanfriendly==10.0
45
+ HyperPyYAML==1.2.3
46
+ idna==3.15
47
+ importlib_metadata==8.7.1
48
+ Jinja2==3.1.6
49
+ jiter==0.14.0
50
+ joblib==1.5.3
51
+ julius==0.2.7
52
+ kiwisolver==1.5.0
53
+ lazy-loader==0.5
54
+ librosa==0.11.0
55
+ lightning==2.6.1
56
+ lightning-utilities==0.15.3
57
+ llvmlite==0.47.0
58
+ Mako==1.3.12
59
+ markdown-it-py==4.2.0
60
+ MarkupSafe==3.0.3
61
+ matplotlib==3.10.9
62
+ mdurl==0.1.2
63
+ more-itertools==11.0.2
64
+ mpmath==1.3.0
65
+ msgpack==1.1.2
66
+ multidict==6.7.1
67
+ networkx==3.4.2
68
+ nltk==3.9.4
69
+ numba==0.65.1
70
+ numpy==1.26.4
71
+ omegaconf==2.3.0
72
+ onnxruntime==1.23.2
73
+ openai-whisper==20250625
74
+ opentelemetry-api==1.41.1
75
+ opentelemetry-exporter-otlp==1.41.1
76
+ opentelemetry-exporter-otlp-proto-common==1.41.1
77
+ opentelemetry-exporter-otlp-proto-grpc==1.41.1
78
+ opentelemetry-exporter-otlp-proto-http==1.41.1
79
+ opentelemetry-proto==1.41.1
80
+ opentelemetry-sdk==1.41.1
81
+ opentelemetry-semantic-conventions==0.62b1
82
+ optuna==4.8.0
83
+ packaging==26.2
84
+ pandas==2.3.3
85
+ pillow==12.2.0
86
+ platformdirs==4.9.6
87
+ pooch==1.9.0
88
+ primePy==1.3
89
+ propcache==0.5.2
90
+ protobuf==6.33.6
91
+ pyannote-core==6.0.1
92
+ pyannote-database==6.1.1
93
+ pyannote-metrics==4.1
94
+ pyannote-pipeline==4.0.0
95
+ pyannote.audio==3.1.1
96
+ pyannoteai-sdk==0.4.0
97
+ pycparser==3.0
98
+ pydantic==2.13.4
99
+ pydantic_core==2.46.4
100
+ pydub==0.25.1
101
+ Pygments==2.20.0
102
+ pyparsing==3.3.2
103
+ python-dateutil==2.9.0.post0
104
+ python-dotenv==1.2.2
105
+ python-multipart==0.0.28
106
+ pytorch-lightning==2.6.1
107
+ pytorch-metric-learning==2.9.0
108
+ pytz==2026.2
109
+ PyYAML==6.0.3
110
+ regex==2026.5.9
111
+ requests==2.34.2
112
+ Resemblyzer==0.1.4
113
+ rich==15.0.0
114
+ ruamel.yaml==0.18.17
115
+ ruamel.yaml.clib==0.2.15
116
+ safetensors==0.7.0
117
+ scikit-learn==1.7.2
118
+ scipy==1.15.3
119
+ semver==3.0.4
120
+ sentencepiece==0.2.1
121
+ shellingham==1.5.4
122
+ six==1.17.0
123
+ sniffio==1.3.1
124
+ sortedcontainers==2.4.0
125
+ soundfile==0.13.1
126
+ soxr==1.1.0
127
+ speechbrain==1.1.0
128
+ SQLAlchemy==2.0.49
129
+ starlette==1.0.0
130
+ sympy==1.13.1
131
+ tensorboardX==2.6.5
132
+ threadpoolctl==3.6.0
133
+ tiktoken==0.13.0
134
+ tokenizers==0.22.2
135
+ tomli==2.4.1
136
+ torch==2.5.1
137
+ torch-audiomentations==0.12.0
138
+ torch_pitch_shift==1.2.5
139
+ torchaudio==2.8.0
140
+ torchcodec==0.7.0
141
+ torchmetrics==1.9.0
142
+ torchvision==0.23.0
143
+ tqdm==4.67.3
144
+ transformers==4.57.6
145
+ typer==0.25.1
146
+ typing==3.7.4.3
147
+ typing-inspection==0.4.2
148
+ typing_extensions==4.15.0
149
+ tzdata==2026.2
150
+ urllib3==2.7.0
151
+ uvicorn==0.47.0
152
+ webrtcvad==2.0.10
153
+ yarl==1.23.0
154
+ zipp==3.23.1
frontend/.gitignore ADDED
@@ -0,0 +1,24 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Logs
2
+ logs
3
+ *.log
4
+ npm-debug.log*
5
+ yarn-debug.log*
6
+ yarn-error.log*
7
+ pnpm-debug.log*
8
+ lerna-debug.log*
9
+
10
+ node_modules
11
+ dist
12
+ dist-ssr
13
+ *.local
14
+
15
+ # Editor directories and files
16
+ .vscode/*
17
+ !.vscode/extensions.json
18
+ .idea
19
+ .DS_Store
20
+ *.suo
21
+ *.ntvs*
22
+ *.njsproj
23
+ *.sln
24
+ *.sw?
frontend/README.md ADDED
@@ -0,0 +1,16 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # React + Vite
2
+
3
+ This template provides a minimal setup to get React working in Vite with HMR and some ESLint rules.
4
+
5
+ Currently, two official plugins are available:
6
+
7
+ - [@vitejs/plugin-react](https://github.com/vitejs/vite-plugin-react/blob/main/packages/plugin-react) uses [Oxc](https://oxc.rs)
8
+ - [@vitejs/plugin-react-swc](https://github.com/vitejs/vite-plugin-react/blob/main/packages/plugin-react-swc) uses [SWC](https://swc.rs/)
9
+
10
+ ## React Compiler
11
+
12
+ The React Compiler is not enabled on this template because of its impact on dev & build performances. To add it, see [this documentation](https://react.dev/learn/react-compiler/installation).
13
+
14
+ ## Expanding the ESLint configuration
15
+
16
+ If you are developing a production application, we recommend using TypeScript with type-aware lint rules enabled. Check out the [TS template](https://github.com/vitejs/vite/tree/main/packages/create-vite/template-react-ts) for information on how to integrate TypeScript and [`typescript-eslint`](https://typescript-eslint.io) in your project.
frontend/eslint.config.js ADDED
@@ -0,0 +1,21 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import js from '@eslint/js'
2
+ import globals from 'globals'
3
+ import reactHooks from 'eslint-plugin-react-hooks'
4
+ import reactRefresh from 'eslint-plugin-react-refresh'
5
+ import { defineConfig, globalIgnores } from 'eslint/config'
6
+
7
+ export default defineConfig([
8
+ globalIgnores(['dist']),
9
+ {
10
+ files: ['**/*.{js,jsx}'],
11
+ extends: [
12
+ js.configs.recommended,
13
+ reactHooks.configs.flat.recommended,
14
+ reactRefresh.configs.vite,
15
+ ],
16
+ languageOptions: {
17
+ globals: globals.browser,
18
+ parserOptions: { ecmaFeatures: { jsx: true } },
19
+ },
20
+ },
21
+ ])
frontend/index.html ADDED
@@ -0,0 +1,13 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ <!doctype html>
2
+ <html lang="en">
3
+ <head>
4
+ <meta charset="UTF-8" />
5
+ <link rel="icon" type="image/svg+xml" href="/favicon.svg" />
6
+ <meta name="viewport" content="width=device-width, initial-scale=1.0" />
7
+ <title>frontend</title>
8
+ </head>
9
+ <body>
10
+ <div id="root"></div>
11
+ <script type="module" src="/src/main.jsx"></script>
12
+ </body>
13
+ </html>
frontend/package-lock.json ADDED
The diff for this file is too large to render. See raw diff
 
frontend/package.json ADDED
@@ -0,0 +1,29 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "name": "frontend",
3
+ "private": true,
4
+ "version": "0.0.0",
5
+ "type": "module",
6
+ "scripts": {
7
+ "dev": "vite",
8
+ "build": "vite build",
9
+ "lint": "eslint .",
10
+ "preview": "vite preview"
11
+ },
12
+ "dependencies": {
13
+ "axios": "^1.16.1",
14
+ "react": "^19.2.6",
15
+ "react-dom": "^19.2.6",
16
+ "recharts": "^3.8.1"
17
+ },
18
+ "devDependencies": {
19
+ "@eslint/js": "^10.0.1",
20
+ "@types/react": "^19.2.14",
21
+ "@types/react-dom": "^19.2.3",
22
+ "@vitejs/plugin-react": "^6.0.1",
23
+ "eslint": "^10.3.0",
24
+ "eslint-plugin-react-hooks": "^7.1.1",
25
+ "eslint-plugin-react-refresh": "^0.5.2",
26
+ "globals": "^17.6.0",
27
+ "vite": "^8.0.12"
28
+ }
29
+ }
frontend/public/favicon.svg ADDED
frontend/public/icons.svg ADDED
frontend/src/App.css ADDED
@@ -0,0 +1,184 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ .counter {
2
+ font-size: 16px;
3
+ padding: 5px 10px;
4
+ border-radius: 5px;
5
+ color: var(--accent);
6
+ background: var(--accent-bg);
7
+ border: 2px solid transparent;
8
+ transition: border-color 0.3s;
9
+ margin-bottom: 24px;
10
+
11
+ &:hover {
12
+ border-color: var(--accent-border);
13
+ }
14
+ &:focus-visible {
15
+ outline: 2px solid var(--accent);
16
+ outline-offset: 2px;
17
+ }
18
+ }
19
+
20
+ .hero {
21
+ position: relative;
22
+
23
+ .base,
24
+ .framework,
25
+ .vite {
26
+ inset-inline: 0;
27
+ margin: 0 auto;
28
+ }
29
+
30
+ .base {
31
+ width: 170px;
32
+ position: relative;
33
+ z-index: 0;
34
+ }
35
+
36
+ .framework,
37
+ .vite {
38
+ position: absolute;
39
+ }
40
+
41
+ .framework {
42
+ z-index: 1;
43
+ top: 34px;
44
+ height: 28px;
45
+ transform: perspective(2000px) rotateZ(300deg) rotateX(44deg) rotateY(39deg)
46
+ scale(1.4);
47
+ }
48
+
49
+ .vite {
50
+ z-index: 0;
51
+ top: 107px;
52
+ height: 26px;
53
+ width: auto;
54
+ transform: perspective(2000px) rotateZ(300deg) rotateX(40deg) rotateY(39deg)
55
+ scale(0.8);
56
+ }
57
+ }
58
+
59
+ #center {
60
+ display: flex;
61
+ flex-direction: column;
62
+ gap: 25px;
63
+ place-content: center;
64
+ place-items: center;
65
+ flex-grow: 1;
66
+
67
+ @media (max-width: 1024px) {
68
+ padding: 32px 20px 24px;
69
+ gap: 18px;
70
+ }
71
+ }
72
+
73
+ #next-steps {
74
+ display: flex;
75
+ border-top: 1px solid var(--border);
76
+ text-align: left;
77
+
78
+ & > div {
79
+ flex: 1 1 0;
80
+ padding: 32px;
81
+ @media (max-width: 1024px) {
82
+ padding: 24px 20px;
83
+ }
84
+ }
85
+
86
+ .icon {
87
+ margin-bottom: 16px;
88
+ width: 22px;
89
+ height: 22px;
90
+ }
91
+
92
+ @media (max-width: 1024px) {
93
+ flex-direction: column;
94
+ text-align: center;
95
+ }
96
+ }
97
+
98
+ #docs {
99
+ border-right: 1px solid var(--border);
100
+
101
+ @media (max-width: 1024px) {
102
+ border-right: none;
103
+ border-bottom: 1px solid var(--border);
104
+ }
105
+ }
106
+
107
+ #next-steps ul {
108
+ list-style: none;
109
+ padding: 0;
110
+ display: flex;
111
+ gap: 8px;
112
+ margin: 32px 0 0;
113
+
114
+ .logo {
115
+ height: 18px;
116
+ }
117
+
118
+ a {
119
+ color: var(--text-h);
120
+ font-size: 16px;
121
+ border-radius: 6px;
122
+ background: var(--social-bg);
123
+ display: flex;
124
+ padding: 6px 12px;
125
+ align-items: center;
126
+ gap: 8px;
127
+ text-decoration: none;
128
+ transition: box-shadow 0.3s;
129
+
130
+ &:hover {
131
+ box-shadow: var(--shadow);
132
+ }
133
+ .button-icon {
134
+ height: 18px;
135
+ width: 18px;
136
+ }
137
+ }
138
+
139
+ @media (max-width: 1024px) {
140
+ margin-top: 20px;
141
+ flex-wrap: wrap;
142
+ justify-content: center;
143
+
144
+ li {
145
+ flex: 1 1 calc(50% - 8px);
146
+ }
147
+
148
+ a {
149
+ width: 100%;
150
+ justify-content: center;
151
+ box-sizing: border-box;
152
+ }
153
+ }
154
+ }
155
+
156
+ #spacer {
157
+ height: 88px;
158
+ border-top: 1px solid var(--border);
159
+ @media (max-width: 1024px) {
160
+ height: 48px;
161
+ }
162
+ }
163
+
164
+ .ticks {
165
+ position: relative;
166
+ width: 100%;
167
+
168
+ &::before,
169
+ &::after {
170
+ content: '';
171
+ position: absolute;
172
+ top: -4.5px;
173
+ border: 5px solid transparent;
174
+ }
175
+
176
+ &::before {
177
+ left: 0;
178
+ border-left-color: var(--border);
179
+ }
180
+ &::after {
181
+ right: 0;
182
+ border-right-color: var(--border);
183
+ }
184
+ }
frontend/src/App.jsx ADDED
@@ -0,0 +1,44 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import { useState } from "react"
2
+ import UploadView from "./components/UploadView"
3
+ import ResultsView from "./components/ResultsView"
4
+ import HistoryView from "./components/HistoryView"
5
+
6
+ export default function App() {
7
+ const [view, setView] = useState("upload")
8
+ const [results, setResults] = useState(null)
9
+
10
+ return (
11
+ <div style={{ maxWidth: 780, margin: "0 auto", padding: 24, fontFamily: "system-ui" }}>
12
+ <div style={{ marginBottom: 28 }}>
13
+ <h1 style={{ fontSize: 22, fontWeight: 700, margin: 0 }}>🪞 Behavioral Mirror</h1>
14
+ <p style={{ color: "#666", marginTop: 4, fontSize: 14, margin: 0 }}>
15
+ Reflective insights from your conversations
16
+ </p>
17
+ </div>
18
+
19
+ <nav style={{ display: "flex", gap: 20, marginBottom: 32,
20
+ borderBottom: "1px solid #eee", paddingBottom: 12 }}>
21
+ {["upload", "history"].map(v => (
22
+ <button key={v} onClick={() => setView(v)}
23
+ style={{ background: "none", border: "none", cursor: "pointer",
24
+ fontWeight: view === v ? 600 : 400, fontSize: 14,
25
+ color: view === v ? "#111" : "#888",
26
+ borderBottom: view === v ? "2px solid #111" : "2px solid transparent",
27
+ paddingBottom: 4, textTransform: "capitalize" }}>
28
+ {v}
29
+ </button>
30
+ ))}
31
+ </nav>
32
+
33
+ {view === "upload" && (
34
+ <UploadView onResults={(r) => { setResults(r); setView("results") }} />
35
+ )}
36
+ {view === "results" && results && (
37
+ <ResultsView results={results} onBack={() => setView("upload")} />
38
+ )}
39
+ {view === "history" && (
40
+ <HistoryView onSelect={(r) => { setResults(r); setView("results") }} />
41
+ )}
42
+ </div>
43
+ )
44
+ }
frontend/src/assets/hero.png ADDED
frontend/src/assets/react.svg ADDED
frontend/src/assets/vite.svg ADDED
frontend/src/components/HistoryView.jsx ADDED
@@ -0,0 +1,90 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import { useState, useEffect } from "react"
2
+ import axios from "axios"
3
+
4
+ export default function HistoryView({ onSelect }) {
5
+ const [sessions, setSessions] = useState([])
6
+ const [loading, setLoading] = useState(true)
7
+
8
+ useEffect(() => {
9
+ axios.get("http://localhost:8000/api/sessions/default_user")
10
+ .then(res => setSessions(res.data))
11
+ .catch(err => console.error(err))
12
+ .finally(() => setLoading(false))
13
+ }, [])
14
+
15
+ if (loading) return (
16
+ <div style={{ textAlign: "center", padding: 48, color: "#888" }}>
17
+ Loading sessions...
18
+ </div>
19
+ )
20
+
21
+ if (sessions.length === 0) return (
22
+ <div style={{ textAlign: "center", padding: 48 }}>
23
+ <div style={{ fontSize: 36, marginBottom: 12 }}>📭</div>
24
+ <p style={{ color: "#888", fontSize: 14 }}>No sessions yet.</p>
25
+ <p style={{ color: "#aaa", fontSize: 13 }}>
26
+ Upload your first conversation to get started.
27
+ </p>
28
+ </div>
29
+ )
30
+
31
+ return (
32
+ <div>
33
+ <p style={{ fontSize: 13, color: "#888", marginBottom: 16 }}>
34
+ {sessions.length} session{sessions.length > 1 ? "s" : ""} recorded
35
+ </p>
36
+
37
+ <div style={{ display: "flex", flexDirection: "column", gap: 10 }}>
38
+ {sessions.map(s => (
39
+ <div key={s.session_id}
40
+ onClick={() => onSelect({ signals: s.signals, insights: s.insights })}
41
+ style={{ border: "1px solid #eee", borderRadius: 10, padding: 16,
42
+ background: "white", cursor: "pointer",
43
+ transition: "border-color 0.2s" }}
44
+ onMouseEnter={e => e.currentTarget.style.borderColor = "#111"}
45
+ onMouseLeave={e => e.currentTarget.style.borderColor = "#eee"}>
46
+
47
+ <div style={{ display: "flex", justifyContent: "space-between",
48
+ alignItems: "center", marginBottom: 8 }}>
49
+ <span style={{ fontSize: 13, fontWeight: 600,
50
+ textTransform: "capitalize", color: "#333" }}>
51
+ {s.context.replace(/_/g, " ")}
52
+ </span>
53
+ <span style={{ fontSize: 11, color: "#aaa" }}>
54
+ {new Date(s.created_at).toLocaleDateString("en-IN", {
55
+ day: "numeric", month: "short", year: "numeric"
56
+ })}
57
+ </span>
58
+ </div>
59
+
60
+ <p style={{ margin: "0 0 10px", fontSize: 13,
61
+ color: "#555", lineHeight: 1.5 }}>
62
+ {s.insights.summary_sentence}
63
+ </p>
64
+
65
+ <div style={{ display: "flex", gap: 16 }}>
66
+ {[
67
+ { label: "Talk ratio", value: `${Math.round(s.signals.talk_ratio.user_ratio * 100)}%` },
68
+ { label: "Speech rate", value: `${s.signals.speech_rate.overall_wpm} wpm` },
69
+ { label: "Duration", value: `${Math.round(s.signals.session_duration_s / 60)}m` },
70
+ ].map(({ label, value }) => (
71
+ <div key={label}>
72
+ <span style={{ fontSize: 11, color: "#aaa" }}>{label}: </span>
73
+ <span style={{ fontSize: 11, fontWeight: 600, color: "#555" }}>{value}</span>
74
+ </div>
75
+ ))}
76
+ </div>
77
+ </div>
78
+ ))}
79
+ </div>
80
+
81
+ {sessions.length >= 3 && (
82
+ <div style={{ marginTop: 20, padding: 14, background: "#f0fdf4",
83
+ border: "1px solid #86efac", borderRadius: 10, fontSize: 13, color: "#166534" }}>
84
+ ✨ You have {sessions.length} sessions — baseline comparisons are now active.
85
+ Future insights will compare against your personal patterns.
86
+ </div>
87
+ )}
88
+ </div>
89
+ )
90
+ }
frontend/src/components/ResultsView.jsx ADDED
@@ -0,0 +1,173 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import { useState } from "react"
2
+ import { LineChart, Line, XAxis, YAxis, Tooltip, ResponsiveContainer } from "recharts"
3
+
4
+ export default function ResultsView({ results, onBack }) {
5
+ const { signals, insights } = results
6
+
7
+ const formatTime = (s) => {
8
+ const m = Math.floor(s / 60)
9
+ const sec = Math.round(s % 60)
10
+ return `${m}:${sec.toString().padStart(2, "0")}`
11
+ }
12
+
13
+ const talkPct = Math.round(signals.talk_ratio.user_ratio * 100)
14
+ const duration = Math.round(signals.session_duration_s / 60)
15
+
16
+ return (
17
+ <div>
18
+ <button onClick={onBack}
19
+ style={{ background: "none", border: "none", cursor: "pointer",
20
+ color: "#666", fontSize: 13, marginBottom: 20, padding: 0 }}>
21
+ ← Back
22
+ </button>
23
+
24
+ {/* Summary */}
25
+ <div style={{ background: "#f8f9fa", borderRadius: 12,
26
+ padding: 20, marginBottom: 24, borderLeft: "4px solid #111" }}>
27
+ <p style={{ fontSize: 15, lineHeight: 1.7, margin: 0, color: "#333" }}>
28
+ {insights.summary_sentence}
29
+ </p>
30
+ </div>
31
+
32
+ {/* Key metrics */}
33
+ <div style={{ display: "grid", gridTemplateColumns: "1fr 1fr 1fr 1fr",
34
+ gap: 10, marginBottom: 28 }}>
35
+ {[
36
+ { label: "Duration", value: `${duration}m`, sub: "total" },
37
+ { label: "You spoke", value: `${talkPct}%`, sub: "of the time" },
38
+ { label: "Speech rate", value: `${signals.speech_rate.overall_wpm}`, sub: "words/min" },
39
+ { label: "Filler words", value: `${signals.filler_words.rate_per_100_words}`, sub: "per 100 words" },
40
+ ].map(({ label, value, sub }) => (
41
+ <div key={label} style={{ textAlign: "center", padding: "14px 8px",
42
+ border: "1px solid #eee", borderRadius: 10, background: "white" }}>
43
+ <div style={{ fontSize: 11, color: "#888", marginBottom: 4 }}>{label}</div>
44
+ <div style={{ fontSize: 22, fontWeight: 700 }}>{value}</div>
45
+ <div style={{ fontSize: 11, color: "#888" }}>{sub}</div>
46
+ </div>
47
+ ))}
48
+ </div>
49
+
50
+ {/* Speech rate timeline */}
51
+ {signals.timeline?.length > 1 && (
52
+ <div style={{ marginBottom: 28 }}>
53
+ <h3 style={{ fontSize: 14, fontWeight: 600, marginBottom: 12, color: "#333" }}>
54
+ Speech Rate Over Time
55
+ </h3>
56
+ <div style={{ background: "white", border: "1px solid #eee",
57
+ borderRadius: 10, padding: 16 }}>
58
+ <ResponsiveContainer width="100%" height={150}>
59
+ <LineChart data={signals.timeline}>
60
+ <XAxis dataKey="window_start_s"
61
+ tickFormatter={formatTime} fontSize={11} />
62
+ <YAxis domain={["auto", "auto"]} fontSize={11} width={35} />
63
+ <Tooltip
64
+ labelFormatter={v => `At ${formatTime(v)}`}
65
+ formatter={v => [`${Math.round(v)} wpm`, "Speech rate"]} />
66
+ <Line type="monotone" dataKey="speech_rate_wpm"
67
+ stroke="#111" strokeWidth={2} dot={false} />
68
+ </LineChart>
69
+ </ResponsiveContainer>
70
+ </div>
71
+ </div>
72
+ )}
73
+
74
+ {/* Interruptions & turns */}
75
+ <div style={{ display: "grid", gridTemplateColumns: "1fr 1fr",
76
+ gap: 10, marginBottom: 28 }}>
77
+ <div style={{ padding: 16, border: "1px solid #eee",
78
+ borderRadius: 10, background: "white" }}>
79
+ <div style={{ fontSize: 12, color: "#888", marginBottom: 8 }}>Interruptions</div>
80
+ <div style={{ fontSize: 13 }}>
81
+ You interrupted: <strong>{signals.interruptions.user_interrupted_other}x</strong>
82
+ </div>
83
+ <div style={{ fontSize: 13, marginTop: 4 }}>
84
+ You were interrupted: <strong>{signals.interruptions.user_was_interrupted}x</strong>
85
+ </div>
86
+ </div>
87
+ <div style={{ padding: 16, border: "1px solid #eee",
88
+ borderRadius: 10, background: "white" }}>
89
+ <div style={{ fontSize: 12, color: "#888", marginBottom: 8 }}>Turn Dynamics</div>
90
+ <div style={{ fontSize: 13 }}>
91
+ Your turns: <strong>{signals.turn_dynamics.user_turns}</strong>
92
+ </div>
93
+ <div style={{ fontSize: 13, marginTop: 4 }}>
94
+ Avg length: <strong>{signals.turn_dynamics.avg_user_turn_length_s}s</strong>
95
+ </div>
96
+ </div>
97
+ </div>
98
+
99
+ {/* Observations */}
100
+ <h3 style={{ fontSize: 14, fontWeight: 600, marginBottom: 12 }}>
101
+ Behavioral Observations
102
+ </h3>
103
+ <div style={{ display: "flex", flexDirection: "column", gap: 12, marginBottom: 24 }}>
104
+ {insights.observations?.map((obs, i) => (
105
+ <ObservationCard key={i} obs={obs} />
106
+ ))}
107
+ </div>
108
+
109
+ {/* Notable pattern */}
110
+ {insights.notable_pattern && (
111
+ <div style={{ background: "#fffbeb", border: "1px solid #fde68a",
112
+ borderRadius: 10, padding: 16, marginBottom: 24 }}>
113
+ <div style={{ fontSize: 12, fontWeight: 600, color: "#92400e", marginBottom: 4 }}>
114
+ Notable Pattern
115
+ </div>
116
+ <p style={{ margin: 0, fontSize: 14, color: "#78350f" }}>
117
+ {insights.notable_pattern}
118
+ </p>
119
+ </div>
120
+ )}
121
+
122
+ {/* Disclaimer */}
123
+ <div style={{ fontSize: 11, color: "#999", padding: 14,
124
+ background: "#fafafa", borderRadius: 8, lineHeight: 1.7 }}>
125
+ <strong>Note:</strong> These observations are probabilistic patterns from audio signals,
126
+ not validated psychological assessments. Use them as prompts for self-reflection only.
127
+ </div>
128
+ </div>
129
+ )
130
+ }
131
+
132
+ function ObservationCard({ obs }) {
133
+ const [resonance, setResonance] = useState(null)
134
+
135
+ return (
136
+ <div style={{ border: "1px solid #eee", borderRadius: 10,
137
+ padding: 16, background: "white" }}>
138
+ <div style={{ fontSize: 11, color: "#888", fontWeight: 600,
139
+ textTransform: "uppercase", letterSpacing: 0.5, marginBottom: 8 }}>
140
+ {obs.signal.replace(/_/g, " ")}
141
+ </div>
142
+ <p style={{ margin: "0 0 10px", fontSize: 14, lineHeight: 1.6, color: "#222" }}>
143
+ {obs.observation}
144
+ </p>
145
+ <p style={{ margin: "0 0 14px", fontSize: 13, color: "#555",
146
+ fontStyle: "italic", lineHeight: 1.5 }}>
147
+ 💭 {obs.resonance_prompt}
148
+ </p>
149
+
150
+ {resonance === null ? (
151
+ <div>
152
+ <div style={{ fontSize: 12, color: "#666", marginBottom: 8 }}>
153
+ Does this resonate?
154
+ </div>
155
+ <div style={{ display: "flex", gap: 8 }}>
156
+ {["Yes", "Somewhat", "No"].map(label => (
157
+ <button key={label} onClick={() => setResonance(label)}
158
+ style={{ padding: "5px 14px", border: "1px solid #ddd",
159
+ borderRadius: 20, background: "white", cursor: "pointer",
160
+ fontSize: 12, color: "#333" }}>
161
+ {label}
162
+ </button>
163
+ ))}
164
+ </div>
165
+ </div>
166
+ ) : (
167
+ <div style={{ fontSize: 12, color: "#888" }}>
168
+ ✓ Noted — thanks for the feedback.
169
+ </div>
170
+ )}
171
+ </div>
172
+ )
173
+ }
frontend/src/components/UploadView.jsx ADDED
@@ -0,0 +1,159 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import { useState } from "react"
2
+ import axios from "axios"
3
+
4
+ const CONTEXTS = [
5
+ { value: "casual", label: "Casual Conversation" },
6
+ { value: "job_interview", label: "Job Interview" },
7
+ { value: "disagreement", label: "Disagreement / Conflict" },
8
+ { value: "presentation", label: "Presentation / Pitch" },
9
+ { value: "meeting", label: "Meeting" },
10
+ ]
11
+
12
+ const STAGES = [
13
+ "Transcribing audio...",
14
+ "Identifying speakers...",
15
+ "Extracting behavioral signals...",
16
+ "Generating reflective insights..."
17
+ ]
18
+
19
+ export default function UploadView({ onResults }) {
20
+ const [file, setFile] = useState(null)
21
+ const [context, setContext] = useState("casual")
22
+ const [numSpeakers, setNumSpeakers] = useState(2)
23
+ const [loading, setLoading] = useState(false)
24
+ const [stageIndex, setStageIndex] = useState(0)
25
+ const [error, setError] = useState("")
26
+
27
+ const handleSubmit = async () => {
28
+ if (!file) return
29
+ setLoading(true)
30
+ setError("")
31
+ setStageIndex(0)
32
+
33
+ const interval = setInterval(() => {
34
+ setStageIndex(prev => Math.min(prev + 1, STAGES.length - 1))
35
+ }, 15000)
36
+
37
+ try {
38
+ const form = new FormData()
39
+ form.append("audio", file)
40
+ form.append("context", context)
41
+ form.append("num_speakers", numSpeakers)
42
+ form.append("user_id", "default_user")
43
+
44
+ const { data } = await axios.post(
45
+ "http://localhost:8000/api/analyze",
46
+ form,
47
+ { headers: { "Content-Type": "multipart/form-data" }, timeout: 300000 }
48
+ )
49
+
50
+ clearInterval(interval)
51
+ onResults(data)
52
+ } catch (e) {
53
+ clearInterval(interval)
54
+ setError(e.response?.data?.detail || "Something went wrong. Check the backend terminal.")
55
+ } finally {
56
+ setLoading(false)
57
+ }
58
+ }
59
+
60
+ if (loading) return (
61
+ <div style={{ textAlign: "center", padding: 64 }}>
62
+ <div style={{ fontSize: 40, marginBottom: 16 }}>⏳</div>
63
+ <p style={{ fontWeight: 500, fontSize: 16 }}>{STAGES[stageIndex]}</p>
64
+ <p style={{ fontSize: 13, color: "#999", marginTop: 8 }}>
65
+ This takes 1–3 minutes depending on audio length
66
+ </p>
67
+ <div style={{ marginTop: 24, background: "#f0f0f0", borderRadius: 8,
68
+ height: 4, overflow: "hidden" }}>
69
+ <div style={{ background: "#111", height: "100%", borderRadius: 8,
70
+ width: `${((stageIndex + 1) / STAGES.length) * 100}%`,
71
+ transition: "width 0.5s ease" }} />
72
+ </div>
73
+ </div>
74
+ )
75
+
76
+ return (
77
+ <div>
78
+ {/* File upload area */}
79
+ <div
80
+ onClick={() => document.getElementById("file-input").click()}
81
+ style={{ border: "2px dashed #ddd", borderRadius: 12, padding: 40,
82
+ textAlign: "center", marginBottom: 20, cursor: "pointer",
83
+ background: file ? "#f0fdf4" : "#fafafa",
84
+ borderColor: file ? "#86efac" : "#ddd",
85
+ transition: "all 0.2s" }}>
86
+ <input id="file-input" type="file" accept="*/*"
87
+ style={{ display: "none" }}
88
+ onChange={e => setFile(e.target.files[0])} />
89
+ {file ? (
90
+ <>
91
+ <div style={{ fontSize: 32 }}>✅</div>
92
+ <p style={{ fontWeight: 600, marginTop: 8 }}>{file.name}</p>
93
+ <p style={{ fontSize: 12, color: "#666" }}>
94
+ {(file.size / 1024 / 1024).toFixed(1)} MB · Click to change
95
+ </p>
96
+ </>
97
+ ) : (
98
+ <>
99
+ <div style={{ fontSize: 36 }}>🎙️</div>
100
+ <p style={{ fontWeight: 500, marginTop: 8 }}>Upload audio or video</p>
101
+ <p style={{ fontSize: 12, color: "#999" }}>MP3, WAV, MP4, M4A · Max 100MB</p>
102
+ </>
103
+ )}
104
+ </div>
105
+
106
+ {/* Context selector */}
107
+ <div style={{ marginBottom: 14 }}>
108
+ <label style={{ display: "block", fontSize: 13,
109
+ fontWeight: 500, marginBottom: 6, color: "#333" }}>
110
+ What kind of conversation was this?
111
+ </label>
112
+ <select value={context} onChange={e => setContext(e.target.value)}
113
+ style={{ width: "100%", padding: "10px 12px", borderRadius: 8,
114
+ border: "1px solid #ddd", fontSize: 14, background: "white" }}>
115
+ {CONTEXTS.map(c => (
116
+ <option key={c.value} value={c.value}>{c.label}</option>
117
+ ))}
118
+ </select>
119
+ </div>
120
+
121
+ {/* Speakers selector */}
122
+ <div style={{ marginBottom: 24 }}>
123
+ <label style={{ display: "block", fontSize: 13,
124
+ fontWeight: 500, marginBottom: 6, color: "#333" }}>
125
+ Number of speakers
126
+ </label>
127
+ <select value={numSpeakers} onChange={e => setNumSpeakers(Number(e.target.value))}
128
+ style={{ width: "100%", padding: "10px 12px", borderRadius: 8,
129
+ border: "1px solid #ddd", fontSize: 14, background: "white" }}>
130
+ <option value={2}>2 speakers</option>
131
+ <option value={3}>3 speakers</option>
132
+ <option value={4}>4 speakers</option>
133
+ </select>
134
+ </div>
135
+
136
+ {/* Error */}
137
+ {error && (
138
+ <div style={{ background: "#fef2f2", border: "1px solid #fca5a5",
139
+ borderRadius: 8, padding: 12, marginBottom: 16,
140
+ fontSize: 13, color: "#991b1b" }}>
141
+ ⚠️ {error}
142
+ </div>
143
+ )}
144
+
145
+ {/* Submit */}
146
+ <button onClick={handleSubmit} disabled={!file}
147
+ style={{ width: "100%", padding: "13px 24px",
148
+ background: file ? "#111" : "#ccc", color: "white",
149
+ border: "none", borderRadius: 8, fontSize: 15,
150
+ cursor: file ? "pointer" : "not-allowed", fontWeight: 500 }}>
151
+ Analyze Conversation
152
+ </button>
153
+
154
+ <p style={{ fontSize: 11, color: "#999", textAlign: "center", marginTop: 10 }}>
155
+ 🔒 Audio is deleted immediately after analysis. Only behavioral signals are stored.
156
+ </p>
157
+ </div>
158
+ )
159
+ }
frontend/src/index.css ADDED
@@ -0,0 +1,125 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ * {
2
+ box-sizing: border-box;
3
+ margin: 0;
4
+ padding: 0;
5
+ }
6
+
7
+ body {
8
+ background: #f5f5f5;
9
+ color: #111;
10
+ -webkit-font-smoothing: antialiased;
11
+ }
12
+
13
+
14
+
15
+ /* :root {
16
+ --text: #6b6375;
17
+ --text-h: #08060d;
18
+ --bg: #fff;
19
+ --border: #e5e4e7;
20
+ --code-bg: #f4f3ec;
21
+ --accent: #aa3bff;
22
+ --accent-bg: rgba(170, 59, 255, 0.1);
23
+ --accent-border: rgba(170, 59, 255, 0.5);
24
+ --social-bg: rgba(244, 243, 236, 0.5);
25
+ --shadow:
26
+ rgba(0, 0, 0, 0.1) 0 10px 15px -3px, rgba(0, 0, 0, 0.05) 0 4px 6px -2px;
27
+
28
+ --sans: system-ui, 'Segoe UI', Roboto, sans-serif;
29
+ --heading: system-ui, 'Segoe UI', Roboto, sans-serif;
30
+ --mono: ui-monospace, Consolas, monospace;
31
+
32
+ font: 18px/145% var(--sans);
33
+ letter-spacing: 0.18px;
34
+ color-scheme: light dark;
35
+ color: var(--text);
36
+ background: var(--bg);
37
+ font-synthesis: none;
38
+ text-rendering: optimizeLegibility;
39
+ -webkit-font-smoothing: antialiased;
40
+ -moz-osx-font-smoothing: grayscale;
41
+
42
+ @media (max-width: 1024px) {
43
+ font-size: 16px;
44
+ }
45
+ }
46
+
47
+ @media (prefers-color-scheme: dark) {
48
+ :root {
49
+ --text: #9ca3af;
50
+ --text-h: #f3f4f6;
51
+ --bg: #16171d;
52
+ --border: #2e303a;
53
+ --code-bg: #1f2028;
54
+ --accent: #c084fc;
55
+ --accent-bg: rgba(192, 132, 252, 0.15);
56
+ --accent-border: rgba(192, 132, 252, 0.5);
57
+ --social-bg: rgba(47, 48, 58, 0.5);
58
+ --shadow:
59
+ rgba(0, 0, 0, 0.4) 0 10px 15px -3px, rgba(0, 0, 0, 0.25) 0 4px 6px -2px;
60
+ }
61
+
62
+ #social .button-icon {
63
+ filter: invert(1) brightness(2);
64
+ }
65
+ }
66
+
67
+ body {
68
+ margin: 0;
69
+ }
70
+
71
+ #root {
72
+ width: 1126px;
73
+ max-width: 100%;
74
+ margin: 0 auto;
75
+ text-align: center;
76
+ border-inline: 1px solid var(--border);
77
+ min-height: 100svh;
78
+ display: flex;
79
+ flex-direction: column;
80
+ box-sizing: border-box;
81
+ }
82
+
83
+ h1,
84
+ h2 {
85
+ font-family: var(--heading);
86
+ font-weight: 500;
87
+ color: var(--text-h);
88
+ }
89
+
90
+ h1 {
91
+ font-size: 56px;
92
+ letter-spacing: -1.68px;
93
+ margin: 32px 0;
94
+ @media (max-width: 1024px) {
95
+ font-size: 36px;
96
+ margin: 20px 0;
97
+ }
98
+ }
99
+ h2 {
100
+ font-size: 24px;
101
+ line-height: 118%;
102
+ letter-spacing: -0.24px;
103
+ margin: 0 0 8px;
104
+ @media (max-width: 1024px) {
105
+ font-size: 20px;
106
+ }
107
+ }
108
+ p {
109
+ margin: 0;
110
+ }
111
+
112
+ code,
113
+ .counter {
114
+ font-family: var(--mono);
115
+ display: inline-flex;
116
+ border-radius: 4px;
117
+ color: var(--text-h);
118
+ }
119
+
120
+ code {
121
+ font-size: 15px;
122
+ line-height: 135%;
123
+ padding: 4px 8px;
124
+ background: var(--code-bg);
125
+ } */
frontend/src/main.jsx ADDED
@@ -0,0 +1,10 @@
 
 
 
 
 
 
 
 
 
 
 
1
+ import { StrictMode } from 'react'
2
+ import { createRoot } from 'react-dom/client'
3
+ import './index.css'
4
+ import App from './App.jsx'
5
+
6
+ createRoot(document.getElementById('root')).render(
7
+ <StrictMode>
8
+ <App />
9
+ </StrictMode>,
10
+ )
frontend/vite.config.js ADDED
@@ -0,0 +1,7 @@
 
 
 
 
 
 
 
 
1
+ import { defineConfig } from 'vite'
2
+ import react from '@vitejs/plugin-react'
3
+
4
+ // https://vite.dev/config/
5
+ export default defineConfig({
6
+ plugins: [react()],
7
+ })