diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000000000000000000000000000000000000..1d62099115d5fc2b2a1237a0c0e9ec61f88eb764 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,9 @@ +.git +.venv +__pycache__ +*.pyc +.env +data/logs +data/processed/embedding_cache.sqlite3 +ml + diff --git a/.gitattributes b/.gitattributes index a6344aac8c09253b3b630fb776ae94478aa0275b..5c8e78301b96354d3a40e9a0d627b079438c74c5 100644 --- a/.gitattributes +++ b/.gitattributes @@ -33,3 +33,10 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text *.zip filter=lfs diff=lfs merge=lfs -text *.zst filter=lfs diff=lfs merge=lfs -text *tfevents* filter=lfs diff=lfs merge=lfs -text +assets/mascots/chat.png filter=lfs diff=lfs merge=lfs -text +assets/mascots/chatvns.png filter=lfs diff=lfs merge=lfs -text +assets/mascots/insight.png filter=lfs diff=lfs merge=lfs -text +assets/mascots/processing.png filter=lfs diff=lfs merge=lfs -text +assets/mascots/trust.png filter=lfs diff=lfs merge=lfs -text +assets/mascots/welcome.png filter=lfs diff=lfs merge=lfs -text +documents/kien-truc-tong-the.png filter=lfs diff=lfs merge=lfs -text diff --git a/Dockerfile b/Dockerfile index 5f51ead59f36f13043e036290df9440e25fe8cbe..320531858d2c06655f95b6621f566dc3ece0bb86 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,20 +1,25 @@ -FROM python:3.13.5-slim +FROM python:3.11-slim -WORKDIR /app +RUN useradd -m -u 1000 user +USER user -RUN apt-get update && apt-get install -y \ - build-essential \ - curl \ - git \ - && rm -rf /var/lib/apt/lists/* +ENV HOME=/home/user \ + PATH=/home/user/.local/bin:$PATH \ + PYTHONUNBUFFERED=1 \ + PYTHONDONTWRITEBYTECODE=1 \ + APP_TICKERS=HPG,FPT,VCB \ + REQUIRE_USER_API_KEYS=1 \ + STREAMLIT_SERVER_HEADLESS=true \ + STREAMLIT_BROWSER_GATHER_USAGE_STATS=false -COPY requirements.txt ./ -COPY src/ ./src/ +WORKDIR $HOME/app -RUN pip3 install -r requirements.txt +COPY --chown=user requirements.txt ./requirements.txt +RUN pip install --no-cache-dir --upgrade pip \ + && pip install --no-cache-dir -r requirements.txt -EXPOSE 8501 +COPY --chown=user . . -HEALTHCHECK CMD curl --fail http://localhost:8501/_stcore/health +EXPOSE 7860 -ENTRYPOINT ["streamlit", "run", "src/streamlit_app.py", "--server.port=8501", "--server.address=0.0.0.0"] \ No newline at end of file +CMD ["sh", "-c", "python hf_bootstrap.py && exec streamlit run app/streamlit_app.py --server.address=0.0.0.0 --server.port=7860"] diff --git a/README.md b/README.md index b1d837fcb6a24cfa86b5e365de962e7226f3fa92..e91da0a4a84c823f6c337214712411f217cb2c4a 100644 --- a/README.md +++ b/README.md @@ -1,19 +1,57 @@ --- -title: Chatvns -emoji: 🚀 -colorFrom: red -colorTo: red +title: ChatVNS +emoji: 📈 +colorFrom: blue +colorTo: indigo sdk: docker -app_port: 8501 -tags: -- streamlit -pinned: false -short_description: a stock chatbot for the Vietnamese market +app_port: 7860 +python_version: 3.11 --- -# Welcome to Streamlit! +# ChatVNS -Edit `/src/streamlit_app.py` to customize this app to your heart's desire. :heart: +Chatbot RAG hỗ trợ phân tích dữ liệu chứng khoán Việt Nam. -If you have any questions, checkout our [documentation](https://docs.streamlit.io) and [community -forums](https://discuss.streamlit.io). +## API key do người dùng cung cấp + +Space không dùng Gemini API key hoặc Hugging Face inference token của chủ Space. +Mỗi người dùng nhập key của họ trong mục **API key của bạn** ở sidebar: + +- Gemini API key: dùng để tạo câu trả lời. +- Hugging Face token: không bắt buộc, dùng cho embedding và reranking. + +Key được giữ trong bộ nhớ của session Streamlit hiện tại, không ghi vào file hoặc +interaction log. Khi thiếu Hugging Face token hoặc Qdrant không khả dụng, ứng dụng +tự fallback sang BM25 trên dữ liệu chunks đi kèm. + +## Cấu hình Space + +Không cấu hình `GEMINI_API_KEY` hoặc `HF_API_KEY` trong Space Secrets. + +Các biến hạ tầng tùy chọn: + +- `QDRANT_URL` và `QDRANT_API_KEY`: Qdrant Cloud dùng chung. +- `CHATVNS_DATASET_REPO`: Hugging Face Dataset repo chứa thư mục `data/`. +- `HF_DATASET_TOKEN`: chỉ cần khi dataset repo ở chế độ private. +- `APP_TICKERS=HPG,FPT,VCB` + +## Tạo bundle để push + +Chạy từ thư mục gốc dự án: + +```powershell +powershell -ExecutionPolicy Bypass -File .\prepare_hf_space.ps1 +``` + +Lệnh mặc định đóng gói mã nguồn và tối đa 500 chunks cho mỗi scope +`HPG`, `FPT`, `VCB`, `MARKET`, đủ để Space chạy ngay bằng BM25. + +Đóng gói toàn bộ dữ liệu local: + +```powershell +powershell -ExecutionPolicy Bypass -File .\prepare_hf_space.ps1 -IncludeLocalData +``` + +Sau đó push toàn bộ nội dung trong `hf-space/` lên Hugging Face Space repository. +Với dữ liệu lớn, nên dùng một Dataset repo và `CHATVNS_DATASET_REPO` thay vì commit +trực tiếp vào Space. diff --git a/__pycache__/app.cpython-314.pyc b/__pycache__/app.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..c850eb0ce9367efe250ad89a63896b5fcf453847 Binary files /dev/null and b/__pycache__/app.cpython-314.pyc differ diff --git a/__pycache__/hf_bootstrap.cpython-314.pyc b/__pycache__/hf_bootstrap.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..687ede2a4df25371d98d0b8c84076c318d897af2 Binary files /dev/null and b/__pycache__/hf_bootstrap.cpython-314.pyc differ diff --git a/app/.streamlit/config.toml b/app/.streamlit/config.toml new file mode 100644 index 0000000000000000000000000000000000000000..4eeffee39f1e60e3d8692034c3a7190e899e7869 --- /dev/null +++ b/app/.streamlit/config.toml @@ -0,0 +1,2 @@ +[client] +showSidebarNavigation = false \ No newline at end of file diff --git a/app/__init__.py b/app/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..86ce6070cf2afbf5e2ba198aef40c3d9cdc56b96 --- /dev/null +++ b/app/__init__.py @@ -0,0 +1 @@ +"""RAG app for the Stock VN pipeline.""" diff --git a/app/__pycache__/__init__.cpython-311.pyc b/app/__pycache__/__init__.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..acb4bbd342260b8155de793ec0c5a7a51de9d706 Binary files /dev/null and b/app/__pycache__/__init__.cpython-311.pyc differ diff --git a/app/__pycache__/__init__.cpython-311.pyc.1973711792576 b/app/__pycache__/__init__.cpython-311.pyc.1973711792576 new file mode 100644 index 0000000000000000000000000000000000000000..acb4bbd342260b8155de793ec0c5a7a51de9d706 Binary files /dev/null and b/app/__pycache__/__init__.cpython-311.pyc.1973711792576 differ diff --git a/app/__pycache__/__init__.cpython-311.pyc.2306772232640 b/app/__pycache__/__init__.cpython-311.pyc.2306772232640 new file mode 100644 index 0000000000000000000000000000000000000000..acb4bbd342260b8155de793ec0c5a7a51de9d706 Binary files /dev/null and b/app/__pycache__/__init__.cpython-311.pyc.2306772232640 differ diff --git a/app/__pycache__/__init__.cpython-311.pyc.2544356157888 b/app/__pycache__/__init__.cpython-311.pyc.2544356157888 new file mode 100644 index 0000000000000000000000000000000000000000..acb4bbd342260b8155de793ec0c5a7a51de9d706 Binary files /dev/null and b/app/__pycache__/__init__.cpython-311.pyc.2544356157888 differ diff --git a/app/__pycache__/__init__.cpython-314.pyc b/app/__pycache__/__init__.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..9d1b4c88cd427e6c67048967150252aa9e8ce5b8 Binary files /dev/null and b/app/__pycache__/__init__.cpython-314.pyc differ diff --git a/app/__pycache__/config.cpython-311.pyc b/app/__pycache__/config.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..b35fb42797e3ef479862c80292706e93bb69b339 Binary files /dev/null and b/app/__pycache__/config.cpython-311.pyc differ diff --git a/app/__pycache__/config.cpython-311.pyc.1580759400608 b/app/__pycache__/config.cpython-311.pyc.1580759400608 new file mode 100644 index 0000000000000000000000000000000000000000..74aff15863baf4903ea59a1e9fa2937d7347a045 Binary files /dev/null and b/app/__pycache__/config.cpython-311.pyc.1580759400608 differ diff --git a/app/__pycache__/config.cpython-311.pyc.1725774067760 b/app/__pycache__/config.cpython-311.pyc.1725774067760 new file mode 100644 index 0000000000000000000000000000000000000000..74aff15863baf4903ea59a1e9fa2937d7347a045 Binary files /dev/null and b/app/__pycache__/config.cpython-311.pyc.1725774067760 differ diff --git a/app/__pycache__/config.cpython-311.pyc.1783676383760 b/app/__pycache__/config.cpython-311.pyc.1783676383760 new file mode 100644 index 0000000000000000000000000000000000000000..6c2ef999d2876e0017c5b78800da242109942f52 Binary files /dev/null and b/app/__pycache__/config.cpython-311.pyc.1783676383760 differ diff --git a/app/__pycache__/config.cpython-311.pyc.1802465591456 b/app/__pycache__/config.cpython-311.pyc.1802465591456 new file mode 100644 index 0000000000000000000000000000000000000000..8774147f8f4f7277403e947e4577bc723202b5f6 Binary files /dev/null and b/app/__pycache__/config.cpython-311.pyc.1802465591456 differ diff --git a/app/__pycache__/config.cpython-311.pyc.1935848577424 b/app/__pycache__/config.cpython-311.pyc.1935848577424 new file mode 100644 index 0000000000000000000000000000000000000000..dd45ef063b0d0d57197f13b1c686cb89b014c139 Binary files /dev/null and b/app/__pycache__/config.cpython-311.pyc.1935848577424 differ diff --git a/app/__pycache__/config.cpython-311.pyc.1962180850192 b/app/__pycache__/config.cpython-311.pyc.1962180850192 new file mode 100644 index 0000000000000000000000000000000000000000..6c2ef999d2876e0017c5b78800da242109942f52 Binary files /dev/null and b/app/__pycache__/config.cpython-311.pyc.1962180850192 differ diff --git a/app/__pycache__/config.cpython-311.pyc.1973720126656 b/app/__pycache__/config.cpython-311.pyc.1973720126656 new file mode 100644 index 0000000000000000000000000000000000000000..46903eecda9d1580fcca1866d5050de011cf2274 Binary files /dev/null and b/app/__pycache__/config.cpython-311.pyc.1973720126656 differ diff --git a/app/__pycache__/config.cpython-311.pyc.1993952126864 b/app/__pycache__/config.cpython-311.pyc.1993952126864 new file mode 100644 index 0000000000000000000000000000000000000000..dd45ef063b0d0d57197f13b1c686cb89b014c139 Binary files /dev/null and b/app/__pycache__/config.cpython-311.pyc.1993952126864 differ diff --git a/app/__pycache__/config.cpython-311.pyc.2050849738064 b/app/__pycache__/config.cpython-311.pyc.2050849738064 new file mode 100644 index 0000000000000000000000000000000000000000..9b1639e87ddf0e7346fc1030e55d4190c6c2e400 Binary files /dev/null and b/app/__pycache__/config.cpython-311.pyc.2050849738064 differ diff --git a/app/__pycache__/config.cpython-311.pyc.2070801723920 b/app/__pycache__/config.cpython-311.pyc.2070801723920 new file mode 100644 index 0000000000000000000000000000000000000000..6c2ef999d2876e0017c5b78800da242109942f52 Binary files /dev/null and b/app/__pycache__/config.cpython-311.pyc.2070801723920 differ diff --git a/app/__pycache__/config.cpython-311.pyc.2151036698784 b/app/__pycache__/config.cpython-311.pyc.2151036698784 new file mode 100644 index 0000000000000000000000000000000000000000..74aff15863baf4903ea59a1e9fa2937d7347a045 Binary files /dev/null and b/app/__pycache__/config.cpython-311.pyc.2151036698784 differ diff --git a/app/__pycache__/config.cpython-311.pyc.2199857928480 b/app/__pycache__/config.cpython-311.pyc.2199857928480 new file mode 100644 index 0000000000000000000000000000000000000000..dd45ef063b0d0d57197f13b1c686cb89b014c139 Binary files /dev/null and b/app/__pycache__/config.cpython-311.pyc.2199857928480 differ diff --git a/app/__pycache__/config.cpython-311.pyc.2306780517680 b/app/__pycache__/config.cpython-311.pyc.2306780517680 new file mode 100644 index 0000000000000000000000000000000000000000..46903eecda9d1580fcca1866d5050de011cf2274 Binary files /dev/null and b/app/__pycache__/config.cpython-311.pyc.2306780517680 differ diff --git a/app/__pycache__/config.cpython-311.pyc.2323301603616 b/app/__pycache__/config.cpython-311.pyc.2323301603616 new file mode 100644 index 0000000000000000000000000000000000000000..dd45ef063b0d0d57197f13b1c686cb89b014c139 Binary files /dev/null and b/app/__pycache__/config.cpython-311.pyc.2323301603616 differ diff --git a/app/__pycache__/config.cpython-311.pyc.2544364459200 b/app/__pycache__/config.cpython-311.pyc.2544364459200 new file mode 100644 index 0000000000000000000000000000000000000000..46903eecda9d1580fcca1866d5050de011cf2274 Binary files /dev/null and b/app/__pycache__/config.cpython-311.pyc.2544364459200 differ diff --git a/app/__pycache__/config.cpython-311.pyc.2779348867024 b/app/__pycache__/config.cpython-311.pyc.2779348867024 new file mode 100644 index 0000000000000000000000000000000000000000..10ffb0036f891a09ecb73769df50204e45d891aa Binary files /dev/null and b/app/__pycache__/config.cpython-311.pyc.2779348867024 differ diff --git a/app/__pycache__/config.cpython-311.pyc.2969461649936 b/app/__pycache__/config.cpython-311.pyc.2969461649936 new file mode 100644 index 0000000000000000000000000000000000000000..0186e5150d2f755cc5b4cb2c9928b3e91d3ed60b Binary files /dev/null and b/app/__pycache__/config.cpython-311.pyc.2969461649936 differ diff --git a/app/__pycache__/config.cpython-314.pyc b/app/__pycache__/config.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..25b936e0a2d82c85f853cab1f1293ff6fb7865ef Binary files /dev/null and b/app/__pycache__/config.cpython-314.pyc differ diff --git a/app/__pycache__/context_compression.cpython-311.pyc b/app/__pycache__/context_compression.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..e7af96129046f6a6c267c73168072699d5d82a95 Binary files /dev/null and b/app/__pycache__/context_compression.cpython-311.pyc differ diff --git a/app/__pycache__/context_compression.cpython-311.pyc.1783676588720 b/app/__pycache__/context_compression.cpython-311.pyc.1783676588720 new file mode 100644 index 0000000000000000000000000000000000000000..f0d37ec065fae927786530e806a52fe00c0bd5b1 Binary files /dev/null and b/app/__pycache__/context_compression.cpython-311.pyc.1783676588720 differ diff --git a/app/__pycache__/context_compression.cpython-311.pyc.1962181054896 b/app/__pycache__/context_compression.cpython-311.pyc.1962181054896 new file mode 100644 index 0000000000000000000000000000000000000000..f0d37ec065fae927786530e806a52fe00c0bd5b1 Binary files /dev/null and b/app/__pycache__/context_compression.cpython-311.pyc.1962181054896 differ diff --git a/app/__pycache__/context_compression.cpython-311.pyc.2070801928624 b/app/__pycache__/context_compression.cpython-311.pyc.2070801928624 new file mode 100644 index 0000000000000000000000000000000000000000..ff1c328eace88c99f3c6a43f49902b0062e81a77 Binary files /dev/null and b/app/__pycache__/context_compression.cpython-311.pyc.2070801928624 differ diff --git a/app/__pycache__/context_compression.cpython-314.pyc b/app/__pycache__/context_compression.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..2f3b00f4ca24285404bcbeb51c14e1449c27a467 Binary files /dev/null and b/app/__pycache__/context_compression.cpython-314.pyc differ diff --git a/app/__pycache__/embedding_cache.cpython-311.pyc b/app/__pycache__/embedding_cache.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..316048b0acafee61083d8acd58c6eca3f0244121 Binary files /dev/null and b/app/__pycache__/embedding_cache.cpython-311.pyc differ diff --git a/app/__pycache__/embedding_cache.cpython-314.pyc b/app/__pycache__/embedding_cache.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..d2e4de821bde1d72ce7a6b5c203a87c4f8fbcb0d Binary files /dev/null and b/app/__pycache__/embedding_cache.cpython-314.pyc differ diff --git a/app/__pycache__/embeddings.cpython-311.pyc b/app/__pycache__/embeddings.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..a381dee5e22e8b8adc0bd9208f88b5db6e521a1b Binary files /dev/null and b/app/__pycache__/embeddings.cpython-311.pyc differ diff --git a/app/__pycache__/embeddings.cpython-311.pyc.1935805179792 b/app/__pycache__/embeddings.cpython-311.pyc.1935805179792 new file mode 100644 index 0000000000000000000000000000000000000000..14ab472e6d11225df1c8b433c607b819fa9e7dd7 Binary files /dev/null and b/app/__pycache__/embeddings.cpython-311.pyc.1935805179792 differ diff --git a/app/__pycache__/embeddings.cpython-311.pyc.1974185286032 b/app/__pycache__/embeddings.cpython-311.pyc.1974185286032 new file mode 100644 index 0000000000000000000000000000000000000000..86b8f15172e0303391db7248e2587d6bc9823784 Binary files /dev/null and b/app/__pycache__/embeddings.cpython-311.pyc.1974185286032 differ diff --git a/app/__pycache__/embeddings.cpython-311.pyc.2199813547920 b/app/__pycache__/embeddings.cpython-311.pyc.2199813547920 new file mode 100644 index 0000000000000000000000000000000000000000..784ae405a98568617e78dd409ee89dc380590d01 Binary files /dev/null and b/app/__pycache__/embeddings.cpython-311.pyc.2199813547920 differ diff --git a/app/__pycache__/embeddings.cpython-311.pyc.2307245939200 b/app/__pycache__/embeddings.cpython-311.pyc.2307245939200 new file mode 100644 index 0000000000000000000000000000000000000000..86b8f15172e0303391db7248e2587d6bc9823784 Binary files /dev/null and b/app/__pycache__/embeddings.cpython-311.pyc.2307245939200 differ diff --git a/app/__pycache__/embeddings.cpython-311.pyc.2323257419664 b/app/__pycache__/embeddings.cpython-311.pyc.2323257419664 new file mode 100644 index 0000000000000000000000000000000000000000..14ab472e6d11225df1c8b433c607b819fa9e7dd7 Binary files /dev/null and b/app/__pycache__/embeddings.cpython-311.pyc.2323257419664 differ diff --git a/app/__pycache__/embeddings.cpython-311.pyc.2542682446224 b/app/__pycache__/embeddings.cpython-311.pyc.2542682446224 new file mode 100644 index 0000000000000000000000000000000000000000..86b8f15172e0303391db7248e2587d6bc9823784 Binary files /dev/null and b/app/__pycache__/embeddings.cpython-311.pyc.2542682446224 differ diff --git a/app/__pycache__/embeddings.cpython-314.pyc b/app/__pycache__/embeddings.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..170bfd9505dd0e7a887a3e3edd18bdf3bafd75ac Binary files /dev/null and b/app/__pycache__/embeddings.cpython-314.pyc differ diff --git a/app/__pycache__/evaluate.cpython-311.pyc b/app/__pycache__/evaluate.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..90eeba8d297f5af1ea31f28861779cc30f0ca5bc Binary files /dev/null and b/app/__pycache__/evaluate.cpython-311.pyc differ diff --git a/app/__pycache__/evaluate.cpython-311.pyc.1949709152400 b/app/__pycache__/evaluate.cpython-311.pyc.1949709152400 new file mode 100644 index 0000000000000000000000000000000000000000..37c5bc32b480a171be7b1ffd1a685b8c4040904c Binary files /dev/null and b/app/__pycache__/evaluate.cpython-311.pyc.1949709152400 differ diff --git a/app/__pycache__/evaluate.cpython-311.pyc.1973714458800 b/app/__pycache__/evaluate.cpython-311.pyc.1973714458800 new file mode 100644 index 0000000000000000000000000000000000000000..c0f13908f5e9a0747caf22f20578c1bca5c47146 Binary files /dev/null and b/app/__pycache__/evaluate.cpython-311.pyc.1973714458800 differ diff --git a/app/__pycache__/evaluate.cpython-311.pyc.1985961518992 b/app/__pycache__/evaluate.cpython-311.pyc.1985961518992 new file mode 100644 index 0000000000000000000000000000000000000000..7a2d6ac3ba6d4fbbc8279d3b7f4d7a41d327923d Binary files /dev/null and b/app/__pycache__/evaluate.cpython-311.pyc.1985961518992 differ diff --git a/app/__pycache__/evaluate.cpython-311.pyc.2076318033808 b/app/__pycache__/evaluate.cpython-311.pyc.2076318033808 new file mode 100644 index 0000000000000000000000000000000000000000..b6c39a54d0f08f935bc03fa77d9ef9a57a2ff968 Binary files /dev/null and b/app/__pycache__/evaluate.cpython-311.pyc.2076318033808 differ diff --git a/app/__pycache__/evaluate.cpython-311.pyc.2157946005952 b/app/__pycache__/evaluate.cpython-311.pyc.2157946005952 new file mode 100644 index 0000000000000000000000000000000000000000..b6c39a54d0f08f935bc03fa77d9ef9a57a2ff968 Binary files /dev/null and b/app/__pycache__/evaluate.cpython-311.pyc.2157946005952 differ diff --git a/app/__pycache__/evaluate.cpython-311.pyc.2264095075824 b/app/__pycache__/evaluate.cpython-311.pyc.2264095075824 new file mode 100644 index 0000000000000000000000000000000000000000..b50bb328f28006006f9a3f62ee4e470b1510ff22 Binary files /dev/null and b/app/__pycache__/evaluate.cpython-311.pyc.2264095075824 differ diff --git a/app/__pycache__/evaluate.cpython-311.pyc.2306774899088 b/app/__pycache__/evaluate.cpython-311.pyc.2306774899088 new file mode 100644 index 0000000000000000000000000000000000000000..f5c261891a4d51a191372746659734de10a54f3b Binary files /dev/null and b/app/__pycache__/evaluate.cpython-311.pyc.2306774899088 differ diff --git a/app/__pycache__/evaluate.cpython-311.pyc.2354630619440 b/app/__pycache__/evaluate.cpython-311.pyc.2354630619440 new file mode 100644 index 0000000000000000000000000000000000000000..f61dd7c2386c9a0ca81ff17970cf24c0ae90896e Binary files /dev/null and b/app/__pycache__/evaluate.cpython-311.pyc.2354630619440 differ diff --git a/app/__pycache__/evaluate.cpython-311.pyc.2544358824224 b/app/__pycache__/evaluate.cpython-311.pyc.2544358824224 new file mode 100644 index 0000000000000000000000000000000000000000..dd158a67a93397684070e0d853416d9c1aba37af Binary files /dev/null and b/app/__pycache__/evaluate.cpython-311.pyc.2544358824224 differ diff --git a/app/__pycache__/evaluate.cpython-311.pyc.2651623843728 b/app/__pycache__/evaluate.cpython-311.pyc.2651623843728 new file mode 100644 index 0000000000000000000000000000000000000000..cd76f5418f255c2bc9ac9908d3aa09bbf4136131 Binary files /dev/null and b/app/__pycache__/evaluate.cpython-311.pyc.2651623843728 differ diff --git a/app/__pycache__/evaluate.cpython-311.pyc.2690257976656 b/app/__pycache__/evaluate.cpython-311.pyc.2690257976656 new file mode 100644 index 0000000000000000000000000000000000000000..a3e4c42d6edf2bb09dcf27a04f412ff8a80d6efa Binary files /dev/null and b/app/__pycache__/evaluate.cpython-311.pyc.2690257976656 differ diff --git a/app/__pycache__/evaluate.cpython-311.pyc.2752630024080 b/app/__pycache__/evaluate.cpython-311.pyc.2752630024080 new file mode 100644 index 0000000000000000000000000000000000000000..73b258682193ed0def51e71ff734982b5e1d031c Binary files /dev/null and b/app/__pycache__/evaluate.cpython-311.pyc.2752630024080 differ diff --git a/app/__pycache__/evaluate.cpython-311.pyc.2794500974480 b/app/__pycache__/evaluate.cpython-311.pyc.2794500974480 new file mode 100644 index 0000000000000000000000000000000000000000..7a2d6ac3ba6d4fbbc8279d3b7f4d7a41d327923d Binary files /dev/null and b/app/__pycache__/evaluate.cpython-311.pyc.2794500974480 differ diff --git a/app/__pycache__/evaluate.cpython-311.pyc.2873389214144 b/app/__pycache__/evaluate.cpython-311.pyc.2873389214144 new file mode 100644 index 0000000000000000000000000000000000000000..73b258682193ed0def51e71ff734982b5e1d031c Binary files /dev/null and b/app/__pycache__/evaluate.cpython-311.pyc.2873389214144 differ diff --git a/app/__pycache__/evaluate.cpython-314.pyc b/app/__pycache__/evaluate.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..d43598b6e21955d6270eee32cdd4b53c29a515f9 Binary files /dev/null and b/app/__pycache__/evaluate.cpython-314.pyc differ diff --git a/app/__pycache__/intermarket.cpython-311.pyc b/app/__pycache__/intermarket.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..5c7b329bb399ed7415942ca58645ff1928bb3478 Binary files /dev/null and b/app/__pycache__/intermarket.cpython-311.pyc differ diff --git a/app/__pycache__/intermarket.cpython-314.pyc b/app/__pycache__/intermarket.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..bb925aecca1a714dd66d60de1c80ea5af6fef443 Binary files /dev/null and b/app/__pycache__/intermarket.cpython-314.pyc differ diff --git a/app/__pycache__/keyword_search.cpython-311.pyc b/app/__pycache__/keyword_search.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..21169e62bc2bd01601b3b269ae19dfbf31cca735 Binary files /dev/null and b/app/__pycache__/keyword_search.cpython-311.pyc differ diff --git a/app/__pycache__/keyword_search.cpython-311.pyc.1973726439200 b/app/__pycache__/keyword_search.cpython-311.pyc.1973726439200 new file mode 100644 index 0000000000000000000000000000000000000000..ba906c5d4943a258a9b4277c5fb243bd3ce4d96d Binary files /dev/null and b/app/__pycache__/keyword_search.cpython-311.pyc.1973726439200 differ diff --git a/app/__pycache__/keyword_search.cpython-311.pyc.2306786879376 b/app/__pycache__/keyword_search.cpython-311.pyc.2306786879376 new file mode 100644 index 0000000000000000000000000000000000000000..ba906c5d4943a258a9b4277c5fb243bd3ce4d96d Binary files /dev/null and b/app/__pycache__/keyword_search.cpython-311.pyc.2306786879376 differ diff --git a/app/__pycache__/keyword_search.cpython-311.pyc.2544366610208 b/app/__pycache__/keyword_search.cpython-311.pyc.2544366610208 new file mode 100644 index 0000000000000000000000000000000000000000..ba906c5d4943a258a9b4277c5fb243bd3ce4d96d Binary files /dev/null and b/app/__pycache__/keyword_search.cpython-311.pyc.2544366610208 differ diff --git a/app/__pycache__/keyword_search.cpython-314.pyc b/app/__pycache__/keyword_search.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..1cf505819a99a24658d09227072f72483ed78810 Binary files /dev/null and b/app/__pycache__/keyword_search.cpython-314.pyc differ diff --git a/app/__pycache__/market_snapshot.cpython-311.pyc b/app/__pycache__/market_snapshot.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..df342194425a5889b37b72fa00378f15531f54d2 Binary files /dev/null and b/app/__pycache__/market_snapshot.cpython-311.pyc differ diff --git a/app/__pycache__/market_snapshot.cpython-311.pyc.1973725346400 b/app/__pycache__/market_snapshot.cpython-311.pyc.1973725346400 new file mode 100644 index 0000000000000000000000000000000000000000..df342194425a5889b37b72fa00378f15531f54d2 Binary files /dev/null and b/app/__pycache__/market_snapshot.cpython-311.pyc.1973725346400 differ diff --git a/app/__pycache__/market_snapshot.cpython-311.pyc.2306785786576 b/app/__pycache__/market_snapshot.cpython-311.pyc.2306785786576 new file mode 100644 index 0000000000000000000000000000000000000000..df342194425a5889b37b72fa00378f15531f54d2 Binary files /dev/null and b/app/__pycache__/market_snapshot.cpython-311.pyc.2306785786576 differ diff --git a/app/__pycache__/market_snapshot.cpython-311.pyc.2544365517408 b/app/__pycache__/market_snapshot.cpython-311.pyc.2544365517408 new file mode 100644 index 0000000000000000000000000000000000000000..df342194425a5889b37b72fa00378f15531f54d2 Binary files /dev/null and b/app/__pycache__/market_snapshot.cpython-311.pyc.2544365517408 differ diff --git a/app/__pycache__/market_snapshot.cpython-314.pyc b/app/__pycache__/market_snapshot.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..f3e252cb74c38d8d549f7358d9a56c1b19630749 Binary files /dev/null and b/app/__pycache__/market_snapshot.cpython-314.pyc differ diff --git a/app/__pycache__/multimodal.cpython-311.pyc b/app/__pycache__/multimodal.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..e0d279f5792c635f58a35a3c38fa6562844d16e7 Binary files /dev/null and b/app/__pycache__/multimodal.cpython-311.pyc differ diff --git a/app/__pycache__/multimodal.cpython-311.pyc.1973725345952 b/app/__pycache__/multimodal.cpython-311.pyc.1973725345952 new file mode 100644 index 0000000000000000000000000000000000000000..e0d279f5792c635f58a35a3c38fa6562844d16e7 Binary files /dev/null and b/app/__pycache__/multimodal.cpython-311.pyc.1973725345952 differ diff --git a/app/__pycache__/multimodal.cpython-311.pyc.2306785786128 b/app/__pycache__/multimodal.cpython-311.pyc.2306785786128 new file mode 100644 index 0000000000000000000000000000000000000000..e0d279f5792c635f58a35a3c38fa6562844d16e7 Binary files /dev/null and b/app/__pycache__/multimodal.cpython-311.pyc.2306785786128 differ diff --git a/app/__pycache__/multimodal.cpython-311.pyc.2544365516960 b/app/__pycache__/multimodal.cpython-311.pyc.2544365516960 new file mode 100644 index 0000000000000000000000000000000000000000..e0d279f5792c635f58a35a3c38fa6562844d16e7 Binary files /dev/null and b/app/__pycache__/multimodal.cpython-311.pyc.2544365516960 differ diff --git a/app/__pycache__/multimodal.cpython-314.pyc b/app/__pycache__/multimodal.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..da10a5a24a414a2e53337ac6a543a626d660adc0 Binary files /dev/null and b/app/__pycache__/multimodal.cpython-314.pyc differ diff --git a/app/__pycache__/observability.cpython-311.pyc b/app/__pycache__/observability.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..cd4a3f5295e767e8f47655b3d45990e1b47c7105 Binary files /dev/null and b/app/__pycache__/observability.cpython-311.pyc differ diff --git a/app/__pycache__/observability.cpython-314.pyc b/app/__pycache__/observability.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..db8768731582fd7c24909fe8eab52ded5f577faf Binary files /dev/null and b/app/__pycache__/observability.cpython-314.pyc differ diff --git a/app/__pycache__/pipeline.cpython-311.pyc b/app/__pycache__/pipeline.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..a369d64200efde71a3344ef1145c0f7b5334c37d Binary files /dev/null and b/app/__pycache__/pipeline.cpython-311.pyc differ diff --git a/app/__pycache__/pipeline.cpython-314.pyc b/app/__pycache__/pipeline.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..80fc17a73fbd4850fc6c01430c00e47284816dea Binary files /dev/null and b/app/__pycache__/pipeline.cpython-314.pyc differ diff --git a/app/__pycache__/prune_raw_runs.cpython-314.pyc b/app/__pycache__/prune_raw_runs.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..4d5f4d929006d35b31ab7ccf99483792d3debace Binary files /dev/null and b/app/__pycache__/prune_raw_runs.cpython-314.pyc differ diff --git a/app/__pycache__/qa_suggestions.cpython-311.pyc b/app/__pycache__/qa_suggestions.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..5eeaca9c4f407c6b9f21625acf5562b39b75c4b0 Binary files /dev/null and b/app/__pycache__/qa_suggestions.cpython-311.pyc differ diff --git a/app/__pycache__/qa_suggestions.cpython-314.pyc b/app/__pycache__/qa_suggestions.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..75122dd44e89f88029a2cbfadb44cfd7784ce66d Binary files /dev/null and b/app/__pycache__/qa_suggestions.cpython-314.pyc differ diff --git a/app/__pycache__/rag.cpython-311.pyc b/app/__pycache__/rag.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..e83d8419f92bd9af3e79d68dfea4bc1df565b04e Binary files /dev/null and b/app/__pycache__/rag.cpython-311.pyc differ diff --git a/app/__pycache__/rag.cpython-311.pyc.1973725346288 b/app/__pycache__/rag.cpython-311.pyc.1973725346288 new file mode 100644 index 0000000000000000000000000000000000000000..860022961fc10e1b792c08dc594016b50e60d4e5 Binary files /dev/null and b/app/__pycache__/rag.cpython-311.pyc.1973725346288 differ diff --git a/app/__pycache__/rag.cpython-311.pyc.2306785786464 b/app/__pycache__/rag.cpython-311.pyc.2306785786464 new file mode 100644 index 0000000000000000000000000000000000000000..860022961fc10e1b792c08dc594016b50e60d4e5 Binary files /dev/null and b/app/__pycache__/rag.cpython-311.pyc.2306785786464 differ diff --git a/app/__pycache__/rag.cpython-311.pyc.2544365517296 b/app/__pycache__/rag.cpython-311.pyc.2544365517296 new file mode 100644 index 0000000000000000000000000000000000000000..860022961fc10e1b792c08dc594016b50e60d4e5 Binary files /dev/null and b/app/__pycache__/rag.cpython-311.pyc.2544365517296 differ diff --git a/app/__pycache__/rag.cpython-314.pyc b/app/__pycache__/rag.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..150b04866f7e37d1e8514637c13b47523bc3f463 Binary files /dev/null and b/app/__pycache__/rag.cpython-314.pyc differ diff --git a/app/__pycache__/reranker.cpython-311.pyc b/app/__pycache__/reranker.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..16b29ef530a718a8fdd2242abb7893f4a37d5be1 Binary files /dev/null and b/app/__pycache__/reranker.cpython-311.pyc differ diff --git a/app/__pycache__/reranker.cpython-311.pyc.2779342828048 b/app/__pycache__/reranker.cpython-311.pyc.2779342828048 new file mode 100644 index 0000000000000000000000000000000000000000..0d4893250f532ee4e8f737abfd6681706298865a Binary files /dev/null and b/app/__pycache__/reranker.cpython-311.pyc.2779342828048 differ diff --git a/app/__pycache__/reranker.cpython-314.pyc b/app/__pycache__/reranker.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..01ffac7bd7255b051e88a6069e9342163e4b5a2e Binary files /dev/null and b/app/__pycache__/reranker.cpython-314.pyc differ diff --git a/app/__pycache__/retriever.cpython-311.pyc b/app/__pycache__/retriever.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..2788421d1b09a34003cc622f920d75fe350c926f Binary files /dev/null and b/app/__pycache__/retriever.cpython-311.pyc differ diff --git a/app/__pycache__/retriever.cpython-311.pyc.1973726438864 b/app/__pycache__/retriever.cpython-311.pyc.1973726438864 new file mode 100644 index 0000000000000000000000000000000000000000..7d63744086a8dfdb41fd98055336f82575ebea42 Binary files /dev/null and b/app/__pycache__/retriever.cpython-311.pyc.1973726438864 differ diff --git a/app/__pycache__/retriever.cpython-311.pyc.2021383385232 b/app/__pycache__/retriever.cpython-311.pyc.2021383385232 new file mode 100644 index 0000000000000000000000000000000000000000..3301d588dbb75b9719bdc1aefdac8c84f58cbee3 Binary files /dev/null and b/app/__pycache__/retriever.cpython-311.pyc.2021383385232 differ diff --git a/app/__pycache__/retriever.cpython-311.pyc.2306786879040 b/app/__pycache__/retriever.cpython-311.pyc.2306786879040 new file mode 100644 index 0000000000000000000000000000000000000000..7d63744086a8dfdb41fd98055336f82575ebea42 Binary files /dev/null and b/app/__pycache__/retriever.cpython-311.pyc.2306786879040 differ diff --git a/app/__pycache__/retriever.cpython-311.pyc.2544366609872 b/app/__pycache__/retriever.cpython-311.pyc.2544366609872 new file mode 100644 index 0000000000000000000000000000000000000000..7d63744086a8dfdb41fd98055336f82575ebea42 Binary files /dev/null and b/app/__pycache__/retriever.cpython-311.pyc.2544366609872 differ diff --git a/app/__pycache__/retriever.cpython-314.pyc b/app/__pycache__/retriever.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..65f5770133e0a54f494b6c7079ad508e923c1921 Binary files /dev/null and b/app/__pycache__/retriever.cpython-314.pyc differ diff --git a/app/__pycache__/runtime_auth.cpython-311.pyc b/app/__pycache__/runtime_auth.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..ab345bfec092ec1dca793ce1fe71e3f0cde5923c Binary files /dev/null and b/app/__pycache__/runtime_auth.cpython-311.pyc differ diff --git a/app/__pycache__/schemas.cpython-311.pyc b/app/__pycache__/schemas.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..b693a596d5f17f0554e46f2f04074d1047a30d92 Binary files /dev/null and b/app/__pycache__/schemas.cpython-311.pyc differ diff --git a/app/__pycache__/schemas.cpython-311.pyc.1973726439648 b/app/__pycache__/schemas.cpython-311.pyc.1973726439648 new file mode 100644 index 0000000000000000000000000000000000000000..f7123465fe5f8adca43250a1aac46a2d23e22d37 Binary files /dev/null and b/app/__pycache__/schemas.cpython-311.pyc.1973726439648 differ diff --git a/app/__pycache__/schemas.cpython-311.pyc.2306786879824 b/app/__pycache__/schemas.cpython-311.pyc.2306786879824 new file mode 100644 index 0000000000000000000000000000000000000000..f7123465fe5f8adca43250a1aac46a2d23e22d37 Binary files /dev/null and b/app/__pycache__/schemas.cpython-311.pyc.2306786879824 differ diff --git a/app/__pycache__/schemas.cpython-311.pyc.2544366610656 b/app/__pycache__/schemas.cpython-311.pyc.2544366610656 new file mode 100644 index 0000000000000000000000000000000000000000..f7123465fe5f8adca43250a1aac46a2d23e22d37 Binary files /dev/null and b/app/__pycache__/schemas.cpython-311.pyc.2544366610656 differ diff --git a/app/__pycache__/schemas.cpython-314.pyc b/app/__pycache__/schemas.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..6ea4991484fc37b87120c0aea9697e9db9c38db3 Binary files /dev/null and b/app/__pycache__/schemas.cpython-314.pyc differ diff --git a/app/__pycache__/source_display.cpython-311.pyc b/app/__pycache__/source_display.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..fef95568b365c3ca30880ccc905da676444797c9 Binary files /dev/null and b/app/__pycache__/source_display.cpython-311.pyc differ diff --git a/app/__pycache__/source_display.cpython-314.pyc b/app/__pycache__/source_display.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..269bec1a83ec9af6c3b4ae92a87877d840b2b070 Binary files /dev/null and b/app/__pycache__/source_display.cpython-314.pyc differ diff --git a/app/__pycache__/streamlit_app.cpython-311.pyc b/app/__pycache__/streamlit_app.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..82430987b15c6c4e03942c918b37f129edbb10f7 Binary files /dev/null and b/app/__pycache__/streamlit_app.cpython-311.pyc differ diff --git a/app/__pycache__/streamlit_app.cpython-314.pyc b/app/__pycache__/streamlit_app.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..f946fde4b7821b12da8c079f19bc7bb02517e701 Binary files /dev/null and b/app/__pycache__/streamlit_app.cpython-314.pyc differ diff --git a/app/__pycache__/technical_analysis.cpython-311.pyc b/app/__pycache__/technical_analysis.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..ecec499c28fb6c06b4498d606cef82b76db00695 Binary files /dev/null and b/app/__pycache__/technical_analysis.cpython-311.pyc differ diff --git a/app/__pycache__/technical_analysis.cpython-311.pyc.1974185415728 b/app/__pycache__/technical_analysis.cpython-311.pyc.1974185415728 new file mode 100644 index 0000000000000000000000000000000000000000..ecec499c28fb6c06b4498d606cef82b76db00695 Binary files /dev/null and b/app/__pycache__/technical_analysis.cpython-311.pyc.1974185415728 differ diff --git a/app/__pycache__/technical_analysis.cpython-311.pyc.2307246068656 b/app/__pycache__/technical_analysis.cpython-311.pyc.2307246068656 new file mode 100644 index 0000000000000000000000000000000000000000..ecec499c28fb6c06b4498d606cef82b76db00695 Binary files /dev/null and b/app/__pycache__/technical_analysis.cpython-311.pyc.2307246068656 differ diff --git a/app/__pycache__/technical_analysis.cpython-311.pyc.2542682559408 b/app/__pycache__/technical_analysis.cpython-311.pyc.2542682559408 new file mode 100644 index 0000000000000000000000000000000000000000..ecec499c28fb6c06b4498d606cef82b76db00695 Binary files /dev/null and b/app/__pycache__/technical_analysis.cpython-311.pyc.2542682559408 differ diff --git a/app/__pycache__/technical_analysis.cpython-314.pyc b/app/__pycache__/technical_analysis.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..6398a7be851f6b389fdbc0a695c218560e7371d8 Binary files /dev/null and b/app/__pycache__/technical_analysis.cpython-314.pyc differ diff --git a/app/__pycache__/ui.cpython-311.pyc b/app/__pycache__/ui.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..f652350f16628abfbfd5bcc0a28ed9587cf39fe6 Binary files /dev/null and b/app/__pycache__/ui.cpython-311.pyc differ diff --git a/app/__pycache__/ui.cpython-314.pyc b/app/__pycache__/ui.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..103951f0d53ce66ca55ba332eb11a2707a00b319 Binary files /dev/null and b/app/__pycache__/ui.cpython-314.pyc differ diff --git a/app/__pycache__/vector_store.cpython-311.pyc b/app/__pycache__/vector_store.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..831868eade2ff43345feb90d78e2b2f617b10680 Binary files /dev/null and b/app/__pycache__/vector_store.cpython-311.pyc differ diff --git a/app/__pycache__/vector_store.cpython-311.pyc.1973726440320 b/app/__pycache__/vector_store.cpython-311.pyc.1973726440320 new file mode 100644 index 0000000000000000000000000000000000000000..767b6f64cdd6e9914c5e65dcfde2ac2085c7339c Binary files /dev/null and b/app/__pycache__/vector_store.cpython-311.pyc.1973726440320 differ diff --git a/app/__pycache__/vector_store.cpython-311.pyc.2306786880496 b/app/__pycache__/vector_store.cpython-311.pyc.2306786880496 new file mode 100644 index 0000000000000000000000000000000000000000..767b6f64cdd6e9914c5e65dcfde2ac2085c7339c Binary files /dev/null and b/app/__pycache__/vector_store.cpython-311.pyc.2306786880496 differ diff --git a/app/__pycache__/vector_store.cpython-311.pyc.2544366611328 b/app/__pycache__/vector_store.cpython-311.pyc.2544366611328 new file mode 100644 index 0000000000000000000000000000000000000000..767b6f64cdd6e9914c5e65dcfde2ac2085c7339c Binary files /dev/null and b/app/__pycache__/vector_store.cpython-311.pyc.2544366611328 differ diff --git a/app/__pycache__/vector_store.cpython-314.pyc b/app/__pycache__/vector_store.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..b9e4f8baff32923e68c8a3a39cf81c5b56734b76 Binary files /dev/null and b/app/__pycache__/vector_store.cpython-314.pyc differ diff --git a/app/config.py b/app/config.py new file mode 100644 index 0000000000000000000000000000000000000000..b8656a798d1309c3a37832831c48190f7ea8e528 --- /dev/null +++ b/app/config.py @@ -0,0 +1,92 @@ +from __future__ import annotations + +import os +from pathlib import Path + +try: + from dotenv import load_dotenv + + load_dotenv() +except Exception: + pass + + +PROJECT_ROOT = Path(__file__).resolve().parent.parent +DATA_DIR = PROJECT_ROOT / "data" +RAW_DIR = DATA_DIR / "raw" +PROCESSED_DIR = DATA_DIR / "processed" +REQUIRE_USER_API_KEYS = os.getenv("REQUIRE_USER_API_KEYS", "").strip().lower() in { + "1", + "true", + "yes", +} + +QDRANT_URL = os.getenv("QDRANT_URL", "http://localhost:6333") +QDRANT_API_KEY = os.getenv("QDRANT_API_KEY", "") +QDRANT_COLLECTION = os.getenv("QDRANT_COLLECTION", "stockqa_chunks") +APP_TICKERS = { + ticker.strip().upper() + for ticker in os.getenv("APP_TICKERS", "").split(",") + if ticker.strip() +} + +HF_API_KEY = ( + "" + if REQUIRE_USER_API_KEYS + else ( + os.getenv("HF_API_KEY") + or os.getenv("HG_API_KEY") + or os.getenv("HUGGINGFACEHUB_API_TOKEN") + or "" + ) +) +if HF_API_KEY: + os.environ.setdefault("HF_TOKEN", HF_API_KEY) + os.environ.setdefault("HUGGING_FACE_HUB_TOKEN", HF_API_KEY) + +EMBEDDING_MODEL = os.getenv( + "EMBEDDING_MODEL", + "BAAI/bge-m3", +) +EMBEDDING_DIM = int(os.getenv("EMBEDDING_DIM", "1024")) +EMBEDDING_PROVIDER = "hf_api" +HF_INFERENCE_PROVIDER = os.getenv("HF_INFERENCE_PROVIDER", "hf-inference") +EMBEDDING_API_URL = os.getenv("EMBEDDING_API_URL", "") +EMBEDDING_API_TIMEOUT = float(os.getenv("EMBEDDING_API_TIMEOUT", "60")) +EMBEDDING_API_RETRIES = int(os.getenv("EMBEDDING_API_RETRIES", "3")) +EMBEDDING_API_RETRY_BACKOFF = float(os.getenv("EMBEDDING_API_RETRY_BACKOFF", "2")) +EMBEDDING_BATCH_SIZE = int(os.getenv("EMBEDDING_BATCH_SIZE", "8")) +EMBEDDING_CACHE_ENABLED = os.getenv("EMBEDDING_CACHE_ENABLED", "1").strip().lower() not in { + "0", + "false", + "no", +} +EMBEDDING_CACHE_PATH = Path(os.getenv("EMBEDDING_CACHE_PATH", str(PROCESSED_DIR / "embedding_cache.sqlite3"))) + +GEMINI_API_KEY = "" if REQUIRE_USER_API_KEYS else os.getenv("GEMINI_API_KEY", "") +GEMINI_MODEL = os.getenv("GEMINI_MODEL", "gemini-2.5-flash") + +DEFAULT_CHUNK_SIZE = int(os.getenv("CHUNK_SIZE", "1200")) +DEFAULT_CHUNK_OVERLAP = int(os.getenv("CHUNK_OVERLAP", "180")) +DEFAULT_TOP_K = int(os.getenv("TOP_K", "5")) + +RERANK_ENABLED = os.getenv("RERANK_ENABLED", "1").strip().lower() not in {"0", "false", "no"} +RERANK_MODEL = os.getenv("RERANK_MODEL", "BAAI/bge-reranker-v2-m3") +RERANK_CANDIDATE_LIMIT = int(os.getenv("RERANK_CANDIDATE_LIMIT", "8")) +RERANK_BATCH_SIZE = int(os.getenv("RERANK_BATCH_SIZE", "16")) +RERANK_API_URL = os.getenv( + "RERANK_API_URL", + f"https://router.huggingface.co/hf-inference/models/{RERANK_MODEL}", +) +RERANK_API_TIMEOUT = float(os.getenv("RERANK_API_TIMEOUT", "60")) +RERANK_API_RETRIES = int(os.getenv("RERANK_API_RETRIES", "2")) +RERANK_API_RETRY_BACKOFF = float(os.getenv("RERANK_API_RETRY_BACKOFF", "1")) + +CONTEXT_COMPRESSION_ENABLED = os.getenv("CONTEXT_COMPRESSION_ENABLED", "1").strip().lower() not in { + "0", + "false", + "no", +} +CONTEXT_MAX_SENTENCES_PER_CHUNK = int(os.getenv("CONTEXT_MAX_SENTENCES_PER_CHUNK", "5")) +CONTEXT_MAX_CHARS_PER_CHUNK = int(os.getenv("CONTEXT_MAX_CHARS_PER_CHUNK", "1400")) +CONTEXT_MIN_SENTENCE_CHARS = int(os.getenv("CONTEXT_MIN_SENTENCE_CHARS", "32")) diff --git a/app/context_compression.py b/app/context_compression.py new file mode 100644 index 0000000000000000000000000000000000000000..ed05072bf95176f68b40f3a4b37cc94c7a27ea89 --- /dev/null +++ b/app/context_compression.py @@ -0,0 +1,131 @@ +from __future__ import annotations + +import re +import unicodedata + +from app.config import ( + CONTEXT_COMPRESSION_ENABLED, + CONTEXT_MAX_CHARS_PER_CHUNK, + CONTEXT_MAX_SENTENCES_PER_CHUNK, + CONTEXT_MIN_SENTENCE_CHARS, +) +from app.schemas import RetrievedChunk + + +FINANCE_TERMS = { + "doanh", + "thu", + "loi", + "nhuan", + "lnst", + "eps", + "roe", + "roa", + "bien", + "lai", + "no", + "vay", + "tang", + "giam", + "gia", + "muc", + "tieu", + "khuyen", + "nghi", + "co", + "phieu", + "rsi", + "macd", + "volume", + "khoi", + "luong", + "thanh", + "khoan", +} + +STOPWORDS = { + "anh", + "bao", + "cho", + "co", + "cua", + "khong", + "hay", + "la", + "mot", + "nhung", + "the", + "thi", + "trong", + "va", + "ve", + "voi", +} + + +def normalize_text(text: str) -> str: + decomposed = unicodedata.normalize("NFD", str(text).lower()) + without_accents = "".join(char for char in decomposed if unicodedata.category(char) != "Mn") + return re.sub(r"\s+", " ", without_accents).strip() + + +def tokens(text: str) -> set[str]: + return { + token + for token in re.findall(r"[\w]+", normalize_text(text), flags=re.UNICODE) + if len(token) > 2 and token not in STOPWORDS + } + + +def sentence_candidates(text: str) -> list[str]: + normalized = re.sub(r"\s+", " ", str(text)).strip() + if not normalized: + return [] + pieces = re.split(r"(?<=[.!?。!?])\s+|\n+|(?<=;)\s+", normalized) + return [piece.strip(" -:\t") for piece in pieces if len(piece.strip()) >= CONTEXT_MIN_SENTENCE_CHARS] + + +def compact_text(text: str, limit: int) -> str: + compact = " ".join(str(text).split()) + if len(compact) <= limit: + return compact + return compact[: limit - 3].rstrip() + "..." + + +def sentence_score(sentence: str, query_tokens: set[str], ticker: str) -> float: + sentence_tokens = tokens(sentence) + if not sentence_tokens: + return 0.0 + + overlap = len(sentence_tokens & query_tokens) + finance_overlap = len(sentence_tokens & FINANCE_TERMS) + score = overlap * 2.0 + finance_overlap * 0.35 + if ticker and ticker.lower() in sentence.lower(): + score += 1.0 + if re.search(r"\d", sentence): + score += 0.5 + return score + + +def compress_chunk_text(query: str, chunk: RetrievedChunk) -> str: + if not CONTEXT_COMPRESSION_ENABLED: + return chunk.text + + sentences = sentence_candidates(chunk.text) + if not sentences: + return compact_text(chunk.text, CONTEXT_MAX_CHARS_PER_CHUNK) + + query_tokens = tokens(query) + ranked = sorted( + enumerate(sentences), + key=lambda item: sentence_score(item[1], query_tokens, chunk.ticker), + reverse=True, + ) + selected_indexes = sorted( + index for index, sentence in ranked[:CONTEXT_MAX_SENTENCES_PER_CHUNK] if sentence_score(sentence, query_tokens, chunk.ticker) > 0 + ) + if not selected_indexes: + return compact_text(chunk.text, CONTEXT_MAX_CHARS_PER_CHUNK) + + compressed = " ".join(sentences[index] for index in selected_indexes) + return compact_text(compressed, CONTEXT_MAX_CHARS_PER_CHUNK) diff --git a/app/embedding_cache.py b/app/embedding_cache.py new file mode 100644 index 0000000000000000000000000000000000000000..36dba18ba7f4a0fc6e7531f2fec9bb01604be2e5 --- /dev/null +++ b/app/embedding_cache.py @@ -0,0 +1,79 @@ +from __future__ import annotations + +import hashlib +import json +import sqlite3 +from pathlib import Path + + +class EmbeddingCache: + def __init__(self, path: Path) -> None: + self.path = path + self.path.parent.mkdir(parents=True, exist_ok=True) + self._connection = sqlite3.connect(self.path) + self._connection.execute( + """ + CREATE TABLE IF NOT EXISTS embeddings ( + cache_key TEXT PRIMARY KEY, + vector_json TEXT NOT NULL, + created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP + ) + """ + ) + self._connection.commit() + + def close(self) -> None: + self._connection.close() + + def get_many(self, keys: list[str]) -> dict[str, list[float]]: + if not keys: + return {} + + found: dict[str, list[float]] = {} + for start in range(0, len(keys), 900): + chunk = keys[start : start + 900] + placeholders = ",".join("?" for _ in chunk) + rows = self._connection.execute( + f"SELECT cache_key, vector_json FROM embeddings WHERE cache_key IN ({placeholders})", + chunk, + ).fetchall() + for cache_key, vector_json in rows: + found[str(cache_key)] = [float(value) for value in json.loads(vector_json)] + return found + + def set_many(self, values: dict[str, list[float]]) -> None: + if not values: + return + + rows = [ + (cache_key, json.dumps(vector, separators=(",", ":"))) + for cache_key, vector in values.items() + ] + self._connection.executemany( + """ + INSERT OR REPLACE INTO embeddings (cache_key, vector_json) + VALUES (?, ?) + """, + rows, + ) + self._connection.commit() + + +def embedding_cache_key( + text: str, + *, + provider: str, + model: str, + dim: int, + api_url: str, +) -> str: + payload = "\n".join( + [ + f"provider={provider}", + f"model={model}", + f"dim={dim}", + f"api_url={api_url}", + text, + ] + ) + return hashlib.sha256(payload.encode("utf-8")).hexdigest() diff --git a/app/embeddings.py b/app/embeddings.py new file mode 100644 index 0000000000000000000000000000000000000000..0925172c47ea8fb8167da0c4c86107bddd906640 --- /dev/null +++ b/app/embeddings.py @@ -0,0 +1,115 @@ +from __future__ import annotations + +import math +import time +from functools import lru_cache + +import httpx +from huggingface_hub import InferenceClient + +from app.config import ( + EMBEDDING_API_RETRIES, + EMBEDDING_API_RETRY_BACKOFF, + EMBEDDING_API_TIMEOUT, + EMBEDDING_API_URL, + EMBEDDING_DIM, + EMBEDDING_MODEL, + EMBEDDING_PROVIDER, + HF_INFERENCE_PROVIDER, +) +from app.runtime_auth import get_hf_api_key + + +class EmbeddingModel: + def __init__(self) -> None: + self.dim = EMBEDDING_DIM + self.provider = EMBEDDING_PROVIDER + + def encode(self, texts: list[str]) -> list[list[float]]: + if self.provider != "hf_api": + raise RuntimeError("Local embedding providers are disabled. Use Hugging Face API only.") + return self._api_embedding(texts) + + def _api_embedding(self, texts: list[str]) -> list[list[float]]: + api_key = get_hf_api_key() + if not api_key: + raise RuntimeError("Enter a Hugging Face token to use API embeddings") + if not texts: + return [] + + client_kwargs = {"api_key": api_key, "timeout": EMBEDDING_API_TIMEOUT} + if EMBEDDING_API_URL: + client = InferenceClient(model=EMBEDDING_API_URL, **client_kwargs) + model = None + else: + client = InferenceClient(provider=HF_INFERENCE_PROVIDER, **client_kwargs) + model = EMBEDDING_MODEL + + payload = self._feature_extraction_with_retry(client, texts, model) + if hasattr(payload, "tolist"): + payload = payload.tolist() + + vectors = self._coerce_api_vectors(payload, expected_count=len(texts)) + return [self._normalize_vector(vector) for vector in vectors] + + def _feature_extraction_with_retry(self, client: InferenceClient, texts: list[str], model: str | None): + attempts = max(1, EMBEDDING_API_RETRIES) + last_error: Exception | None = None + for attempt in range(1, attempts + 1): + try: + return client.feature_extraction(texts, model=model) + except (httpx.TimeoutException, httpx.TransportError) as exc: + last_error = exc + if attempt == attempts: + break + time.sleep(EMBEDDING_API_RETRY_BACKOFF * attempt) + raise RuntimeError( + "Hugging Face embedding request timed out. " + "Try lowering EMBEDDING_BATCH_SIZE or increasing EMBEDDING_API_TIMEOUT." + ) from last_error + + def _coerce_api_vectors(self, payload, expected_count: int) -> list[list[float]]: + if not isinstance(payload, list): + raise RuntimeError(f"Unexpected embedding API response: {type(payload).__name__}") + if expected_count == 1 and self._is_vector(payload): + return [self._fit_dimension([float(value) for value in payload])] + if len(payload) != expected_count: + raise RuntimeError(f"Expected {expected_count} embeddings, received {len(payload)}") + + vectors = [] + for item in payload: + if self._is_vector(item): + vectors.append(self._fit_dimension([float(value) for value in item])) + elif isinstance(item, list) and item and all(self._is_vector(token_vector) for token_vector in item): + vectors.append(self._fit_dimension(self._mean_pool(item))) + else: + raise RuntimeError("Unexpected embedding vector shape from API") + return vectors + + def _is_vector(self, value) -> bool: + return isinstance(value, list) and all(isinstance(item, int | float) for item in value) + + def _mean_pool(self, token_vectors: list[list[float]]) -> list[float]: + width = len(token_vectors[0]) + pooled = [] + for index in range(width): + pooled.append(sum(float(vector[index]) for vector in token_vectors) / len(token_vectors)) + return pooled + + def _fit_dimension(self, vector: list[float]) -> list[float]: + if len(vector) == self.dim: + return vector + if len(vector) > self.dim: + return vector[: self.dim] + return vector + [0.0] * (self.dim - len(vector)) + + def _normalize_vector(self, vector: list[float]) -> list[float]: + norm = math.sqrt(sum(value * value for value in vector)) + if norm == 0: + return vector + return [value / norm for value in vector] + + +@lru_cache(maxsize=1) +def get_embedding_model() -> EmbeddingModel: + return EmbeddingModel() diff --git a/app/evaluate.py b/app/evaluate.py new file mode 100644 index 0000000000000000000000000000000000000000..79af3e9d801a65619bd2b3100e97ba7f2c392465 --- /dev/null +++ b/app/evaluate.py @@ -0,0 +1,1019 @@ +from __future__ import annotations + +import argparse +import json +import math +import os +import re +import statistics +import time +from datetime import datetime, timezone +from pathlib import Path + +from app.config import DATA_DIR, GEMINI_API_KEY, GEMINI_MODEL +from app.multimodal import multimodal_artifacts +from app.rag import answer_question +from app.retriever import hybrid_retrieve + + +EVAL_CASES_PATH = DATA_DIR / "evaluation" / "eval_cases.json" +EVAL_REPORT_DIR = DATA_DIR / "evaluation" / "reports" +SUGGESTED_QUESTIONS_PATH = DATA_DIR / "processed" / "q&a.json" +TICKER_PATTERN = re.compile(r"\b[A-Z]{2,5}\b") +NUMBER_PATTERN = re.compile(r"[-+]?\d+(?:[.,]\d+)*(?:%|x)?") +STOPWORDS = { + "a", + "an", + "and", + "are", + "as", + "at", + "be", + "by", + "cho", + "co", + "cua", + "da", + "duoc", + "gi", + "hay", + "khi", + "khong", + "la", + "mot", + "neu", + "nhung", + "noi", + "nua", + "or", + "tai", + "the", + "thi", + "this", + "to", + "tren", + "tu", + "va", + "ve", + "voi", +} + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Evaluate retrieval, generation and performance.") + parser.add_argument("--cases", default=str(EVAL_CASES_PATH), help="Path to evaluation cases JSON.") + parser.add_argument("--top-k", type=int, default=5, help="Top-k retrieval cutoff.") + parser.add_argument( + "--repeats", + type=int, + default=3, + help="Number of repeated runs for latency measurements.", + ) + parser.add_argument( + "--output-dir", + default=str(EVAL_REPORT_DIR), + help="Directory to write JSON and Markdown reports.", + ) + parser.add_argument( + "--eval-model", + default=None, + help="Optional DeepEval model name. Defaults to GEMINI_MODEL when GEMINI_API_KEY is set.", + ) + parser.add_argument( + "--deepeval-threshold", + type=float, + default=0.5, + help="Passing threshold for DeepEval metrics.", + ) + parser.add_argument( + "--include-reason", + action="store_true", + help="Include DeepEval metric reasons in the JSON/Markdown reports.", + ) + return parser.parse_args() + + +def normalize_text(text: str) -> str: + return " ".join(str(text).lower().replace("\n", " ").split()) + + +def tokenize(text: str) -> list[str]: + cleaned = [] + current = [] + for char in normalize_text(text): + if char.isalnum(): + current.append(char) + else: + if current: + cleaned.append("".join(current)) + current = [] + if current: + cleaned.append("".join(current)) + return cleaned + + +def informative_tokens(text: str) -> list[str]: + return [token for token in tokenize(text) if len(token) > 2 and token not in STOPWORDS] + + +def token_set(text: str) -> set[str]: + return set(informative_tokens(text)) + + +def overlap_score(candidate: str, reference: str) -> float: + reference_tokens = token_set(reference) + if not reference_tokens: + return 0.0 + candidate_tokens = token_set(candidate) + return len(candidate_tokens & reference_tokens) / len(reference_tokens) + + +def mean_or_zero(values: list[float]) -> float: + if not values: + return 0.0 + return round(statistics.mean(values), 3) + + +def numeric_values(results: list[dict], key: str) -> list[float]: + values = [] + for result in results: + value = result.get(key) + if isinstance(value, (int, float)) and not isinstance(value, bool): + values.append(float(value)) + return values + + +def preview(text: str, limit: int = 180) -> str: + compact = " ".join(str(text).split()) + if len(compact) <= limit: + return compact + return compact[: limit - 3] + "..." + + +def percentile(values: list[float], pct: float) -> float: + if not values: + return 0.0 + ordered = sorted(values) + if len(ordered) == 1: + return ordered[0] + rank = pct * (len(ordered) - 1) + lower = math.floor(rank) + upper = math.ceil(rank) + if lower == upper: + return ordered[lower] + fraction = rank - lower + return ordered[lower] * (1 - fraction) + ordered[upper] * fraction + + +def latency_summary(latencies_ms: list[float]) -> dict: + if not latencies_ms: + return {"count": 0, "avg_ms": 0.0, "p95_ms": 0.0, "min_ms": 0.0, "max_ms": 0.0} + return { + "count": len(latencies_ms), + "avg_ms": round(statistics.mean(latencies_ms), 2), + "p95_ms": round(percentile(latencies_ms, 0.95), 2), + "min_ms": round(min(latencies_ms), 2), + "max_ms": round(max(latencies_ms), 2), + } + + +def infer_case_ticker(question: str) -> str | None: + for match in TICKER_PATTERN.findall(question.upper()): + if match in {"HPG", "FPT", "VCB"}: + return match + return None + + +def build_default_eval_cases(limit: int = 9) -> list[dict]: + payload = {} + if SUGGESTED_QUESTIONS_PATH.exists(): + payload = json.loads(SUGGESTED_QUESTIONS_PATH.read_text(encoding="utf-8")) + + questions = [] + for group in payload.get("suggested_questions", []): + if not isinstance(group, dict): + continue + for question in group.get("questions", []): + if isinstance(question, str) and question.strip(): + questions.append(question.strip()) + + if not questions: + questions = [ + "Tom tat nhanh co phieu HPG hien tai", + "FPT co nhung dong luc tang truong nao?", + "VCB co diem manh va rui ro gi?", + ] + + cases = [] + for index, question in enumerate(questions[:limit], start=1): + cases.append( + { + "id": f"auto_{index:03d}", + "question": question, + "ticker": infer_case_ticker(question), + "expected_chunks": [], + "expected_answer_keywords": [], + "expected_source_keywords": [], + } + ) + return cases + + +def write_default_eval_cases(path: Path, cases: list[dict]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + payload = { + "note": ( + "Auto-generated starter cases. Add expected_chunks, expected_answer_keywords " + "and expected_source_keywords for stricter evaluation." + ), + "cases": cases, + } + path.write_text(json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8") + + +def load_eval_cases(path: Path) -> list[dict]: + if not path.exists(): + cases = build_default_eval_cases() + write_default_eval_cases(path, cases) + print(f"Created starter eval cases: {path.as_posix()}") + return cases + + payload = json.loads(path.read_text(encoding="utf-8")) + if isinstance(payload, dict): + cases = payload.get("cases", []) + elif isinstance(payload, list): + cases = payload + else: + cases = [] + return [case for case in cases if isinstance(case, dict)] + + +def chunk_matches_expectation(chunk, expectation: dict) -> bool: + source_path = normalize_text(getattr(chunk, "source_path", "")) + heading_path = normalize_text(" ".join(getattr(chunk, "heading_path", []))) + text = normalize_text(getattr(chunk, "text", "")) + modality = normalize_text(getattr(chunk, "modality", "")) + ticker = normalize_text(getattr(chunk, "ticker", "")) + scope = normalize_text(getattr(chunk, "scope", "") or getattr(chunk, "ticker", "")) + + if expectation.get("ticker") and ticker != normalize_text(expectation["ticker"]): + return False + if expectation.get("scope") and scope != normalize_text(expectation["scope"]): + return False + if expectation.get("modality") and modality != normalize_text(expectation["modality"]): + return False + if expectation.get("source_path_contains"): + if normalize_text(expectation["source_path_contains"]) not in source_path: + return False + if expectation.get("heading_contains_any"): + if not any(normalize_text(value) in heading_path for value in expectation["heading_contains_any"]): + return False + if expectation.get("text_contains_any"): + if not any(normalize_text(value) in text for value in expectation["text_contains_any"]): + return False + return True + + +def expected_context_text(case: dict) -> str: + values = [] + for key in ["expected_output", "expected_answer", "reference_answer"]: + if case.get(key): + values.append(str(case[key])) + values.extend(str(value) for value in case.get("expected_answer_keywords", [])) + values.extend(str(value) for value in case.get("expected_source_keywords", [])) + for expectation in case.get("expected_chunks", []): + values.extend(str(value) for value in expectation.get("text_contains_any", [])) + values.extend(str(value) for value in expectation.get("heading_contains_any", [])) + if expectation.get("source_path_contains"): + values.append(str(expectation["source_path_contains"])) + return " ".join(values) + + +def chunk_relevance_flags(case: dict, chunks) -> list[bool]: + expectations = case.get("expected_chunks", []) + if expectations: + return [ + any(chunk_matches_expectation(chunk, expectation) for expectation in expectations) + for chunk in chunks + ] + + reference = expected_context_text(case) or case["question"] + return [overlap_score(chunk.text, reference) > 0 for chunk in chunks] + + +def evaluate_retrieval_case(case: dict, top_k: int) -> dict: + started_at = time.perf_counter() + try: + hits = hybrid_retrieve(case["question"], top_k=top_k, ticker=case.get("ticker")) + latency_ms = (time.perf_counter() - started_at) * 1000 + except Exception as exc: # noqa: BLE001 + return { + "case_id": case["id"], + "question": case["question"], + "ticker": case.get("ticker"), + "latency_ms": round((time.perf_counter() - started_at) * 1000, 2), + "top_k": top_k, + "expected_evidence_count": len(case.get("expected_chunks", [])), + "matched_evidence_count": 0, + "strict_evaluation": bool(case.get("expected_chunks")), + "recall_at_k": None, + "precision_at_k": None, + "hit_rate_at_k": None, + "first_relevant_rank": None, + "mrr": None, + "qualitative_top_chunks": [], + "error": str(exc), + } + + expectations = case.get("expected_chunks", []) + matched_ranks: list[int] = [] + for expectation in expectations: + matched_rank = None + for rank, chunk in enumerate(hits, start=1): + if chunk_matches_expectation(chunk, expectation): + matched_rank = rank + break + if matched_rank is not None: + matched_ranks.append(matched_rank) + + expected_count = len(expectations) + strict_evaluation = expected_count > 0 + first_relevant_rank = min(matched_ranks) if matched_ranks else None + coverage = (len(matched_ranks) / expected_count) if strict_evaluation else None + relevance_flags = chunk_relevance_flags(case, hits) + if strict_evaluation and relevance_flags and first_relevant_rank is None: + first_relevant_rank = next((rank for rank, flag in enumerate(relevance_flags, start=1) if flag), None) + + evaluated_flags = relevance_flags[:top_k] + precision_denominator = len(evaluated_flags) + relevant_retrieved_count = sum(evaluated_flags) if strict_evaluation else 0 + precision_at_k = ( + relevant_retrieved_count / precision_denominator + if strict_evaluation and precision_denominator > 0 + else (0.0 if strict_evaluation else None) + ) + hit_rate_at_k = ( + float(any(evaluated_flags)) if strict_evaluation else None + ) + + return { + "case_id": case["id"], + "question": case["question"], + "ticker": case.get("ticker"), + "latency_ms": round(latency_ms, 2), + "top_k": top_k, + "strict_evaluation": strict_evaluation, + "expected_evidence_count": expected_count, + "matched_evidence_count": len(matched_ranks), + "relevant_retrieved_count": relevant_retrieved_count if strict_evaluation else None, + "recall_at_k": round(coverage, 3) if coverage is not None else None, + "precision_at_k": round(precision_at_k, 3) if precision_at_k is not None else None, + "hit_rate_at_k": round(hit_rate_at_k, 3) if hit_rate_at_k is not None else None, + "first_relevant_rank": first_relevant_rank, + "mrr": round(1 / first_relevant_rank, 4) if first_relevant_rank else None, + "qualitative_top_chunks": [ + { + "rank": rank, + "score": round(chunk.score, 4), + "ticker": chunk.ticker, + "scope": chunk.scope, + "modality": chunk.modality, + "source_path": chunk.source_path, + "heading_path": chunk.heading_path, + "preview": preview(chunk.text), + } + for rank, chunk in enumerate(hits[: min(3, len(hits))], start=1) + ], + } + + +def source_keyword_match_score(sources: list[dict], expected_keywords: list[str]) -> float: + if not expected_keywords: + return 1.0 + haystack = normalize_text( + " ".join( + f"{source.get('source_path', '')} {source.get('artifact_path', '')} {source.get('url', '')}" + for source in sources + ) + ) + hits = sum(1 for keyword in expected_keywords if normalize_text(keyword) in haystack) + return hits / len(expected_keywords) + + +def context_grounding_score(answer: str, retrieved_chunks) -> float: + answer_tokens = informative_tokens(answer) + if not answer_tokens: + return 0.0 + context_tokens: set[str] = set() + for chunk in retrieved_chunks: + context_tokens.update(informative_tokens(chunk.text)) + if not context_tokens: + return 0.0 + shared = sum(1 for token in answer_tokens if token in context_tokens) + return shared / len(answer_tokens) + + +def lexical_answer_relevancy_score(question: str, answer: str) -> float: + question_tokens = token_set(question) + answer_tokens = token_set(answer) + if not question_tokens: + return 0.0 + return len(question_tokens & answer_tokens) / len(question_tokens) + + +def extract_numbers(text: str) -> list[float]: + values = [] + for raw_value in NUMBER_PATTERN.findall(text): + parsed = parse_number_token(raw_value) + if parsed is None: + continue + values.append(parsed) + return values + + +def parse_number_token(raw_value: str) -> float | None: + value = raw_value.strip().rstrip("%xX") + if not value: + return None + + sign = "" + if value[0] in {"+", "-"}: + sign, value = value[0], value[1:] + + if "," in value and "." in value: + last_comma = value.rfind(",") + last_dot = value.rfind(".") + if last_comma > last_dot: + normalized = value.replace(".", "").replace(",", ".") + else: + normalized = value.replace(",", "") + elif "," in value: + parts = value.split(",") + if len(parts) > 2 or len(parts[-1]) == 3: + normalized = "".join(parts) + else: + normalized = value.replace(",", ".") + elif "." in value: + parts = value.split(".") + if len(parts) > 2: + normalized = "".join(parts) + elif len(parts[-1]) == 3 and len(parts[0]) > 2: + normalized = "".join(parts) + else: + normalized = value + else: + normalized = value + + try: + return float(f"{sign}{normalized}") + except ValueError: + return None + + +def numerical_accuracy_score(answer: str, case: dict) -> float | None: + expected_numbers = case.get("expected_numbers") + if expected_numbers is None: + return None + else: + expected_numbers = [float(value) for value in expected_numbers] + if not expected_numbers: + return None + + answer_numbers = extract_numbers(answer) + if not answer_numbers: + return 0.0 + + matched = 0 + remaining = answer_numbers[:] + for expected in expected_numbers: + tolerance = max(abs(expected) * 0.01, 0.01) + match_index = next( + (index for index, actual in enumerate(remaining) if math.isclose(actual, expected, abs_tol=tolerance)), + None, + ) + if match_index is not None: + matched += 1 + remaining.pop(match_index) + return matched / len(expected_numbers) + + +def citation_accuracy_score(sources: list[dict], case: dict) -> float | None: + expected_keywords = case.get("expected_source_keywords", []) + if expected_keywords: + return source_keyword_match_score(sources, expected_keywords) + + expectations = case.get("expected_chunks", []) + if not expectations: + return None + + matched = 0 + for expectation in expectations: + expected_path = normalize_text(expectation.get("source_path_contains", "")) + expected_text = normalize_text(expectation.get("text_contains", "")) + for source in sources: + source_text = normalize_text( + " ".join( + str(source.get(key, "")) + for key in ["source_path", "artifact_path", "url", "title", "structure_type"] + ) + ) + if expected_path and expected_path in source_text: + matched += 1 + break + if expected_text and expected_text in source_text: + matched += 1 + break + return matched / len(expectations) + + +def evaluate_generation_case(case: dict, top_k: int) -> dict: + started_at = time.perf_counter() + try: + result = answer_question(case["question"], ticker=case.get("ticker"), top_k=top_k) + latency_ms = (time.perf_counter() - started_at) * 1000 + except Exception as exc: # noqa: BLE001 + return { + "case_id": case["id"], + "question": case["question"], + "ticker": case.get("ticker"), + "latency_ms": round((time.perf_counter() - started_at) * 1000, 2), + "source_count": 0, + "numerical_accuracy": None, + "citation_accuracy": None, + "has_sources": False, + "answer_preview": "", + "source_preview": [], + "error": str(exc), + } + + answer = str(result.get("answer", "")) + sources = list(result.get("sources", [])) + numerical_accuracy = numerical_accuracy_score(answer, case) + citation_accuracy = citation_accuracy_score(sources, case) + fallback_metrics = {} + if not case.get("expected_numbers") and not extract_numbers(expected_output_for_deepeval(case)): + fallback_metrics["numerical_accuracy"] = "no_expected_numbers" + if not case.get("expected_source_keywords") and not case.get("expected_chunks"): + fallback_metrics["citation_accuracy"] = "sources_present_without_expected_citations" + + return { + "case_id": case["id"], + "question": case["question"], + "ticker": case.get("ticker"), + "latency_ms": round(latency_ms, 2), + "source_count": len(sources), + "numerical_accuracy": round(numerical_accuracy, 3) if numerical_accuracy is not None else None, + "citation_accuracy": round(citation_accuracy, 3) if citation_accuracy is not None else None, + "fallback_metrics": fallback_metrics, + "has_sources": bool(sources), + "answer": answer, + "answer_preview": preview(answer, limit=260), + "source_preview": [ + { + "ticker": source.get("ticker"), + "modality": source.get("modality"), + "structure_type": source.get("structure_type"), + "source_path": source.get("source_path"), + "url": source.get("url"), + } + for source in sources[:3] + ], + } + + +def expected_output_for_deepeval(case: dict) -> str: + return str( + case.get("expected_output") + or case.get("expected_answer") + or case.get("reference_answer") + or expected_context_text(case) + or "" + ) + + +def deepeval_metric_kwargs(eval_model: str | None, threshold: float, include_reason: bool) -> dict: + kwargs = { + "threshold": threshold, + "include_reason": include_reason, + } + if eval_model and not eval_model.lower().startswith("gemini"): + kwargs["model"] = eval_model + elif GEMINI_API_KEY: + from deepeval.models.llms.gemini_model import GeminiModel + + kwargs["model"] = GeminiModel( + model=eval_model or GEMINI_MODEL, + api_key=GEMINI_API_KEY or None, + temperature=0, + ) + return kwargs + + +def effective_eval_model(eval_model: str | None) -> str | None: + if eval_model: + return eval_model + if GEMINI_API_KEY: + return GEMINI_MODEL + return None + + +def measure_deepeval_metric(metric, test_case) -> dict: + metric.measure(test_case) + return { + "score": round(float(getattr(metric, "score", 0.0) or 0.0), 3), + "reason": getattr(metric, "reason", None), + "success": bool(getattr(metric, "success", False)), + } + + +def apply_deepeval_generation_scores( + case: dict, + result: dict, + chunks, + answer: str, + eval_model: str | None, + threshold: float, + include_reason: bool, +) -> dict: + try: + from deepeval.metrics import AnswerRelevancyMetric, FaithfulnessMetric + from deepeval.test_case import LLMTestCase + except ImportError as exc: + result["deepeval_error"] = ( + "DeepEval is not installed. Install the optional dependency with " + "`pip install deepeval` before running evaluation." + ) + result["deepeval_import_error"] = str(exc) + return result + + retrieval_context = [chunk.text for chunk in chunks] + test_case_kwargs = { + "input": case["question"], + "actual_output": answer, + "retrieval_context": retrieval_context, + } + expected_output = expected_output_for_deepeval(case) + if expected_output: + test_case_kwargs["expected_output"] = expected_output + test_case = LLMTestCase(**test_case_kwargs) + metric_kwargs = deepeval_metric_kwargs(eval_model, threshold, include_reason) + scores = {} + + for key, metric_cls in [ + ("answer_relevancy", AnswerRelevancyMetric), + ("faithfulness", FaithfulnessMetric), + ]: + try: + scores[key] = measure_deepeval_metric(metric_cls(**metric_kwargs), test_case) + result[key] = scores[key]["score"] + result.get("fallback_metrics", {}).pop(key, None) + except Exception as exc: # noqa: BLE001 + scores[key] = {"error": str(exc)} + + result["deepeval"] = scores + return result + + +def evaluate_generation_case_with_deepeval( + case: dict, + top_k: int, + eval_model: str | None, + threshold: float, + include_reason: bool, +) -> dict: + result = evaluate_generation_case(case, top_k) + if result.get("error"): + return result + + try: + retrieval_chunks = hybrid_retrieve(case["question"], top_k=top_k, ticker=case.get("ticker")) + except Exception as exc: # noqa: BLE001 + result["deepeval_error"] = str(exc) + return result + + fallback_scores = { + "faithfulness": round(context_grounding_score(str(result.get("answer", "")), retrieval_chunks), 3), + "answer_relevancy": round( + lexical_answer_relevancy_score(case["question"], str(result.get("answer", ""))), + 3, + ), + } + result.setdefault("fallback_metrics", {}).update( + { + "faithfulness": "lexical_context_grounding", + "answer_relevancy": "question_answer_token_overlap", + } + ) + + result = apply_deepeval_generation_scores( + case, + result, + retrieval_chunks, + str(result.get("answer", "")), + eval_model, + threshold, + include_reason, + ) + for metric_name, fallback_score in fallback_scores.items(): + if result.get(metric_name) is None: + result[metric_name] = fallback_score + return result + + +def evaluate_multimodal_readiness(cases: list[dict]) -> dict: + tickers = sorted({str(case.get("ticker", "")).upper() for case in cases if case.get("ticker")}) + inventory = [] + lookup_latencies = [] + for ticker in tickers: + started_at = time.perf_counter() + artifacts = multimodal_artifacts(ticker) + lookup_latencies.append((time.perf_counter() - started_at) * 1000) + inventory.append( + { + "ticker": ticker, + "has_chart": bool(artifacts["chart"]), + "table_count": len(artifacts["tables"]), + "pdf_count": len(artifacts["pdfs"]), + } + ) + + ready_all_three = sum( + 1 + for item in inventory + if item["has_chart"] and item["table_count"] > 0 and item["pdf_count"] > 0 + ) + return { + "tickers_evaluated": tickers, + "inventory": inventory, + "tickers_with_chart_table_pdf": ready_all_three, + "artifact_lookup_latency_ms": latency_summary(lookup_latencies), + "note": ( + "Current system surfaces image/csv/pdf artifacts in UI and sources, " + "but retrieval is still primarily text+dense/BM25 rather than true multimodal embedding." + ), + } + + +def evaluate_performance(cases: list[dict], top_k: int, repeats: int) -> dict: + retrieval_latencies = [] + generation_latencies = [] + retrieval_errors: dict[str, int] = {} + generation_errors: dict[str, int] = {} + per_case = [] + + for case in cases: + case_retrieval_latencies = [] + case_generation_latencies = [] + for _ in range(repeats): + try: + retrieval_started_at = time.perf_counter() + hybrid_retrieve(case["question"], top_k=top_k, ticker=case.get("ticker")) + case_retrieval_latencies.append((time.perf_counter() - retrieval_started_at) * 1000) + except Exception as exc: # noqa: BLE001 + message = str(exc) + retrieval_errors[message] = retrieval_errors.get(message, 0) + 1 + + try: + generation_started_at = time.perf_counter() + answer_question(case["question"], ticker=case.get("ticker"), top_k=top_k) + case_generation_latencies.append((time.perf_counter() - generation_started_at) * 1000) + except Exception as exc: # noqa: BLE001 + message = str(exc) + generation_errors[message] = generation_errors.get(message, 0) + 1 + + retrieval_latencies.extend(case_retrieval_latencies) + generation_latencies.extend(case_generation_latencies) + per_case.append( + { + "case_id": case["id"], + "question": case["question"], + "ticker": case.get("ticker"), + "retrieval_latency_ms": latency_summary(case_retrieval_latencies), + "generation_latency_ms": latency_summary(case_generation_latencies), + } + ) + + return { + "retrieval_latency_ms": latency_summary(retrieval_latencies), + "generation_latency_ms": latency_summary(generation_latencies), + "retrieval_failure_count": sum(retrieval_errors.values()), + "generation_failure_count": sum(generation_errors.values()), + "retrieval_errors": retrieval_errors, + "generation_errors": generation_errors, + "multimodal_readiness": evaluate_multimodal_readiness(cases), + "per_case": per_case, + } + + +def summarize_retrieval(results: list[dict]) -> dict: + total = len(results) + error_count = sum(1 for result in results if result.get("error")) + strict_results = [result for result in results if result.get("strict_evaluation")] + mrr_values = numeric_values(strict_results, "mrr") + recall_at_k_values = numeric_values(strict_results, "recall_at_k") + precision_at_k_values = numeric_values(strict_results, "precision_at_k") + hit_rate_at_k_values = numeric_values(strict_results, "hit_rate_at_k") + latencies = [result["latency_ms"] for result in results] + return { + "case_count": total, + "error_count": error_count, + "strict_case_count": len(strict_results), + "smoke_case_count": total - len(strict_results), + "mean_mrr": round(statistics.mean(mrr_values), 3) if mrr_values else None, + "recall_at_k": mean_or_zero(recall_at_k_values) if recall_at_k_values else None, + "precision_at_k": mean_or_zero(precision_at_k_values) if precision_at_k_values else None, + "hit_rate_at_k": mean_or_zero(hit_rate_at_k_values) if hit_rate_at_k_values else None, + "latency_ms": latency_summary(latencies), + } + + +def summarize_generation(results: list[dict]) -> dict: + total = len(results) + error_count = sum(1 for result in results if result.get("error")) + answer_relevancy_values = numeric_values(results, "answer_relevancy") + faithfulness_values = numeric_values(results, "faithfulness") + numerical_accuracy_values = numeric_values(results, "numerical_accuracy") + citation_accuracy_values = numeric_values(results, "citation_accuracy") + latencies = [result["latency_ms"] for result in results] + return { + "case_count": total, + "error_count": error_count, + "latency_ms": latency_summary(latencies), + "faithfulness": mean_or_zero(faithfulness_values) if faithfulness_values else None, + "answer_relevancy": mean_or_zero(answer_relevancy_values) if answer_relevancy_values else None, + "numerical_case_count": len(numerical_accuracy_values), + "numerical_accuracy": mean_or_zero(numerical_accuracy_values) if numerical_accuracy_values else None, + "citation_case_count": len(citation_accuracy_values), + "citation_accuracy": mean_or_zero(citation_accuracy_values) if citation_accuracy_values else None, + } + + +def build_markdown_report(report: dict) -> str: + retrieval_summary = report["retrieval"]["summary"] + generation_summary = report["generation"]["summary"] + performance_summary = report["performance"] + + lines = [ + "# Danh gia he thong", + "", + f"Thoi gian tao bao cao: {report['generated_at_utc']}", + f"DeepEval model: {report.get('eval_model') or 'N/A'}", + f"Top-k: {report['top_k']}", + f"So case: {report['case_count']}", + "", + "## 4.x.1 Danh gia Retrieval", + f"- Strict cases: {retrieval_summary.get('strict_case_count', 0)}", + f"- Smoke-only cases: {retrieval_summary.get('smoke_case_count', 0)}", + f"- Mean MRR: {retrieval_summary['mean_mrr']}", + f"- Recall@{report['top_k']}: {retrieval_summary['recall_at_k']}", + f"- Precision@{report['top_k']}: {retrieval_summary['precision_at_k']}", + f"- Hit Rate@{report['top_k']}: {retrieval_summary['hit_rate_at_k']}", + "", + "### Qualitative examples", + ] + + for case in report["retrieval"]["cases"][:3]: + lines.extend( + [ + f"- Case `{case['case_id']}`: {case['question']}", + ( + f" strict={case.get('strict_evaluation', False)} " + f"mrr={case['mrr']} recall_at_k={case.get('recall_at_k', 'N/A')} " + f"precision_at_k={case.get('precision_at_k', 'N/A')} " + f"hit_rate_at_k={case.get('hit_rate_at_k', 'N/A')}" + ), + ] + ) + if case.get("deepeval_error"): + lines.append(f" deepeval_error: {preview(case['deepeval_error'], 220)}") + for chunk in case["qualitative_top_chunks"]: + lines.append( + f" top{chunk['rank']}: {chunk['source_path']} | score={chunk['score']} | {chunk['preview']}" + ) + + lines.extend( + [ + "", + "## 4.x.2 Danh gia Generation", + f"- Faithfulness: {generation_summary.get('faithfulness', 'N/A')}", + f"- Answer relevancy: {generation_summary.get('answer_relevancy', 'N/A')}", + f"- Numerical cases: {generation_summary.get('numerical_case_count', 0)}", + f"- Numerical accuracy: {generation_summary.get('numerical_accuracy', 'N/A')}", + f"- Citation cases: {generation_summary.get('citation_case_count', 0)}", + f"- Citation accuracy: {generation_summary.get('citation_accuracy', 'N/A')}", + "", + "### Qualitative examples", + ] + ) + + for case in report["generation"]["cases"][:3]: + lines.extend( + [ + ( + f"- Case `{case['case_id']}`: source_count={case['source_count']} " + f"numerical_accuracy={case.get('numerical_accuracy', 'N/A')} " + f"citation_accuracy={case.get('citation_accuracy', 'N/A')}" + ), + f" answer: {case['answer_preview']}", + ] + ) + lines.append( + f" answer_relevancy={case.get('answer_relevancy', 'N/A')} " + f"faithfulness={case.get('faithfulness', 'N/A')}" + ) + if case.get("deepeval_error"): + lines.append(f" deepeval_error: {preview(case['deepeval_error'], 220)}") + for metric_name in ["answer_relevancy", "faithfulness"]: + reason = case.get("deepeval", {}).get(metric_name, {}).get("reason") + if reason: + lines.append(f" {metric_name}_reason: {preview(reason, 220)}") + for source in case["source_preview"]: + lines.append( + f" source: {source['source_path']} | modality={source['modality']} | ticker={source['ticker']}" + ) + + multimodal = performance_summary["multimodal_readiness"] + lines.extend( + [ + "", + "## 4.x.3 Danh gia hieu nang he thong", + f"- Retrieval P95 latency (ms): {performance_summary['retrieval_latency_ms']['p95_ms']}", + f"- Answer P95 latency (ms): {performance_summary['generation_latency_ms']['p95_ms']}", + f"- Retrieval failures: {performance_summary.get('retrieval_failure_count', 0)}", + f"- Answer failures: {performance_summary.get('generation_failure_count', 0)}", + f"- Tickers co du chart + table + pdf: {multimodal['tickers_with_chart_table_pdf']}", + f"- Artifact lookup latency avg/p95 (ms): {multimodal['artifact_lookup_latency_ms']['avg_ms']} / {multimodal['artifact_lookup_latency_ms']['p95_ms']}", + f"- Ghi chu multimodal: {multimodal['note']}", + "", + "### Multimodal inventory", + ] + ) + + for item in multimodal["inventory"]: + lines.append( + f"- {item['ticker']}: chart={item['has_chart']} tables={item['table_count']} pdfs={item['pdf_count']}" + ) + + return "\n".join(lines) + "\n" + + +def ensure_output_dir(path: Path) -> None: + path.mkdir(parents=True, exist_ok=True) + + +def main() -> int: + args = parse_args() + cases_path = Path(args.cases) + output_dir = Path(args.output_dir) + ensure_output_dir(output_dir) + + cases = load_eval_cases(cases_path) + retrieval_cases = [evaluate_retrieval_case(case, args.top_k) for case in cases] + generation_cases = [ + evaluate_generation_case_with_deepeval( + case, + args.top_k, + args.eval_model, + args.deepeval_threshold, + args.include_reason, + ) + for case in cases + ] + performance = evaluate_performance(cases, args.top_k, args.repeats) + + report = { + "generated_at_utc": datetime.now(timezone.utc).isoformat(), + "eval_model": effective_eval_model(args.eval_model), + "deepeval_threshold": args.deepeval_threshold, + "top_k": args.top_k, + "repeats": args.repeats, + "case_count": len(cases), + "cases_path": cases_path.as_posix(), + "retrieval": { + "summary": summarize_retrieval(retrieval_cases), + "cases": retrieval_cases, + }, + "generation": { + "summary": summarize_generation(generation_cases), + "cases": generation_cases, + }, + "performance": performance, + } + + timestamp = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ") + json_path = output_dir / f"evaluation_report_{timestamp}.json" + md_path = output_dir / f"evaluation_report_{timestamp}.md" + + json_path.write_text(json.dumps(report, ensure_ascii=False, indent=2), encoding="utf-8") + md_path.write_text(build_markdown_report(report), encoding="utf-8") + + print(f"Saved JSON report: {json_path.as_posix()}") + print(f"Saved Markdown report: {md_path.as_posix()}") + print(json.dumps(report["retrieval"]["summary"], ensure_ascii=False, indent=2)) + print(json.dumps(report["generation"]["summary"], ensure_ascii=False, indent=2)) + print(json.dumps(report["performance"]["retrieval_latency_ms"], ensure_ascii=False, indent=2)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/app/intermarket.py b/app/intermarket.py new file mode 100644 index 0000000000000000000000000000000000000000..7bfc88b37bf976b39c2d27b6b23cf232c560131f --- /dev/null +++ b/app/intermarket.py @@ -0,0 +1,134 @@ +from __future__ import annotations + +import re +from dataclasses import dataclass +from pathlib import Path + +from app.config import RAW_DIR + + +@dataclass(frozen=True) +class IntermarketIndex: + symbol: str + name: str + price: str + change: str + change_percent: str + updated_at: str + + @property + def trend(self) -> str: + if self.change.strip().startswith("-") or self.change_percent.strip().startswith("(-"): + return "decreasing" + if self.change.strip().startswith("+") or self.change_percent.strip().startswith("(+"): + return "increasing" + return "flat/unclear" + + +TARGET_INDEX_SYMBOLS = ["US 30", "Hang Seng", "Shanghai", "Nikkei 225", "KOSPI"] +INTERMARKET_QUERY_TERMS = [ + "lien thi truong", + "the gioi", + "thi truong chung", + "vn-index", + "vnindex", + "dow", + "hang seng", + "shanghai", + "shang hai", + "nikkei", + "kospi", +] + + +def normalize_query(text: str) -> str: + import unicodedata + + decomposed = unicodedata.normalize("NFD", str(text).lower()) + without_accents = "".join(char for char in decomposed if unicodedata.category(char) != "Mn") + return re.sub(r"\s+", " ", without_accents).strip() + + +def is_intermarket_query(question: str) -> bool: + normalized = normalize_query(question) + return any(term in normalized for term in INTERMARKET_QUERY_TERMS) + + +def latest_world_market_text() -> tuple[Path | None, str]: + market_dir = RAW_DIR / "text" / "market" + if not market_dir.exists(): + return None, "" + + files = sorted( + market_dir.glob("*world_market*.txt"), + key=lambda path: path.stat().st_mtime, + reverse=True, + ) + if not files: + return None, "" + + path = files[0] + return path, path.read_text(encoding="utf-8", errors="ignore") + + +def parse_world_market_indices(text: str) -> list[IntermarketIndex]: + lines = [line.strip() for line in str(text).splitlines() if line.strip()] + indexes: list[IntermarketIndex] = [] + for symbol in TARGET_INDEX_SYMBOLS: + try: + start = lines.index(symbol) + except ValueError: + continue + + row = lines[start : start + 10] + if len(row) < 6: + continue + updated_at = next((value for value in row[6:] if re.match(r"^\d{1,2}:\d{2}:\d{2}$", value)), "") + indexes.append( + IntermarketIndex( + symbol=row[0], + name=row[2], + price=row[3], + change=row[4], + change_percent=row[5], + updated_at=updated_at, + ) + ) + return indexes + + +def build_intermarket_context(question: str) -> tuple[str, dict | None]: + if not is_intermarket_query(question): + return "", None + + path, text = latest_world_market_text() + indexes = parse_world_market_indices(text) + if not path or not indexes: + return "", None + + lines = [ + "Intermarket context from 24HMoney world market page.", + ( + "Use this as macro/market-background context only, not as a stock ticker. " + "For Vietnam market inference, treat broad weakness in US/Asia indexes as risk-off pressure, " + "and broad strength as supportive sentiment." + ), + ] + for item in indexes: + lines.append( + f"- {item.symbol} ({item.name}): {item.price}, {item.change} {item.change_percent}, " + f"trend={item.trend}, updated={item.updated_at}" + ) + + source = { + "score": 1.0, + "ticker": "", + "scope": "market", + "modality": "text", + "structure_type": "intermarket_context", + "heading_path": [], + "source_path": path.relative_to(RAW_DIR.parent).as_posix(), + "url": "https://24hmoney.vn/world", + "artifact_path": path.relative_to(RAW_DIR.parent).as_posix(), + } + return "\n".join(lines), source diff --git a/app/keyword_search.py b/app/keyword_search.py new file mode 100644 index 0000000000000000000000000000000000000000..c83fb62cbae4d7731e57566b866e25999b6528b1 --- /dev/null +++ b/app/keyword_search.py @@ -0,0 +1,133 @@ +from __future__ import annotations + +import json +import math +import re + +from app.config import PROCESSED_DIR +from app.schemas import RetrievedChunk + + +TOKEN_PATTERN = re.compile(r"[\wÀ-ỹ]+", flags=re.UNICODE) + + +def tokenize(text: str) -> list[str]: + return TOKEN_PATTERN.findall(text.lower()) + + +def load_processed_chunks(ticker: str | None = None) -> list[dict]: + chunk_root = PROCESSED_DIR / "chunks" + if not chunk_root.exists(): + return [] + + paths = [] + if ticker: + paths = [chunk_root / ticker.upper() / "chunks.jsonl"] + else: + paths = list(chunk_root.glob("*/chunks.jsonl")) + if (chunk_root / "market" / "chunks.jsonl").exists(): + paths = [path for path in paths if path.parent.name != "MARKET"] + + chunks: list[dict] = [] + for path in paths: + if not path.exists(): + continue + with path.open("r", encoding="utf-8") as handle: + for line in handle: + line = line.strip() + if not line: + continue + try: + chunks.append(json.loads(line)) + except json.JSONDecodeError: + continue + return chunks + + +def fallback_bm25_scores(query_tokens: list[str], corpus_tokens: list[list[str]]) -> list[float]: + if not corpus_tokens: + return [] + + doc_count = len(corpus_tokens) + doc_freq: dict[str, int] = {} + for tokens in corpus_tokens: + for token in set(tokens): + doc_freq[token] = doc_freq.get(token, 0) + 1 + + scores: list[float] = [] + for tokens in corpus_tokens: + token_count = len(tokens) or 1 + term_freq: dict[str, int] = {} + for token in tokens: + term_freq[token] = term_freq.get(token, 0) + 1 + + score = 0.0 + for token in query_tokens: + if token not in term_freq: + continue + idf = math.log((doc_count - doc_freq.get(token, 0) + 0.5) / (doc_freq.get(token, 0) + 0.5) + 1) + score += idf * (term_freq[token] / token_count) + scores.append(score) + return scores + + +def bm25_scores(query: str, chunks: list[dict]) -> list[float]: + query_tokens = tokenize(query) + corpus_tokens = [tokenize(str(chunk.get("text", ""))) for chunk in chunks] + if not query_tokens or not corpus_tokens: + return [0.0] * len(chunks) + + try: + from rank_bm25 import BM25Okapi + + bm25 = BM25Okapi(corpus_tokens) + return [float(score) for score in bm25.get_scores(query_tokens)] + except Exception: + return fallback_bm25_scores(query_tokens, corpus_tokens) + + +def normalize_scores(scores: list[float]) -> list[float]: + if not scores: + return [] + min_score = min(scores) + max_score = max(scores) + if max_score == min_score: + return [0.0 if max_score == 0 else 1.0 for _ in scores] + return [(score - min_score) / (max_score - min_score) for score in scores] + + +def chunk_to_retrieved(chunk: dict, score: float) -> RetrievedChunk: + source_path = str(chunk.get("source_path", "")) + metadata = dict(chunk.get("metadata") or {}) + raw_ticker = str(chunk.get("ticker", "")) + scope = str(chunk.get("scope") or metadata.get("scope") or raw_ticker or "") + if raw_ticker.upper() == "MARKET" or "world_market" in source_path or "/market/" in source_path.replace("\\", "/"): + raw_ticker = "" + scope = "market" + return RetrievedChunk( + id=str(chunk.get("id", "")), + text=str(chunk.get("text", "")), + score=score, + ticker=raw_ticker, + modality=str(chunk.get("modality", "")), + source_path=source_path, + structure_type=str(chunk.get("structure_type", "")), + heading_path=list(chunk.get("heading_path") or []), + metadata=metadata, + scope=scope, + ) + + +def keyword_search(query: str, top_k: int, ticker: str | None = None) -> list[RetrievedChunk]: + chunks = load_processed_chunks(ticker=ticker) + scores = normalize_scores(bm25_scores(query, chunks)) + ranked = sorted( + zip(chunks, scores), + key=lambda item: item[1], + reverse=True, + ) + return [ + chunk_to_retrieved(chunk, score) + for chunk, score in ranked[:top_k] + if score > 0 + ] diff --git a/app/market_snapshot.py b/app/market_snapshot.py new file mode 100644 index 0000000000000000000000000000000000000000..7441911367a80a8a520522ab60eb131838ae37c9 --- /dev/null +++ b/app/market_snapshot.py @@ -0,0 +1,262 @@ +from __future__ import annotations + +import csv +import re +import unicodedata +from dataclasses import dataclass +from pathlib import Path + +from app.config import RAW_DIR + + +@dataclass(frozen=True) +class MarketSnapshot: + ticker: str + path: Path + row: dict[str, str] + + +def normalize_query(text: str) -> str: + decomposed = unicodedata.normalize("NFD", str(text).lower()) + without_accents = "".join(char for char in decomposed if unicodedata.category(char) != "Mn") + return re.sub(r"\s+", " ", without_accents).strip() + + +def is_market_snapshot_query(question: str) -> bool: + normalized = normalize_query(question) + terms = [ + "gia", + "price", + "hom nay", + "hien tai", + "dang tang", + "dang giam", + "khoi luong", + "thanh khoan", + ] + return any(term in normalized for term in terms) + + +def is_quick_summary_query(question: str) -> bool: + normalized = normalize_query(question) + terms = ["tom tat", "summary", "quick", "nhanh", "tong quan"] + return any(term in normalized for term in terms) + + +def load_latest_market_snapshot(ticker: str | None) -> MarketSnapshot | None: + if not ticker: + return None + + path = RAW_DIR / "csv" / ticker.upper() / "stock_overview_timeseries.csv" + if not path.exists(): + return None + + with path.open("r", encoding="utf-8-sig", newline="") as handle: + rows = list(csv.DictReader(handle)) + if not rows: + return None + + return MarketSnapshot(ticker=ticker.upper(), path=path, row=rows[-1]) + + +def build_market_snapshot_context(snapshot: MarketSnapshot | None) -> str: + if not snapshot: + return "" + + row = snapshot.row + fields = [ + ("ticker", "Ticker"), + ("company_name", "Company"), + ("exchange", "Exchange"), + ("price", "Current price"), + ("change", "Change"), + ("change_percent", "Change percent"), + ("volume", "Volume"), + ("updated_at", "Updated at"), + ("day_high", "Day high"), + ("day_low", "Day low"), + ("reference_price", "Reference price"), + ("price_ceiling", "Ceiling price"), + ("price_floor", "Floor price"), + ("foreign_buy_volume", "Foreign buy volume"), + ("foreign_sell_volume", "Foreign sell volume"), + ("bid_1_price", "Best bid"), + ("offer_1_price", "Best offer"), + ] + + lines = [ + f"Market snapshot source: {snapshot.path.relative_to(RAW_DIR.parent).as_posix()}", + ( + "Use this snapshot as the primary source for questions about current/latest price, " + "change, volume or liquidity. Do not confuse current price with analyst target price." + ), + ] + for key, label in fields: + value = row.get(key) + if value: + lines.append(f"{label}: {value}") + return "\n".join(lines) + + +def compact_text(text: str, limit: int = 160) -> str: + compact = " ".join(str(text).split()) + if len(compact) <= limit: + return compact + return compact[: limit - 3].rstrip() + "..." + + +def sentence_candidates(text: str) -> list[str]: + normalized = re.sub(r"\s+", " ", str(text)).strip() + if not normalized: + return [] + return [ + sentence.strip(" -:\t") + for sentence in re.split(r"(?<=[.!?])\s+|\n+", normalized) + if len(sentence.strip()) >= 40 + ] + + +def normalized_contains_any(text: str, terms: list[str]) -> bool: + normalized = normalize_query(text) + return any(term in normalized for term in terms) + + +def extract_bullets(chunks, terms: list[str], limit: int = 2) -> list[str]: + bullets: list[str] = [] + seen: set[str] = set() + for chunk in chunks or []: + for sentence in sentence_candidates(getattr(chunk, "text", "")): + if not normalized_contains_any(sentence, terms): + continue + bullet = compact_text(sentence) + key = bullet.lower() + if key in seen: + continue + seen.add(key) + bullets.append(bullet) + if len(bullets) >= limit: + return bullets + return bullets + + +def source_bullets(snapshot: MarketSnapshot | None, chunks, limit: int = 4) -> list[str]: + sources: list[str] = [] + seen: set[str] = set() + if snapshot: + snapshot_source = snapshot.path.relative_to(RAW_DIR.parent).as_posix() + sources.append(f"{snapshot_source} | https://24hmoney.vn/stock/{snapshot.ticker}") + seen.add(snapshot_source) + + for chunk in chunks or []: + source_path = str(getattr(chunk, "source_path", "")) + if not source_path or source_path in seen: + continue + seen.add(source_path) + sources.append(source_path) + if len(sources) >= limit: + break + return sources + + +def compressed_quick_summary_context(snapshot: MarketSnapshot | None, chunks, max_chunks: int = 3) -> str: + lines: list[str] = [] + if snapshot: + lines.append("Market snapshot:") + lines.append(build_market_snapshot_context(snapshot)) + + selected_chunks = list(chunks or [])[:max_chunks] + if selected_chunks: + lines.append("Retrieved context previews:") + for index, chunk in enumerate(selected_chunks, start=1): + heading = " > ".join(getattr(chunk, "heading_path", []) or []) + lines.extend( + [ + f"[{index}] ticker={getattr(chunk, 'ticker', '')} score={getattr(chunk, 'score', 0):.4f}", + f"source={getattr(chunk, 'source_path', '')}", + f"heading={heading}", + compact_text(getattr(chunk, "text", ""), limit=900), + ] + ) + return "\n".join(lines) + + +def trend_from_snapshot(row: dict[str, str]) -> str: + change = (row.get("change") or "").strip() + change_percent = (row.get("change_percent") or "").strip() + if change.startswith("-") or change_percent.startswith("-"): + return "Ngắn hạn đang giảm theo snapshot gần nhất." + if change and not change.startswith("0"): + return "Ngắn hạn đang tăng theo snapshot gần nhất." + return "Ngắn hạn đi ngang/chưa có biến động rõ theo snapshot gần nhất." + + +def format_quick_stock_summary(snapshot: MarketSnapshot | None, chunks=None, ticker: str | None = None) -> str: + resolved_ticker = (ticker or (snapshot.ticker if snapshot else "")).upper() + row = snapshot.row if snapshot else {} + price = row.get("price") or "chưa có dữ liệu" + change_percent = row.get("change_percent") or "" + volume = row.get("volume") or "chưa có dữ liệu" + price_line = f"{price} {change_percent}" if change_percent else price + + highlights = extract_bullets( + chunks, + ["tang truong", "trien vong", "khuyen nghi", "loi nhuan", "doanh thu", "dong luc", "tich cuc", "muc tieu", "du an"], + ) + risks = extract_bullets( + chunks, + ["rui ro", "ap luc", "kho khan", "suy giam", "bien dong", "canh tranh", "no", "chi phi"], + ) + + if not highlights: + highlights = [ + compact_text(getattr(chunk, "text", ""), limit=150) + for chunk in (chunks or [])[:2] + if compact_text(getattr(chunk, "text", ""), limit=150) + ] + if not highlights: + highlights = ["Chưa tìm thấy điểm nổi bật rõ ràng trong context retrieved."] + if not risks: + risks = ["Context hiện tại chưa nêu rủi ro cụ thể; cần đọc thêm báo cáo/tin tức liên quan."] + + lines = [ + f"## {resolved_ticker} - Tóm tắt nhanh", + "", + f"- Giá hiện tại: {price_line}", + f"- Thanh khoản: {volume}", + f"- Xu hướng ngắn hạn: {trend_from_snapshot(row)}", + "- Điểm nổi bật:", + ] + lines.extend(f" - {bullet}" for bullet in highlights) + lines.append("- Rủi ro cần lưu ý:") + lines.extend(f" - {bullet}" for bullet in risks) + lines.append("- Nguồn:") + lines.extend(f" - {source}" for source in source_bullets(snapshot, chunks)) + return "\n".join(lines) + + +def format_market_snapshot_answer(snapshot: MarketSnapshot) -> str: + row = snapshot.row + ticker = snapshot.ticker + price = row.get("price") or "chưa có" + change = row.get("change") or "" + change_percent = row.get("change_percent") or "" + volume = row.get("volume") or "" + updated_at = row.get("updated_at") or row.get("crawled_at_utc") or "" + company_name = row.get("company_name") or ticker + + direction = "" + if change.strip().startswith("-"): + direction = "giảm" + elif change.strip() and not change.strip().startswith("0"): + direction = "tăng" + + parts = [f"{ticker} ({company_name}) đang có giá {price}."] + if direction or change or change_percent: + movement = " ".join(part for part in [direction, change, change_percent] if part) + parts.append(f"Biến động: {movement}.") + if volume: + parts.append(f"Khối lượng: {volume}.") + if updated_at: + parts.append(f"Dữ liệu cập nhật: {updated_at}.") + parts.append(f"Nguồn: https://24hmoney.vn/stock/{ticker}.") + return "\n\n".join(parts) diff --git a/app/multimodal.py b/app/multimodal.py new file mode 100644 index 0000000000000000000000000000000000000000..72be63b80871d57d605cb574989f3ffa8af0cee6 --- /dev/null +++ b/app/multimodal.py @@ -0,0 +1,85 @@ +from __future__ import annotations + +from pathlib import Path + +from app.config import RAW_DIR + + +def latest_file(paths: list[Path]) -> Path | None: + if not paths: + return None + return max(paths, key=lambda path: path.stat().st_mtime) + + +def latest_chart_image(ticker: str | None) -> Path | None: + if not ticker: + return None + image_dir = RAW_DIR / "images" / ticker.upper() + if not image_dir.exists(): + return None + candidates = [ + path + for path in image_dir.glob("*") + if path.is_file() + and path.suffix.lower() in {".png", ".jpg", ".jpeg", ".webp"} + and ("chart" in path.stem.lower() or "tradingview" in path.stem.lower()) + ] + return latest_file(candidates) + + +def ticker_tables(ticker: str | None, limit: int = 5) -> list[Path]: + if not ticker: + return [] + csv_dir = RAW_DIR / "csv" / ticker.upper() + if not csv_dir.exists(): + return [] + candidates = [ + path + for path in csv_dir.glob("*.csv") + if path.is_file() and has_meaningful_csv_content(path) + ] + return sorted(candidates, key=lambda path: path.stat().st_mtime, reverse=True)[:limit] + + +def has_meaningful_csv_content(path: Path) -> bool: + try: + rows = path.read_text(encoding="utf-8-sig", errors="ignore").splitlines() + except OSError: + return False + non_empty_rows = [row for row in rows if row.strip()] + if len(non_empty_rows) < 2: + return False + return any(row.count(",") >= 2 for row in non_empty_rows[:3]) + + +def artifact_label(path: Path) -> str: + stem = path.stem.lower() + if "stock_overview_timeseries" in stem: + return "Bảng giá và giao dịch" + if "financial_document" in stem or "financial_documents" in stem: + return "Báo cáo tài chính" + if "analysis_report" in stem: + return "Báo cáo phân tích" + if "ticker_news" in stem or "news_events" in stem: + return "Tin tức và sự kiện" + if "chart" in stem or "tradingview" in stem: + return "Biểu đồ TradingView" + return path.name + + +def ticker_pdfs(ticker: str | None, limit: int = 5) -> list[Path]: + if not ticker: + return [] + pdf_dir = RAW_DIR / "pdf" / ticker.upper() + if not pdf_dir.exists(): + return [] + candidates = [path for path in pdf_dir.glob("*.pdf") if path.is_file()] + return sorted(candidates, key=lambda path: path.stat().st_mtime, reverse=True)[:limit] + + +def multimodal_artifacts(ticker: str | None) -> dict[str, object]: + return { + "chart": latest_chart_image(ticker), + "tables": ticker_tables(ticker), + "pdfs": ticker_pdfs(ticker), + } diff --git a/app/observability.py b/app/observability.py new file mode 100644 index 0000000000000000000000000000000000000000..5c9f675bf3b074a5c9699af460df85be9286a27e --- /dev/null +++ b/app/observability.py @@ -0,0 +1,43 @@ +from __future__ import annotations + +import json +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + +from app.config import DATA_DIR + + +INTERACTION_LOG_PATH = DATA_DIR / "logs" / "chatbot_interactions.jsonl" + + +def append_interaction_log(event: dict[str, Any], path: Path = INTERACTION_LOG_PATH) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + payload = { + "timestamp_utc": datetime.now(timezone.utc).isoformat(), + **event, + } + with path.open("a", encoding="utf-8") as file: + file.write(json.dumps(payload, ensure_ascii=False) + "\n") + + +def load_interaction_logs(path: Path = INTERACTION_LOG_PATH, limit: int | None = None) -> list[dict[str, Any]]: + if not path.exists(): + return [] + + rows: list[dict[str, Any]] = [] + with path.open("r", encoding="utf-8", errors="ignore") as file: + for line in file: + line = line.strip() + if not line: + continue + try: + payload = json.loads(line) + except json.JSONDecodeError: + continue + if isinstance(payload, dict): + rows.append(payload) + + if limit is None: + return rows + return rows[-limit:] diff --git a/app/pages/1_Dashboard.py b/app/pages/1_Dashboard.py new file mode 100644 index 0000000000000000000000000000000000000000..ab529d5d890c5d7f1ae7204413575545ada69db7 --- /dev/null +++ b/app/pages/1_Dashboard.py @@ -0,0 +1,381 @@ +from __future__ import annotations + +import json +import sys +from datetime import datetime +from pathlib import Path + +import pandas as pd +import streamlit as st + +PROJECT_ROOT = Path(__file__).resolve().parents[2] +if str(PROJECT_ROOT) not in sys.path: + sys.path.insert(0, str(PROJECT_ROOT)) + +from app.config import APP_TICKERS, DATA_DIR, PROCESSED_DIR, RAW_DIR +from app.evaluate import EVAL_CASES_PATH, EVAL_REPORT_DIR +from app.observability import INTERACTION_LOG_PATH, load_interaction_logs +from app.ui import ( + inject_app_css, + mascot_data_uri, + mascot_path, + render_disclaimer, +) + +try: + from app.ui import apply_metric_card_style, render_section_header +except ImportError: + def apply_metric_card_style() -> None: + pass + + def render_section_header(label: str, description: str = "") -> None: + st.subheader(label, divider="blue") + if description: + st.caption(description) + + +st.set_page_config(page_title="Bảng điều khiển ChatVNS", page_icon=str(mascot_path("insight")), layout="wide") +inject_app_css() +apply_metric_card_style() + + +def count_files(root: Path, pattern: str = "*") -> int: + if not root.exists(): + return 0 + return sum(1 for path in root.rglob(pattern) if path.is_file()) + + +def count_jsonl_lines(path: Path) -> int: + if not path.exists(): + return 0 + with path.open("r", encoding="utf-8", errors="ignore") as file: + return sum(1 for line in file if line.strip()) + + +def load_json(path: Path) -> dict | list | None: + if not path.exists(): + return None + try: + return json.loads(path.read_text(encoding="utf-8")) + except Exception: + return None + + +def latest_file(root: Path, pattern: str) -> Path | None: + if not root.exists(): + return None + files = [path for path in root.glob(pattern) if path.is_file()] + if not files: + return None + return max(files, key=lambda path: path.stat().st_mtime) + + +def format_time(path: Path | None) -> str: + if not path or not path.exists(): + return "N/A" + return datetime.fromtimestamp(path.stat().st_mtime).strftime("%Y-%m-%d %H:%M") + + +def available_tickers() -> list[str]: + tickers: set[str] = set(APP_TICKERS) + for category in ["html", "text", "csv", "pdf", "images", "metadata"]: + root = RAW_DIR / category + if not root.exists(): + continue + for path in root.iterdir(): + if path.is_dir() and path.name.lower() != "market": + tickers.add(path.name.upper()) + return sorted(tickers) + + +def raw_inventory_frame() -> pd.DataFrame: + rows = [] + categories = ["html", "text", "csv", "pdf", "images", "metadata"] + for ticker in available_tickers(): + row = {"ticker": ticker} + for category in categories: + row[category] = count_files(RAW_DIR / category / ticker) + row["chunks"] = count_jsonl_lines(PROCESSED_DIR / "chunks" / ticker / "chunks.jsonl") + rows.append(row) + + market_chunks = count_jsonl_lines(PROCESSED_DIR / "chunks" / "market" / "chunks.jsonl") + if not market_chunks: + market_chunks = count_jsonl_lines(PROCESSED_DIR / "chunks" / "MARKET" / "chunks.jsonl") + if market_chunks or count_files(RAW_DIR / "text" / "market"): + rows.append( + { + "ticker": "", + "scope": "market", + "html": count_files(RAW_DIR / "html" / "market"), + "text": count_files(RAW_DIR / "text" / "market"), + "csv": count_files(RAW_DIR / "csv" / "market"), + "pdf": count_files(RAW_DIR / "pdf" / "market"), + "images": count_files(RAW_DIR / "images" / "market"), + "metadata": count_files(RAW_DIR / "metadata" / "market"), + "chunks": market_chunks, + } + ) + + return pd.DataFrame(rows) + + +def latest_evaluation_report() -> tuple[Path | None, dict | None]: + if not EVAL_REPORT_DIR.exists(): + return None, None + + files = sorted(EVAL_REPORT_DIR.glob("evaluation_report_*.json"), key=lambda item: item.stat().st_mtime) + for path in reversed(files): + payload = load_json(path) + if isinstance(payload, dict): + return path, payload + return None, None + + +def eval_case_count() -> int: + payload = load_json(EVAL_CASES_PATH) + if isinstance(payload, dict): + cases = payload.get("cases", []) + elif isinstance(payload, list): + cases = payload + else: + cases = [] + return len([case for case in cases if isinstance(case, dict)]) + + +def run_summary() -> dict | None: + path = latest_file(DATA_DIR / "logs" / "crawl_logs", "run_summary_*.json") + payload = load_json(path) if path else None + if isinstance(payload, dict): + payload["_path"] = path + return payload + return None + + +def recent_processed_files(limit: int = 12) -> pd.DataFrame: + root = PROCESSED_DIR / "text" + if not root.exists(): + return pd.DataFrame() + + files = sorted( + [path for path in root.rglob("*.txt") if path.is_file()], + key=lambda path: path.stat().st_mtime, + reverse=True, + )[:limit] + return pd.DataFrame( + [ + { + "scope": path.parent.name.upper(), + "file": path.name, + "updated": format_time(path), + "size_kb": round(path.stat().st_size / 1024, 1), + } + for path in files + ] + ) + + +def metric_value(summary: dict | None, *keys: str, default: str = "N/A") -> str | float: + value = summary + for key in keys: + if not isinstance(value, dict) or key not in value: + return default + value = value[key] + return value + + +def interaction_frame(limit: int = 100) -> pd.DataFrame: + rows = load_interaction_logs(limit=limit) + if not rows: + return pd.DataFrame() + frame = pd.DataFrame(rows) + expected_columns = [ + "timestamp_utc", + "ticker", + "question", + "latency_ms", + "source_count", + "answer_chars", + "top_k", + ] + for column in expected_columns: + if column not in frame.columns: + frame[column] = None + return frame[expected_columns].sort_values("timestamp_utc", ascending=False) + + +def mean_value(frame: pd.DataFrame, column: str) -> float: + if frame.empty or column not in frame: + return 0.0 + values = pd.to_numeric(frame[column], errors="coerce").dropna() + if values.empty: + return 0.0 + return round(float(values.mean()), 2) + + +def render_report_header(path: Path | None, report: dict, label: str) -> None: + if path: + st.caption(f"{label}: `{path.relative_to(PROJECT_ROOT).as_posix()}`") + st.caption(f"Cập nhật: {format_time(path)}") + st.caption(f"Top-k: `{report.get('top_k', 'N/A')}` | Số tình huống: `{report.get('case_count', 'N/A')}`") + + +def render_evaluation_report(path: Path | None, report: dict | None) -> None: + if not report: + st.info("Chưa có báo cáo đánh giá. Chạy `python -m app.evaluate` để tạo báo cáo đầy đủ.") + return + + retrieval = report.get("retrieval", {}).get("summary", {}) + generation = report.get("generation", {}).get("summary", {}) + performance = report.get("performance", {}) + + render_report_header(path, report, "Báo cáo đánh giá") + st.caption(f"Mô hình DeepEval: `{report.get('eval_model') or 'N/A'}`") + + st.write("Truy xuất") + top_k = report.get("top_k", 5) + r1, r2, r3, r4 = st.columns(4) + r1.metric(f"Recall@{top_k}", metric_value(retrieval, "recall_at_k")) + r2.metric(f"Precision@{top_k}", metric_value(retrieval, "precision_at_k")) + r3.metric(f"Hit Rate@{top_k}", metric_value(retrieval, "hit_rate_at_k")) + r4.metric("MRR", metric_value(retrieval, "mean_mrr")) + + st.write("Sinh câu trả lời") + g1, g2 = st.columns(2) + g1.metric("Tính trung thực", metric_value(generation, "faithfulness")) + g2.metric("Độ liên quan câu trả lời", metric_value(generation, "answer_relevancy")) + + st.write("Độ chính xác tài chính") + f1, f2 = st.columns(2) + f1.metric("Độ chính xác số liệu", metric_value(generation, "numerical_accuracy")) + f2.metric("Độ chính xác trích nguồn", metric_value(generation, "citation_accuracy")) + + st.write("Hiệu năng") + p1, p2 = st.columns(2) + p1.metric("Truy xuất p95 (ms)", metric_value(performance, "retrieval_latency_ms", "p95_ms")) + p2.metric("Trả lời p95 (ms)", metric_value(performance, "generation_latency_ms", "p95_ms")) + + retrieval_cases = report.get("retrieval", {}).get("cases", []) + if retrieval_cases: + st.write("Chi tiết tình huống truy xuất") + case_frame = pd.DataFrame(retrieval_cases) + columns = ["case_id", "ticker", "mrr", "recall_at_k", "precision_at_k", "hit_rate_at_k"] + available_columns = [column for column in columns if column in case_frame.columns] + st.dataframe(case_frame[available_columns], use_container_width=True, hide_index=True) + + generation_cases = report.get("generation", {}).get("cases", []) + if generation_cases: + st.write("Chi tiết tình huống sinh câu trả lời") + case_frame = pd.DataFrame(generation_cases) + columns = ["case_id", "ticker", "faithfulness", "answer_relevancy", "numerical_accuracy", "citation_accuracy"] + available_columns = [column for column in columns if column in case_frame.columns] + st.dataframe(case_frame[available_columns], use_container_width=True, hide_index=True) + + +with st.sidebar: + st.image(str(mascot_path("insight")), width=118) + st.title("ChatVNS") + st.caption("Bảng điều khiển chất lượng hệ thống") + st.link_button("💬 Trợ lý phân tích", "/", use_container_width=True) + st.link_button("📊 Bảng điều khiển", "/1_Dashboard", use_container_width=True) + +_dashboard_mascot = mascot_data_uri("insight") +st.markdown( + f""" + + + + Giám sát và chất lượng + Bảng điều khiển hệ thống + Theo dõi dữ liệu đã thu thập, mức độ xử lý, chất lượng truy xuất, câu trả lời và hoạt động chatbot. + + Kho dữ liệu + Đánh giá RAG + Sức khỏe thu thập dữ liệu + + + + """, + unsafe_allow_html=True, +) + +inventory = raw_inventory_frame() +evaluation_path, evaluation_report = latest_evaluation_report() +summary = run_summary() +interactions = interaction_frame() + +total_chunks = int(inventory["chunks"].sum()) if not inventory.empty else 0 +total_raw_files = sum(count_files(RAW_DIR / category) for category in ["html", "text", "csv", "pdf", "images"]) + +col_a, col_b, col_c, col_d, col_e = st.columns(5) +col_a.metric("Mã cổ phiếu", len([ticker for ticker in available_tickers()])) +col_b.metric("Tệp dữ liệu thô", total_raw_files) +col_c.metric("Đoạn đã lập chỉ mục", total_chunks) +col_d.metric("Tình huống đánh giá", eval_case_count()) +col_e.metric("Lượt hội thoại", 0 if interactions.empty else len(interactions)) + +st.divider() + +left, right = st.columns([1.15, 0.85]) + +with left: + render_section_header("Kho dữ liệu", "Số lượng tệp và đoạn dữ liệu theo từng mã cổ phiếu") + if inventory.empty: + st.info("Chưa tìm thấy dữ liệu thô hoặc dữ liệu đã xử lý.") + else: + st.dataframe( + inventory.rename(columns={ + "ticker": "Mã", "scope": "Phạm vi", "html": "HTML", + "text": "Văn bản", "csv": "CSV", "pdf": "PDF", + "images": "Hình ảnh", "metadata": "Siêu dữ liệu", "chunks": "Đoạn dữ liệu", + }), + use_container_width=True, + hide_index=True, + ) + + render_section_header("Văn bản xử lý gần đây", "Các tệp văn bản mới nhất đã đi qua quy trình xử lý") + recent_files = recent_processed_files() + if recent_files.empty: + st.info("Chưa có văn bản đã xử lý.") + else: + st.dataframe( + recent_files.rename(columns={ + "scope": "Phạm vi", "file": "Tệp", "updated": "Cập nhật", "size_kb": "Dung lượng (KB)", + }), + use_container_width=True, + hide_index=True, + ) + + render_section_header("Nhật ký chatbot", "Theo dõi độ trễ và mức độ sử dụng nguồn") + if interactions.empty: + st.info("Chưa có nhật ký tương tác. Hãy hỏi chatbot một câu để bảng điều khiển bắt đầu có dữ liệu.") + else: + l1, l2, l3 = st.columns(3) + l1.metric("Độ trễ TB (ms)", mean_value(interactions, "latency_ms")) + l2.metric("Số nguồn TB", mean_value(interactions, "source_count")) + l3.metric("Độ dài trả lời TB", mean_value(interactions, "answer_chars")) + st.caption(f"Nguồn: `{INTERACTION_LOG_PATH.relative_to(PROJECT_ROOT).as_posix()}`") + st.dataframe( + interactions.head(25).rename(columns={ + "timestamp_utc": "Thời gian UTC", "ticker": "Mã", "question": "Câu hỏi", + "latency_ms": "Độ trễ (ms)", "source_count": "Số nguồn", + "answer_chars": "Độ dài trả lời", "top_k": "Top-k", + }), + use_container_width=True, + hide_index=True, + ) + +with right: + render_section_header("Đánh giá RAG", "Chất lượng truy xuất, sinh câu trả lời và số liệu tài chính") + render_evaluation_report(evaluation_path, evaluation_report) + + render_section_header("Lần thu thập gần nhất", "Trạng thái phiên thu thập dữ liệu mới nhất") + if not summary: + st.info("Chưa có tổng kết phiên thu thập dữ liệu.") + else: + summary_path = summary.get("_path") + st.caption(f"Nguồn: `{summary_path.relative_to(PROJECT_ROOT).as_posix()}`") + display_summary = {key: value for key, value in summary.items() if key != "_path"} + st.json(display_summary, expanded=False) + + +render_disclaimer() diff --git a/app/pages/__pycache__/1_Dashboard.cpython-311.pyc b/app/pages/__pycache__/1_Dashboard.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..712f1aa112d03cd48a5c121a8dae0992b1e66eef Binary files /dev/null and b/app/pages/__pycache__/1_Dashboard.cpython-311.pyc differ diff --git a/app/pages/__pycache__/1_Dashboard.cpython-314.pyc b/app/pages/__pycache__/1_Dashboard.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..ae42437fdac9dea315064ae3ae7e25fe25183e45 Binary files /dev/null and b/app/pages/__pycache__/1_Dashboard.cpython-314.pyc differ diff --git a/app/pipeline.py b/app/pipeline.py new file mode 100644 index 0000000000000000000000000000000000000000..1dd3d036b5c49fb4103e15fcd8c77896059e6961 --- /dev/null +++ b/app/pipeline.py @@ -0,0 +1,243 @@ +from __future__ import annotations + +import argparse +import json +import subprocess +import sys +import time +from pathlib import Path + +from app.processing import process_raw_data_with_summary +from app.qa_suggestions import export_suggested_questions_json +from app.vector_store import index_chunks + + +PROJECT_ROOT = Path(__file__).resolve().parent.parent + + +class PipelineConsole: + def __init__(self) -> None: + self._started_at = time.perf_counter() + + def intro(self, command: str) -> None: + print("=" * 72) + print("ChatVNS pipeline") + print(f"Command: {command}") + print("Flow: collect raw data -> process/chunk -> embed/index Qdrant -> Streamlit") + print("=" * 72) + + def stage(self, number: int, total: int, title: str, detail: str | None = None) -> float: + print() + print(f"[{number}/{total}] {title}") + if detail: + print(f" {detail}") + return time.perf_counter() + + def done(self, started_at: float, detail: str | None = None) -> None: + duration = time.perf_counter() - started_at + suffix = f" | {detail}" if detail else "" + print(f" DONE in {duration:.1f}s{suffix}") + + def note(self, message: str) -> None: + print(f" {message}") + + def finish(self) -> None: + total = time.perf_counter() - self._started_at + print() + print(f"Pipeline ready in {total:.1f}s") + + +def format_counts(values: dict[str, int]) -> str: + if not values: + return "none" + return ", ".join(f"{key}={value}" for key, value in values.items()) + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Run ChatVNS pipeline commands.") + subparsers = parser.add_subparsers(dest="command", required=True) + + collect = subparsers.add_parser("collect", help="Collect raw data into data/raw.") + add_collect_args(collect) + + index = subparsers.add_parser("index", help="Process raw data and index chunks into Qdrant.") + index.add_argument("--skip-qdrant", action="store_true") + + chat = subparsers.add_parser("chat", help="Open the Streamlit chatbot.") + chat.add_argument("--server-port", default="8501") + + all_cmd = subparsers.add_parser("all", help="Collect, index, then open Streamlit.") + add_collect_args(all_cmd) + all_cmd.add_argument("--server-port", default="8501") + all_cmd.add_argument("--skip-qdrant", action="store_true") + return parser.parse_args() + + +def add_collect_args(parser: argparse.ArgumentParser) -> None: + parser.add_argument("--tickers", nargs="+", default=["HPG", "FPT", "VCB"]) + parser.add_argument("--include-news", action="store_true") + parser.add_argument("--include-charts", action="store_true") + parser.add_argument("--max-reports", type=int, default=5) + parser.add_argument("--delay-seconds", type=float, default=1.5) + parser.add_argument("--timeout", type=int, default=30) + parser.add_argument("--min-crawl-interval-minutes", type=float, default=30) + + +def run_collect(args: argparse.Namespace) -> None: + command = [ + sys.executable, + str(PROJECT_ROOT / "bot-collect-data" / "crawl_raw_data.py"), + "--tickers", + *args.tickers, + "--max-reports", + str(args.max_reports), + "--delay-seconds", + str(args.delay_seconds), + "--timeout", + str(args.timeout), + "--min-crawl-interval-minutes", + str(args.min_crawl_interval_minutes), + ] + if args.include_news: + command.append("--include-news") + if args.include_charts: + command.append("--include-charts") + subprocess.run(command, cwd=PROJECT_ROOT, check=True) + + +def ensure_qdrant() -> None: + subprocess.run( + ["docker", "compose", "up", "-d", "qdrant"], + cwd=PROJECT_ROOT, + check=True, + ) + + +def run_index() -> None: + suggestions = export_suggested_questions_json() + print(f" Synced Q&A suggestions: groups={len(suggestions)}") + chunks, summary = process_raw_data_with_summary() + print( + " Processed raw data: " + f"documents={summary['document_count']}, " + f"chunks={summary['chunk_count']}, " + f"tickers={', '.join(summary['tickers']) or 'none'}" + ) + print(f" Modalities: {format_counts(summary['modalities'])}") + print(f" Chunks by ticker: {format_counts(summary['chunks_by_ticker'])}") + print(f" Chunks by scope: {format_counts(summary['chunks_by_scope'])}") + print(f" Structures: {format_counts(summary['structures'])}") + + def on_index_progress(event: dict) -> None: + action = "Embedding" if event["stage"] == "embedding" else "Upserted" + cache_detail = "" + if event["stage"] == "embedding" and "cache_hits" in event: + cache_detail = f", cache_hits={event['cache_hits']}, api_chunks={event['cache_misses']}" + print( + " " + f"{action} batch {event['batch_number']}/{event['total_batches']} " + f"({event['batch_size']} chunks, indexed={event['indexed_so_far']}/{event['total_chunks']}" + f"{cache_detail})" + ) + + indexed_count = index_chunks(chunks, progress_callback=on_index_progress) + print( + json.dumps( + { + "chunk_count": len(chunks), + "indexed_count": indexed_count, + "documents_by_ticker": summary["documents_by_ticker"], + "chunks_by_ticker": summary["chunks_by_ticker"], + "documents_by_scope": summary["documents_by_scope"], + "chunks_by_scope": summary["chunks_by_scope"], + }, + ensure_ascii=False, + indent=2, + ) + ) + + +def run_streamlit(server_port: str) -> None: + subprocess.run( + [ + sys.executable, + "-m", + "streamlit", + "run", + str(PROJECT_ROOT / "app" / "streamlit_app.py"), + "--server.port", + server_port, + ], + cwd=PROJECT_ROOT, + check=True, + ) + + +def main() -> int: + args = parse_args() + console = PipelineConsole() + console.intro(args.command) + if args.command == "collect": + stage = console.stage( + 1, + 1, + "Collect raw data", + f"tickers={', '.join(args.tickers)} | news={args.include_news} | charts={args.include_charts}", + ) + run_collect(args) + console.done(stage) + console.finish() + return 0 + if args.command == "index": + if not args.skip_qdrant: + stage = console.stage(1, 2, "Start Qdrant", "docker compose up -d qdrant") + ensure_qdrant() + console.done(stage) + stage = console.stage(2 if not args.skip_qdrant else 1, 2 if not args.skip_qdrant else 1, "Process and index", "parse raw files -> chunk -> embed -> upsert") + run_index() + console.done(stage) + console.finish() + return 0 + if args.command == "chat": + stage = console.stage(1, 1, "Open Streamlit", f"http://localhost:{args.server_port}") + console.note("Launching Streamlit app. Keep this terminal open while using the UI.") + console.done(stage, f"url=http://localhost:{args.server_port}") + run_streamlit(args.server_port) + return 0 + if args.command == "all": + if not args.skip_qdrant: + stage = console.stage(1, 4, "Start Qdrant", "docker compose up -d qdrant") + ensure_qdrant() + console.done(stage) + collect_stage = console.stage( + 2 if not args.skip_qdrant else 1, + 4 if not args.skip_qdrant else 3, + "Collect raw data", + f"tickers={', '.join(args.tickers)} | news={args.include_news} | charts={args.include_charts}", + ) + run_collect(args) + console.done(collect_stage) + index_stage = console.stage( + 3 if not args.skip_qdrant else 2, + 4 if not args.skip_qdrant else 3, + "Process and index", + "parse raw files -> chunk -> embed -> upsert", + ) + run_index() + console.done(index_stage) + streamlit_stage = console.stage( + 4 if not args.skip_qdrant else 3, + 4 if not args.skip_qdrant else 3, + "Open Streamlit", + f"http://localhost:{args.server_port}", + ) + console.note("Launching Streamlit app. Keep this terminal open while using the UI.") + console.done(streamlit_stage, f"url=http://localhost:{args.server_port}") + console.finish() + run_streamlit(args.server_port) + return 0 + return 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/app/processing/__init__.py b/app/processing/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..94ffbc38ff53c2aa5d2767767e4affce35e7e1ab --- /dev/null +++ b/app/processing/__init__.py @@ -0,0 +1,67 @@ +from __future__ import annotations + +from app.processing.chunking import chunk_documents, chunk_blocks, split_block_by_tokens +from app.processing.documents import iter_raw_documents, modality_from_path, text_dump_exists, ticker_from_path +from app.processing.outputs import summarize_processed_data, write_processed_outputs +from app.processing.readers import load_metadata_for_artifact, read_csv_rows, read_pdf_text, read_raw_file +from app.processing.structures import HTMLStructureParser, parse_document_structures, parse_text_blocks +from app.processing.text_utils import ( + clean_document_text, + detokenize, + is_noise_line, + looks_like_heading, + looks_like_table, + looks_like_widget, + normalize_text, + rows_to_table_text, + stable_id, + token_count, + tokenize, +) + + +def process_raw_data(): + documents = iter_raw_documents() + chunks = chunk_documents(documents) + write_processed_outputs(documents, chunks) + return chunks + + +def process_raw_data_with_summary(): + documents = iter_raw_documents() + chunks = chunk_documents(documents) + write_processed_outputs(documents, chunks) + return chunks, summarize_processed_data(documents, chunks) + + +__all__ = [ + "HTMLStructureParser", + "chunk_blocks", + "chunk_documents", + "clean_document_text", + "detokenize", + "is_noise_line", + "iter_raw_documents", + "load_metadata_for_artifact", + "looks_like_heading", + "looks_like_table", + "looks_like_widget", + "modality_from_path", + "normalize_text", + "parse_document_structures", + "parse_text_blocks", + "process_raw_data", + "process_raw_data_with_summary", + "read_csv_rows", + "read_pdf_text", + "read_raw_file", + "rows_to_table_text", + "split_block_by_tokens", + "stable_id", + "summarize_processed_data", + "text_dump_exists", + "ticker_from_path", + "token_count", + "tokenize", + "write_processed_outputs", +] diff --git a/app/processing/__pycache__/__init__.cpython-311.pyc b/app/processing/__pycache__/__init__.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..ee10da753a95f7a651d3af7ddfc055cdb0fd16ce Binary files /dev/null and b/app/processing/__pycache__/__init__.cpython-311.pyc differ diff --git a/app/processing/__pycache__/__init__.cpython-314.pyc b/app/processing/__pycache__/__init__.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..e608e189ee1b76a085650e0be0826337f6f9c38f Binary files /dev/null and b/app/processing/__pycache__/__init__.cpython-314.pyc differ diff --git a/app/processing/__pycache__/chunking.cpython-311.pyc b/app/processing/__pycache__/chunking.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..2276cae76c6bd676e293ce83b91e57a09477c202 Binary files /dev/null and b/app/processing/__pycache__/chunking.cpython-311.pyc differ diff --git a/app/processing/__pycache__/chunking.cpython-314.pyc b/app/processing/__pycache__/chunking.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..3c761d786fb221381795f1d11d96835616a59e0c Binary files /dev/null and b/app/processing/__pycache__/chunking.cpython-314.pyc differ diff --git a/app/processing/__pycache__/constants.cpython-311.pyc b/app/processing/__pycache__/constants.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..ac7fe0a38cee43801b90b4fc4812b789f07cd91a Binary files /dev/null and b/app/processing/__pycache__/constants.cpython-311.pyc differ diff --git a/app/processing/__pycache__/constants.cpython-314.pyc b/app/processing/__pycache__/constants.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..cc0fcc28b231d83a76ad4e689f81ea229919040b Binary files /dev/null and b/app/processing/__pycache__/constants.cpython-314.pyc differ diff --git a/app/processing/__pycache__/documents.cpython-311.pyc b/app/processing/__pycache__/documents.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..eba1ea10bc5033d60b7396d2b2ebbd03696d05d5 Binary files /dev/null and b/app/processing/__pycache__/documents.cpython-311.pyc differ diff --git a/app/processing/__pycache__/documents.cpython-314.pyc b/app/processing/__pycache__/documents.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..72d1b01d8ddb2a136446fa9715f053871f2ef02a Binary files /dev/null and b/app/processing/__pycache__/documents.cpython-314.pyc differ diff --git a/app/processing/__pycache__/outputs.cpython-311.pyc b/app/processing/__pycache__/outputs.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..d36793f30791a7a73eafc989b1d3bd586be65c38 Binary files /dev/null and b/app/processing/__pycache__/outputs.cpython-311.pyc differ diff --git a/app/processing/__pycache__/outputs.cpython-314.pyc b/app/processing/__pycache__/outputs.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..c199a27b19168d0627b73fbcc2ad1f20bc52066b Binary files /dev/null and b/app/processing/__pycache__/outputs.cpython-314.pyc differ diff --git a/app/processing/__pycache__/readers.cpython-311.pyc b/app/processing/__pycache__/readers.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..f8cae876f6403bb938cbbd6fca7697f4e262134d Binary files /dev/null and b/app/processing/__pycache__/readers.cpython-311.pyc differ diff --git a/app/processing/__pycache__/readers.cpython-311.pyc.1580763090224 b/app/processing/__pycache__/readers.cpython-311.pyc.1580763090224 new file mode 100644 index 0000000000000000000000000000000000000000..c76b9cc902775643ed0d44be30cb36d2365563d4 Binary files /dev/null and b/app/processing/__pycache__/readers.cpython-311.pyc.1580763090224 differ diff --git a/app/processing/__pycache__/readers.cpython-311.pyc.1802469281200 b/app/processing/__pycache__/readers.cpython-311.pyc.1802469281200 new file mode 100644 index 0000000000000000000000000000000000000000..f8cae876f6403bb938cbbd6fca7697f4e262134d Binary files /dev/null and b/app/processing/__pycache__/readers.cpython-311.pyc.1802469281200 differ diff --git a/app/processing/__pycache__/readers.cpython-311.pyc.2050855911088 b/app/processing/__pycache__/readers.cpython-311.pyc.2050855911088 new file mode 100644 index 0000000000000000000000000000000000000000..eeb74749cb3e47fcbad9e0b67af27f2fbc013d9d Binary files /dev/null and b/app/processing/__pycache__/readers.cpython-311.pyc.2050855911088 differ diff --git a/app/processing/__pycache__/readers.cpython-311.pyc.2151040388528 b/app/processing/__pycache__/readers.cpython-311.pyc.2151040388528 new file mode 100644 index 0000000000000000000000000000000000000000..6ee83aac862d267db5eb759e5656c09e6d908cc7 Binary files /dev/null and b/app/processing/__pycache__/readers.cpython-311.pyc.2151040388528 differ diff --git a/app/processing/__pycache__/readers.cpython-314.pyc b/app/processing/__pycache__/readers.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..5fbef29cfab6710750d4300f2d5b25bde7d8e1fe Binary files /dev/null and b/app/processing/__pycache__/readers.cpython-314.pyc differ diff --git a/app/processing/__pycache__/structures.cpython-311.pyc b/app/processing/__pycache__/structures.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..3b25d3b8afeedc8470fcdf46f3c894353c451e4b Binary files /dev/null and b/app/processing/__pycache__/structures.cpython-311.pyc differ diff --git a/app/processing/__pycache__/structures.cpython-314.pyc b/app/processing/__pycache__/structures.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..26583eedfffb4b9d3bba826501610090ce25384e Binary files /dev/null and b/app/processing/__pycache__/structures.cpython-314.pyc differ diff --git a/app/processing/__pycache__/text_utils.cpython-311.pyc b/app/processing/__pycache__/text_utils.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..254093fbe7f06b9391476e44ba1fcb1798e79f76 Binary files /dev/null and b/app/processing/__pycache__/text_utils.cpython-311.pyc differ diff --git a/app/processing/__pycache__/text_utils.cpython-314.pyc b/app/processing/__pycache__/text_utils.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..606d99fc845f0ef2a491e0d842e227840bf31abe Binary files /dev/null and b/app/processing/__pycache__/text_utils.cpython-314.pyc differ diff --git a/app/processing/chunking.py b/app/processing/chunking.py new file mode 100644 index 0000000000000000000000000000000000000000..9ec1fb1b59ad5e5caa21a29e56dc899468e297b3 --- /dev/null +++ b/app/processing/chunking.py @@ -0,0 +1,112 @@ +from __future__ import annotations + +from app.config import DEFAULT_CHUNK_OVERLAP, DEFAULT_CHUNK_SIZE, RAW_DIR +from app.processing.readers import read_csv_rows +from app.processing.structures import parse_document_structures +from app.processing.text_utils import detokenize, stable_id, token_count, tokenize +from app.schemas import Chunk, RawDocument, StructureBlock + + +def split_block_by_tokens(block: StructureBlock, max_tokens: int, overlap: int) -> list[StructureBlock]: + tokens = tokenize(block.text) + if len(tokens) <= max_tokens: + return [block] + + blocks: list[StructureBlock] = [] + start = 0 + part_index = 0 + while start < len(tokens): + end = min(start + max_tokens, len(tokens)) + text = detokenize(tokens[start:end]) + blocks.append( + StructureBlock( + text=text, + structure_type=block.structure_type, + heading_path=block.heading_path, + metadata={**block.metadata, "split_part": part_index}, + ) + ) + if end >= len(tokens): + break + start = max(end - overlap, start + 1) + part_index += 1 + return blocks + + +def chunk_blocks( + blocks: list[StructureBlock], + max_tokens: int, + overlap: int, +) -> list[tuple[str, str, list[str], int, dict]]: + chunks: list[tuple[str, str, list[str], int, dict]] = [] + current_blocks: list[StructureBlock] = [] + current_tokens = 0 + + def flush() -> None: + nonlocal current_blocks, current_tokens + if not current_blocks: + return + text = "\n\n".join(block.text for block in current_blocks) + structure_types = [block.structure_type for block in current_blocks] + heading_path = current_blocks[-1].heading_path + metadata = { + "structure_types": structure_types, + "primary_structure_type": structure_types[0], + "block_count": len(current_blocks), + "block_metadata": [block.metadata for block in current_blocks], + } + chunks.append((text, structure_types[0], heading_path, token_count(text), metadata)) + current_blocks = [] + current_tokens = 0 + + for block in blocks: + for part in split_block_by_tokens(block, max_tokens, overlap): + part_tokens = token_count(part.text) + if current_blocks and current_tokens + part_tokens > max_tokens: + flush() + current_blocks.append(part) + current_tokens += part_tokens + + flush() + return chunks + + +def enrich_chunk_metadata(document: RawDocument, block_metadata: dict) -> dict: + return { + **document.metadata, + "document_id": document.id, + "source_file": document.source_path.name, + "parser": "structure-aware-token-chunker", + **block_metadata, + } + + +def chunk_documents( + documents: list[RawDocument], + chunk_size: int = DEFAULT_CHUNK_SIZE, + overlap: int = DEFAULT_CHUNK_OVERLAP, +) -> list[Chunk]: + chunks: list[Chunk] = [] + for document in documents: + source_path = document.source_path.relative_to(RAW_DIR.parent).as_posix() + csv_rows = read_csv_rows(document.source_path) if document.source_path.suffix.lower() == ".csv" else None + blocks = parse_document_structures(document, csv_rows=csv_rows) + for index, (text, structure_type, heading_path, tokens, block_metadata) in enumerate( + chunk_blocks(blocks, chunk_size, overlap) + ): + chunks.append( + Chunk( + id=stable_id(document.id, str(index), text[:120]), + text=text, + ticker=document.ticker, + modality=document.modality, + source_path=source_path, + chunk_index=index, + structure_type=structure_type, + heading_path=heading_path, + token_count=tokens, + metadata=enrich_chunk_metadata(document, block_metadata), + scope=document.scope, + ) + ) + return chunks diff --git a/app/processing/constants.py b/app/processing/constants.py new file mode 100644 index 0000000000000000000000000000000000000000..02ba990acc117cfa2f07be071e00cef2688180f7 --- /dev/null +++ b/app/processing/constants.py @@ -0,0 +1,29 @@ +from __future__ import annotations + +import re + + +TEXT_EXTENSIONS = {".txt", ".html", ".csv", ".json", ".md"} +PDF_EXTENSIONS = {".pdf"} +IMAGE_EXTENSIONS = {".png", ".jpg", ".jpeg", ".webp"} +TOKEN_PATTERN = re.compile(r"[\wÀ-ỹ]+|[^\w\s]", flags=re.UNICODE) +BOILERPLATE_LINES = { + "viết bài", + "tiện ích", + "sách", + "cá nhân hóa", + "tùy chỉnh", + "dành cho bạn", + "live", + "xem thêm", + "tải về", + "đang tải", + "bình luận", + "video", + "mới", + "hot nhất", + "mới nhất", + "liên hệ", + "về chúng tôi", + "chính sách và điều khoản", +} diff --git a/app/processing/documents.py b/app/processing/documents.py new file mode 100644 index 0000000000000000000000000000000000000000..3a7cd79b9c5ca8e1b99a2d8808dabbf534e13b19 --- /dev/null +++ b/app/processing/documents.py @@ -0,0 +1,81 @@ +from __future__ import annotations + +from pathlib import Path + +from app.config import RAW_DIR +from app.processing.constants import IMAGE_EXTENSIONS, PDF_EXTENSIONS, TEXT_EXTENSIONS +from app.processing.readers import load_metadata_for_artifact, read_raw_file +from app.processing.text_utils import clean_document_text, normalize_text, stable_id +from app.schemas import RawDocument + + +def ticker_from_path(path: Path) -> str: + scope = scope_from_path(path) + if scope.lower() == "market": + return "" + return scope.upper() + + +def scope_from_path(path: Path) -> str: + try: + relative = path.relative_to(RAW_DIR) + except ValueError: + return "unknown" + parts = relative.parts + return parts[1] if len(parts) > 1 else "unknown" + + +def modality_from_path(path: Path) -> str: + try: + category = path.relative_to(RAW_DIR).parts[0] + except (ValueError, IndexError): + category = path.parent.name + if category == "csv": + return "table" + if category == "pdf": + return "pdf" + if category == "images": + return "image" + return "text" + + +def text_dump_exists(path: Path) -> bool: + scope = scope_from_path(path) + text_path = RAW_DIR / "text" / scope / f"{path.stem}.txt" + return text_path.exists() + + +def iter_raw_documents(raw_dir: Path = RAW_DIR) -> list[RawDocument]: + documents: list[RawDocument] = [] + allowed_extensions = TEXT_EXTENSIONS | PDF_EXTENSIONS | IMAGE_EXTENSIONS + + for path in raw_dir.rglob("*"): + if not path.is_file(): + continue + if "metadata" in path.parts: + continue + if path.suffix.lower() == ".html" and text_dump_exists(path): + continue + if path.suffix.lower() not in allowed_extensions: + continue + + text = clean_document_text(normalize_text(read_raw_file(path))) + if not text: + continue + + ticker = ticker_from_path(path) + scope = scope_from_path(path) + relative = path.relative_to(raw_dir.parent) + metadata = load_metadata_for_artifact(path, scope) + documents.append( + RawDocument( + id=stable_id(relative.as_posix()), + text=text, + source_path=path, + modality=modality_from_path(path), + ticker=ticker, + metadata=metadata, + scope=scope, + ) + ) + return documents diff --git a/app/processing/outputs.py b/app/processing/outputs.py new file mode 100644 index 0000000000000000000000000000000000000000..540d8c9508ab8554da9b4964e14099014697a304 --- /dev/null +++ b/app/processing/outputs.py @@ -0,0 +1,86 @@ +from __future__ import annotations + +import json + +from app.config import DEFAULT_CHUNK_OVERLAP, DEFAULT_CHUNK_SIZE, PROCESSED_DIR +from app.schemas import Chunk, RawDocument + + +def output_scope(value) -> str: + return (getattr(value, "scope", "") or getattr(value, "ticker", "") or "unknown").lower() + + +def write_processed_outputs(documents: list[RawDocument], chunks: list[Chunk]) -> None: + for directory in [PROCESSED_DIR / "text", PROCESSED_DIR / "chunks", PROCESSED_DIR / "metadata"]: + directory.mkdir(parents=True, exist_ok=True) + + for document in documents: + out_dir = PROCESSED_DIR / "text" / output_scope(document) + out_dir.mkdir(parents=True, exist_ok=True) + name = document.source_path.stem + ".txt" + (out_dir / name).write_text(document.text, encoding="utf-8") + + chunks_by_scope: dict[str, list[Chunk]] = {} + for chunk in chunks: + chunks_by_scope.setdefault(output_scope(chunk), []).append(chunk) + + for scope, scope_chunks in chunks_by_scope.items(): + out_dir = PROCESSED_DIR / "chunks" / scope + out_dir.mkdir(parents=True, exist_ok=True) + out_path = out_dir / "chunks.jsonl" + with out_path.open("w", encoding="utf-8") as handle: + for chunk in scope_chunks: + handle.write(json.dumps(chunk.__dict__, ensure_ascii=False) + "\n") + + manifest = { + "document_count": len(documents), + "chunk_count": len(chunks), + "tickers": sorted({chunk.ticker for chunk in chunks if chunk.ticker}), + "scopes": sorted({output_scope(chunk) for chunk in chunks}), + "chunking": { + "strategy": "structure-aware-token-based", + "max_tokens": DEFAULT_CHUNK_SIZE, + "overlap_tokens": DEFAULT_CHUNK_OVERLAP, + "structures": ["headings", "tables", "widgets", "paragraphs"], + }, + } + (PROCESSED_DIR / "metadata" / "manifest.json").write_text( + json.dumps(manifest, ensure_ascii=False, indent=2), + encoding="utf-8", + ) + + +def summarize_processed_data(documents: list[RawDocument], chunks: list[Chunk]) -> dict: + modality_counts: dict[str, int] = {} + ticker_doc_counts: dict[str, int] = {} + ticker_chunk_counts: dict[str, int] = {} + scope_doc_counts: dict[str, int] = {} + scope_chunk_counts: dict[str, int] = {} + structure_counts: dict[str, int] = {} + + for document in documents: + modality_counts[document.modality] = modality_counts.get(document.modality, 0) + 1 + scope = output_scope(document) + scope_doc_counts[scope] = scope_doc_counts.get(scope, 0) + 1 + if document.ticker: + ticker_doc_counts[document.ticker] = ticker_doc_counts.get(document.ticker, 0) + 1 + + for chunk in chunks: + scope = output_scope(chunk) + scope_chunk_counts[scope] = scope_chunk_counts.get(scope, 0) + 1 + if chunk.ticker: + ticker_chunk_counts[chunk.ticker] = ticker_chunk_counts.get(chunk.ticker, 0) + 1 + structure_counts[chunk.structure_type] = structure_counts.get(chunk.structure_type, 0) + 1 + + return { + "document_count": len(documents), + "chunk_count": len(chunks), + "tickers": sorted(ticker_chunk_counts), + "scopes": sorted(scope_chunk_counts), + "modalities": dict(sorted(modality_counts.items())), + "documents_by_ticker": dict(sorted(ticker_doc_counts.items())), + "chunks_by_ticker": dict(sorted(ticker_chunk_counts.items())), + "documents_by_scope": dict(sorted(scope_doc_counts.items())), + "chunks_by_scope": dict(sorted(scope_chunk_counts.items())), + "structures": dict(sorted(structure_counts.items())), + } diff --git a/app/processing/readers.py b/app/processing/readers.py new file mode 100644 index 0000000000000000000000000000000000000000..a99cd6d87130235ee73a4b6c804d95c16f5d186a --- /dev/null +++ b/app/processing/readers.py @@ -0,0 +1,52 @@ +from __future__ import annotations + +import csv +import json +from pathlib import Path + +from app.config import RAW_DIR +from app.processing.constants import IMAGE_EXTENSIONS, PDF_EXTENSIONS +from app.processing.text_utils import rows_to_table_text + + +def read_csv_rows(path: Path) -> list[list[str]]: + with path.open("r", encoding="utf-8-sig", newline="") as handle: + return [row for row in csv.reader(handle)] + + +def read_pdf_text(path: Path) -> str: + try: + import fitz + + document = fitz.open(str(path)) + try: + text = "\n".join(page.get_text("text") or "" for page in document) + finally: + document.close() + except Exception as exc: # noqa: BLE001 + return f"[PDF artifact without extracted text] {path.name} | pymupdf_error={exc}" + if text.strip(): + return text + return f"[PDF artifact without extracted text] {path.name}" + + +def read_raw_file(path: Path) -> str: + suffix = path.suffix.lower() + if suffix == ".csv": + return rows_to_table_text(read_csv_rows(path)) + if suffix in PDF_EXTENSIONS: + return read_pdf_text(path) + if suffix in IMAGE_EXTENSIONS: + return f"[Image artifact] {path.name}" + return path.read_text(encoding="utf-8", errors="ignore") + + +def load_metadata_for_artifact(path: Path, ticker: str) -> dict: + stem = path.stem + candidates = sorted((RAW_DIR / "metadata" / ticker).glob(f"{stem}.metadata.json")) + if not candidates: + return {} + try: + return json.loads(candidates[0].read_text(encoding="utf-8-sig")) + except (OSError, json.JSONDecodeError): + return {} diff --git a/app/processing/structures.py b/app/processing/structures.py new file mode 100644 index 0000000000000000000000000000000000000000..e2fb703d198a264891e968287c7c4ca5b76f8db2 --- /dev/null +++ b/app/processing/structures.py @@ -0,0 +1,210 @@ +from __future__ import annotations + +from html.parser import HTMLParser + +from app.processing.text_utils import ( + looks_like_heading, + looks_like_table, + looks_like_widget, + normalize_text, + rows_to_table_text, +) +from app.schemas import RawDocument, StructureBlock + + +class HTMLStructureParser(HTMLParser): + def __init__(self) -> None: + super().__init__() + self.blocks: list[StructureBlock] = [] + self.heading_path: list[str] = [] + self._tag_stack: list[str] = [] + self._capture_tag: str | None = None + self._capture_parts: list[str] = [] + self._table_depth = 0 + self._table_rows: list[list[str]] = [] + self._current_row: list[str] | None = None + self._current_cell: list[str] | None = None + + def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None: + self._tag_stack.append(tag) + if tag == "table": + self._flush_capture() + self._table_depth += 1 + self._table_rows = [] + elif self._table_depth and tag == "tr": + self._current_row = [] + elif self._table_depth and tag in {"td", "th"}: + self._current_cell = [] + elif tag in {"h1", "h2", "h3", "h4", "h5", "h6", "p", "li"}: + self._flush_capture() + self._capture_tag = tag + self._capture_parts = [] + elif tag == "br" and self._capture_tag: + self._capture_parts.append("\n") + + def handle_endtag(self, tag: str) -> None: + if self._table_depth and tag in {"td", "th"} and self._current_cell is not None: + cell = normalize_text(" ".join(self._current_cell)) + if self._current_row is not None: + self._current_row.append(cell) + self._current_cell = None + elif self._table_depth and tag == "tr" and self._current_row is not None: + if any(cell for cell in self._current_row): + self._table_rows.append(self._current_row) + self._current_row = None + elif tag == "table" and self._table_depth: + self._table_depth -= 1 + table_text = rows_to_table_text(self._table_rows) + if table_text: + self.blocks.append( + StructureBlock( + text=table_text, + structure_type="table", + heading_path=list(self.heading_path), + metadata={"row_count": len(self._table_rows)}, + ) + ) + self._table_rows = [] + elif tag == self._capture_tag: + self._flush_capture() + + if self._tag_stack and self._tag_stack[-1] == tag: + self._tag_stack.pop() + + def handle_data(self, data: str) -> None: + if self._current_cell is not None: + self._current_cell.append(data) + elif self._capture_tag: + self._capture_parts.append(data) + + def _flush_capture(self) -> None: + if not self._capture_tag: + return + + text = normalize_text(" ".join(self._capture_parts)) + tag = self._capture_tag + self._capture_tag = None + self._capture_parts = [] + if not text: + return + + if tag.startswith("h"): + level = int(tag[1]) + self.heading_path = self.heading_path[: level - 1] + [text] + self.blocks.append( + StructureBlock( + text=text, + structure_type="heading", + heading_path=list(self.heading_path), + metadata={"heading_level": level}, + ) + ) + return + + self.blocks.append( + StructureBlock( + text=text, + structure_type="paragraph", + heading_path=list(self.heading_path), + metadata={"html_tag": tag}, + ) + ) + + +def parse_text_blocks(text: str) -> list[StructureBlock]: + blocks: list[StructureBlock] = [] + heading_path: list[str] = [] + paragraph_lines: list[str] = [] + table_lines: list[str] = [] + widget_lines: list[str] = [] + + def flush_paragraph() -> None: + nonlocal paragraph_lines + paragraph = normalize_text("\n".join(paragraph_lines)) + paragraph_lines = [] + if paragraph: + blocks.append(StructureBlock(paragraph, "paragraph", list(heading_path), {})) + + def flush_table() -> None: + nonlocal table_lines + table = normalize_text("\n".join(table_lines)) + table_lines = [] + if table: + blocks.append(StructureBlock(table, "table", list(heading_path), {})) + + def flush_widget() -> None: + nonlocal widget_lines + widget = normalize_text("\n".join(widget_lines)) + widget_lines = [] + if widget: + blocks.append(StructureBlock(widget, "widget", list(heading_path), {})) + + for raw_line in text.splitlines(): + line = normalize_text(raw_line) + if not line: + flush_paragraph() + flush_table() + flush_widget() + continue + if looks_like_heading(line): + flush_paragraph() + flush_table() + flush_widget() + heading = line.lstrip("#").strip() + heading_path = [heading] + blocks.append( + StructureBlock( + heading, + "heading", + list(heading_path), + {"heading_level": 1}, + ) + ) + elif looks_like_table(line): + flush_paragraph() + flush_widget() + table_lines.append(line) + elif looks_like_widget(line): + flush_paragraph() + flush_table() + widget_lines.append(line) + else: + flush_table() + flush_widget() + paragraph_lines.append(line) + + flush_paragraph() + flush_table() + flush_widget() + return blocks + + +def parse_document_structures(document: RawDocument, csv_rows: list[list[str]] | None = None) -> list[StructureBlock]: + suffix = document.source_path.suffix.lower() + if suffix == ".html": + parser = HTMLStructureParser() + parser.feed(document.text) + parser._flush_capture() + if parser.blocks: + return parser.blocks + if suffix == ".csv": + rows = csv_rows or [] + table = rows_to_table_text(rows) + return [ + StructureBlock( + table, + "table", + [], + {"row_count": len(rows), "source_format": "csv"}, + ) + ] + if suffix in {".png", ".jpg", ".jpeg", ".webp"}: + return [ + StructureBlock( + document.text, + "widget", + [], + {"source_format": "image"}, + ) + ] + return parse_text_blocks(document.text) diff --git a/app/processing/text_utils.py b/app/processing/text_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..de9493b91549ccd6e2645b648f80debb268de17a --- /dev/null +++ b/app/processing/text_utils.py @@ -0,0 +1,91 @@ +from __future__ import annotations + +import hashlib +import html +import re +import uuid + +from app.processing.constants import BOILERPLATE_LINES, TOKEN_PATTERN + + +def stable_id(*parts: str) -> str: + joined = "|".join(parts) + digest = hashlib.sha1(joined.encode("utf-8")).hexdigest() + return str(uuid.uuid5(uuid.NAMESPACE_URL, digest)) + + +def tokenize(text: str) -> list[str]: + return TOKEN_PATTERN.findall(text) + + +def token_count(text: str) -> int: + return len(tokenize(text)) + + +def detokenize(tokens: list[str]) -> str: + text = " ".join(tokens) + text = re.sub(r"\s+([,.;:!?%)\]\}])", r"\1", text) + text = re.sub(r"([\(\[\{])\s+", r"\1", text) + return text.strip() + + +def normalize_text(text: str) -> str: + text = html.unescape(text) + text = re.sub(r"\r\n?", "\n", text) + text = re.sub(r"[ \t]+", " ", text) + text = re.sub(r"\n{3,}", "\n\n", text) + return text.strip() + + +def rows_to_table_text(rows: list[list[str]]) -> str: + lines = [] + for row in rows: + cleaned = [normalize_text(cell) for cell in row if normalize_text(cell)] + if cleaned: + lines.append(" | ".join(cleaned)) + return "\n".join(lines) + + +def looks_like_heading(line: str) -> bool: + if line.startswith("#"): + return True + if len(line) > 90 or len(tokenize(line)) > 14: + return False + if re.match(r"^\d+[\).\s-]+", line): + return True + letters = re.sub(r"[^A-Za-zÀ-ỹ]", "", line) + return bool(letters) and letters.upper() == letters and len(letters) >= 3 + + +def looks_like_table(line: str) -> bool: + return line.count("|") >= 2 or line.count(",") >= 4 or "\t" in line + + +def looks_like_widget(line: str) -> bool: + key_value = bool(re.search(r"[::]\s*\S+", line)) + numeric_dense = len(re.findall(r"\d+(?:[.,]\d+)?%?", line)) >= 3 + return key_value or numeric_dense + + +def is_noise_line(line: str) -> bool: + lowered = line.strip().lower() + if not lowered: + return True + if lowered in BOILERPLATE_LINES: + return True + if lowered.startswith(("window[", "function ", "var ", "const ", "let ")): + return True + if "googletagmanager.com" in lowered or "_gtm_" in lowered: + return True + if lowered.startswith(("{", "};", "])", "", "
Theo dõi dữ liệu đã thu thập, mức độ xử lý, chất lượng truy xuất, câu trả lời và hoạt động chatbot.