Spaces:
Runtime error
Runtime error
Merge pull request #3 from Youranalyst-coder/codex/enhance-and-deploy-project-for-interview-c6c16v
Browse files- .github/workflows/ci.yml +31 -0
- .gitignore +12 -0
- Makefile +22 -0
- README.md +170 -22
- app/app.py +54 -78
- app/logistic_model.pkl +0 -0
- app/tfidf_vectorizer.pkl +0 -0
- config/settings.yaml +59 -0
- deployment/Dockerfile +17 -0
- deployment/github_live_validation.md +65 -0
- deployment/oracle_cloud.md +70 -0
- deployment/streamlit_cloud.md +57 -0
- deployment/vercel/index.html +42 -0
- deployment/vercel_redirect.md +61 -0
- docs/architecture.md +42 -0
- logistic_model.pkl +0 -0
- model +0 -1
- requirements.txt +7 -6
- scripts/__init__.py +0 -0
- scripts/check_live_app.py +117 -0
- scripts/predict.py +38 -0
- scripts/train.py +50 -0
- scripts/update_vercel_redirect.py +102 -0
- sentiment_model.pkl +0 -0
- src/twitter_sentiment/__init__.py +10 -0
- src/twitter_sentiment/config.py +76 -0
- src/twitter_sentiment/data_loader.py +80 -0
- src/twitter_sentiment/logging_utils.py +39 -0
- src/twitter_sentiment/modeling.py +161 -0
- src/twitter_sentiment/predictor.py +69 -0
- src/twitter_sentiment/preprocessing.py +64 -0
- tests/conftest.py +30 -0
- tests/test_check_live_app.py +103 -0
- tests/test_predictor.py +19 -0
- tests/test_preprocessing.py +24 -0
- tests/test_update_vercel_redirect.py +37 -0
.github/workflows/ci.yml
ADDED
|
@@ -0,0 +1,31 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
name: CI
|
| 2 |
+
|
| 3 |
+
on:
|
| 4 |
+
push:
|
| 5 |
+
branches: [ main ]
|
| 6 |
+
pull_request:
|
| 7 |
+
branches: [ main ]
|
| 8 |
+
|
| 9 |
+
jobs:
|
| 10 |
+
build-and-test:
|
| 11 |
+
runs-on: ubuntu-latest
|
| 12 |
+
|
| 13 |
+
steps:
|
| 14 |
+
- name: Checkout repository
|
| 15 |
+
uses: actions/checkout@v4
|
| 16 |
+
|
| 17 |
+
- name: Set up Python
|
| 18 |
+
uses: actions/setup-python@v5
|
| 19 |
+
with:
|
| 20 |
+
python-version: '3.11'
|
| 21 |
+
|
| 22 |
+
- name: Install dependencies
|
| 23 |
+
run: |
|
| 24 |
+
python -m pip install --upgrade pip
|
| 25 |
+
pip install -r requirements.txt
|
| 26 |
+
|
| 27 |
+
- name: Run unit tests
|
| 28 |
+
run: pytest -q
|
| 29 |
+
|
| 30 |
+
- name: Lint package
|
| 31 |
+
run: python -m compileall src
|
.gitignore
ADDED
|
@@ -0,0 +1,12 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
.venv/
|
| 2 |
+
__pycache__/
|
| 3 |
+
*.pyc
|
| 4 |
+
artifacts/metrics.json
|
| 5 |
+
artifacts/*.log
|
| 6 |
+
artifacts/*.joblib
|
| 7 |
+
artifacts/*.pkl
|
| 8 |
+
logistic_model.pkl
|
| 9 |
+
model
|
| 10 |
+
.env
|
| 11 |
+
.DS_Store
|
| 12 |
+
.ipynb_checkpoints/
|
Makefile
ADDED
|
@@ -0,0 +1,22 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
.PHONY: install test train app format verify-live vercel-redirect
|
| 2 |
+
|
| 3 |
+
install:
|
| 4 |
+
python -m venv .venv && . .venv/bin/activate && pip install --upgrade pip && pip install -r requirements.txt
|
| 5 |
+
|
| 6 |
+
test:
|
| 7 |
+
pytest -q
|
| 8 |
+
|
| 9 |
+
train:
|
| 10 |
+
python scripts/train.py
|
| 11 |
+
|
| 12 |
+
app:
|
| 13 |
+
streamlit run app/app.py
|
| 14 |
+
|
| 15 |
+
format:
|
| 16 |
+
python -m compileall src
|
| 17 |
+
|
| 18 |
+
verify-live:
|
| 19 |
+
python scripts/check_live_app.py $(if $(URL),--url $(URL),)
|
| 20 |
+
|
| 21 |
+
vercel-redirect:
|
| 22 |
+
python scripts/update_vercel_redirect.py $(if $(URL),--url $(URL),)
|
README.md
CHANGED
|
@@ -1,37 +1,185 @@
|
|
| 1 |
-
|
| 2 |
|
| 3 |
-
A
|
|
|
|
| 4 |
|
| 5 |
-
|
| 6 |
|
| 7 |
-
|
| 8 |
-
- TF-IDF Vectorization
|
| 9 |
-
- Multiclass Logistic Regression
|
| 10 |
-
- Streamlit (Web App UI)
|
| 11 |
|
| 12 |
-
|
| 13 |
|
| 14 |
-
|
| 15 |
-
- Trained logistic regression pipeline for three-way sentiment detection
|
| 16 |
-
- Streamlit UI that displays the predicted class and class probabilities
|
| 17 |
|
| 18 |
-
|
| 19 |
|
| 20 |
-
|
| 21 |
|
| 22 |
-
|
| 23 |
|
| 24 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 25 |
pip install -r requirements.txt
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 26 |
streamlit run app/app.py
|
| 27 |
-
```
|
| 28 |
|
| 29 |
-
## 🧠 Training Artifacts
|
| 30 |
|
| 31 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 32 |
|
| 33 |
-
|
| 34 |
-
- `app/tfidf_vectorizer.pkl` – Fitted TF-IDF vectorizer compatible with the model
|
| 35 |
-
- `sentiment_model.pkl` – End-to-end Scikit-learn pipeline (vectorizer + classifier)
|
| 36 |
|
| 37 |
-
|
|
|
|
|
|
|
|
|
| 1 |
+
💼 Deloitte-Ready Twitter Sentiment Intelligence
|
| 2 |
|
| 3 |
+
A production-grade sentiment analysis accelerator designed to showcase the end-to-end delivery skills expected from a Deloitte India Oracle Analyst.
|
| 4 |
+
The solution spans data ingestion, preprocessing, model training, governance, deployment readiness, and business storytelling — all within a lightweight, reproducible ML pipeline.
|
| 5 |
|
| 6 |
+
🧰 Technology Stack & Analytics Capabilities
|
| 7 |
|
| 8 |
+
Python, Pandas, Scikit-learn, NLTK – rapid experimentation & production-ready sentiment pipelines
|
|
|
|
|
|
|
|
|
|
| 9 |
|
| 10 |
+
TF-IDF Vectorization + Multiclass Logistic Regression – explainable predictions across positive, neutral, negative classes
|
| 11 |
|
| 12 |
+
Streamlit UI (app/app.py) – displays class probabilities & narrative insights for stakeholders
|
|
|
|
|
|
|
| 13 |
|
| 14 |
+
Notebook-driven exploration (notebooks/) – retraining lineage and model interpretability
|
| 15 |
|
| 16 |
+
Container-ready CI/CD pipeline – automates tests, linting, and retraining
|
| 17 |
|
| 18 |
+
🧭 Why This Project Stands Out
|
| 19 |
|
| 20 |
+
Oracle ecosystem alignment – optional ingestion from Oracle Autonomous Database with OCI deployment playbooks
|
| 21 |
+
|
| 22 |
+
Consulting-grade engineering – modular Python package, logging, configuration management, GitHub Actions CI
|
| 23 |
+
|
| 24 |
+
Business storytelling – dashboards, KPIs, and executive-friendly insights translating data into measurable ROI
|
| 25 |
+
|
| 26 |
+
🔗 Live Resources & Quick Links
|
| 27 |
+
Asset Purpose Link
|
| 28 |
+
Live Streamlit Demo Interview-ready interactive dashboard https://<your-app>.streamlit.app
|
| 29 |
+
|
| 30 |
+
Architecture Blueprint Deep dive on data, model & DevOps layers docs/architecture.md
|
| 31 |
+
|
| 32 |
+
OCI Deployment Playbook Containerized rollout on Oracle Cloud deployment/oracle_cloud.md
|
| 33 |
+
|
| 34 |
+
Streamlit Cloud Guide Launch public URL in minutes deployment/streamlit_cloud.md
|
| 35 |
+
|
| 36 |
+
Vercel Redirect Playbook Vanity domain / QR-friendly link deployment/vercel_redirect.md
|
| 37 |
+
|
| 38 |
+
GitHub Live Validation Verify live deployment health deployment/github_live_validation.md
|
| 39 |
+
|
| 40 |
+
Training Data Sample Quick-start dataset for retraining data/twitter_training.csv
|
| 41 |
+
|
| 42 |
+
✅ After publishing, replace the placeholder URL above and embed it directly in your resume / interview slides.
|
| 43 |
+
|
| 44 |
+
🏗️ Architecture Overview
|
| 45 |
+
flowchart LR
|
| 46 |
+
subgraph Data_Layer
|
| 47 |
+
A[Oracle Autonomous DB] -- optional --> B[(CSV Data)]
|
| 48 |
+
end
|
| 49 |
+
subgraph Processing_Layer
|
| 50 |
+
B --> C[Data Loader]
|
| 51 |
+
C --> D[Text Preprocessor]
|
| 52 |
+
D --> E[Scikit-learn Pipeline]
|
| 53 |
+
E --> F[Model Metrics]
|
| 54 |
+
end
|
| 55 |
+
subgraph Experience_Layer
|
| 56 |
+
E --> G[Streamlit Dashboard]
|
| 57 |
+
E --> H[CLI Automation]
|
| 58 |
+
F --> G
|
| 59 |
+
F --> I[Reporting / GitHub Pages]
|
| 60 |
+
end
|
| 61 |
+
subgraph DevOps_Layer
|
| 62 |
+
J[GitHub Actions CI]
|
| 63 |
+
J --> E
|
| 64 |
+
J --> G
|
| 65 |
+
end
|
| 66 |
+
|
| 67 |
+
📁 Repository Structure
|
| 68 |
+
Path Description
|
| 69 |
+
app/ Streamlit app for demos
|
| 70 |
+
artifacts/ Generated model artifacts (gitignored; created via scripts/train.py)
|
| 71 |
+
config/ Central YAML settings controlling ingestion & deployment toggles
|
| 72 |
+
data/ Sample labelled tweets
|
| 73 |
+
deployment/ Multi-cloud deployment guides
|
| 74 |
+
docs/ Architecture diagrams & KPI catalogue
|
| 75 |
+
scripts/ Automation scripts for training & inference
|
| 76 |
+
src/twitter_sentiment/ Core reusable Python package
|
| 77 |
+
tests/ Pytest-based unit tests
|
| 78 |
+
.github/workflows/ CI pipeline (lint + tests)
|
| 79 |
+
🚀 Quick Start
|
| 80 |
+
# 1️⃣ Create virtual environment
|
| 81 |
+
python -m venv .venv
|
| 82 |
+
source .venv/bin/activate # Windows: .venv\Scripts\activate
|
| 83 |
+
|
| 84 |
+
# 2️⃣ Install dependencies
|
| 85 |
+
pip install --upgrade pip
|
| 86 |
pip install -r requirements.txt
|
| 87 |
+
|
| 88 |
+
# 3️⃣ Train pipeline & generate artifacts
|
| 89 |
+
python scripts/train.py
|
| 90 |
+
|
| 91 |
+
# 4️⃣ Launch Deloitte Storytelling Dashboard
|
| 92 |
streamlit run app/app.py
|
|
|
|
| 93 |
|
|
|
|
| 94 |
|
| 95 |
+
Note: Artifacts are excluded from source control.
|
| 96 |
+
Run python scripts/train.py whenever cloning the repo to recreate artifacts/sentiment_pipeline.joblib.
|
| 97 |
+
|
| 98 |
+
🧪 Quality Gates
|
| 99 |
+
Check Command
|
| 100 |
+
Unit tests pytest -q
|
| 101 |
+
Package compile check python -m compileall src
|
| 102 |
+
Model retrain python scripts/train.py
|
| 103 |
+
|
| 104 |
+
CI runs automatically on every push or PR to main.
|
| 105 |
+
|
| 106 |
+
🧩 Feature Highlights
|
| 107 |
+
|
| 108 |
+
Config-driven ingestion switch (CSV ⇄ Oracle DB)
|
| 109 |
+
|
| 110 |
+
Reusable text-cleaning module (URL, mention, punctuation handling)
|
| 111 |
+
|
| 112 |
+
Auditable Scikit-learn pipeline with persisted metrics
|
| 113 |
+
|
| 114 |
+
Streamlit dashboard → probabilities + governance tab
|
| 115 |
+
|
| 116 |
+
CLI utilities for automated workflows
|
| 117 |
+
|
| 118 |
+
Pytest suite + GitHub Actions CI/CD
|
| 119 |
+
|
| 120 |
+
☁️ Cloud Deployment
|
| 121 |
+
|
| 122 |
+
See deployment/oracle_cloud.md
|
| 123 |
+
for the Oracle Cloud Infrastructure (OCI) guide:
|
| 124 |
+
|
| 125 |
+
Containerize Streamlit app using OCI Container Instances
|
| 126 |
+
|
| 127 |
+
Automate retraining via OCI Data Science jobs / GitHub Actions
|
| 128 |
+
|
| 129 |
+
Connect to Autonomous DB with wallet credentials
|
| 130 |
+
|
| 131 |
+
Or follow deployment/streamlit_cloud.md
|
| 132 |
+
for a free Streamlit Cloud public link—perfect for interviews.
|
| 133 |
+
|
| 134 |
+
🗃️ Oracle Database Integration (Optional)
|
| 135 |
+
oracle_integration:
|
| 136 |
+
enabled: true
|
| 137 |
+
wallet_location: /path/to/wallet
|
| 138 |
+
user: DATA_ENGINEER
|
| 139 |
+
dsn: myadb_high
|
| 140 |
+
sql_query: |
|
| 141 |
+
SELECT text, sentiment
|
| 142 |
+
FROM analytics.twitter_training_data
|
| 143 |
+
WHERE created_at >= SYSDATE - 30
|
| 144 |
+
|
| 145 |
+
📊 Storytelling in Interviews
|
| 146 |
+
|
| 147 |
+
Business Impact – sentiment tracking → improved ROI & retention
|
| 148 |
+
|
| 149 |
+
Oracle Expertise – wallet connectivity, SQL modeling, OCI integration
|
| 150 |
+
|
| 151 |
+
Engineering Rigor – modular design, testing, CI/CD
|
| 152 |
+
|
| 153 |
+
Consulting Mindset – executive-ready Streamlit deliverable
|
| 154 |
+
|
| 155 |
+
📦 GitHub Best Practices
|
| 156 |
+
|
| 157 |
+
Use feature branches (e.g., feature/oracle-ingestion)
|
| 158 |
+
|
| 159 |
+
Raise PRs with CI status + screenshots
|
| 160 |
+
|
| 161 |
+
Track backlog via GitHub Projects
|
| 162 |
+
|
| 163 |
+
Tag releases (e.g., v1.0.0) post-deployment
|
| 164 |
+
|
| 165 |
+
🧠 Training Artifacts
|
| 166 |
+
|
| 167 |
+
Generated after running scripts/train.py:
|
| 168 |
+
|
| 169 |
+
sentiment_pipeline.joblib – Full TF-IDF + Logistic Regression pipeline
|
| 170 |
+
|
| 171 |
+
Legacy (compatible) files:
|
| 172 |
+
|
| 173 |
+
app/logistic_model.pkl
|
| 174 |
+
|
| 175 |
+
app/tfidf_vectorizer.pkl
|
| 176 |
+
|
| 177 |
+
sentiment_model.pkl
|
| 178 |
+
|
| 179 |
+
Use these for quick demos or backward comparisons.
|
| 180 |
|
| 181 |
+
🤝 Contributing
|
|
|
|
|
|
|
| 182 |
|
| 183 |
+
Pull requests welcome!
|
| 184 |
+
Please open an issue describing any enhancement or bugfix before major changes.
|
| 185 |
+
|
app/app.py
CHANGED
|
@@ -1,87 +1,63 @@
|
|
|
|
|
|
|
|
| 1 |
from __future__ import annotations
|
| 2 |
|
|
|
|
|
|
|
| 3 |
from pathlib import Path
|
| 4 |
-
from typing import Dict
|
| 5 |
|
| 6 |
-
import joblib
|
| 7 |
import pandas as pd
|
| 8 |
import streamlit as st
|
| 9 |
|
| 10 |
-
|
| 11 |
-
|
| 12 |
-
|
| 13 |
-
|
| 14 |
-
|
| 15 |
-
|
| 16 |
-
|
| 17 |
-
|
| 18 |
-
|
| 19 |
-
|
| 20 |
-
|
| 21 |
-
|
| 22 |
-
|
| 23 |
-
|
| 24 |
-
|
| 25 |
-
|
| 26 |
-
|
| 27 |
-
|
| 28 |
-
top_score = class_probabilities[top_label]
|
| 29 |
-
return top_label, top_score, class_probabilities
|
| 30 |
-
|
| 31 |
-
|
| 32 |
-
SENTIMENT_DISPLAY = {
|
| 33 |
-
"positive": {
|
| 34 |
-
"title": "Positive",
|
| 35 |
-
"emoji": "😊",
|
| 36 |
-
"description": "The tweet expresses a favourable or uplifting sentiment.",
|
| 37 |
-
},
|
| 38 |
-
"neutral": {
|
| 39 |
-
"title": "Neutral",
|
| 40 |
-
"emoji": "😐",
|
| 41 |
-
"description": "The tweet is balanced without a clear positive or negative leaning.",
|
| 42 |
-
},
|
| 43 |
-
"negative": {
|
| 44 |
-
"title": "Negative",
|
| 45 |
-
"emoji": "😞",
|
| 46 |
-
"description": "The tweet carries criticism, frustration, or an unfavourable view.",
|
| 47 |
-
},
|
| 48 |
-
}
|
| 49 |
-
|
| 50 |
-
|
| 51 |
-
st.set_page_config(page_title="Twitter Sentiment Analysis", page_icon="💬")
|
| 52 |
-
st.title("💬 Twitter Sentiment Analysis")
|
| 53 |
-
st.write(
|
| 54 |
-
"Enter a tweet below and the model will classify it as positive, neutral, or negative."
|
| 55 |
-
)
|
| 56 |
-
|
| 57 |
-
user_input = st.text_area(
|
| 58 |
-
"Tweet text",
|
| 59 |
-
placeholder="I love how friendly everyone is on Twitter today!",
|
| 60 |
-
height=150,
|
| 61 |
)
|
| 62 |
|
| 63 |
-
|
| 64 |
-
|
| 65 |
-
|
| 66 |
-
|
| 67 |
-
|
| 68 |
-
|
| 69 |
-
|
| 70 |
-
|
| 71 |
-
|
| 72 |
-
|
| 73 |
-
|
| 74 |
-
|
| 75 |
-
|
| 76 |
-
|
| 77 |
-
|
| 78 |
-
|
| 79 |
-
|
| 80 |
-
|
| 81 |
-
|
| 82 |
-
)
|
| 83 |
-
|
| 84 |
-
|
| 85 |
-
|
| 86 |
-
|
| 87 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Streamlit front-end for the Deloitte-ready Twitter Sentiment Intelligence dashboard."""
|
| 2 |
+
|
| 3 |
from __future__ import annotations
|
| 4 |
|
| 5 |
+
import json
|
| 6 |
+
import sys
|
| 7 |
from pathlib import Path
|
| 8 |
+
from typing import Dict
|
| 9 |
|
|
|
|
| 10 |
import pandas as pd
|
| 11 |
import streamlit as st
|
| 12 |
|
| 13 |
+
# -------------------------------------------------------------------------
|
| 14 |
+
# Path setup to include the local src/ package for imports
|
| 15 |
+
# -------------------------------------------------------------------------
|
| 16 |
+
ROOT = Path(__file__).resolve().parents[1]
|
| 17 |
+
SRC_PATH = ROOT / "src"
|
| 18 |
+
if str(SRC_PATH) not in sys.path:
|
| 19 |
+
sys.path.insert(0, str(SRC_PATH))
|
| 20 |
+
|
| 21 |
+
from twitter_sentiment.config import load_config
|
| 22 |
+
from twitter_sentiment.predictor import load_artifacts, predict_with_threshold
|
| 23 |
+
|
| 24 |
+
# -------------------------------------------------------------------------
|
| 25 |
+
# Streamlit App Configuration
|
| 26 |
+
# -------------------------------------------------------------------------
|
| 27 |
+
st.set_page_config(
|
| 28 |
+
page_title="Twitter Sentiment Intelligence",
|
| 29 |
+
page_icon="💼",
|
| 30 |
+
layout="wide",
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 31 |
)
|
| 32 |
|
| 33 |
+
# -------------------------------------------------------------------------
|
| 34 |
+
# Cached resource loading (config, pipeline, metrics)
|
| 35 |
+
# -------------------------------------------------------------------------
|
| 36 |
+
@st.cache_resource(show_spinner=False)
|
| 37 |
+
def _load_dependencies():
|
| 38 |
+
"""Load configuration, trained pipeline, and metrics from artifacts."""
|
| 39 |
+
config = load_config()
|
| 40 |
+
pipeline, metrics = load_artifacts(config)
|
| 41 |
+
return config, pipeline, metrics
|
| 42 |
+
|
| 43 |
+
|
| 44 |
+
# -------------------------------------------------------------------------
|
| 45 |
+
# Helper function to format prediction probabilities
|
| 46 |
+
# -------------------------------------------------------------------------
|
| 47 |
+
def format_probabilities(probabilities: Dict[str, float]) -> pd.DataFrame:
|
| 48 |
+
"""Convert prediction probabilities to a styled DataFrame for display."""
|
| 49 |
+
return (
|
| 50 |
+
pd.DataFrame.from_dict(probabilities, orient="index", columns=["confidence"])
|
| 51 |
+
.sort_values("confidence", ascending=False)
|
| 52 |
+
.style.format({"confidence": "{:.2%}"})
|
| 53 |
+
)
|
| 54 |
+
|
| 55 |
+
|
| 56 |
+
# -------------------------------------------------------------------------
|
| 57 |
+
# Main Streamlit Application
|
| 58 |
+
# -------------------------------------------------------------------------
|
| 59 |
+
def main() -> None:
|
| 60 |
+
"""Render the Deloitte-ready Twitter Sentiment Intelligence Dashboard."""
|
| 61 |
+
config, pipeline, metrics = _load_dependencies()
|
| 62 |
+
|
| 63 |
+
# -----------------------------------------------------
|
app/logistic_model.pkl
DELETED
|
Binary file (1.52 kB)
|
|
|
app/tfidf_vectorizer.pkl
DELETED
|
Binary file (1.6 kB)
|
|
|
config/settings.yaml
ADDED
|
@@ -0,0 +1,59 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"project": {
|
| 3 |
+
"name": "Twitter Sentiment Analysis",
|
| 4 |
+
"description": "End-to-end sentiment analysis workflow aligned with Deloitte India Oracle Analyst capabilities.",
|
| 5 |
+
"author": "Your Name"
|
| 6 |
+
},
|
| 7 |
+
"live_app": {
|
| 8 |
+
"streamlit_url": "https://<your-app>.streamlit.app",
|
| 9 |
+
"vercel_redirect_domain": "https://<your-project>.vercel.app"
|
| 10 |
+
},
|
| 11 |
+
"data": {
|
| 12 |
+
"path": "data/twitter.csv",
|
| 13 |
+
"text_column": "text",
|
| 14 |
+
"target_column": "sentiment",
|
| 15 |
+
"class_order": ["negative", "neutral", "positive"]
|
| 16 |
+
},
|
| 17 |
+
"preprocessing": {
|
| 18 |
+
"lowercase": true,
|
| 19 |
+
"strip_urls": true,
|
| 20 |
+
"strip_mentions": true,
|
| 21 |
+
"strip_hashtags": false,
|
| 22 |
+
"remove_punctuation": true,
|
| 23 |
+
"normalize_whitespace": true,
|
| 24 |
+
"stemming": false,
|
| 25 |
+
"lemmatize": false
|
| 26 |
+
},
|
| 27 |
+
"model": {
|
| 28 |
+
"artifact_dir": "artifacts",
|
| 29 |
+
"pipeline_filename": "sentiment_pipeline.joblib",
|
| 30 |
+
"vectorizer_filename": "tfidf_vectorizer.joblib",
|
| 31 |
+
"model_filename": "logistic_model.joblib",
|
| 32 |
+
"test_size": 0.2,
|
| 33 |
+
"random_state": 42,
|
| 34 |
+
"max_features": 5000,
|
| 35 |
+
"ngram_range": [1, 2],
|
| 36 |
+
"class_weight": "balanced"
|
| 37 |
+
},
|
| 38 |
+
"training": {
|
| 39 |
+
"cv_folds": 5,
|
| 40 |
+
"scoring": "f1_macro",
|
| 41 |
+
"n_jobs": -1,
|
| 42 |
+
"verbose": 1,
|
| 43 |
+
"probability_thresholds": {
|
| 44 |
+
"positive": 0.55,
|
| 45 |
+
"negative": 0.45
|
| 46 |
+
}
|
| 47 |
+
},
|
| 48 |
+
"monitoring": {
|
| 49 |
+
"enable_model_drift_checks": true,
|
| 50 |
+
"psi_threshold": 0.2
|
| 51 |
+
},
|
| 52 |
+
"oracle_integration": {
|
| 53 |
+
"enabled": false,
|
| 54 |
+
"wallet_location": "~/.oci/wallet",
|
| 55 |
+
"user": "",
|
| 56 |
+
"dsn": "",
|
| 57 |
+
"sql_query": "SELECT text, sentiment FROM twitter_training_data"
|
| 58 |
+
}
|
| 59 |
+
}
|
deployment/Dockerfile
ADDED
|
@@ -0,0 +1,17 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
FROM python:3.11-slim
|
| 2 |
+
|
| 3 |
+
WORKDIR /app
|
| 4 |
+
|
| 5 |
+
COPY requirements.txt ./
|
| 6 |
+
RUN pip install --no-cache-dir --upgrade pip \
|
| 7 |
+
&& pip install --no-cache-dir -r requirements.txt
|
| 8 |
+
|
| 9 |
+
COPY . .
|
| 10 |
+
|
| 11 |
+
ENV STREAMLIT_SERVER_PORT=8501 \
|
| 12 |
+
STREAMLIT_SERVER_HEADLESS=true \
|
| 13 |
+
STREAMLIT_BROWSER_GATHER_USAGE_STATS=false
|
| 14 |
+
|
| 15 |
+
EXPOSE 8501
|
| 16 |
+
|
| 17 |
+
CMD ["streamlit", "run", "app/app.py"]
|
deployment/github_live_validation.md
ADDED
|
@@ -0,0 +1,65 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# GitHub Connection & Live App Validation Guide
|
| 2 |
+
|
| 3 |
+
This runbook walks you through connecting the local project to your GitHub repository, wiring the repo to Streamlit Community Cloud (or another host), and verifying that the live dashboard is reachable from any device.
|
| 4 |
+
|
| 5 |
+
## 1. Connect the local repo to GitHub
|
| 6 |
+
|
| 7 |
+
1. Create a new **empty** repository on GitHub (no README or licence yet) – e.g. `deloitte-twitter-sentiment`.
|
| 8 |
+
2. From your local workstation, authenticate with GitHub if you have not already:
|
| 9 |
+
```bash
|
| 10 |
+
gh auth login # or use `git config credential.helper` if you prefer HTTPS tokens
|
| 11 |
+
```
|
| 12 |
+
3. Add the remote origin and push the current branch:
|
| 13 |
+
```bash
|
| 14 |
+
git remote add origin git@github.com:<your-handle>/deloitte-twitter-sentiment.git
|
| 15 |
+
# or use https://github.com/<your-handle>/deloitte-twitter-sentiment.git if you prefer HTTPS
|
| 16 |
+
git push -u origin work
|
| 17 |
+
```
|
| 18 |
+
4. (Optional) Protect the `main` branch and enable required status checks (`pytest`, `python -m compileall`) from the **Settings ▸ Branches** tab for interview-ready governance.
|
| 19 |
+
|
| 20 |
+
## 2. Enable Streamlit Community Cloud deployment
|
| 21 |
+
|
| 22 |
+
1. Visit [share.streamlit.io](https://share.streamlit.io) and sign in with GitHub.
|
| 23 |
+
2. Grant Streamlit access to the new repository when prompted.
|
| 24 |
+
3. Configure the deployment with the following options:
|
| 25 |
+
- **Repository**: `<your-handle>/deloitte-twitter-sentiment`
|
| 26 |
+
- **Branch**: `work` (or `main` after merge)
|
| 27 |
+
- **Main file path**: `app/app.py`
|
| 28 |
+
- **Python version**: 3.10+
|
| 29 |
+
- **Secrets** (optional): Add Oracle wallet paths, API keys, etc.
|
| 30 |
+
4. Click **Deploy** and wait for the build log to show `🎈 You can now view your Streamlit app` with the generated `https://<slug>.streamlit.app` URL.
|
| 31 |
+
|
| 32 |
+
## 3. Validate the live URL locally
|
| 33 |
+
|
| 34 |
+
Once the deployment completes, store the URL in an environment variable or `.env` file:
|
| 35 |
+
|
| 36 |
+
```bash
|
| 37 |
+
export STREAMLIT_APP_URL="https://your-app.streamlit.app"
|
| 38 |
+
```
|
| 39 |
+
|
| 40 |
+
Then run the verification helper included in this repository:
|
| 41 |
+
|
| 42 |
+
```bash
|
| 43 |
+
python scripts/check_live_app.py --url "$STREAMLIT_APP_URL"
|
| 44 |
+
# or equivalently
|
| 45 |
+
make verify-live URL=$STREAMLIT_APP_URL
|
| 46 |
+
```
|
| 47 |
+
|
| 48 |
+
The script sends a quick HTTP request, checks for a `200 OK` response, and reports latency. This makes it easy to confirm that the dashboard is reachable from phones or tablets before an interview.
|
| 49 |
+
|
| 50 |
+
## 4. Update README quick links
|
| 51 |
+
|
| 52 |
+
1. Edit the **Live resources & quick links** table in `README.md` and replace the placeholder domain with your actual Streamlit URL.
|
| 53 |
+
2. Commit and push the change:
|
| 54 |
+
```bash
|
| 55 |
+
git commit -am "docs: add live app URL" && git push
|
| 56 |
+
```
|
| 57 |
+
3. Verify the README renders correctly on GitHub and that the link opens on mobile.
|
| 58 |
+
|
| 59 |
+
## 5. Keep the deployment healthy
|
| 60 |
+
|
| 61 |
+
- Re-deploy automatically by enabling **App settings ▸ Watch for changes** in Streamlit Cloud, which rebuilds when you push to the configured branch.
|
| 62 |
+
- Periodically run `make test` and `make train` locally so the CI workflow and Streamlit Cloud dependency cache stay fresh.
|
| 63 |
+
- Monitor the Streamlit Cloud logs (⚙️ ▸ App settings ▸ Logs) for errors after each push.
|
| 64 |
+
|
| 65 |
+
Following this sequence guarantees the repo is connected to GitHub, the cloud deployment is online, and you can confidently demonstrate the live app during interviews.
|
deployment/oracle_cloud.md
ADDED
|
@@ -0,0 +1,70 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Oracle Cloud Infrastructure Deployment Runbook
|
| 2 |
+
|
| 3 |
+
This guide describes how to host the Streamlit dashboard and model artifacts on Oracle Cloud Infrastructure (OCI) in a way that resonates with Deloitte India Oracle Analyst expectations.
|
| 4 |
+
|
| 5 |
+
## 1. Prerequisites
|
| 6 |
+
|
| 7 |
+
- OCI tenancy with permissions for Container Registry, Container Instances, and Object Storage.
|
| 8 |
+
- Oracle Autonomous Database (ATP/ADW) with wallet credentials downloaded.
|
| 9 |
+
- GitHub repository with this project (recommended branching: `main`, `develop`, feature branches).
|
| 10 |
+
|
| 11 |
+
## 2. Build the container image
|
| 12 |
+
|
| 13 |
+
```bash
|
| 14 |
+
otc login # OCI CLI authentication
|
| 15 |
+
export OCIR_REGION=iad.ocir.io # replace with your region
|
| 16 |
+
export OCIR_NAMESPACE=<tenancy-namespace>
|
| 17 |
+
export IMAGE_NAME=twitter-sentiment
|
| 18 |
+
|
| 19 |
+
# Build and tag the image
|
| 20 |
+
podman build -t $IMAGE_NAME -f deployment/Dockerfile .
|
| 21 |
+
podman tag $IMAGE_NAME $OCIR_REGION/$OCIR_NAMESPACE/$IMAGE_NAME:latest
|
| 22 |
+
|
| 23 |
+
# Push to OCI Registry
|
| 24 |
+
podman push $OCIR_REGION/$OCIR_NAMESPACE/$IMAGE_NAME:latest
|
| 25 |
+
```
|
| 26 |
+
|
| 27 |
+
> Tip: Use GitHub Actions to automate image builds on every tagged release.
|
| 28 |
+
|
| 29 |
+
## 3. Provision infrastructure
|
| 30 |
+
|
| 31 |
+
1. **Container Instance** – create via OCI Console, referencing the pushed image.
|
| 32 |
+
2. **Networking** – open HTTPS ingress on the public subnet and map to port `8501`.
|
| 33 |
+
3. **Object Storage bucket** – store model artifacts (`artifacts/sentiment_pipeline.joblib`) and metrics.
|
| 34 |
+
4. **Vault** – manage secrets for database credentials and wallet passwords.
|
| 35 |
+
|
| 36 |
+
## 4. Configure environment variables
|
| 37 |
+
|
| 38 |
+
| Variable | Description |
|
| 39 |
+
| --- | --- |
|
| 40 |
+
| `MODEL_ARTIFACT_URI` | OCI Object Storage pre-authenticated URL to the pipeline joblib |
|
| 41 |
+
| `CONFIG_PATH` | Path to mounted configuration file (e.g., `/app/config/settings.yaml`) |
|
| 42 |
+
| `ORACLE_WALLET_DIR` | Mount point for wallet files to enable database connectivity |
|
| 43 |
+
| `STREAMLIT_SERVER_PORT` | Set to `8501` |
|
| 44 |
+
|
| 45 |
+
Mount the wallet as a secret volume in the container instance for secure connections.
|
| 46 |
+
|
| 47 |
+
## 5. Automate retraining
|
| 48 |
+
|
| 49 |
+
- Schedule `scripts/train.py` via OCI Data Science jobs or GitHub Actions workflow.
|
| 50 |
+
- Upload the refreshed `sentiment_pipeline.joblib` to Object Storage.
|
| 51 |
+
- Trigger a rolling restart of the container instance to pick up the new artifact.
|
| 52 |
+
|
| 53 |
+
## 6. Observability & governance
|
| 54 |
+
|
| 55 |
+
- Enable OCI Logging for container stdout/err to capture Streamlit logs.
|
| 56 |
+
- Connect to OCI Application Performance Monitoring for latency insights.
|
| 57 |
+
- Track model performance drift by comparing live predictions vs. labelled feedback in Autonomous Database.
|
| 58 |
+
|
| 59 |
+
## 7. Extend to Deloitte storytelling
|
| 60 |
+
|
| 61 |
+
- Embed the public Streamlit URL in Oracle APEX or Deloitte's internal portals.
|
| 62 |
+
- Prepare a KPI dashboard using Oracle Analytics Cloud for executive presentations.
|
| 63 |
+
- Document the runbook and operational SLAs in Confluence or Deloitte knowledge base.
|
| 64 |
+
|
| 65 |
+
## 8. Multi-cloud notes
|
| 66 |
+
|
| 67 |
+
- **Azure App Service** – deploy the same container, store artifacts in Azure Blob Storage, and manage secrets in Azure Key Vault.
|
| 68 |
+
- **AWS App Runner** – push the container image to ECR, configure environment variables, and mount AWS Secrets Manager for database credentials.
|
| 69 |
+
|
| 70 |
+
With these steps you can confidently demonstrate cloud deployment proficiency, automation best practices, and Oracle ecosystem fluency during interviews.
|
deployment/streamlit_cloud.md
ADDED
|
@@ -0,0 +1,57 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Streamlit Community Cloud Launch Guide
|
| 2 |
+
|
| 3 |
+
Get a production-style link for the Deloitte-ready dashboard in under 10 minutes. Once deployed, the URL will look like `https://<your-app>.streamlit.app` and is reachable from any phone, tablet, or laptop.
|
| 4 |
+
|
| 5 |
+
## 1. Prepare the repository
|
| 6 |
+
|
| 7 |
+
1. Fork or push this project to your personal GitHub account.
|
| 8 |
+
2. Ensure the latest model artifact exists at `artifacts/sentiment_pipeline.joblib` (run `python scripts/train.py` if needed).
|
| 9 |
+
3. Commit and push any changes so Streamlit Cloud can clone the repo.
|
| 10 |
+
|
| 11 |
+
## 2. Create the Streamlit app
|
| 12 |
+
|
| 13 |
+
1. Navigate to [https://share.streamlit.io](https://share.streamlit.io) and sign in with GitHub.
|
| 14 |
+
2. Click **New app** and select the repository and branch containing this project.
|
| 15 |
+
3. Set the main file path to `app/app.py` and the Python version to **3.10** (or higher).
|
| 16 |
+
|
| 17 |
+
## 3. Configure secrets & environment variables
|
| 18 |
+
|
| 19 |
+
Add the following environment settings under **Advanced settings → Secrets**:
|
| 20 |
+
|
| 21 |
+
```toml
|
| 22 |
+
MODEL_ARTIFACT_PATH = "artifacts/sentiment_pipeline.joblib"
|
| 23 |
+
CONFIG_PATH = "config/settings.yaml"
|
| 24 |
+
```
|
| 25 |
+
|
| 26 |
+
If you plan to stream data from Oracle Autonomous Database, create an Object Storage bucket or GitHub release that hosts the wallet zip and reference it via an additional variable, for example:
|
| 27 |
+
|
| 28 |
+
```toml
|
| 29 |
+
ORACLE_WALLET_ZIP_URL = "https://objectstorage.<region>.oraclecloud.com/.../wallet.zip"
|
| 30 |
+
```
|
| 31 |
+
|
| 32 |
+
> ℹ️ Secrets are encrypted by Streamlit Cloud and can be rotated without redeploying the app.
|
| 33 |
+
|
| 34 |
+
## 4. Manage Python dependencies
|
| 35 |
+
|
| 36 |
+
1. Keep `requirements.txt` in the project root. Streamlit Cloud will automatically install the listed packages.
|
| 37 |
+
2. If you add Oracle-specific drivers such as `oracledb`, ensure the package is listed in `requirements.txt` and that the wallet files are accessible via the secret or public URL.
|
| 38 |
+
|
| 39 |
+
## 5. Deploy and validate
|
| 40 |
+
|
| 41 |
+
1. Click **Deploy**. The platform will build the environment and serve the Streamlit app.
|
| 42 |
+
2. Copy the generated URL (e.g., `https://deloitte-sentiment.streamlit.app`).
|
| 43 |
+
3. Open the link on a mobile phone to validate responsive behaviour and confirm the app loads the cached model artifact.
|
| 44 |
+
4. Update the **Live resources** table in [`README.md`](../README.md) with the new URL.
|
| 45 |
+
|
| 46 |
+
## 6. Keep the link fresh
|
| 47 |
+
|
| 48 |
+
- When you push updates to `main`, Streamlit Cloud redeploys automatically.
|
| 49 |
+
- Use feature branches for new features and merge into `main` once CI passes so the live link always reflects production-ready code.
|
| 50 |
+
- Capture screenshots of the deployed app (mobile and desktop) and store them in `docs/screenshots/` for interview-ready collateral.
|
| 51 |
+
|
| 52 |
+
## 7. Optional: Custom domain & analytics
|
| 53 |
+
|
| 54 |
+
- Map a custom domain via Streamlit Cloud's **Domain management** settings to share a memorable URL during interviews.
|
| 55 |
+
- Integrate Google Analytics or Microsoft Clarity to showcase user engagement metrics. Add the script via `st.components.v1.html` in `app/app.py`.
|
| 56 |
+
|
| 57 |
+
With this setup you can hand over a polished, always-on demo link that demonstrates cloud readiness and Deloitte-style delivery excellence.
|
deployment/vercel/index.html
ADDED
|
@@ -0,0 +1,42 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
<!DOCTYPE html>
|
| 2 |
+
<html lang="en">
|
| 3 |
+
<head>
|
| 4 |
+
<meta charset="utf-8" />
|
| 5 |
+
<title>Configure your Streamlit redirect</title>
|
| 6 |
+
<meta name="viewport" content="width=device-width, initial-scale=1" />
|
| 7 |
+
<style>
|
| 8 |
+
body {
|
| 9 |
+
font-family: system-ui, -apple-system, BlinkMacSystemFont, 'Segoe UI', sans-serif;
|
| 10 |
+
display: flex;
|
| 11 |
+
min-height: 100vh;
|
| 12 |
+
align-items: center;
|
| 13 |
+
justify-content: center;
|
| 14 |
+
background: #020617;
|
| 15 |
+
color: #f8fafc;
|
| 16 |
+
padding: 2rem;
|
| 17 |
+
text-align: center;
|
| 18 |
+
}
|
| 19 |
+
main {
|
| 20 |
+
max-width: 36rem;
|
| 21 |
+
}
|
| 22 |
+
code {
|
| 23 |
+
background: rgba(148, 163, 184, 0.2);
|
| 24 |
+
padding: 0.25rem 0.4rem;
|
| 25 |
+
border-radius: 0.25rem;
|
| 26 |
+
}
|
| 27 |
+
</style>
|
| 28 |
+
</head>
|
| 29 |
+
<body>
|
| 30 |
+
<main>
|
| 31 |
+
<h1>Vercel redirect not yet configured</h1>
|
| 32 |
+
<p>
|
| 33 |
+
Generate the production redirect page by running
|
| 34 |
+
<code>python scripts/update_vercel_redirect.py --url https://your-app.streamlit.app</code> locally and committing the
|
| 35 |
+
resulting file. This placeholder ensures Vercel deploys successfully even before the Streamlit link is ready.
|
| 36 |
+
</p>
|
| 37 |
+
<p>
|
| 38 |
+
Need detailed instructions? Read <a href="../vercel_redirect.md">deployment/vercel_redirect.md</a> in the repository.
|
| 39 |
+
</p>
|
| 40 |
+
</main>
|
| 41 |
+
</body>
|
| 42 |
+
</html>
|
deployment/vercel_redirect.md
ADDED
|
@@ -0,0 +1,61 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Vercel redirect setup for the Streamlit dashboard
|
| 2 |
+
|
| 3 |
+
The Streamlit experience in this repository is designed to run as a long-lived Python process. Vercel is optimised for static
|
| 4 |
+
content and serverless functions, so deploying Streamlit directly on Vercel typically results in a **404 Not Found** error like
|
| 5 |
+
the one shown in the screenshot you shared.
|
| 6 |
+
|
| 7 |
+
To turn your Vercel link into a reliable launchpad for interviews, use it as a lightweight redirect to the publicly hosted
|
| 8 |
+
Streamlit application (for example, a Streamlit Community Cloud deployment). The repository now ships with automation to
|
| 9 |
+
generate that redirect.
|
| 10 |
+
|
| 11 |
+
## 1. Configure the Streamlit URL
|
| 12 |
+
|
| 13 |
+
Update `live_app.streamlit_url` inside [`config/settings.yaml`](../config/settings.yaml) once your Streamlit deployment is
|
| 14 |
+
live. This value is used for both CLI validation and the Vercel redirect generator.
|
| 15 |
+
|
| 16 |
+
```json
|
| 17 |
+
"live_app": {
|
| 18 |
+
"streamlit_url": "https://your-sentiment-app.streamlit.app",
|
| 19 |
+
"vercel_redirect_domain": "https://twitter-sentiment.vercel.app"
|
| 20 |
+
}
|
| 21 |
+
```
|
| 22 |
+
|
| 23 |
+
## 2. Generate the redirect artefact
|
| 24 |
+
|
| 25 |
+
Run the helper script to create `deployment/vercel/index.html` with a zero-second redirect to your Streamlit dashboard:
|
| 26 |
+
|
| 27 |
+
```bash
|
| 28 |
+
python scripts/update_vercel_redirect.py
|
| 29 |
+
# or override explicitly
|
| 30 |
+
python scripts/update_vercel_redirect.py --url https://your-sentiment-app.streamlit.app
|
| 31 |
+
```
|
| 32 |
+
|
| 33 |
+
Commit the generated file so Vercel serves it as the project root.
|
| 34 |
+
|
| 35 |
+
## 3. Deploy on Vercel
|
| 36 |
+
|
| 37 |
+
1. Create (or update) a Vercel project pointing to this GitHub repository.
|
| 38 |
+
2. Set the **Framework Preset** to **Other** so Vercel treats the repo as static content.
|
| 39 |
+
3. Add an environment variable `STREAMLIT_PUBLIC_URL` with the Streamlit link (optional if you bake it into the config).
|
| 40 |
+
4. Deploy. Vercel will host the generated `deployment/vercel/index.html`. If you prefer the URL to live at the root, set the
|
| 41 |
+
project output directory to `deployment/vercel` under **Build & Output Settings**.
|
| 42 |
+
|
| 43 |
+
Visiting your Vercel domain will now redirect immediately to the live Streamlit experience—perfect for resume QR codes or
|
| 44 |
+
mobile demos.
|
| 45 |
+
|
| 46 |
+
## 4. Validate the setup
|
| 47 |
+
|
| 48 |
+
After deployment, run the live check CLI without arguments. It now reads from the config and confirms the Streamlit app is
|
| 49 |
+
reachable:
|
| 50 |
+
|
| 51 |
+
```bash
|
| 52 |
+
python scripts/check_live_app.py
|
| 53 |
+
```
|
| 54 |
+
|
| 55 |
+
If the CLI reports a 404, double-check that the generated HTML is committed and that Vercel is configured to serve the
|
| 56 |
+
`deployment/vercel` directory. The CLI surfaces this hint directly when a 404 occurs.
|
| 57 |
+
|
| 58 |
+
---
|
| 59 |
+
|
| 60 |
+
With these changes in place, both the Streamlit Cloud link and your Vercel vanity URL work seamlessly from any phone or
|
| 61 |
+
browser.
|
docs/architecture.md
ADDED
|
@@ -0,0 +1,42 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Architecture & Solution Blueprint
|
| 2 |
+
|
| 3 |
+
## High-level flow
|
| 4 |
+
|
| 5 |
+
1. **Ingest**: CSV files for local dev, Oracle Autonomous Database for enterprise deployments.
|
| 6 |
+
2. **Process**: Config-driven preprocessing with reusable Python package.
|
| 7 |
+
3. **Model**: Scikit-learn pipeline with TF-IDF + Logistic Regression.
|
| 8 |
+
4. **Serve**: Streamlit dashboard and CLI automation.
|
| 9 |
+
5. **Operate**: GitHub Actions CI, retraining script, and OCI deployment path.
|
| 10 |
+
|
| 11 |
+
```mermaid
|
| 12 |
+
sequenceDiagram
|
| 13 |
+
participant User
|
| 14 |
+
participant Streamlit
|
| 15 |
+
participant Predictor
|
| 16 |
+
participant Pipeline
|
| 17 |
+
participant OracleDB
|
| 18 |
+
|
| 19 |
+
User->>Streamlit: Input tweet
|
| 20 |
+
Streamlit->>Predictor: call predict_with_threshold
|
| 21 |
+
Predictor->>Pipeline: transform + predict_proba
|
| 22 |
+
Pipeline-->>Predictor: labels & probabilities
|
| 23 |
+
Predictor-->>Streamlit: curated response
|
| 24 |
+
Streamlit-->>User: sentiment insights & KPIs
|
| 25 |
+
Predictor->>OracleDB: (optional) pull latest training data
|
| 26 |
+
```
|
| 27 |
+
|
| 28 |
+
## Key metrics & KPIs
|
| 29 |
+
|
| 30 |
+
| KPI | Description | Target |
|
| 31 |
+
| --- | --- | --- |
|
| 32 |
+
| Macro F1 | Balanced view across positive/neutral/negative | ≥ 0.80 |
|
| 33 |
+
| Prediction latency | Streamlit inference response time | < 200 ms |
|
| 34 |
+
| Data freshness | Time since last Oracle sync | < 24 hours |
|
| 35 |
+
| Model drift PSI | Population stability index | < 0.2 |
|
| 36 |
+
|
| 37 |
+
## Extensibility roadmap
|
| 38 |
+
|
| 39 |
+
- Plug-in architecture for additional languages.
|
| 40 |
+
- OCI Data Science jobs for scheduled retraining.
|
| 41 |
+
- Oracle APEX dashboard embedding the Streamlit app.
|
| 42 |
+
- Integration with Deloitte's accelerators for risk & compliance logging.
|
logistic_model.pkl
DELETED
|
Binary file (1.52 kB)
|
|
|
model
DELETED
|
@@ -1 +0,0 @@
|
|
| 1 |
-
|
|
|
|
|
|
requirements.txt
CHANGED
|
@@ -1,6 +1,7 @@
|
|
| 1 |
-
|
| 2 |
-
|
| 3 |
-
scikit-learn
|
| 4 |
-
|
| 5 |
-
|
| 6 |
-
|
|
|
|
|
|
| 1 |
+
pandas>=2.2.0
|
| 2 |
+
numpy>=1.26.0
|
| 3 |
+
scikit-learn>=1.4.0
|
| 4 |
+
streamlit>=1.39.0
|
| 5 |
+
joblib>=1.3.0
|
| 6 |
+
pytest>=8.3.0
|
| 7 |
+
requests>=2.31.0
|
scripts/__init__.py
ADDED
|
File without changes
|
scripts/check_live_app.py
ADDED
|
@@ -0,0 +1,117 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Utility for verifying the deployed Streamlit app is reachable."""
|
| 2 |
+
|
| 3 |
+
from __future__ import annotations
|
| 4 |
+
|
| 5 |
+
import argparse
|
| 6 |
+
import sys
|
| 7 |
+
import time
|
| 8 |
+
from dataclasses import dataclass
|
| 9 |
+
from typing import Optional
|
| 10 |
+
|
| 11 |
+
import requests
|
| 12 |
+
|
| 13 |
+
from src.twitter_sentiment.config import load_config
|
| 14 |
+
|
| 15 |
+
|
| 16 |
+
@dataclass
|
| 17 |
+
class LiveCheckResult:
|
| 18 |
+
url: str
|
| 19 |
+
status_code: Optional[int]
|
| 20 |
+
elapsed_seconds: Optional[float]
|
| 21 |
+
error: Optional[str] = None
|
| 22 |
+
|
| 23 |
+
@property
|
| 24 |
+
def is_success(self) -> bool:
|
| 25 |
+
return self.status_code == 200 and self.error is None
|
| 26 |
+
|
| 27 |
+
|
| 28 |
+
def check_live_app(url: str, timeout: float = 10.0) -> LiveCheckResult:
|
| 29 |
+
start = time.perf_counter()
|
| 30 |
+
try:
|
| 31 |
+
response = requests.get(url, timeout=timeout)
|
| 32 |
+
elapsed = time.perf_counter() - start
|
| 33 |
+
return LiveCheckResult(
|
| 34 |
+
url=url,
|
| 35 |
+
status_code=response.status_code,
|
| 36 |
+
elapsed_seconds=elapsed,
|
| 37 |
+
)
|
| 38 |
+
except requests.RequestException as exc: # pragma: no cover - network errors vary
|
| 39 |
+
elapsed = time.perf_counter() - start
|
| 40 |
+
return LiveCheckResult(
|
| 41 |
+
url=url,
|
| 42 |
+
status_code=getattr(exc.response, "status_code", None),
|
| 43 |
+
elapsed_seconds=elapsed,
|
| 44 |
+
error=str(exc),
|
| 45 |
+
)
|
| 46 |
+
|
| 47 |
+
|
| 48 |
+
def parse_args(argv: Optional[list[str]] = None) -> argparse.Namespace:
|
| 49 |
+
parser = argparse.ArgumentParser(description=__doc__)
|
| 50 |
+
parser.add_argument(
|
| 51 |
+
"--url",
|
| 52 |
+
required=False,
|
| 53 |
+
help="Public Streamlit app URL to validate. Defaults to config live_app.streamlit_url",
|
| 54 |
+
)
|
| 55 |
+
parser.add_argument(
|
| 56 |
+
"--config",
|
| 57 |
+
default="config/settings.yaml",
|
| 58 |
+
help="Path to the project configuration file",
|
| 59 |
+
)
|
| 60 |
+
parser.add_argument(
|
| 61 |
+
"--timeout",
|
| 62 |
+
type=float,
|
| 63 |
+
default=10.0,
|
| 64 |
+
help="Seconds to wait for the server response (default: 10)",
|
| 65 |
+
)
|
| 66 |
+
return parser.parse_args(argv)
|
| 67 |
+
|
| 68 |
+
|
| 69 |
+
def _resolve_target_url(url_argument: Optional[str], config_path: str) -> str:
|
| 70 |
+
if url_argument:
|
| 71 |
+
return url_argument
|
| 72 |
+
|
| 73 |
+
config = load_config(config_path)
|
| 74 |
+
target_url = config.live_app.get("streamlit_url")
|
| 75 |
+
|
| 76 |
+
if not target_url or "<" in target_url or "your-app" in target_url:
|
| 77 |
+
raise ValueError(
|
| 78 |
+
"No live Streamlit URL configured. Provide --url or update live_app.streamlit_url in the config."
|
| 79 |
+
)
|
| 80 |
+
|
| 81 |
+
return target_url
|
| 82 |
+
|
| 83 |
+
|
| 84 |
+
def main(argv: Optional[list[str]] = None) -> int:
|
| 85 |
+
args = parse_args(argv)
|
| 86 |
+
try:
|
| 87 |
+
target_url = _resolve_target_url(args.url, args.config)
|
| 88 |
+
except ValueError as error:
|
| 89 |
+
print(f"⚠️ {error}", file=sys.stderr)
|
| 90 |
+
return 1
|
| 91 |
+
|
| 92 |
+
result = check_live_app(target_url, timeout=args.timeout)
|
| 93 |
+
|
| 94 |
+
if result.is_success:
|
| 95 |
+
print(
|
| 96 |
+
f"✅ Live app reachable at {result.url} (status {result.status_code}, "
|
| 97 |
+
f"latency {result.elapsed_seconds:.2f}s)"
|
| 98 |
+
)
|
| 99 |
+
return 0
|
| 100 |
+
|
| 101 |
+
message = (
|
| 102 |
+
f"❌ Unable to confirm app availability at {result.url}."
|
| 103 |
+
f" Status: {result.status_code or 'N/A'}."
|
| 104 |
+
)
|
| 105 |
+
if result.error:
|
| 106 |
+
message += f" Error: {result.error}"
|
| 107 |
+
elif result.status_code == 404:
|
| 108 |
+
message += (
|
| 109 |
+
" The server returned 404. Verify your Vercel redirect or update the"
|
| 110 |
+
" Streamlit deployment link as described in deployment/vercel_redirect.md."
|
| 111 |
+
)
|
| 112 |
+
print(message, file=sys.stderr)
|
| 113 |
+
return 1
|
| 114 |
+
|
| 115 |
+
|
| 116 |
+
if __name__ == "__main__": # pragma: no cover - script entry point
|
| 117 |
+
raise SystemExit(main())
|
scripts/predict.py
ADDED
|
@@ -0,0 +1,38 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""CLI helper for scoring ad-hoc text snippets."""
|
| 2 |
+
|
| 3 |
+
from __future__ import annotations
|
| 4 |
+
|
| 5 |
+
import argparse
|
| 6 |
+
import json
|
| 7 |
+
import sys
|
| 8 |
+
from pathlib import Path
|
| 9 |
+
|
| 10 |
+
ROOT = Path(__file__).resolve().parents[1]
|
| 11 |
+
SRC = ROOT / "src"
|
| 12 |
+
if str(SRC) not in sys.path:
|
| 13 |
+
sys.path.insert(0, str(SRC))
|
| 14 |
+
|
| 15 |
+
from twitter_sentiment.config import load_config
|
| 16 |
+
from twitter_sentiment.predictor import predict_with_threshold
|
| 17 |
+
|
| 18 |
+
|
| 19 |
+
def parse_args() -> argparse.Namespace:
|
| 20 |
+
parser = argparse.ArgumentParser(description="Score a tweet or message with the trained sentiment model")
|
| 21 |
+
parser.add_argument("text", help="The text to analyse")
|
| 22 |
+
parser.add_argument(
|
| 23 |
+
"--config",
|
| 24 |
+
default="config/settings.yaml",
|
| 25 |
+
help="Path to the YAML configuration file",
|
| 26 |
+
)
|
| 27 |
+
return parser.parse_args()
|
| 28 |
+
|
| 29 |
+
|
| 30 |
+
def main() -> None:
|
| 31 |
+
args = parse_args()
|
| 32 |
+
config = load_config(args.config)
|
| 33 |
+
label, probabilities = predict_with_threshold(args.text, config)
|
| 34 |
+
print(json.dumps({"label": label, "probabilities": probabilities}, indent=2))
|
| 35 |
+
|
| 36 |
+
|
| 37 |
+
if __name__ == "__main__":
|
| 38 |
+
main()
|
scripts/train.py
ADDED
|
@@ -0,0 +1,50 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Command line utility for training the sentiment analysis pipeline."""
|
| 2 |
+
|
| 3 |
+
from __future__ import annotations
|
| 4 |
+
|
| 5 |
+
import argparse
|
| 6 |
+
import json
|
| 7 |
+
import sys
|
| 8 |
+
from pathlib import Path
|
| 9 |
+
|
| 10 |
+
ROOT = Path(__file__).resolve().parents[1]
|
| 11 |
+
SRC = ROOT / "src"
|
| 12 |
+
if str(SRC) not in sys.path:
|
| 13 |
+
sys.path.insert(0, str(SRC))
|
| 14 |
+
|
| 15 |
+
from twitter_sentiment.config import load_config
|
| 16 |
+
from twitter_sentiment.modeling import persist_artifacts, train_and_evaluate
|
| 17 |
+
|
| 18 |
+
|
| 19 |
+
def parse_args() -> argparse.Namespace:
|
| 20 |
+
parser = argparse.ArgumentParser(description="Train the Deloitte-aligned Twitter sentiment model")
|
| 21 |
+
parser.add_argument(
|
| 22 |
+
"--config",
|
| 23 |
+
default="config/settings.yaml",
|
| 24 |
+
help="Path to the YAML configuration file",
|
| 25 |
+
)
|
| 26 |
+
parser.add_argument(
|
| 27 |
+
"--metrics",
|
| 28 |
+
default="artifacts/metrics.json",
|
| 29 |
+
help="Optional path to persist evaluation metrics as JSON",
|
| 30 |
+
)
|
| 31 |
+
return parser.parse_args()
|
| 32 |
+
|
| 33 |
+
|
| 34 |
+
def main() -> None:
|
| 35 |
+
args = parse_args()
|
| 36 |
+
config = load_config(args.config)
|
| 37 |
+
|
| 38 |
+
pipeline, metrics_summary = train_and_evaluate(config)
|
| 39 |
+
artifact_path = persist_artifacts(pipeline, config, metrics_summary)
|
| 40 |
+
|
| 41 |
+
metrics_path = Path(args.metrics)
|
| 42 |
+
metrics_path.parent.mkdir(parents=True, exist_ok=True)
|
| 43 |
+
metrics_path.write_text(json.dumps(metrics_summary, indent=2), encoding="utf-8")
|
| 44 |
+
|
| 45 |
+
print(f"Artifacts saved to {artifact_path}")
|
| 46 |
+
print(f"Metrics saved to {metrics_path}")
|
| 47 |
+
|
| 48 |
+
|
| 49 |
+
if __name__ == "__main__":
|
| 50 |
+
main()
|
scripts/update_vercel_redirect.py
ADDED
|
@@ -0,0 +1,102 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Generate a Vercel-friendly redirect page that points to the live Streamlit app."""
|
| 2 |
+
|
| 3 |
+
from __future__ import annotations
|
| 4 |
+
|
| 5 |
+
import argparse
|
| 6 |
+
from pathlib import Path
|
| 7 |
+
from typing import Optional
|
| 8 |
+
|
| 9 |
+
from src.twitter_sentiment.config import load_config
|
| 10 |
+
|
| 11 |
+
REDIRECT_TEMPLATE = """<!DOCTYPE html>
|
| 12 |
+
<html lang=\"en\">
|
| 13 |
+
<head>
|
| 14 |
+
<meta charset=\"utf-8\" />
|
| 15 |
+
<title>Redirecting to Streamlit dashboard</title>
|
| 16 |
+
<meta http-equiv=\"refresh\" content=\"0; url={target}\" />
|
| 17 |
+
<meta name=\"viewport\" content=\"width=device-width, initial-scale=1\" />
|
| 18 |
+
<style>
|
| 19 |
+
body {{
|
| 20 |
+
font-family: system-ui, -apple-system, BlinkMacSystemFont, 'Segoe UI', sans-serif;
|
| 21 |
+
display: flex;
|
| 22 |
+
min-height: 100vh;
|
| 23 |
+
align-items: center;
|
| 24 |
+
justify-content: center;
|
| 25 |
+
background: #0f172a;
|
| 26 |
+
color: #f8fafc;
|
| 27 |
+
padding: 2rem;
|
| 28 |
+
text-align: center;
|
| 29 |
+
}}
|
| 30 |
+
a {{ color: #38bdf8; }}
|
| 31 |
+
</style>
|
| 32 |
+
</head>
|
| 33 |
+
<body>
|
| 34 |
+
<main>
|
| 35 |
+
<h1>Taking you to the Deloitte-ready sentiment dashboard…</h1>
|
| 36 |
+
<p>If you are not redirected automatically, <a href=\"{target}\">click here to open the Streamlit app</a>.</p>
|
| 37 |
+
</main>
|
| 38 |
+
<script>
|
| 39 |
+
window.location.replace('{target}');
|
| 40 |
+
</script>
|
| 41 |
+
</body>
|
| 42 |
+
</html>
|
| 43 |
+
"""
|
| 44 |
+
|
| 45 |
+
|
| 46 |
+
def parse_args(argv: Optional[list[str]] = None) -> argparse.Namespace:
|
| 47 |
+
parser = argparse.ArgumentParser(description=__doc__)
|
| 48 |
+
parser.add_argument(
|
| 49 |
+
"--config",
|
| 50 |
+
default="config/settings.yaml",
|
| 51 |
+
help="Path to the configuration file containing live_app settings",
|
| 52 |
+
)
|
| 53 |
+
parser.add_argument(
|
| 54 |
+
"--output",
|
| 55 |
+
default="deployment/vercel/index.html",
|
| 56 |
+
help="HTML file path to write (defaults to deployment/vercel/index.html)",
|
| 57 |
+
)
|
| 58 |
+
parser.add_argument(
|
| 59 |
+
"--url",
|
| 60 |
+
help="Override URL for the redirect. If omitted, use live_app.streamlit_url from the config",
|
| 61 |
+
)
|
| 62 |
+
return parser.parse_args(argv)
|
| 63 |
+
|
| 64 |
+
|
| 65 |
+
def resolve_streamlit_url(args: argparse.Namespace) -> str:
|
| 66 |
+
if args.url:
|
| 67 |
+
return args.url
|
| 68 |
+
|
| 69 |
+
config = load_config(args.config)
|
| 70 |
+
streamlit_url = config.live_app.get("streamlit_url")
|
| 71 |
+
if not streamlit_url or "<" in streamlit_url or "your-app" in streamlit_url:
|
| 72 |
+
raise ValueError(
|
| 73 |
+
"Configure live_app.streamlit_url in config/settings.yaml or pass --url to set the redirect target."
|
| 74 |
+
)
|
| 75 |
+
|
| 76 |
+
return streamlit_url
|
| 77 |
+
|
| 78 |
+
|
| 79 |
+
def write_redirect_html(output_path: str, target_url: str) -> Path:
|
| 80 |
+
html_content = REDIRECT_TEMPLATE.format(target=target_url)
|
| 81 |
+
destination = Path(output_path)
|
| 82 |
+
destination.parent.mkdir(parents=True, exist_ok=True)
|
| 83 |
+
destination.write_text(html_content, encoding="utf-8")
|
| 84 |
+
return destination
|
| 85 |
+
|
| 86 |
+
|
| 87 |
+
def main(argv: Optional[list[str]] = None) -> int:
|
| 88 |
+
args = parse_args(argv)
|
| 89 |
+
|
| 90 |
+
try:
|
| 91 |
+
url = resolve_streamlit_url(args)
|
| 92 |
+
except ValueError as error:
|
| 93 |
+
print(f"⚠️ {error}")
|
| 94 |
+
return 1
|
| 95 |
+
|
| 96 |
+
destination = write_redirect_html(args.output, url)
|
| 97 |
+
print(f"✅ Wrote Vercel redirect page pointing to {url} at {destination}")
|
| 98 |
+
return 0
|
| 99 |
+
|
| 100 |
+
|
| 101 |
+
if __name__ == "__main__": # pragma: no cover - script entry point
|
| 102 |
+
raise SystemExit(main())
|
sentiment_model.pkl
DELETED
|
Binary file (2.93 kB)
|
|
|
src/twitter_sentiment/__init__.py
ADDED
|
@@ -0,0 +1,10 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Core package for the Deloitte-aligned Twitter sentiment analysis solution."""
|
| 2 |
+
|
| 3 |
+
from importlib.metadata import version, PackageNotFoundError
|
| 4 |
+
|
| 5 |
+
try:
|
| 6 |
+
__version__ = version("twitter-sentiment-analysis")
|
| 7 |
+
except PackageNotFoundError: # pragma: no cover - best effort
|
| 8 |
+
__version__ = "0.1.0"
|
| 9 |
+
|
| 10 |
+
__all__ = ["__version__"]
|
src/twitter_sentiment/config.py
ADDED
|
@@ -0,0 +1,76 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Utilities for loading and validating project configuration."""
|
| 2 |
+
|
| 3 |
+
from __future__ import annotations
|
| 4 |
+
|
| 5 |
+
import importlib
|
| 6 |
+
import json
|
| 7 |
+
from dataclasses import dataclass
|
| 8 |
+
from pathlib import Path
|
| 9 |
+
from typing import Any, Dict
|
| 10 |
+
|
| 11 |
+
_yaml_spec = importlib.util.find_spec("yaml")
|
| 12 |
+
_yaml_module = importlib.import_module("yaml") if _yaml_spec else None
|
| 13 |
+
|
| 14 |
+
|
| 15 |
+
@dataclass
|
| 16 |
+
class Config:
|
| 17 |
+
"""Typed representation of the project configuration."""
|
| 18 |
+
|
| 19 |
+
raw: Dict[str, Any]
|
| 20 |
+
|
| 21 |
+
@property
|
| 22 |
+
def data(self) -> Dict[str, Any]:
|
| 23 |
+
return self.raw.get("data", {})
|
| 24 |
+
|
| 25 |
+
@property
|
| 26 |
+
def preprocessing(self) -> Dict[str, Any]:
|
| 27 |
+
return self.raw.get("preprocessing", {})
|
| 28 |
+
|
| 29 |
+
@property
|
| 30 |
+
def model(self) -> Dict[str, Any]:
|
| 31 |
+
return self.raw.get("model", {})
|
| 32 |
+
|
| 33 |
+
@property
|
| 34 |
+
def training(self) -> Dict[str, Any]:
|
| 35 |
+
return self.raw.get("training", {})
|
| 36 |
+
|
| 37 |
+
@property
|
| 38 |
+
def project(self) -> Dict[str, Any]:
|
| 39 |
+
return self.raw.get("project", {})
|
| 40 |
+
|
| 41 |
+
@property
|
| 42 |
+
def monitoring(self) -> Dict[str, Any]:
|
| 43 |
+
return self.raw.get("monitoring", {})
|
| 44 |
+
|
| 45 |
+
@property
|
| 46 |
+
def oracle_integration(self) -> Dict[str, Any]:
|
| 47 |
+
return self.raw.get("oracle_integration", {})
|
| 48 |
+
|
| 49 |
+
@property
|
| 50 |
+
def live_app(self) -> Dict[str, Any]:
|
| 51 |
+
return self.raw.get("live_app", {})
|
| 52 |
+
|
| 53 |
+
|
| 54 |
+
def _deserialize_config(content: str) -> Dict[str, Any]:
|
| 55 |
+
if _yaml_module:
|
| 56 |
+
return _yaml_module.safe_load(content)
|
| 57 |
+
return json.loads(content)
|
| 58 |
+
|
| 59 |
+
|
| 60 |
+
def load_config(path: str | Path = "config/settings.yaml") -> Config:
|
| 61 |
+
"""Load configuration from YAML (or JSON) file."""
|
| 62 |
+
|
| 63 |
+
config_path = Path(path)
|
| 64 |
+
if not config_path.exists():
|
| 65 |
+
raise FileNotFoundError(f"Configuration file not found at {config_path!s}")
|
| 66 |
+
|
| 67 |
+
content = config_path.read_text(encoding="utf-8")
|
| 68 |
+
raw_config = _deserialize_config(content)
|
| 69 |
+
|
| 70 |
+
if not isinstance(raw_config, dict):
|
| 71 |
+
raise ValueError("Configuration file must define a dictionary at the top level")
|
| 72 |
+
|
| 73 |
+
return Config(raw=raw_config)
|
| 74 |
+
|
| 75 |
+
|
| 76 |
+
__all__ = ["Config", "load_config"]
|
src/twitter_sentiment/data_loader.py
ADDED
|
@@ -0,0 +1,80 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Data loading utilities supporting CSV and Oracle Autonomous DB sources."""
|
| 2 |
+
|
| 3 |
+
from __future__ import annotations
|
| 4 |
+
|
| 5 |
+
import importlib
|
| 6 |
+
from pathlib import Path
|
| 7 |
+
from typing import Optional
|
| 8 |
+
|
| 9 |
+
import pandas as pd
|
| 10 |
+
|
| 11 |
+
from .config import Config
|
| 12 |
+
from .logging_utils import get_logger
|
| 13 |
+
|
| 14 |
+
LOGGER = get_logger(__name__)
|
| 15 |
+
|
| 16 |
+
|
| 17 |
+
def _load_from_csv(csv_path: Path, text_column: str, target_column: str) -> pd.DataFrame:
|
| 18 |
+
if not csv_path.exists():
|
| 19 |
+
raise FileNotFoundError(f"Training data not found at {csv_path!s}")
|
| 20 |
+
|
| 21 |
+
df = pd.read_csv(csv_path)
|
| 22 |
+
if text_column not in df.columns or target_column not in df.columns:
|
| 23 |
+
raise KeyError(
|
| 24 |
+
f"CSV file must contain '{text_column}' and '{target_column}' columns. Found: {df.columns.tolist()}"
|
| 25 |
+
)
|
| 26 |
+
|
| 27 |
+
LOGGER.info("Loaded %d records from %s", len(df), csv_path)
|
| 28 |
+
return df[[text_column, target_column]].dropna()
|
| 29 |
+
|
| 30 |
+
|
| 31 |
+
def _load_from_oracle(settings: dict) -> pd.DataFrame:
|
| 32 |
+
spec = importlib.util.find_spec("oracledb")
|
| 33 |
+
if spec is None:
|
| 34 |
+
raise ModuleNotFoundError(
|
| 35 |
+
"oracledb package is required for Oracle Autonomous Database ingestion. Install via `pip install oracledb`."
|
| 36 |
+
)
|
| 37 |
+
|
| 38 |
+
oracledb = importlib.import_module("oracledb")
|
| 39 |
+
wallet_location = settings.get("wallet_location")
|
| 40 |
+
user = settings.get("user")
|
| 41 |
+
dsn = settings.get("dsn")
|
| 42 |
+
query = settings.get("sql_query")
|
| 43 |
+
|
| 44 |
+
if not all([wallet_location, user, dsn, query]):
|
| 45 |
+
raise ValueError("Oracle configuration must define wallet_location, user, dsn and sql_query")
|
| 46 |
+
|
| 47 |
+
connection = oracledb.connect(user=user, dsn=dsn, config_dir=wallet_location)
|
| 48 |
+
try:
|
| 49 |
+
df = pd.read_sql(query, con=connection)
|
| 50 |
+
finally:
|
| 51 |
+
connection.close()
|
| 52 |
+
|
| 53 |
+
LOGGER.info("Loaded %d records from Oracle Autonomous Database", len(df))
|
| 54 |
+
return df
|
| 55 |
+
|
| 56 |
+
|
| 57 |
+
def load_dataset(config: Config, limit: Optional[int] = None) -> pd.DataFrame:
|
| 58 |
+
"""Load dataset based on configuration.
|
| 59 |
+
|
| 60 |
+
If the oracle integration is enabled the function attempts to load data from
|
| 61 |
+
Oracle Autonomous Database, otherwise it falls back to CSV ingestion.
|
| 62 |
+
"""
|
| 63 |
+
|
| 64 |
+
data_config = config.data
|
| 65 |
+
oracle_config = config.oracle_integration
|
| 66 |
+
|
| 67 |
+
if oracle_config.get("enabled"):
|
| 68 |
+
df = _load_from_oracle(oracle_config)
|
| 69 |
+
else:
|
| 70 |
+
csv_path = Path(data_config.get("path", ""))
|
| 71 |
+
df = _load_from_csv(csv_path, data_config.get("text_column", "text"), data_config.get("target_column", "sentiment"))
|
| 72 |
+
|
| 73 |
+
if limit:
|
| 74 |
+
df = df.head(limit)
|
| 75 |
+
LOGGER.info("Sampling first %d records for experimentation", limit)
|
| 76 |
+
|
| 77 |
+
return df
|
| 78 |
+
|
| 79 |
+
|
| 80 |
+
__all__ = ["load_dataset"]
|
src/twitter_sentiment/logging_utils.py
ADDED
|
@@ -0,0 +1,39 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Centralised logging configuration for the project."""
|
| 2 |
+
|
| 3 |
+
from __future__ import annotations
|
| 4 |
+
|
| 5 |
+
import logging
|
| 6 |
+
from logging import Logger
|
| 7 |
+
from pathlib import Path
|
| 8 |
+
from typing import Optional
|
| 9 |
+
|
| 10 |
+
|
| 11 |
+
def get_logger(name: str, log_file: Optional[str] = None) -> Logger:
|
| 12 |
+
"""Return a configured logger instance."""
|
| 13 |
+
|
| 14 |
+
logger = logging.getLogger(name)
|
| 15 |
+
if logger.handlers:
|
| 16 |
+
return logger
|
| 17 |
+
|
| 18 |
+
logger.setLevel(logging.INFO)
|
| 19 |
+
formatter = logging.Formatter(
|
| 20 |
+
fmt="%(asctime)s | %(levelname)s | %(name)s | %(message)s",
|
| 21 |
+
datefmt="%Y-%m-%d %H:%M:%S",
|
| 22 |
+
)
|
| 23 |
+
|
| 24 |
+
stream_handler = logging.StreamHandler()
|
| 25 |
+
stream_handler.setFormatter(formatter)
|
| 26 |
+
logger.addHandler(stream_handler)
|
| 27 |
+
|
| 28 |
+
if log_file:
|
| 29 |
+
log_path = Path(log_file)
|
| 30 |
+
log_path.parent.mkdir(parents=True, exist_ok=True)
|
| 31 |
+
file_handler = logging.FileHandler(log_path, encoding="utf-8")
|
| 32 |
+
file_handler.setFormatter(formatter)
|
| 33 |
+
logger.addHandler(file_handler)
|
| 34 |
+
|
| 35 |
+
logger.propagate = False
|
| 36 |
+
return logger
|
| 37 |
+
|
| 38 |
+
|
| 39 |
+
__all__ = ["get_logger"]
|
src/twitter_sentiment/modeling.py
ADDED
|
@@ -0,0 +1,161 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Model training utilities."""
|
| 2 |
+
|
| 3 |
+
from __future__ import annotations
|
| 4 |
+
|
| 5 |
+
from pathlib import Path
|
| 6 |
+
from typing import Dict, Tuple
|
| 7 |
+
|
| 8 |
+
import joblib
|
| 9 |
+
import numpy as np
|
| 10 |
+
from sklearn import metrics
|
| 11 |
+
from sklearn.feature_extraction.text import TfidfVectorizer
|
| 12 |
+
from sklearn.linear_model import LogisticRegression
|
| 13 |
+
from sklearn.model_selection import StratifiedKFold, cross_val_score, train_test_split
|
| 14 |
+
from sklearn.pipeline import Pipeline
|
| 15 |
+
|
| 16 |
+
from .config import Config
|
| 17 |
+
from .logging_utils import get_logger
|
| 18 |
+
from .preprocessing import preprocess_dataframe
|
| 19 |
+
|
| 20 |
+
LOGGER = get_logger(__name__)
|
| 21 |
+
|
| 22 |
+
MetricsDict = Dict[str, float]
|
| 23 |
+
|
| 24 |
+
|
| 25 |
+
def build_pipeline(config: Config) -> Pipeline:
|
| 26 |
+
model_settings = config.model
|
| 27 |
+
|
| 28 |
+
vectorizer = TfidfVectorizer(
|
| 29 |
+
max_features=model_settings.get("max_features", 5000),
|
| 30 |
+
ngram_range=tuple(model_settings.get("ngram_range", (1, 1))),
|
| 31 |
+
stop_words="english",
|
| 32 |
+
)
|
| 33 |
+
|
| 34 |
+
estimator = LogisticRegression(
|
| 35 |
+
class_weight=model_settings.get("class_weight"),
|
| 36 |
+
max_iter=1000,
|
| 37 |
+
multi_class="auto",
|
| 38 |
+
solver="lbfgs",
|
| 39 |
+
)
|
| 40 |
+
|
| 41 |
+
pipeline = Pipeline(
|
| 42 |
+
steps=[
|
| 43 |
+
("tfidf", vectorizer),
|
| 44 |
+
("clf", estimator),
|
| 45 |
+
]
|
| 46 |
+
)
|
| 47 |
+
|
| 48 |
+
return pipeline
|
| 49 |
+
|
| 50 |
+
|
| 51 |
+
def _resolve_test_size(config: Config, target) -> float | int:
|
| 52 |
+
test_size = config.model.get("test_size", 0.2)
|
| 53 |
+
n_samples = len(target)
|
| 54 |
+
n_classes = int(getattr(target, "nunique", lambda: len(set(target)))())
|
| 55 |
+
|
| 56 |
+
if isinstance(test_size, float):
|
| 57 |
+
computed = max(1, int(round(test_size * n_samples)))
|
| 58 |
+
if computed < n_classes and n_samples > 0:
|
| 59 |
+
adjusted = max(n_classes / n_samples, test_size)
|
| 60 |
+
test_size = min(adjusted, 0.5)
|
| 61 |
+
LOGGER.warning(
|
| 62 |
+
"Adjusted test_size to %.2f to ensure each class appears in the test split",
|
| 63 |
+
test_size,
|
| 64 |
+
)
|
| 65 |
+
else:
|
| 66 |
+
if test_size < n_classes:
|
| 67 |
+
test_size = n_classes
|
| 68 |
+
LOGGER.warning(
|
| 69 |
+
"Adjusted test_size to %d to ensure each class appears in the test split",
|
| 70 |
+
test_size,
|
| 71 |
+
)
|
| 72 |
+
|
| 73 |
+
return test_size
|
| 74 |
+
|
| 75 |
+
|
| 76 |
+
def train_and_evaluate(config: Config) -> Tuple[Pipeline, MetricsDict]:
|
| 77 |
+
df = preprocess_dataframe(load_dataset(config), config)
|
| 78 |
+
|
| 79 |
+
data_settings = config.data
|
| 80 |
+
text_column = data_settings.get("text_column", "text")
|
| 81 |
+
target_column = data_settings.get("target_column", "sentiment")
|
| 82 |
+
|
| 83 |
+
y_full = df[target_column].astype(str)
|
| 84 |
+
test_size = _resolve_test_size(config, y_full)
|
| 85 |
+
|
| 86 |
+
X_train, X_test, y_train, y_test = train_test_split(
|
| 87 |
+
df[text_column],
|
| 88 |
+
y_full,
|
| 89 |
+
test_size=test_size,
|
| 90 |
+
random_state=config.model.get("random_state", 42),
|
| 91 |
+
stratify=y_full,
|
| 92 |
+
)
|
| 93 |
+
|
| 94 |
+
pipeline = build_pipeline(config)
|
| 95 |
+
LOGGER.info("Starting model training on %d samples", X_train.shape[0])
|
| 96 |
+
pipeline.fit(X_train, y_train)
|
| 97 |
+
|
| 98 |
+
predictions = pipeline.predict(X_test)
|
| 99 |
+
classification_report = metrics.classification_report(y_test, predictions, output_dict=True)
|
| 100 |
+
|
| 101 |
+
metrics_summary = {
|
| 102 |
+
"accuracy": metrics.accuracy_score(y_test, predictions),
|
| 103 |
+
"f1_macro": metrics.f1_score(y_test, predictions, average="macro"),
|
| 104 |
+
"precision_macro": metrics.precision_score(y_test, predictions, average="macro"),
|
| 105 |
+
"recall_macro": metrics.recall_score(y_test, predictions, average="macro"),
|
| 106 |
+
}
|
| 107 |
+
LOGGER.info("Evaluation metrics: %s", metrics_summary)
|
| 108 |
+
|
| 109 |
+
cv_folds = int(config.training.get("cv_folds", 0))
|
| 110 |
+
if cv_folds > 1:
|
| 111 |
+
value_counts = getattr(y_full, 'value_counts', lambda: [])()
|
| 112 |
+
min_class_count = int(value_counts.min()) if len(value_counts) else 0
|
| 113 |
+
if min_class_count < cv_folds:
|
| 114 |
+
cv_folds = max(min_class_count, 1)
|
| 115 |
+
LOGGER.warning("Reduced cv_folds to %d based on smallest class size", cv_folds)
|
| 116 |
+
if cv_folds > 1:
|
| 117 |
+
cv = StratifiedKFold(
|
| 118 |
+
n_splits=cv_folds,
|
| 119 |
+
shuffle=True,
|
| 120 |
+
random_state=config.model.get("random_state", 42),
|
| 121 |
+
)
|
| 122 |
+
cv_scores = cross_val_score(
|
| 123 |
+
pipeline,
|
| 124 |
+
df[text_column],
|
| 125 |
+
y_full,
|
| 126 |
+
cv=cv,
|
| 127 |
+
scoring=config.training.get("scoring", "f1_macro"),
|
| 128 |
+
)
|
| 129 |
+
metrics_summary["cv_mean"] = float(np.mean(cv_scores))
|
| 130 |
+
metrics_summary["cv_std"] = float(np.std(cv_scores))
|
| 131 |
+
LOGGER.info(
|
| 132 |
+
"Cross-validation %s: mean=%.4f std=%.4f",
|
| 133 |
+
config.training.get("scoring", "f1_macro"),
|
| 134 |
+
metrics_summary["cv_mean"],
|
| 135 |
+
metrics_summary["cv_std"],
|
| 136 |
+
)
|
| 137 |
+
|
| 138 |
+
class_order = config.data.get("class_order", [])
|
| 139 |
+
for label in class_order:
|
| 140 |
+
if label in classification_report:
|
| 141 |
+
metrics_summary[f"{label}_f1"] = classification_report[label]["f1-score"]
|
| 142 |
+
|
| 143 |
+
return pipeline, metrics_summary
|
| 144 |
+
|
| 145 |
+
|
| 146 |
+
def persist_artifacts(pipeline: Pipeline, config: Config, metrics_summary: MetricsDict) -> Path:
|
| 147 |
+
artifact_dir = Path(config.model.get("artifact_dir", "artifacts"))
|
| 148 |
+
artifact_dir.mkdir(parents=True, exist_ok=True)
|
| 149 |
+
|
| 150 |
+
artifact_path = artifact_dir / config.model.get("pipeline_filename", "sentiment_pipeline.joblib")
|
| 151 |
+
joblib.dump({"pipeline": pipeline, "metrics": metrics_summary}, artifact_path)
|
| 152 |
+
|
| 153 |
+
LOGGER.info("Saved pipeline and metrics to %s", artifact_path)
|
| 154 |
+
return artifact_path
|
| 155 |
+
|
| 156 |
+
|
| 157 |
+
# Avoid circular import by late import
|
| 158 |
+
from .data_loader import load_dataset # noqa: E402 pylint: disable=wrong-import-position
|
| 159 |
+
|
| 160 |
+
|
| 161 |
+
__all__ = ["build_pipeline", "train_and_evaluate", "persist_artifacts"]
|
src/twitter_sentiment/predictor.py
ADDED
|
@@ -0,0 +1,69 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Utilities to load persisted artifacts and score new text."""
|
| 2 |
+
|
| 3 |
+
from __future__ import annotations
|
| 4 |
+
|
| 5 |
+
from pathlib import Path
|
| 6 |
+
from typing import Dict, List, Tuple
|
| 7 |
+
|
| 8 |
+
import joblib
|
| 9 |
+
|
| 10 |
+
from .config import Config, load_config
|
| 11 |
+
from .logging_utils import get_logger
|
| 12 |
+
|
| 13 |
+
LOGGER = get_logger(__name__)
|
| 14 |
+
|
| 15 |
+
|
| 16 |
+
def load_artifacts(config: Config | None = None):
|
| 17 |
+
if config is None:
|
| 18 |
+
config = load_config()
|
| 19 |
+
|
| 20 |
+
artifact_dir = Path(config.model.get("artifact_dir", "artifacts"))
|
| 21 |
+
artifact_path = artifact_dir / config.model.get("pipeline_filename", "sentiment_pipeline.joblib")
|
| 22 |
+
|
| 23 |
+
if not artifact_path.exists():
|
| 24 |
+
raise FileNotFoundError(
|
| 25 |
+
f"Model artifact not found at {artifact_path!s}. Train the model by running `python scripts/train.py`."
|
| 26 |
+
)
|
| 27 |
+
|
| 28 |
+
data = joblib.load(artifact_path)
|
| 29 |
+
pipeline = data["pipeline"] if isinstance(data, dict) and "pipeline" in data else data
|
| 30 |
+
metrics: Dict[str, float] = data.get("metrics", {}) if isinstance(data, dict) else {}
|
| 31 |
+
|
| 32 |
+
LOGGER.info("Loaded pipeline from %s", artifact_path)
|
| 33 |
+
return pipeline, metrics
|
| 34 |
+
|
| 35 |
+
|
| 36 |
+
def predict(texts: List[str], config: Config | None = None) -> Tuple[List[str], List[Dict[str, float]]]:
|
| 37 |
+
pipeline, _ = load_artifacts(config)
|
| 38 |
+
probabilities = pipeline.predict_proba(texts)
|
| 39 |
+
labels = pipeline.classes_
|
| 40 |
+
|
| 41 |
+
predicted_labels = pipeline.predict(texts)
|
| 42 |
+
probability_dicts: List[Dict[str, float]] = []
|
| 43 |
+
for prob in probabilities:
|
| 44 |
+
probability_dicts.append({label: float(score) for label, score in zip(labels, prob)})
|
| 45 |
+
|
| 46 |
+
return predicted_labels.tolist(), probability_dicts
|
| 47 |
+
|
| 48 |
+
|
| 49 |
+
def predict_with_threshold(text: str, config: Config | None = None) -> Tuple[str, Dict[str, float]]:
|
| 50 |
+
config = config or load_config()
|
| 51 |
+
labels, probabilities = predict([text], config)
|
| 52 |
+
probability = probabilities[0]
|
| 53 |
+
|
| 54 |
+
thresholds = config.training.get("probability_thresholds", {})
|
| 55 |
+
positive_threshold = thresholds.get("positive", 0.5)
|
| 56 |
+
negative_threshold = thresholds.get("negative", 0.5)
|
| 57 |
+
|
| 58 |
+
selected_label = labels[0]
|
| 59 |
+
if probability.get("positive", 0.0) >= positive_threshold:
|
| 60 |
+
selected_label = "positive"
|
| 61 |
+
elif probability.get("negative", 0.0) >= negative_threshold:
|
| 62 |
+
selected_label = "negative"
|
| 63 |
+
else:
|
| 64 |
+
selected_label = "neutral"
|
| 65 |
+
|
| 66 |
+
return selected_label, probability
|
| 67 |
+
|
| 68 |
+
|
| 69 |
+
__all__ = ["load_artifacts", "predict", "predict_with_threshold"]
|
src/twitter_sentiment/preprocessing.py
ADDED
|
@@ -0,0 +1,64 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Reusable text cleaning utilities."""
|
| 2 |
+
|
| 3 |
+
from __future__ import annotations
|
| 4 |
+
|
| 5 |
+
import re
|
| 6 |
+
import string
|
| 7 |
+
from typing import Iterable
|
| 8 |
+
|
| 9 |
+
import pandas as pd
|
| 10 |
+
|
| 11 |
+
from .config import Config
|
| 12 |
+
from .logging_utils import get_logger
|
| 13 |
+
|
| 14 |
+
LOGGER = get_logger(__name__)
|
| 15 |
+
|
| 16 |
+
URL_PATTERN = re.compile(r"https?://\S+|www\.\S+")
|
| 17 |
+
MENTION_PATTERN = re.compile(r"@[A-Za-z0-9_]+")
|
| 18 |
+
HASHTAG_PATTERN = re.compile(r"#[A-Za-z0-9_]+")
|
| 19 |
+
|
| 20 |
+
|
| 21 |
+
def clean_text(text: str, config: Config) -> str:
|
| 22 |
+
settings = config.preprocessing
|
| 23 |
+
processed = text
|
| 24 |
+
|
| 25 |
+
if settings.get("lowercase", True):
|
| 26 |
+
processed = processed.lower()
|
| 27 |
+
|
| 28 |
+
if settings.get("strip_urls", True):
|
| 29 |
+
processed = URL_PATTERN.sub("", processed)
|
| 30 |
+
|
| 31 |
+
if settings.get("strip_mentions", True):
|
| 32 |
+
processed = MENTION_PATTERN.sub("", processed)
|
| 33 |
+
|
| 34 |
+
if settings.get("strip_hashtags", False):
|
| 35 |
+
processed = HASHTAG_PATTERN.sub("", processed)
|
| 36 |
+
|
| 37 |
+
if settings.get("remove_punctuation", True):
|
| 38 |
+
processed = processed.translate(str.maketrans("", "", string.punctuation))
|
| 39 |
+
|
| 40 |
+
if settings.get("normalize_whitespace", True):
|
| 41 |
+
processed = re.sub(r"\s+", " ", processed).strip()
|
| 42 |
+
|
| 43 |
+
return processed
|
| 44 |
+
|
| 45 |
+
|
| 46 |
+
def preprocess_dataframe(df: pd.DataFrame, config: Config) -> pd.DataFrame:
|
| 47 |
+
"""Apply text cleaning and reorder target labels."""
|
| 48 |
+
|
| 49 |
+
data_settings = config.data
|
| 50 |
+
text_column = data_settings.get("text_column", "text")
|
| 51 |
+
target_column = data_settings.get("target_column", "sentiment")
|
| 52 |
+
|
| 53 |
+
df = df.copy()
|
| 54 |
+
df[text_column] = df[text_column].astype(str).apply(lambda text: clean_text(text, config))
|
| 55 |
+
|
| 56 |
+
class_order: Iterable[str] = data_settings.get("class_order") or config.model.get("class_order")
|
| 57 |
+
if class_order:
|
| 58 |
+
df[target_column] = pd.Categorical(df[target_column], categories=list(class_order), ordered=True)
|
| 59 |
+
|
| 60 |
+
LOGGER.info("Completed preprocessing for %d records", len(df))
|
| 61 |
+
return df
|
| 62 |
+
|
| 63 |
+
|
| 64 |
+
__all__ = ["clean_text", "preprocess_dataframe"]
|
tests/conftest.py
ADDED
|
@@ -0,0 +1,30 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from __future__ import annotations
|
| 2 |
+
|
| 3 |
+
import sys
|
| 4 |
+
from pathlib import Path
|
| 5 |
+
|
| 6 |
+
ROOT = Path(__file__).resolve().parents[1]
|
| 7 |
+
SRC = ROOT / "src"
|
| 8 |
+
if str(SRC) not in sys.path:
|
| 9 |
+
sys.path.insert(0, str(SRC))
|
| 10 |
+
|
| 11 |
+
import pytest
|
| 12 |
+
|
| 13 |
+
from twitter_sentiment.config import load_config
|
| 14 |
+
from twitter_sentiment.modeling import persist_artifacts, train_and_evaluate
|
| 15 |
+
|
| 16 |
+
|
| 17 |
+
@pytest.fixture(scope="session", autouse=True)
|
| 18 |
+
def ensure_artifacts():
|
| 19 |
+
"""Train and persist the pipeline if artifacts are absent."""
|
| 20 |
+
|
| 21 |
+
config = load_config()
|
| 22 |
+
artifact_dir = Path(config.model.get("artifact_dir", "artifacts"))
|
| 23 |
+
artifact_path = artifact_dir / config.model.get("pipeline_filename", "sentiment_pipeline.joblib")
|
| 24 |
+
|
| 25 |
+
if artifact_path.exists():
|
| 26 |
+
return
|
| 27 |
+
|
| 28 |
+
pipeline, metrics = train_and_evaluate(config)
|
| 29 |
+
persist_artifacts(pipeline, config, metrics)
|
| 30 |
+
|
tests/test_check_live_app.py
ADDED
|
@@ -0,0 +1,103 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from __future__ import annotations
|
| 2 |
+
|
| 3 |
+
import json
|
| 4 |
+
import sys
|
| 5 |
+
from pathlib import Path
|
| 6 |
+
from types import SimpleNamespace
|
| 7 |
+
|
| 8 |
+
import requests
|
| 9 |
+
|
| 10 |
+
sys.path.append(str(Path(__file__).resolve().parents[1]))
|
| 11 |
+
|
| 12 |
+
from scripts.check_live_app import LiveCheckResult, check_live_app, main
|
| 13 |
+
|
| 14 |
+
|
| 15 |
+
class DummyResponse(SimpleNamespace):
|
| 16 |
+
def __init__(self, status_code: int):
|
| 17 |
+
super().__init__(status_code=status_code)
|
| 18 |
+
|
| 19 |
+
|
| 20 |
+
def test_check_live_app_success(monkeypatch):
|
| 21 |
+
def fake_get(url, timeout):
|
| 22 |
+
assert url == "https://example.com"
|
| 23 |
+
assert timeout == 5
|
| 24 |
+
return DummyResponse(status_code=200)
|
| 25 |
+
|
| 26 |
+
monkeypatch.setattr("scripts.check_live_app.requests.get", fake_get)
|
| 27 |
+
|
| 28 |
+
result = check_live_app("https://example.com", timeout=5)
|
| 29 |
+
assert isinstance(result, LiveCheckResult)
|
| 30 |
+
assert result.is_success
|
| 31 |
+
assert result.status_code == 200
|
| 32 |
+
assert result.error is None
|
| 33 |
+
|
| 34 |
+
|
| 35 |
+
def test_check_live_app_failure(monkeypatch):
|
| 36 |
+
class DummyException(requests.RequestException):
|
| 37 |
+
def __init__(self, message: str):
|
| 38 |
+
super().__init__(message)
|
| 39 |
+
self.response = DummyResponse(status_code=503)
|
| 40 |
+
|
| 41 |
+
def fake_get(url, timeout): # pragma: no cover - executed via exception
|
| 42 |
+
raise DummyException("service unavailable")
|
| 43 |
+
|
| 44 |
+
monkeypatch.setattr("scripts.check_live_app.requests.get", fake_get)
|
| 45 |
+
|
| 46 |
+
result = check_live_app("https://example.com")
|
| 47 |
+
assert not result.is_success
|
| 48 |
+
assert result.status_code == 503
|
| 49 |
+
assert "service unavailable" in result.error
|
| 50 |
+
|
| 51 |
+
|
| 52 |
+
def test_main_success(monkeypatch, capsys):
|
| 53 |
+
def fake_check(url, timeout):
|
| 54 |
+
return LiveCheckResult(url=url, status_code=200, elapsed_seconds=0.12, error=None)
|
| 55 |
+
|
| 56 |
+
monkeypatch.setattr("scripts.check_live_app.check_live_app", fake_check)
|
| 57 |
+
|
| 58 |
+
exit_code = main(["--url", "https://example.com"])
|
| 59 |
+
captured = capsys.readouterr()
|
| 60 |
+
|
| 61 |
+
assert exit_code == 0
|
| 62 |
+
assert "✅" in captured.out
|
| 63 |
+
|
| 64 |
+
|
| 65 |
+
def test_main_failure(monkeypatch, capsys):
|
| 66 |
+
def fake_check(url, timeout):
|
| 67 |
+
return LiveCheckResult(url=url, status_code=500, elapsed_seconds=0.12, error="boom")
|
| 68 |
+
|
| 69 |
+
monkeypatch.setattr("scripts.check_live_app.check_live_app", fake_check)
|
| 70 |
+
|
| 71 |
+
exit_code = main(["--url", "https://example.com"])
|
| 72 |
+
captured = capsys.readouterr()
|
| 73 |
+
|
| 74 |
+
assert exit_code == 1
|
| 75 |
+
assert "❌" in captured.err
|
| 76 |
+
|
| 77 |
+
|
| 78 |
+
def test_main_uses_config_when_url_missing(tmp_path, monkeypatch, capsys):
|
| 79 |
+
config_path = tmp_path / "settings.json"
|
| 80 |
+
config_path.write_text(json.dumps({"live_app": {"streamlit_url": "https://config-app.streamlit.app"}}))
|
| 81 |
+
|
| 82 |
+
def fake_check(url, timeout):
|
| 83 |
+
assert url == "https://config-app.streamlit.app"
|
| 84 |
+
return LiveCheckResult(url=url, status_code=200, elapsed_seconds=0.05, error=None)
|
| 85 |
+
|
| 86 |
+
monkeypatch.setattr("scripts.check_live_app.check_live_app", fake_check)
|
| 87 |
+
|
| 88 |
+
exit_code = main(["--config", str(config_path)])
|
| 89 |
+
captured = capsys.readouterr()
|
| 90 |
+
|
| 91 |
+
assert exit_code == 0
|
| 92 |
+
assert "✅" in captured.out
|
| 93 |
+
|
| 94 |
+
|
| 95 |
+
def test_main_warns_when_config_missing_url(tmp_path, capsys):
|
| 96 |
+
config_path = tmp_path / "settings.json"
|
| 97 |
+
config_path.write_text(json.dumps({"live_app": {"streamlit_url": "https://<your-app>.streamlit.app"}}))
|
| 98 |
+
|
| 99 |
+
exit_code = main(["--config", str(config_path)])
|
| 100 |
+
captured = capsys.readouterr()
|
| 101 |
+
|
| 102 |
+
assert exit_code == 1
|
| 103 |
+
assert "⚠️" in captured.err
|
tests/test_predictor.py
ADDED
|
@@ -0,0 +1,19 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from __future__ import annotations
|
| 2 |
+
|
| 3 |
+
from twitter_sentiment.config import load_config
|
| 4 |
+
from twitter_sentiment.predictor import load_artifacts, predict_with_threshold
|
| 5 |
+
|
| 6 |
+
|
| 7 |
+
def test_load_artifacts_returns_pipeline():
|
| 8 |
+
config = load_config()
|
| 9 |
+
pipeline, metrics = load_artifacts(config)
|
| 10 |
+
assert hasattr(pipeline, "predict")
|
| 11 |
+
assert isinstance(metrics, dict)
|
| 12 |
+
|
| 13 |
+
|
| 14 |
+
def test_predict_with_threshold_returns_label_and_probabilities():
|
| 15 |
+
config = load_config()
|
| 16 |
+
pipeline, _ = load_artifacts(config)
|
| 17 |
+
label, probabilities = predict_with_threshold("I love this experience", config)
|
| 18 |
+
assert label in {"positive", "negative", "neutral"}
|
| 19 |
+
assert set(probabilities.keys()) == set(pipeline.classes_)
|
tests/test_preprocessing.py
ADDED
|
@@ -0,0 +1,24 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from __future__ import annotations
|
| 2 |
+
|
| 3 |
+
import pandas as pd
|
| 4 |
+
|
| 5 |
+
from twitter_sentiment.config import load_config, Config
|
| 6 |
+
from twitter_sentiment.preprocessing import clean_text, preprocess_dataframe
|
| 7 |
+
|
| 8 |
+
|
| 9 |
+
def test_clean_text_removes_urls_mentions(tmp_path):
|
| 10 |
+
config = load_config()
|
| 11 |
+
result = clean_text("Check this https://example.com @user", config)
|
| 12 |
+
assert "http" not in result
|
| 13 |
+
assert "@" not in result
|
| 14 |
+
|
| 15 |
+
|
| 16 |
+
def test_preprocess_dataframe_returns_expected_columns(tmp_path):
|
| 17 |
+
config = load_config()
|
| 18 |
+
df = pd.DataFrame({
|
| 19 |
+
config.data["text_column"]: ["Great flight!"],
|
| 20 |
+
config.data["target_column"]: ["positive"],
|
| 21 |
+
})
|
| 22 |
+
processed = preprocess_dataframe(df, config)
|
| 23 |
+
assert processed.shape == (1, 2)
|
| 24 |
+
assert processed[config.data["text_column"]].iloc[0] == "great flight"
|
tests/test_update_vercel_redirect.py
ADDED
|
@@ -0,0 +1,37 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from __future__ import annotations
|
| 2 |
+
|
| 3 |
+
import json
|
| 4 |
+
import sys
|
| 5 |
+
from pathlib import Path
|
| 6 |
+
|
| 7 |
+
sys.path.append(str(Path(__file__).resolve().parents[1]))
|
| 8 |
+
|
| 9 |
+
from scripts.update_vercel_redirect import main
|
| 10 |
+
|
| 11 |
+
|
| 12 |
+
def test_main_generates_redirect(tmp_path, capsys):
|
| 13 |
+
config_path = tmp_path / "settings.json"
|
| 14 |
+
config_path.write_text(json.dumps({"live_app": {"streamlit_url": "https://demo.streamlit.app"}}))
|
| 15 |
+
output_path = tmp_path / "index.html"
|
| 16 |
+
|
| 17 |
+
exit_code = main(["--config", str(config_path), "--output", str(output_path)])
|
| 18 |
+
captured = capsys.readouterr()
|
| 19 |
+
|
| 20 |
+
assert exit_code == 0
|
| 21 |
+
assert "✅" in captured.out
|
| 22 |
+
html = output_path.read_text(encoding="utf-8")
|
| 23 |
+
assert "https://demo.streamlit.app" in html
|
| 24 |
+
assert "window.location.replace" in html
|
| 25 |
+
|
| 26 |
+
|
| 27 |
+
def test_main_errors_with_placeholder_url(tmp_path, capsys):
|
| 28 |
+
config_path = tmp_path / "settings.json"
|
| 29 |
+
config_path.write_text(json.dumps({"live_app": {"streamlit_url": "https://<your-app>.streamlit.app"}}))
|
| 30 |
+
output_path = tmp_path / "index.html"
|
| 31 |
+
|
| 32 |
+
exit_code = main(["--config", str(config_path), "--output", str(output_path)])
|
| 33 |
+
captured = capsys.readouterr()
|
| 34 |
+
|
| 35 |
+
assert exit_code == 1
|
| 36 |
+
assert "⚠️" in captured.out
|
| 37 |
+
assert not output_path.exists()
|