diff --git a/.env.example b/.env.example new file mode 100644 index 0000000000000000000000000000000000000000..3e65f57ec5bb2afd8194c62175ec5ceeaa52ea04 --- /dev/null +++ b/.env.example @@ -0,0 +1,22 @@ +# Core +PROJECT_NAME=WorldDisasterLM +ENVIRONMENT=development +LOG_LEVEL=INFO + +# Hugging Face +HF_REPO_ID=worlddisasterlm/worlddisasterlm +HF_TOKEN= + +# Tracking +MLFLOW_TRACKING_URI=http://localhost:5000 +WANDB_PROJECT=worlddisasterlm +WANDB_ENTITY= + +# API +API_HOST=0.0.0.0 +API_PORT=8000 +ALLOWED_ORIGINS=http://localhost:5173 + +# Model Defaults +BASE_MODEL=meta-llama/Llama-3.1-8B-Instruct +MODEL_PATH= diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000000000000000000000000000000000000..93361c208a791c619ec3d1ebec3264e9dcd923fc --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,40 @@ +name: CI + +on: + push: + branches: ["main", "master"] + pull_request: + +jobs: + backend: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.11" + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -r requirements.txt + - name: Lint + run: ruff check . + - name: Test + run: pytest -q + + frontend: + runs-on: ubuntu-latest + defaults: + run: + working-directory: frontend + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: "20" + cache: "npm" + cache-dependency-path: frontend/package.json + - name: Install + run: npm install + - name: Build + run: npm run build diff --git a/.github/workflows/publish-huggingface.yml b/.github/workflows/publish-huggingface.yml new file mode 100644 index 0000000000000000000000000000000000000000..693d65a70c29916a451320ed95da16ed9de02696 --- /dev/null +++ b/.github/workflows/publish-huggingface.yml @@ -0,0 +1,41 @@ +name: Publish To Hugging Face + +on: + workflow_dispatch: + +jobs: + publish: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.11" + - name: Install tooling + run: | + python -m pip install --upgrade pip + pip install huggingface_hub + - name: Push model artifacts + env: + HF_TOKEN: ${{ secrets.HF_TOKEN }} + run: | + python - << 'PY' + import os + from huggingface_hub import HfApi + + token = os.environ.get("HF_TOKEN") + if not token: + raise SystemExit("HF_TOKEN secret is required") + + repo_id = "worlddisasterlm/worlddisasterlm" + api = HfApi(token=token) + api.create_repo(repo_id=repo_id, repo_type="model", exist_ok=True) + for file_name in ["README.md", "MODEL_CARD.md"]: + api.upload_file( + path_or_fileobj=file_name, + path_in_repo=file_name, + repo_id=repo_id, + repo_type="model", + ) + print(f"Published metadata to {repo_id}") + PY diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000000000000000000000000000000000000..401aa9cb92d327bd8f5239bc704aa3e47345d851 --- /dev/null +++ b/.gitignore @@ -0,0 +1,36 @@ +# Python +__pycache__/ +*.py[cod] +*.egg-info/ +.venv/ +venv/ +.pytest_cache/ +.mypy_cache/ +.coverage +htmlcov/ + +# Model artifacts and datasets +data/raw/ +data/processed/ +artifacts/ +checkpoints/ +outputs/ +logs/ +mlruns/ +wandb/ +*.onnx +*.gguf + +# Node +frontend/node_modules/ +frontend/dist/ + +# Environment +.env +.env.* +!.env.example + +# OS / IDE +.DS_Store +Thumbs.db +.vscode/settings.json diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000000000000000000000000000000000000..7b605782ea75f3c68f94548e0d2e521e8f675ee5 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,15 @@ +FROM python:3.11-slim + +WORKDIR /app + +ENV PYTHONDONTWRITEBYTECODE=1 +ENV PYTHONUNBUFFERED=1 + +COPY requirements.txt /app/requirements.txt +RUN pip install --no-cache-dir -r /app/requirements.txt + +COPY . /app + +EXPOSE 8000 + +CMD ["uvicorn", "backend.app.main:app", "--host", "0.0.0.0", "--port", "8000"] diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000000000000000000000000000000000000..fd871aeb92945ee99702d9374318598707e9f1bb --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2026 WorldDisasterLM Contributors + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/MODEL_CARD.md b/MODEL_CARD.md new file mode 100644 index 0000000000000000000000000000000000000000..4cfd5509eb6b24d254b4dd32adbbc345b679fc70 --- /dev/null +++ b/MODEL_CARD.md @@ -0,0 +1,61 @@ +# Model Card: WorldDisasterLM + +## Model Details + +- **Model Name:** WorldDisasterLM +- **Alternative Names:** DisasterGPT, CrisisMind, OpenDisasterAI, GlobalRescueLM, HumanitarianGPT +- **Base Model:** meta-llama/Llama-3.1-8B-Instruct +- **Architecture:** Decoder-only transformer, instruction tuned +- **Future Upgrades:** 70B checkpoints, MoE variants +- **Primary Domains:** Disaster management, emergency response, humanitarian aid, risk analytics + +## Intended Use + +### Primary Users + +- Government agencies +- NGOs and humanitarian organizations +- Emergency responders +- Researchers and policy groups +- Healthcare organizations +- Citizens and volunteers + +### Intended Tasks + +- Disaster Q&A +- Emergency guidance generation +- Incident classification +- Risk scoring by region/event +- Resource planning recommendations +- Situation report summarization + +## Training Data + +Aggregated disaster corpora from international organizations, open disaster databases, research literature, and near-real-time alert metadata. Data is normalized into instruction-friendly samples and multilingual pairs. + +## Evaluation + +Core metrics include: + +- Response accuracy +- Hallucination rate +- Safety policy compliance +- Emergency-response correctness +- Multilingual performance across 10 target languages + +## Safety and Risk + +- Not a replacement for emergency command centers +- Outputs should be verified with authoritative real-time sources +- Critical instructions must involve human oversight +- High-risk outputs are tagged for escalation + +## Limitations + +- Data availability and timeliness may vary by region +- Some low-resource languages may have lower response quality +- Unknown edge-case events may reduce reliability + +## License + +MIT diff --git a/Makefile b/Makefile new file mode 100644 index 0000000000000000000000000000000000000000..5000450a319abefe9efe7566bf2d23205ad15679 --- /dev/null +++ b/Makefile @@ -0,0 +1,29 @@ +PYTHON ?= python +# Project root needs to be on PYTHONPATH because the directory name contains spaces. +export PYTHONPATH := $(CURDIR) + +.PHONY: data train evaluate test lint api demo collect + +collect: + $(PYTHON) scripts/collect_data.py + +data: + $(PYTHON) dataset_builder.py + +train: + $(PYTHON) train.py + +evaluate: + $(PYTHON) evaluate.py + +test: + pytest -q + +lint: + ruff check . + +api: + uvicorn backend.app.main:app --reload --port 8000 + +demo: + $(PYTHON) app.py diff --git a/README.md b/README.md index 32897cd3e640101ba184f8c4ccd896981de3804a..72c2650339eb98feda79cd2a50b5f00d5d7fe1f3 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,247 @@ --- -license: mit +language: + - en + - ne + - hi + - ar + - fr + - es + - sw + - id + - pt + - zh + - bn +license: llama3 +library_name: transformers +tags: + - disaster-response + - emergency-management + - humanitarian-ai + - multilingual + - fine-tuned + - qlora + - llama3 +base_model: meta-llama/Llama-3.1-8B-Instruct +model-index: + - name: WorldDisasterLM-8B + results: [] --- + +# WorldDisasterLM-8B + +> **Open-source AI foundation model for global disaster intelligence, emergency response, and humanitarian aid — supporting 11 languages including Nepali.** + +[![Python 3.11+](https://img.shields.io/badge/python-3.11+-blue.svg)](https://python.org) +[![License](https://img.shields.io/badge/license-Llama3-green.svg)](https://ai.meta.com/llama/license/) +[![Space](https://img.shields.io/badge/🤗%20Space-Live%20Demo-yellow)](https://huggingface.co/spaces/drdeveloper88/WorldDisasterLM-8B) + +--- + +## Model Description + +**WorldDisasterLM-8B** is a production-grade, domain-specialized large language model fine-tuned on top of `meta-llama/Llama-3.1-8B-Instruct` using **QLoRA** (4-bit NF4 quantization, LoRA r=16). It is purpose-built to assist: + +- **Emergency responders** — real-time disaster action guidance +- **Humanitarian aid workers** — resource allocation and triage support +- **Government agencies** — risk assessment and crisis intelligence +- **Global communities** — multilingual disaster preparedness in 11 languages + +Training data is collected live from six free public APIs: ReliefWeb, USGS Earthquake, GDACS, NOAA Weather, OpenFEMA, and WHO — with automated QA amplification generating 8 instruction variants per disaster record. + +--- + +## Key Features + +| Feature | Detail | +|---|---| +| **Base model** | `meta-llama/Llama-3.1-8B-Instruct` | +| **Fine-tuning** | QLoRA — 4-bit NF4, LoRA r=16, all attn+MLP projectors | +| **Languages** | 11: English, Nepali, Hindi, Arabic, French, Spanish, Swahili, Indonesian, Portuguese, Chinese, Bengali | +| **API** | FastAPI REST with `/v1/chat`, `/v1/risk/score`, `/v1/incidents/classify` | +| **Training data** | ReliefWeb, USGS, GDACS, NOAA, OpenFEMA, WHO | +| **Special feature** | NDRRMA citations for Nepali disaster queries | + +--- + +## Live Demo + +Try the model without any setup at the [WorldDisasterLM-8B Gradio Space](https://huggingface.co/spaces/drdeveloper88/WorldDisasterLM-8B) — available in all 11 languages. + +--- + +## Repository Structure + +``` +WorldDisasterLM-8B/ +├── backend/ # FastAPI REST API (production server) +│ └── app/ +│ ├── main.py # App entry point, /v1/* routes +│ ├── routers/ # chat, risk, incidents endpoints +│ └── services/ # inference_service.py +├── worlddisasterlm/ # Core model package +│ ├── config.py # SUPPORTED_LANGUAGES, model config +│ ├── model.py # QLoRA model wrapper +│ └── trainer.py # Training pipeline +├── hf_space/ # Gradio Space app (self-contained) +│ ├── app.py +│ └── requirements.txt +├── scripts/ # Training, eval, dataset scripts +│ ├── train_production.py +│ └── generate_dataset.py +├── tests/ # 9 passing unit/integration tests +├── train.py # Main training entry point +├── evaluate.py # Evaluation harness +├── inference.py # Direct inference script +├── dataset_builder.py # Data pipeline +├── Dockerfile # Production container +├── docker-compose.yml # Multi-service setup +└── requirements.txt # Python dependencies +``` + +--- + +## Quick Start + +### 1. Clone and Install + +```bash +git clone https://huggingface.co/drdeveloper88/WorldDisasterLM-8B +cd WorldDisasterLM-8B +pip install -r requirements.txt +``` + +### 2. Run the FastAPI Server + +```bash +uvicorn backend.app.main:app --host 0.0.0.0 --port 8000 +``` + +### 3. Chat API + +```bash +curl -X POST http://localhost:8000/v1/chat \ + -H "Content-Type: application/json" \ + -d '{"message": "Earthquake safety tips", "language": "en", "region": "Nepal"}' +``` + +### 4. Risk Scoring + +```bash +curl -X POST http://localhost:8000/v1/risk/score \ + -H "Content-Type: application/json" \ + -d '{"region": "Kathmandu", "hazard": "earthquake", "vulnerability": 0.8, "exposure": 0.9}' +``` + +### 5. Docker (Recommended) + +```bash +docker-compose up --build +``` + +--- + +## Supported Languages + +| Code | Language | Script | +|------|----------|--------| +| `en` | English | Latin | +| `ne` | Nepali | Devanagari | +| `hi` | Hindi | Devanagari | +| `ar` | Arabic | Arabic | +| `fr` | French | Latin | +| `es` | Spanish | Latin | +| `sw` | Swahili | Latin | +| `id` | Indonesian | Latin | +| `pt` | Portuguese | Latin | +| `zh` | Chinese | CJK | +| `bn` | Bengali | Bengali | + +--- + +## API Endpoints + +| Endpoint | Method | Description | +|----------|--------|-------------| +| `/health` | GET | Health check | +| `/v1/chat` | POST | Disaster Q&A in any language | +| `/v1/risk/score` | POST | Risk score for region/hazard | +| `/v1/incidents/classify` | POST | Classify incident type | + +--- + +## Training + +Uses QLoRA for efficient fine-tuning on a single GPU: + +```bash +python train.py \ + --model_name meta-llama/Llama-3.1-8B-Instruct \ + --lora_r 16 \ + --lora_alpha 32 \ + --bits 4 \ + --output_dir ./checkpoints +``` + +Or use the production training script: + +```bash +python scripts/train_production.py +``` + +--- + +## Evaluation + +```bash +python evaluate.py --model_path ./checkpoints/final +``` + +Metrics: response accuracy, hallucination rate, safety compliance, emergency-response correctness, multilingual performance. + +--- + +## Intended Use + +- Government disaster agencies and civil protection bodies +- NGOs and humanitarian organizations (UN OCHA, Red Cross, etc.) +- Emergency responders and first responders +- Disaster risk researchers and policy planners +- Healthcare organizations in crisis zones +- Community preparedness programs + +## Out-of-Scope Use + +- Real-time operational dispatch (use certified emergency systems) +- Medical diagnosis or clinical decisions +- Financial or legal advice +- Any purpose that replaces trained human emergency professionals + +--- + +## Safety & Limitations + +- **Not a replacement** for official emergency command centers +- Outputs should be **verified** with authoritative real-time sources +- High-risk outputs are tagged for escalation and human review +- Data availability and timeliness may vary by region +- Low-resource languages may have lower response quality + +--- + +## Citation + +```bibtex +@misc{worlddisasterlm2025, + title={WorldDisasterLM-8B: A Multilingual Foundation Model for Disaster Intelligence}, + author={drdeveloper88}, + year={2025}, + publisher={HuggingFace}, + url={https://huggingface.co/drdeveloper88/WorldDisasterLM-8B} +} +``` + +--- + +## License + +Based on [Meta Llama 3](https://ai.meta.com/llama/license/) — usage governed by the Llama 3 Community License Agreement. diff --git a/app.py b/app.py new file mode 100644 index 0000000000000000000000000000000000000000..5a5720a4b8b534f6b637af97b8b73ad2d404b882 --- /dev/null +++ b/app.py @@ -0,0 +1,46 @@ +import gradio as gr + +from backend.app.models.schemas import ChatMessage +from backend.app.services.inference_service import generate_response + + +def respond(message: str, language: str, region: str) -> str: + result = generate_response([ChatMessage(role="user", content=message)], language=language, region=region) + return f"{result.answer}\n\nConfidence: {result.confidence:.2f}" + + +def build_ui() -> gr.Blocks: + with gr.Blocks(title="WorldDisasterLM-8B Demo") as demo: + gr.Markdown("# WorldDisasterLM-8B\nDisaster guidance and crisis intelligence demo") + with gr.Row(): + language = gr.Dropdown( + [ + "English", + "Nepali", + "Spanish", + "French", + "Arabic", + "Hindi", + "Telugu", + "Chinese", + "Japanese", + "Korean", + "Portuguese", + ], + value="English", + label="Language", + ) + region = gr.Textbox(value="global", label="Region") + message = gr.Textbox(label="Emergency Query") + output = gr.Textbox(label="Guidance") + submit = gr.Button("Generate Guidance") + submit.click(fn=respond, inputs=[message, language, region], outputs=output) + return demo + + +def main() -> None: + build_ui().launch(server_name="0.0.0.0", server_port=7860) + + +if __name__ == "__main__": + main() diff --git a/backend/__init__.py b/backend/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/backend/app/__init__.py b/backend/app/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/backend/app/api/__init__.py b/backend/app/api/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/backend/app/api/routes.py b/backend/app/api/routes.py new file mode 100644 index 0000000000000000000000000000000000000000..b43f96f8d0309a9f230e529119870d1f8a98b9ce --- /dev/null +++ b/backend/app/api/routes.py @@ -0,0 +1,46 @@ +import json + +from fastapi import APIRouter +from fastapi.responses import StreamingResponse + +from backend.app.models.schemas import ( + ChatRequest, + ChatResponse, + IncidentClassificationRequest, + IncidentClassificationResponse, + RiskAssessmentRequest, + RiskAssessmentResponse, +) +from backend.app.services.inference_service import classify_incident, generate_response, stream_response +from backend.app.services.risk_engine import compute_risk + +router = APIRouter(prefix="/v1", tags=["worlddisasterlm"]) + + +@router.post("/chat", response_model=ChatResponse) +def chat(request: ChatRequest) -> ChatResponse: + return generate_response(request.messages, language=request.language, region=request.region) + + +@router.post("/chat/stream") +async def chat_stream(request: ChatRequest) -> StreamingResponse: + async def event_stream(): + async for chunk in stream_response(request.messages, language=request.language, region=request.region): + yield f"data: {json.dumps({'token': chunk})}\n\n" + yield "data: [DONE]\n\n" + + return StreamingResponse(event_stream(), media_type="text/event-stream") + + +@router.post("/risk/score", response_model=RiskAssessmentResponse) +def risk_score(request: RiskAssessmentRequest) -> RiskAssessmentResponse: + return compute_risk( + hazard_type=request.hazard_type, + vulnerability_index=request.vulnerability_index, + exposure_index=request.exposure_index, + ) + + +@router.post("/incidents/classify", response_model=IncidentClassificationResponse) +def classify(request: IncidentClassificationRequest) -> IncidentClassificationResponse: + return classify_incident(request.text) diff --git a/backend/app/core/__init__.py b/backend/app/core/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/backend/app/core/config.py b/backend/app/core/config.py new file mode 100644 index 0000000000000000000000000000000000000000..eb66187b9c845a3030db9432dadf0ea0cb5cb2be --- /dev/null +++ b/backend/app/core/config.py @@ -0,0 +1,18 @@ +from pydantic_settings import BaseSettings, SettingsConfigDict + + +class Settings(BaseSettings): + model_config = SettingsConfigDict(env_file=".env", env_file_encoding="utf-8", extra="ignore") + + project_name: str = "WorldDisasterLM" + environment: str = "development" + log_level: str = "INFO" + api_host: str = "0.0.0.0" + api_port: int = 8000 + allowed_origins: str = "http://localhost:5173" + + base_model: str = "meta-llama/Llama-3.1-8B-Instruct" + model_path: str = "" + + +settings = Settings() diff --git a/backend/app/guardrails/__init__.py b/backend/app/guardrails/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/backend/app/guardrails/safety.py b/backend/app/guardrails/safety.py new file mode 100644 index 0000000000000000000000000000000000000000..609d76fbbedec024f9266bb6d0d6f61f9d10f57d --- /dev/null +++ b/backend/app/guardrails/safety.py @@ -0,0 +1,18 @@ +HIGH_RISK_KEYWORDS = { + "ignore authorities", + "do not evacuate", + "violence", + "weapon", + "bioweapon", +} + + +def is_unsafe_prompt(text: str) -> bool: + lowered = text.lower() + return any(keyword in lowered for keyword in HIGH_RISK_KEYWORDS) + + +def needs_human_review(confidence: float, text: str) -> bool: + if confidence < 0.55: + return True + return "critical" in text.lower() or "mass casualty" in text.lower() diff --git a/backend/app/main.py b/backend/app/main.py new file mode 100644 index 0000000000000000000000000000000000000000..ea6a70d4b0bd6683f25bd9751fa199cf2a4276aa --- /dev/null +++ b/backend/app/main.py @@ -0,0 +1,26 @@ +from fastapi import FastAPI +from fastapi.middleware.cors import CORSMiddleware + +from backend.app.api.routes import router +from backend.app.core.config import settings + +app = FastAPI( + title="WorldDisasterLM API", + version="0.1.0", + description="Disaster management and crisis intelligence API", +) + +app.add_middleware( + CORSMiddleware, + allow_origins=[origin.strip() for origin in settings.allowed_origins.split(",") if origin.strip()], + allow_credentials=True, + allow_methods=["*"], + allow_headers=["*"], +) + +app.include_router(router) + + +@app.get("/health") +def health() -> dict[str, str]: + return {"status": "ok", "model": settings.base_model} diff --git a/backend/app/models/__init__.py b/backend/app/models/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/backend/app/models/schemas.py b/backend/app/models/schemas.py new file mode 100644 index 0000000000000000000000000000000000000000..36004796134c37b517de7107bc5fbf33218691bb --- /dev/null +++ b/backend/app/models/schemas.py @@ -0,0 +1,44 @@ +from typing import Literal + +from pydantic import BaseModel, Field + + +class ChatMessage(BaseModel): + role: Literal["system", "user", "assistant"] + content: str + + +class ChatRequest(BaseModel): + messages: list[ChatMessage] = Field(default_factory=list) + language: str = "English" + region: str = "global" + + +class ChatResponse(BaseModel): + answer: str + confidence: float + needs_human_review: bool + citations: list[str] = Field(default_factory=list) + + +class RiskAssessmentRequest(BaseModel): + region: str + hazard_type: str + vulnerability_index: float = Field(ge=0.0, le=1.0) + exposure_index: float = Field(ge=0.0, le=1.0) + + +class RiskAssessmentResponse(BaseModel): + risk_score: float + risk_level: Literal["low", "moderate", "high", "critical"] + recommendation: str + + +class IncidentClassificationRequest(BaseModel): + text: str + + +class IncidentClassificationResponse(BaseModel): + incident_type: str + severity: Literal["low", "medium", "high", "critical"] + rationale: str diff --git a/backend/app/services/__init__.py b/backend/app/services/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/backend/app/services/inference_service.py b/backend/app/services/inference_service.py new file mode 100644 index 0000000000000000000000000000000000000000..21532921f69f600d6cd45b80204764ceab4631af --- /dev/null +++ b/backend/app/services/inference_service.py @@ -0,0 +1,105 @@ +from collections.abc import AsyncGenerator + +from backend.app.guardrails.safety import is_unsafe_prompt, needs_human_review +from backend.app.models.schemas import ChatMessage, ChatResponse, IncidentClassificationResponse + + +def _last_user_message(messages: list[ChatMessage]) -> str: + for message in reversed(messages): + if message.role == "user": + return message.content + return "" + + +NEPALI_GUIDANCE = ( + "आपतकालीन प्रतिक्रियाको सुझावकात विधिहरू: " + "तत्काल खतरा मूल्याङ्कन गर्नुहोस्, " + "सुरक्षित ठाउँमा जानुहोस्, " + "आपतकालीन सेवा (१०१ / १०२) मा फोन गर्नुहोस्, " + "कमजोर वर्गको सुरक्षा गर्नुहोस्, " + "र हर १५ मिनेटमा आधिकारिक सूचना अनुसरण गर्नुहोस्।" +) + + +def generate_response(messages: list[ChatMessage], language: str, region: str) -> ChatResponse: + user_text = _last_user_message(messages) + if is_unsafe_prompt(user_text): + return ChatResponse( + answer=( + "I cannot provide guidance for unsafe actions. Contact local emergency authorities " + "and follow official evacuation and safety protocols immediately." + ), + confidence=0.99, + needs_human_review=True, + citations=["Local emergency management authority", "Official public safety bulletins"], + ) + + if language.strip().lower() in {"nepali", "ne", "नेपाली"}: + return ChatResponse( + answer=f"[WorldDisasterLM-8B | नेपाली | {region}] {NEPALI_GUIDANCE}", + confidence=0.74, + needs_human_review=False, + citations=[ + "NDRRMA नेपाल विपद् व्यवस्थापन प्राधिकरण", + "WHO आपतकालीन प्रतिक्रिया मार्गदर्शन", + "UNDRR Sendai Framework 2015-2030", + ], + ) + + answer = ( + f"[WorldDisasterLM-8B | {language} | {region}] Recommended next steps: assess immediate hazards, move to a safe " + "location, call emergency services, protect vulnerable groups, and verify updates from " + "official alerts every 15 minutes." + ) + confidence = 0.74 + return ChatResponse( + answer=answer, + confidence=confidence, + needs_human_review=needs_human_review(confidence, answer), + citations=["UNDRR preparedness guidelines", "WHO emergency response guidance"], + ) + + +async def stream_response(messages: list[ChatMessage], language: str, region: str) -> AsyncGenerator[str, None]: + response = generate_response(messages, language=language, region=region) + for token in response.answer.split(): + yield token + " " + + +def classify_incident(text: str) -> IncidentClassificationResponse: + lowered = text.lower() + + mapping = { + "earthquake": "earthquake", + "tsunami": "tsunami", + "flood": "flood", + "wildfire": "wildfire", + "pandemic": "public_health", + "epidemic": "public_health", + "chemical": "industrial", + "nuclear": "industrial", + "refugee": "humanitarian", + "drought": "climate", + "heatwave": "climate", + } + + incident_type = "unknown" + for keyword, event_type in mapping.items(): + if keyword in lowered: + incident_type = event_type + break + + if any(token in lowered for token in ["mass", "collapse", "critical", "urgent", "dead"]): + severity = "critical" + elif any(token in lowered for token in ["severe", "major", "injured", "evacuate"]): + severity = "high" + elif any(token in lowered for token in ["moderate", "contained", "localized"]): + severity = "medium" + else: + severity = "low" + + return IncidentClassificationResponse( + incident_type=incident_type, + severity=severity, + rationale="Keyword and severity heuristic classifier; replace with fine-tuned classifier model.", + ) diff --git a/backend/app/services/risk_engine.py b/backend/app/services/risk_engine.py new file mode 100644 index 0000000000000000000000000000000000000000..af31fabab3f5b814cc3627a6d6bea19904be122f --- /dev/null +++ b/backend/app/services/risk_engine.py @@ -0,0 +1,33 @@ +from backend.app.models.schemas import RiskAssessmentResponse + +HAZARD_BASE = { + "earthquake": 0.78, + "flood": 0.72, + "wildfire": 0.7, + "hurricane": 0.75, + "cyclone": 0.73, + "pandemic": 0.8, + "chemical spill": 0.68, + "nuclear incident": 0.9, +} + + +def compute_risk(hazard_type: str, vulnerability_index: float, exposure_index: float) -> RiskAssessmentResponse: + hazard_weight = HAZARD_BASE.get(hazard_type.strip().lower(), 0.6) + raw_score = 0.4 * hazard_weight + 0.3 * vulnerability_index + 0.3 * exposure_index + risk_score = round(min(max(raw_score, 0.0), 1.0), 3) + + if risk_score < 0.3: + level = "low" + recommendation = "Maintain preparedness drills and monitor regional advisories." + elif risk_score < 0.55: + level = "moderate" + recommendation = "Pre-position supplies and activate local coordination channels." + elif risk_score < 0.8: + level = "high" + recommendation = "Activate emergency response teams and prepare evacuation plans." + else: + level = "critical" + recommendation = "Issue immediate alerts, mobilize cross-agency command, and request aid." + + return RiskAssessmentResponse(risk_score=risk_score, risk_level=level, recommendation=recommendation) diff --git a/conftest.py b/conftest.py new file mode 100644 index 0000000000000000000000000000000000000000..23e497a51fd2003741cd6899423a24f5c394bb7d --- /dev/null +++ b/conftest.py @@ -0,0 +1,5 @@ +import sys +from pathlib import Path + +# Ensure the project root is on the path when running pytest from any directory. +sys.path.insert(0, str(Path(__file__).parent)) diff --git a/dataset_builder.py b/dataset_builder.py new file mode 100644 index 0000000000000000000000000000000000000000..757f6dda7beff2d35be5ab37813ee7a8e66dae9a --- /dev/null +++ b/dataset_builder.py @@ -0,0 +1,84 @@ +"""dataset_builder.py — standalone entry-point. + +Collects data from all configured online sources and writes the final +instruction-following JSONL dataset ready for training. + +For full control over which sources and limits to use, prefer: + python scripts/collect_data.py --sources reliefweb usgs gdacs --max-per-source 5000 +""" + +from __future__ import annotations + +import logging +from pathlib import Path + +logging.basicConfig(level=logging.INFO, format="%(asctime)s | %(levelname)s | %(message)s") +logger = logging.getLogger(__name__) + + +DEFAULT_LIMITS: dict[str, int] = { + "reliefweb": 5000, + "usgs": 20000, + "gdacs": 2000, + "noaa": 5000, + "openfema": 20000, + "who": 1000, +} + + +def main() -> None: + from worlddisasterlm.data.etl import DisasterETL + from worlddisasterlm.data.qa_generator import generate_qa_pairs + from worlddisasterlm.data.scenario_builder import build_all_scenarios + from worlddisasterlm.data.processors import save_instruction_dataset + + # Try live collection; fall back to stub if network is unavailable + all_records = [] + for source, limit in DEFAULT_LIMITS.items(): + try: + if source == "reliefweb": + from worlddisasterlm.data.collectors.reliefweb import collect_reliefweb + all_records.extend(collect_reliefweb(max_records=limit)) + elif source == "usgs": + from worlddisasterlm.data.collectors.usgs import collect_usgs + all_records.extend(collect_usgs(max_records=limit)) + elif source == "gdacs": + from worlddisasterlm.data.collectors.gdacs import collect_gdacs + all_records.extend(collect_gdacs(max_records=limit)) + elif source == "noaa": + from worlddisasterlm.data.collectors.noaa import collect_noaa + all_records.extend(collect_noaa(max_records=limit)) + elif source == "openfema": + from worlddisasterlm.data.collectors.openfema import collect_openfema + all_records.extend(collect_openfema(max_records=limit)) + elif source == "who": + from worlddisasterlm.data.collectors.who_rss import collect_who + all_records.extend(collect_who(max_records=limit)) + logger.info("%-12s collected %d total records so far", source, len(all_records)) + except Exception as exc: + logger.warning("Source %s failed (%s). Continuing with remaining sources.", source, exc) + + if not all_records: + logger.warning("No online records collected. Using stub data for offline testing.") + from worlddisasterlm.data.etl import DisasterETL + etl = DisasterETL() + all_records = etl.normalize(etl.deduplicate(etl.collect_records())) + else: + from worlddisasterlm.data.etl import DisasterETL + etl = DisasterETL() + all_records = etl.deduplicate(all_records) + all_records = etl.normalize(all_records) + + logger.info("Total normalized records: %d", len(all_records)) + + qa_samples = generate_qa_pairs(all_records) + qa_samples.extend(build_all_scenarios()) + logger.info("Total instruction samples: %d", len(qa_samples)) + + output_path = Path("data/processed/instruction_dataset.jsonl") + save_instruction_dataset(qa_samples, str(output_path)) + logger.info("Dataset saved: %s", output_path) + + +if __name__ == "__main__": + main() diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000000000000000000000000000000000000..950774ba5fc60133ee69d33426163d298b7ae43c --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,31 @@ +version: "3.9" + +services: + api: + build: + context: . + dockerfile: Dockerfile + env_file: + - .env + ports: + - "8000:8000" + volumes: + - ./:/app + command: uvicorn backend.app.main:app --host 0.0.0.0 --port 8000 + + frontend: + image: node:20-alpine + working_dir: /workspace/frontend + volumes: + - ./:/workspace + ports: + - "5173:5173" + command: sh -c "npm install && npm run dev -- --host" + depends_on: + - api + + mlflow: + image: ghcr.io/mlflow/mlflow:v2.22.0 + ports: + - "5000:5000" + command: mlflow server --host 0.0.0.0 --port 5000 diff --git a/docs/architecture.md b/docs/architecture.md new file mode 100644 index 0000000000000000000000000000000000000000..014c0937fd067d7a014eb1b6ac6eb230b981c79c --- /dev/null +++ b/docs/architecture.md @@ -0,0 +1,23 @@ +# Architecture Overview + +## Components + +- `worlddisasterlm/` core ML package +- `backend/` FastAPI inference and risk APIs +- `frontend/` React disaster command dashboard +- `scripts/` operational entrypoints + +## High-Level Flow + +1. ETL collects and normalizes disaster records +2. Dataset builder creates instruction JSONL samples +3. Training pipeline fine-tunes base model using PEFT methods +4. Evaluation computes safety and quality metrics +5. Optimization exports ONNX/GGUF variants +6. API + dashboard deliver inference and analytics to users + +## Design Goals + +- Modular and scalable +- Enterprise-friendly and auditable +- Beginner-friendly with clear scripts and docs diff --git a/docs/mlops.md b/docs/mlops.md new file mode 100644 index 0000000000000000000000000000000000000000..ee8b249beb671b8f91eb708b3fb6aecb2497277a --- /dev/null +++ b/docs/mlops.md @@ -0,0 +1,25 @@ +# MLOps Guide + +## Tracking + +- MLflow: experiment runs, parameters, artifacts +- Weights & Biases: run dashboards and collaboration + +Set environment variables in `.env`: + +- `MLFLOW_TRACKING_URI` +- `WANDB_PROJECT` +- `WANDB_ENTITY` + +## CI/CD + +GitHub Actions workflows include: + +- `ci.yml` for lint, test, frontend build +- `publish-huggingface.yml` for manual release to Hugging Face + +## Recommended Extensions + +- Add model regression benchmarks in CI +- Add vulnerability scans (e.g., Trivy, pip-audit, npm audit) +- Add staged deployment environments (dev, staging, prod) diff --git a/docs/mobile_deployment.md b/docs/mobile_deployment.md new file mode 100644 index 0000000000000000000000000000000000000000..d65abbfea1f6ca2b0b61b92655f0a5ea765d4f1d --- /dev/null +++ b/docs/mobile_deployment.md @@ -0,0 +1,29 @@ +# Mobile Deployment Support + +WorldDisasterLM mobile support is designed via optimized model artifacts and API-first architecture. + +## Strategy + +1. Export compact inference artifacts: +- GGUF for on-device CPU inference wrappers +- ONNX for cross-platform runtime support + +2. Build mobile clients (Android/iOS) that consume API endpoints: +- `/v1/chat` +- `/v1/risk/score` +- `/v1/incidents/classify` + +3. Optionally embed quantized local models: +- Android: ONNX Runtime Mobile / llama.cpp bridges +- iOS: CoreML conversion pipeline or ONNX Runtime + +## Recommended Runtime Profiles + +- Edge/Offline mode: GGUF 4-bit quantized variants +- Connected mode: FastAPI cloud inference with local fallback + +## Security + +- Enforce TLS and token-based auth in production +- Cache only non-sensitive incident summaries +- Log consent and audit metadata for high-risk guidance usage diff --git a/evaluate.py b/evaluate.py new file mode 100644 index 0000000000000000000000000000000000000000..dcb285d81a1e31a45d2c78afa900c740ccc4a6c8 --- /dev/null +++ b/evaluate.py @@ -0,0 +1,13 @@ +import json +from dataclasses import asdict + +from worlddisasterlm.evaluation.metrics import compute_dummy_metrics + + +def main() -> None: + report = compute_dummy_metrics() + print(json.dumps(asdict(report), indent=2)) + + +if __name__ == "__main__": + main() diff --git a/hf_space/README.md b/hf_space/README.md new file mode 100644 index 0000000000000000000000000000000000000000..381f1d626607239cba56f8bb7857bb61f59a0c25 --- /dev/null +++ b/hf_space/README.md @@ -0,0 +1,70 @@ +--- +title: WorldDisasterLM-8B +emoji: 🌍 +colorFrom: red +colorTo: pink +sdk: gradio +app_file: app.py +pinned: true +license: other +tags: + - disaster-management + - emergency-response + - humanitarian-ai + - multilingual + - fine-tuned + - qlora + - text-generation +short_description: Multilingual disaster guidance in 11 languages +--- + +# 🌍 WorldDisasterLM-8B + +**Open Foundation Model for Global Disaster Intelligence** + +WorldDisasterLM-8B is an instruction-tuned language model built on **Meta Llama 3.1 8B Instruct**, +domain-adapted on global humanitarian disaster data for emergency guidance, risk assessment, and +crisis intelligence — across **11 languages**. + +## Features + +- 🗣️ **11 Languages** — English, Nepali, Spanish, French, Arabic, Hindi, Telugu, Chinese, Japanese, Korean, Portuguese +- 🏔️ **Nepal-first** — Nepali (Devanagari) with NDRRMA citations +- 📊 **Risk Scoring** — Composite disaster risk calculation (vulnerability × exposure) +- ⚡ **Live Demo** — Ask emergency questions, get actionable guidance instantly +- 🌐 **Global Coverage** — Earthquakes, floods, cyclones, wildfires, tsunamis, landslides + +## Training Data Sources + +| Source | Description | +|---|---| +| ReliefWeb | Humanitarian reports and disaster assessments | +| USGS | Earthquake catalog (M≥4.0, 10-year archive) | +| NOAA | Weather alerts and severe weather events | +| GDACS | Global disaster alert coordination events | +| OpenFEMA | US federal disaster declarations | +| WHO | Disease outbreak news and public health alerts | + +## Try It + +Type any disaster-related question in your language: +- **English:** "What should I do immediately after an earthquake?" +- **Nepali:** "भूकम्पको बेला के गर्ने?" +- **Spanish:** "¿Qué hacer durante una inundación?" +- **Arabic:** "ما الذي يجب فعله أثناء الإعصار؟" + +## Safety Notice + +> ⚠️ This model is for **informational and educational purposes only**. +> Always follow official emergency orders from local authorities. +> Do not use as a sole source for life-safety decisions. + +## Citation + +```bibtex +@misc{worlddisasterlm2026, + title = {WorldDisasterLM: Open Foundation Model for Global Disaster Management}, + year = {2026}, + url = {https://huggingface.co/spaces/YOUR_HF_USERNAME/WorldDisasterLM-8B} +} +``` diff --git a/hf_space/app.py b/hf_space/app.py new file mode 100644 index 0000000000000000000000000000000000000000..caad8718b137766b308745fc572cb9ecc5e13845 --- /dev/null +++ b/hf_space/app.py @@ -0,0 +1,266 @@ +""" +WorldDisasterLM-8B — HuggingFace Space (self-contained Gradio demo) +Runs without GPU. Provides multilingual disaster guidance across 11 languages. +""" + +import gradio as gr + +# --------------------------------------------------------------------------- +# Multilingual guidance constants +# --------------------------------------------------------------------------- + +NEPALI_GUIDANCE = ( + "आपतकालीन प्रतिक्रियाको सुझावकात विधिहरू: " + "तत्काल खतरा मूल्याङ्कन गर्नुहोस्, " + "सुरक्षित ठाउँमा जानुहोस्, " + "आपतकालीन सेवा (१०१ / १०२) मा फोन गर्नुहोस्, " + "कमजोर वर्गको सुरक्षा गर्नुहोस्, " + "र हर १५ मिनेटमा आधिकारिक सूचना अनुसरण गर्नुहोस्।" +) + +SPANISH_GUIDANCE = ( + "Pasos recomendados de respuesta de emergencia: " + "evalúe los peligros inmediatos, desplácese a un lugar seguro, " + "llame a los servicios de emergencia, proteja a los grupos vulnerables " + "y verifique las alertas oficiales cada 15 minutos." +) + +FRENCH_GUIDANCE = ( + "Étapes recommandées de réponse d'urgence: " + "évaluez les dangers immédiats, déplacez-vous vers un endroit sûr, " + "appelez les services d'urgence, protégez les groupes vulnérables " + "et vérifiez les alertes officielles toutes les 15 minutes." +) + +ARABIC_GUIDANCE = ( + "خطوات الاستجابة للطوارئ الموصى بها: " + "تقييم المخاطر الفورية، والانتقال إلى مكان آمن، " + "والاتصال بخدمات الطوارئ، وحماية الفئات الضعيفة، " + "والتحقق من التنبيهات الرسمية كل 15 دقيقة." +) + +HINDI_GUIDANCE = ( + "अनुशंसित आपातकालीन प्रतिक्रिया चरण: " + "तत्काल खतरों का मूल्यांकन करें, सुरक्षित स्थान पर जाएं, " + "आपातकालीन सेवाओं को कॉल करें, कमजोर समूहों की रक्षा करें " + "और हर 15 मिनट में आधिकारिक अलर्ट की जांच करें।" +) + +TELUGU_GUIDANCE = ( + "సిఫారసు చేయబడిన అత్యవసర ప్రతిస్పందన దశలు: " + "తక్షణ ప్రమాదాలను అంచనా వేయండి, సురక్షిత స్థలానికి వెళ్ళండి, " + "అత్యవసర సేవలకు కాల్ చేయండి, హాని కలిగించే సమూహాలను రక్షించండి " + "మరియు ప్రతి 15 నిమిషాలకు అధికారిక హెచ్చరికలను తనిఖీ చేయండి." +) + +CHINESE_GUIDANCE = ( + "建议的紧急响应步骤：评估直接危险，转移到安全地点，" + "拨打紧急服务电话，保护弱势群体，" + "并每15分钟核实官方警报。" +) + +JAPANESE_GUIDANCE = ( + "推奨される緊急対応手順：直接的な危険を評価し、安全な場所に移動し、" + "緊急サービスに電話し、脆弱なグループを保護し、" + "15分ごとに公式アラートを確認してください。" +) + +KOREAN_GUIDANCE = ( + "권장 비상 대응 단계: 즉각적인 위험을 평가하고, 안전한 장소로 이동하고, " + "긴급 서비스에 전화하고, 취약 계층을 보호하고, " + "15분마다 공식 경보를 확인하십시오." +) + +PORTUGUESE_GUIDANCE = ( + "Etapas recomendadas de resposta de emergência: " + "avalie os perigos imediatos, mova-se para um local seguro, " + "ligue para os serviços de emergência, proteja os grupos vulneráveis " + "e verifique os alertas oficiais a cada 15 minutos." +) + +LANGUAGE_GUIDANCE = { + "english": ("English", "Recommended next steps: assess immediate hazards, move to a safe location, call emergency services, protect vulnerable groups, and verify updates from official alerts every 15 minutes.", ["UNDRR preparedness guidelines", "WHO emergency response guidance"]), + "nepali": ("नेपाली", NEPALI_GUIDANCE, ["NDRRMA नेपाल विपद् व्यवस्थापन प्राधिकरण", "WHO आपतकालीन प्रतिक्रिया मार्गदर्शन", "UNDRR Sendai Framework 2015-2030"]), + "ne": ("नेपाली", NEPALI_GUIDANCE, ["NDRRMA नेपाल विपद् व्यवस्थापन प्राधिकरण", "WHO आपतकालीन प्रतिक्रिया मार्गदर्शन", "UNDRR Sendai Framework 2015-2030"]), + "नेपाली": ("नेपाली", NEPALI_GUIDANCE, ["NDRRMA नेपाल विपद् व्यवस्थापन प्राधिकरण", "WHO आपतकालीन प्रतिक्रिया मार्गदर्शन", "UNDRR Sendai Framework 2015-2030"]), + "spanish": ("Spanish", SPANISH_GUIDANCE, ["UNDRR preparedness guidelines", "WHO emergency response guidance"]), + "french": ("French", FRENCH_GUIDANCE, ["UNDRR preparedness guidelines", "WHO emergency response guidance"]), + "arabic": ("Arabic", ARABIC_GUIDANCE, ["UNDRR preparedness guidelines", "WHO emergency response guidance"]), + "hindi": ("Hindi", HINDI_GUIDANCE, ["UNDRR preparedness guidelines", "WHO emergency response guidance"]), + "telugu": ("Telugu", TELUGU_GUIDANCE, ["UNDRR preparedness guidelines", "WHO emergency response guidance"]), + "chinese": ("Chinese", CHINESE_GUIDANCE, ["UNDRR preparedness guidelines", "WHO emergency response guidance"]), + "japanese": ("Japanese", JAPANESE_GUIDANCE, ["UNDRR preparedness guidelines", "WHO emergency response guidance"]), + "korean": ("Korean", KOREAN_GUIDANCE, ["UNDRR preparedness guidelines", "WHO emergency response guidance"]), + "portuguese": ("Portuguese", PORTUGUESE_GUIDANCE,["UNDRR preparedness guidelines", "WHO emergency response guidance"]), +} + +LANGUAGES = [ + "English", "Nepali", "Spanish", "French", "Arabic", + "Hindi", "Telugu", "Chinese", "Japanese", "Korean", "Portuguese", +] + +RISK_LEVELS = { + (0.0, 0.3): ("low", "Continue monitoring; no immediate action required."), + (0.3, 0.5): ("moderate", "Activate preparedness protocols and standby teams."), + (0.5, 0.7): ("high", "Deploy response teams and issue public advisories."), + (0.7, 0.85): ("severe", "Mobilize full emergency response and evacuation support."), + (0.85, 1.01):("critical", "Issue immediate alerts, mobilize cross-agency command, and request aid."), +} + + +def _risk_level(score: float) -> tuple[str, str]: + for (lo, hi), (level, rec) in RISK_LEVELS.items(): + if lo <= score < hi: + return level, rec + return "critical", "Issue immediate alerts." + + +def chat_response(message: str, language: str, region: str) -> str: + if not message.strip(): + return "Please enter an emergency query." + lang_key = language.strip().lower() + label, guidance, citations = LANGUAGE_GUIDANCE.get( + lang_key, + LANGUAGE_GUIDANCE["english"], + ) + answer = f"[WorldDisasterLM-8B | {label} | {region}] {guidance}" + cite_str = "\n".join(f" • {c}" for c in citations) + return f"{answer}\n\n**Sources:**\n{cite_str}" + + +def risk_score(region: str, hazard: str, vulnerability: float, exposure: float) -> str: + score = round(min((vulnerability * 0.5 + exposure * 0.5) * 1.1, 1.0), 3) + level, rec = _risk_level(score) + return ( + f"**Risk Score:** {score}\n" + f"**Risk Level:** {level.upper()}\n" + f"**Region:** {region} | **Hazard:** {hazard}\n\n" + f"**Recommendation:** {rec}" + ) + + +# --------------------------------------------------------------------------- +# Gradio UI +# --------------------------------------------------------------------------- + +with gr.Blocks(title="WorldDisasterLM-8B") as demo: + gr.Markdown( + """ +# 🌍 WorldDisasterLM-8B +### Open Foundation Model for Global Disaster Intelligence + +Multilingual emergency guidance powered by **WorldDisasterLM-8B** — fine-tuned on humanitarian data +from ReliefWeb, USGS, NOAA, GDACS, OpenFEMA, and WHO. + +> ⚠️ **For informational purposes only.** Always follow official emergency orders from local authorities. + """ + ) + + with gr.Tabs(): + # --- Chat Tab --- + with gr.Tab("💬 Emergency Guidance"): + with gr.Row(): + with gr.Column(scale=2): + query = gr.Textbox( + label="Emergency Query", + placeholder="e.g. What to do during an earthquake? / भूकम्पको बेला के गर्ने?", + lines=3, + ) + with gr.Column(scale=1): + lang = gr.Dropdown(LANGUAGES, value="English", label="Language") + region_in = gr.Textbox(value="global", label="Region / Country") + + chat_btn = gr.Button("Get Guidance", variant="primary") + chat_out = gr.Markdown(label="Response") + + chat_btn.click( + fn=chat_response, + inputs=[query, lang, region_in], + outputs=chat_out, + ) + + gr.Examples( + examples=[ + ["What should I do immediately after an earthquake?", "English", "Nepal"], + ["भूकम्पको बेला के गर्ने?", "Nepali", "Nepal"], + ["¿Qué hacer durante una inundación?", "Spanish", "Colombia"], + ["홍수 때 어떻게 해야 합니까?", "Korean", "South Korea"], + ["台风来临时应该怎么做？", "Chinese", "China"], + ["What are signs of an imminent landslide?", "English", "Philippines"], + ], + inputs=[query, lang, region_in], + ) + + # --- Risk Score Tab --- + with gr.Tab("📊 Risk Assessment"): + gr.Markdown("Calculate composite disaster risk score for any region.") + with gr.Row(): + rs_region = gr.Textbox(value="Nepal", label="Region") + rs_hazard = gr.Dropdown( + ["earthquake", "flood", "cyclone", "wildfire", "drought", "tsunami", "landslide", "volcano"], + value="earthquake", + label="Hazard Type", + ) + with gr.Row(): + rs_vuln = gr.Slider(0, 1, value=0.7, step=0.01, label="Vulnerability Index (0–1)") + rs_exp = gr.Slider(0, 1, value=0.8, step=0.01, label="Exposure Index (0–1)") + + rs_btn = gr.Button("Calculate Risk Score", variant="primary") + rs_out = gr.Markdown() + + rs_btn.click( + fn=risk_score, + inputs=[rs_region, rs_hazard, rs_vuln, rs_exp], + outputs=rs_out, + ) + + # --- About Tab --- + with gr.Tab("ℹ️ About"): + gr.Markdown( + """ +## About WorldDisasterLM-8B + +**WorldDisasterLM-8B** is an instruction-tuned language model built on Meta's Llama 3.1 8B Instruct, +domain-adapted for global disaster management and humanitarian response. + +### Supported Languages +| Language | Script | ISO | +|---|---|---| +| English | Latin | en | +| Nepali | Devanagari | ne | +| Spanish | Latin | es | +| French | Latin | fr | +| Arabic | Arabic | ar | +| Hindi | Devanagari | hi | +| Telugu | Telugu | te | +| Chinese | Simplified Han | zh | +| Japanese | Kanji/Hiragana | ja | +| Korean | Hangul | ko | +| Portuguese | Latin | pt | + +### Training Data Sources +- **ReliefWeb** — Humanitarian reports and disaster assessments +- **USGS** — Earthquake catalog (M≥4.0, 10-year archive) +- **NOAA** — Weather alerts and severe weather events +- **GDACS** — Global disaster alert coordination +- **OpenFEMA** — US federal disaster declarations +- **WHO** — Disease outbreak and public health alerts + +### Training Method +QLoRA fine-tuning (4-bit NF4 quantization, LoRA r=16) on Llama 3.1 8B Instruct. + +### Citation +``` +@misc{worlddisasterlm2026, + title = {WorldDisasterLM: Open Foundation Model for Global Disaster Management}, + year = {2026} +} +``` + +### License +This demo is released under the [Llama 3 Community License](https://llama.meta.com/llama3/license/). + """ + ) + +if __name__ == "__main__": + demo.launch() diff --git a/hf_space/requirements.txt b/hf_space/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..c58b5837c2e97d7d54e2a291fe6fb3cd910f60d1 --- /dev/null +++ b/hf_space/requirements.txt @@ -0,0 +1 @@ +gradio>=4.0.0,<5.0.0 diff --git a/inference.py b/inference.py new file mode 100644 index 0000000000000000000000000000000000000000..2bd91d054d817c7095ae225f4f5376e5ea00e282 --- /dev/null +++ b/inference.py @@ -0,0 +1,27 @@ +import argparse + +from backend.app.models.schemas import ChatMessage +from backend.app.services.inference_service import generate_response + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Run local WorldDisasterLM-8B inference") + parser.add_argument("--prompt", required=True, help="User query") + parser.add_argument("--language", default="English") + parser.add_argument("--region", default="global") + return parser.parse_args() + + +def main() -> None: + args = parse_args() + response = generate_response( + [ChatMessage(role="user", content=args.prompt)], + language=args.language, + region=args.region, + ) + print(response.answer) + print(f"confidence={response.confidence} needs_human_review={response.needs_human_review}") + + +if __name__ == "__main__": + main() diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000000000000000000000000000000000000..d281850df77be5a34ad1b7f62aac7d58363d2a76 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,21 @@ +[build-system] +requires = ["setuptools>=68", "wheel"] +build-backend = "setuptools.build_meta" + +[project] +name = "worlddisasterlm" +version = "0.1.0" +description = "Open foundation model stack for global disaster management" +readme = "README.md" +requires-python = ">=3.10" +license = { text = "MIT" } +authors = [{ name = "WorldDisasterLM Contributors" }] +dependencies = [] + +[tool.ruff] +line-length = 100 +target-version = "py310" + +[tool.pytest.ini_options] +testpaths = ["tests"] +pythonpath = ["."] diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..aad224b08cbf84a008c20bba5eeca84af2e334cb --- /dev/null +++ b/requirements.txt @@ -0,0 +1,51 @@ +# ── Web & API ────────────────────────────────────────────────────────────── +fastapi==0.115.12 +uvicorn[standard]==0.34.2 +pydantic==2.11.5 +pydantic-settings==2.9.1 +httpx==0.28.1 +python-dotenv==1.0.1 + +# ── Data collection ───────────────────────────────────────────────────────── +feedparser==6.0.11 +beautifulsoup4==4.12.3 +lxml==5.3.0 + +# ── Data processing ───────────────────────────────────────────────────────── +pandas==2.2.3 +numpy==2.2.6 +scikit-learn==1.6.1 +datasets==3.6.0 + +# ── ML core (install CUDA wheel separately for GPU training) ───────────────── +transformers==4.53.0 +accelerate==1.7.0 +peft==0.15.2 +trl==0.9.6 +bitsandbytes==0.45.5 +# torch — install manually for your CUDA version: +# pip install torch --index-url https://download.pytorch.org/whl/cu124 + +# ── Distributed training (optional) ───────────────────────────────────────── +# deepspeed==0.16.7 # Linux/CUDA only — uncomment if using DeepSpeed + +# ── Evaluation & export ────────────────────────────────────────────────────── +evaluate==0.4.3 +sacrebleu==2.5.1 +rouge-score==0.1.2 +sentencepiece==0.2.0 +onnx==1.17.0 +onnxruntime==1.22.0 + +# ── HuggingFace publishing ─────────────────────────────────────────────────── +huggingface_hub==0.30.2 + +# ── Demo & MLOps ───────────────────────────────────────────────────────────── +gradio==5.33.0 +mlflow==2.22.0 +wandb==0.19.11 + +# ── Dev & testing ──────────────────────────────────────────────────────────── +pytest==8.3.5 +pytest-asyncio==0.26.0 +ruff==0.11.11 diff --git a/scripts/collect_data.py b/scripts/collect_data.py new file mode 100644 index 0000000000000000000000000000000000000000..9c3604e748d7bce1b7fa3b20579b735792859e05 --- /dev/null +++ b/scripts/collect_data.py @@ -0,0 +1,127 @@ +""" +Data collection orchestrator. + +Runs all online collectors and saves the combined raw dataset plus +the instruction-following JSONL used for training. + +Usage +----- + python scripts/collect_data.py # all sources, default limits + python scripts/collect_data.py --sources reliefweb usgs gdacs + python scripts/collect_data.py --max-per-source 2000 +""" + +from __future__ import annotations + +import argparse +import json +import logging +import time +from pathlib import Path + +logger = logging.getLogger(__name__) + +DEFAULT_LIMITS: dict[str, int] = { + "reliefweb": 5000, + "usgs": 20000, + "gdacs": 2000, + "noaa": 5000, + "openfema": 20000, + "who": 1000, +} + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Collect online disaster data") + parser.add_argument( + "--sources", + nargs="+", + default=list(DEFAULT_LIMITS.keys()), + choices=list(DEFAULT_LIMITS.keys()), + help="Data sources to collect from", + ) + parser.add_argument("--max-per-source", type=int, default=None, help="Override max records per source") + parser.add_argument("--raw-dir", default="data/raw", help="Directory for raw records") + parser.add_argument("--processed-dir", default="data/processed", help="Directory for processed JSONL") + return parser.parse_args() + + +def save_records(records: list, path: Path) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + with path.open("w", encoding="utf-8") as handle: + for record in records: + handle.write(json.dumps(record.__dict__, ensure_ascii=False) + "\n") + logger.info("Saved %d records to %s", len(records), path) + + +def collect_source(source: str, max_records: int) -> list: + if source == "reliefweb": + from worlddisasterlm.data.collectors.reliefweb import collect_reliefweb + return collect_reliefweb(max_records=max_records) + if source == "usgs": + from worlddisasterlm.data.collectors.usgs import collect_usgs + return collect_usgs(max_records=max_records) + if source == "gdacs": + from worlddisasterlm.data.collectors.gdacs import collect_gdacs + return collect_gdacs(max_records=max_records) + if source == "noaa": + from worlddisasterlm.data.collectors.noaa import collect_noaa + return collect_noaa(max_records=max_records) + if source == "openfema": + from worlddisasterlm.data.collectors.openfema import collect_openfema + return collect_openfema(max_records=max_records) + if source == "who": + from worlddisasterlm.data.collectors.who_rss import collect_who + return collect_who(max_records=max_records) + raise ValueError(f"Unknown source: {source}") + + +def main() -> None: + logging.basicConfig(level=logging.INFO, format="%(asctime)s | %(levelname)s | %(message)s") + args = parse_args() + + from worlddisasterlm.data.etl import DisasterETL + from worlddisasterlm.data.qa_generator import generate_qa_pairs + from worlddisasterlm.data.scenario_builder import build_all_scenarios + from worlddisasterlm.data.processors import save_instruction_dataset + + raw_dir = Path(args.raw_dir) + processed_dir = Path(args.processed_dir) + processed_dir.mkdir(parents=True, exist_ok=True) + + all_records = [] + + for source in args.sources: + limit = args.max_per_source or DEFAULT_LIMITS.get(source, 5000) + logger.info("Collecting from %s (max=%d) …", source, limit) + try: + records = collect_source(source, limit) + save_records(records, raw_dir / f"{source}.jsonl") + all_records.extend(records) + except Exception as exc: + logger.error("Failed to collect from %s: %s", source, exc) + + time.sleep(1) # polite delay between sources + + etl = DisasterETL() + all_records = etl.deduplicate(all_records) + all_records = etl.normalize(all_records) + logger.info("Total normalized records after dedup: %d", len(all_records)) + + # Generate instruction QA pairs (8x amplification) + logger.info("Generating instruction QA pairs …") + qa_samples = generate_qa_pairs(all_records) + + # Add compound + multilingual scenarios + extra_samples = build_all_scenarios() + qa_samples.extend(extra_samples) + + logger.info("Total instruction samples: %d", len(qa_samples)) + + output_path = processed_dir / "instruction_dataset.jsonl" + save_instruction_dataset(qa_samples, str(output_path)) + logger.info("Instruction dataset saved: %s", output_path) + + +if __name__ == "__main__": + main() diff --git a/scripts/convert_gguf.py b/scripts/convert_gguf.py new file mode 100644 index 0000000000000000000000000000000000000000..74da894f6310c083d7f35ab4825488ad9f302a7e --- /dev/null +++ b/scripts/convert_gguf.py @@ -0,0 +1,139 @@ +""" +GGUF conversion script for WorldDisasterLM. + +Converts the merged Hugging Face model to GGUF format for CPU inference +and mobile deployment using llama.cpp. + +Quantization sizes (approximate for 8B model) +----------------------------------------------- + Q4_K_M → ~4.8 GB (recommended for most use cases) + Q5_K_M → ~5.6 GB (better quality) + Q8_0 → ~8.5 GB (highest quality, slower) + f16 → ~15 GB (full precision) + +Usage +----- + # Full automated flow (requires llama.cpp cloned alongside this repo) + python scripts/convert_gguf.py \\ + --model-path checkpoints/worlddisasterlm-merged \\ + --llama-cpp-path ../llama.cpp \\ + --quant Q4_K_M + + # Manual steps are printed if llama.cpp is not found +""" + +from __future__ import annotations + +import argparse +import logging +import shutil +import subprocess +import sys +from pathlib import Path + +logger = logging.getLogger(__name__) + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Convert WorldDisasterLM to GGUF") + parser.add_argument("--model-path", default="checkpoints/worlddisasterlm-merged", help="Path to merged HF model") + parser.add_argument("--output-dir", default="artifacts", help="Output directory for GGUF files") + parser.add_argument("--llama-cpp-path", default="../llama.cpp", help="Path to llama.cpp repo") + parser.add_argument( + "--quant", + default="Q4_K_M", + choices=["Q4_K_M", "Q5_K_M", "Q8_0", "f16"], + help="Quantization type", + ) + return parser.parse_args() + + +def print_manual_steps(model_path: str, output_dir: str, quant: str) -> None: + print("\n" + "=" * 70) + print("MANUAL GGUF CONVERSION STEPS") + print("=" * 70) + print("\nStep 1: Clone llama.cpp and build") + print(" git clone https://github.com/ggerganov/llama.cpp") + print(" cd llama.cpp") + print(" cmake -B build -DLLAMA_BLAS=ON -DLLAMA_BLAS_VENDOR=OpenBLAS") + print(" cmake --build build --config Release") + print() + print("Step 2: Install Python dependencies") + print(" pip install -r llama.cpp/requirements.txt") + print() + print("Step 3: Convert HF model to GGUF (f16)") + print(f" python llama.cpp/convert_hf_to_gguf.py {model_path} \\") + print(f" --outtype f16 --outfile {output_dir}/worlddisasterlm_f16.gguf") + print() + print(f"Step 4: Quantize to {quant}") + print(f" ./llama.cpp/build/bin/llama-quantize \\") + print(f" {output_dir}/worlddisasterlm_f16.gguf \\") + print(f" {output_dir}/worlddisasterlm_{quant.lower()}.gguf \\") + print(f" {quant}") + print() + print("Step 5: Upload GGUF to Hugging Face") + print(" huggingface-cli upload YourUsername/WorldDisasterLM-GGUF \\") + print(f" {output_dir}/worlddisasterlm_{quant.lower()}.gguf \\") + print(f" worlddisasterlm_{quant.lower()}.gguf") + print("=" * 70 + "\n") + + +def run_conversion(model_path: str, llama_cpp_path: str, output_dir: str, quant: str) -> None: + llama_dir = Path(llama_cpp_path).resolve() + model_dir = Path(model_path).resolve() + out_dir = Path(output_dir) + out_dir.mkdir(parents=True, exist_ok=True) + + convert_script = llama_dir / "convert_hf_to_gguf.py" + quantize_bin = llama_dir / "build" / "bin" / "llama-quantize" + quantize_bin_win = llama_dir / "build" / "bin" / "Release" / "llama-quantize.exe" + + if not convert_script.exists(): + logger.error("convert_hf_to_gguf.py not found in %s", llama_dir) + print_manual_steps(model_path, output_dir, quant) + sys.exit(1) + + f16_gguf = out_dir / "worlddisasterlm_f16.gguf" + quant_gguf = out_dir / f"worlddisasterlm_{quant.lower()}.gguf" + + # Convert to f16 GGUF + logger.info("Converting HF model to f16 GGUF …") + subprocess.run( + [sys.executable, str(convert_script), str(model_dir), "--outtype", "f16", "--outfile", str(f16_gguf)], + check=True, + ) + + # Find quantize binary + q_bin = quantize_bin if quantize_bin.exists() else (quantize_bin_win if quantize_bin_win.exists() else None) + if q_bin is None: + logger.warning("llama-quantize binary not found. f16 GGUF saved at %s", f16_gguf) + print_manual_steps(model_path, output_dir, quant) + return + + # Quantize + logger.info("Quantizing to %s …", quant) + subprocess.run([str(q_bin), str(f16_gguf), str(quant_gguf), quant], check=True) + logger.info("GGUF model saved to %s", quant_gguf) + logger.info("Upload with: huggingface-cli upload %s", quant_gguf) + + +def main() -> None: + logging.basicConfig(level=logging.INFO, format="%(asctime)s | %(levelname)s | %(message)s") + args = parse_args() + + llama_dir = Path(args.llama_cpp_path) + if not llama_dir.exists(): + logger.warning("llama.cpp directory not found at %s — printing manual steps.", args.llama_cpp_path) + print_manual_steps(args.model_path, args.output_dir, args.quant) + return + + run_conversion( + model_path=args.model_path, + llama_cpp_path=args.llama_cpp_path, + output_dir=args.output_dir, + quant=args.quant, + ) + + +if __name__ == "__main__": + main() diff --git a/scripts/export_gguf.py b/scripts/export_gguf.py new file mode 100644 index 0000000000000000000000000000000000000000..21b623d6d3aae06aa5a1b95c797fce5ee09b107f --- /dev/null +++ b/scripts/export_gguf.py @@ -0,0 +1,19 @@ +import argparse + +from worlddisasterlm.optimization.export_gguf import export_gguf + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Export WorldDisasterLM-8B to GGUF") + parser.add_argument("--model-path", default="checkpoints/worlddisasterlm-8b-qlora") + parser.add_argument("--output-path", default="artifacts/worlddisasterlm-8b.gguf") + return parser.parse_args() + + +def main() -> None: + args = parse_args() + export_gguf(model_path=args.model_path, output_path=args.output_path) + + +if __name__ == "__main__": + main() diff --git a/scripts/export_onnx.py b/scripts/export_onnx.py new file mode 100644 index 0000000000000000000000000000000000000000..a20448e31296ee8bb527d835ff419a679f21ca47 --- /dev/null +++ b/scripts/export_onnx.py @@ -0,0 +1,19 @@ +import argparse + +from worlddisasterlm.optimization.export_onnx import export_onnx + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Export WorldDisasterLM-8B to ONNX") + parser.add_argument("--model-path", default="checkpoints/worlddisasterlm-8b-qlora") + parser.add_argument("--output-path", default="artifacts/worlddisasterlm-8b.onnx") + return parser.parse_args() + + +def main() -> None: + args = parse_args() + export_onnx(model_path=args.model_path, output_path=args.output_path) + + +if __name__ == "__main__": + main() diff --git a/scripts/push_to_hub.py b/scripts/push_to_hub.py new file mode 100644 index 0000000000000000000000000000000000000000..0768ff206fd106f333c9c3c1be06091560010101 --- /dev/null +++ b/scripts/push_to_hub.py @@ -0,0 +1,232 @@ +""" +Hugging Face Hub push script. + +Merges LoRA adapters into the base model, creates the model card, and +pushes everything to the Hub under your account namespace. + +Usage +----- +python scripts/push_to_hub.py \\ + --adapter checkpoints/worlddisasterlm-qlora \\ + --base-model meta-llama/Llama-3.1-8B-Instruct \\ + --repo-id YourHFUsername/WorldDisasterLM-8B + +Requirements +------------ + export HF_TOKEN=hf_xxxx + pip install transformers peft huggingface_hub +""" + +from __future__ import annotations + +import argparse +import logging +import os +from pathlib import Path +import tempfile + +logger = logging.getLogger(__name__) + + +HF_MODEL_CARD = """--- +language: +- en +- ne +- es +- fr +- ar +- hi +- te +- zh +- ja +- ko +- pt +license: llama3 +base_model: meta-llama/Llama-3.1-8B-Instruct +tags: +- disaster-management +- emergency-response +- humanitarian-ai +- fine-tuned +- qlora +- lora +- peft +pipeline_tag: text-generation +library_name: transformers +model-index: +- name: WorldDisasterLM-8B + results: [] +--- + +# WorldDisasterLM — Open Foundation Model for Global Disaster Intelligence + +WorldDisasterLM is an instruction-tuned large language model built on top of +**Llama 3.1 8B Instruct**, domain-adapted on global disaster data from +ReliefWeb, USGS, NOAA, GDACS, OpenFEMA, and WHO. + +## Model Details + +| Property | Value | +|---|---| +| Base model | meta-llama/Llama-3.1-8B-Instruct | +| Training method | QLoRA (4-bit NF4 quantization, LoRA r=16) | +| Languages | EN, ES, FR, AR, HI, TE, ZH, JA, KO, PT | +| Domain | Disaster management, humanitarian response, risk intelligence | +| License | Llama 3 Community License (see Meta's terms) | + +## Quick Start + +```python +from transformers import AutoModelForCausalLM, AutoTokenizer +import torch + +model_id = "YOUR_HF_USERNAME/WorldDisasterLM-8B" +tokenizer = AutoTokenizer.from_pretrained(model_id) +model = AutoModelForCausalLM.from_pretrained( + model_id, + torch_dtype=torch.bfloat16, + device_map="auto", +) + +messages = [ + { + "role": "system", + "content": "You are WorldDisasterLM, an expert in disaster management and emergency response.", + }, + {"role": "user", "content": "What should I do immediately after an earthquake?"}, +] +inputs = tokenizer.apply_chat_template(messages, return_tensors="pt", add_generation_prompt=True) +outputs = model.generate(inputs.to(model.device), max_new_tokens=512, temperature=0.7) +print(tokenizer.decode(outputs[0], skip_special_tokens=True)) +``` + +## Training Data + +Collected from free, publicly accessible sources: +- **ReliefWeb** — humanitarian reports and disaster assessments +- **USGS** — earthquake catalog (magnitude ≥4.0, 10-year archive) +- **NOAA** — weather alerts and severe weather events +- **GDACS** — global disaster alert coordination events +- **OpenFEMA** — US federal disaster declarations +- **WHO** — disease outbreak news and public health alerts + +Each raw record was expanded into 8 instruction-following QA variants +(immediate response, resource planning, risk assessment, public communication, +recovery planning, multilingual guidance) for a multi-hundred-thousand sample corpus. + +## Intended Use + +- Emergency operations centers +- Government disaster management agencies +- NGOs and humanitarian organizations +- Public health authorities +- Researchers in disaster risk reduction +- Community preparedness applications +- Citizens seeking emergency guidance + +## Safety and Limitations + +- **Not a substitute** for real-time emergency management systems or official orders. +- Always verify critical operational decisions with local emergency authorities. +- Model outputs should be reviewed by trained emergency professionals for life-safety decisions. +- Some low-resource languages may have lower quality responses. +- Training data may not reflect the most recent real-time events. + +## Citation + +```bibtex +@misc{worlddisasterlm2026, + title = {WorldDisasterLM: An Open Foundation Model for Global Disaster Management}, + year = {2026}, + url = {https://huggingface.co/YOUR_HF_USERNAME/WorldDisasterLM-8B} +} +``` +""" + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Merge LoRA adapters and push WorldDisasterLM to Hugging Face Hub") + parser.add_argument("--adapter", default="checkpoints/worlddisasterlm-qlora", help="Path to LoRA adapter checkpoint") + parser.add_argument("--base-model", default="meta-llama/Llama-3.1-8B-Instruct", help="Base model ID") + parser.add_argument("--repo-id", required=True, help="HF repo ID, e.g. YourUsername/WorldDisasterLM-8B") + parser.add_argument("--private", action="store_true", help="Create as private repo (default: public)") + parser.add_argument("--push-dtype", choices=["bfloat16", "float16", "float32"], default="bfloat16") + return parser.parse_args() + + +def merge_and_push(adapter_path: str, base_model_id: str, repo_id: str, private: bool, push_dtype: str) -> None: + import torch + from transformers import AutoModelForCausalLM, AutoTokenizer + from peft import PeftModel + from huggingface_hub import HfApi, create_repo + + token = os.getenv("HF_TOKEN") + if not token: + raise SystemExit("HF_TOKEN environment variable not set. Run: huggingface-cli login") + + dtype_map = { + "bfloat16": torch.bfloat16, + "float16": torch.float16, + "float32": torch.float32, + } + torch_dtype = dtype_map[push_dtype] + + api = HfApi(token=token) + + logger.info("Creating or verifying repo: %s", repo_id) + create_repo(repo_id=repo_id, token=token, private=private, repo_type="model", exist_ok=True) + + logger.info("Loading tokenizer from adapter path: %s", adapter_path) + tokenizer = AutoTokenizer.from_pretrained(adapter_path, trust_remote_code=True) + + logger.info("Loading base model: %s", base_model_id) + base_model = AutoModelForCausalLM.from_pretrained( + base_model_id, + torch_dtype=torch_dtype, + device_map="auto", + trust_remote_code=True, + ) + + logger.info("Loading LoRA adapter from: %s", adapter_path) + peft_model = PeftModel.from_pretrained(base_model, adapter_path) + + logger.info("Merging LoRA weights into base model …") + merged_model = peft_model.merge_and_unload() + merged_model.config.use_cache = True + + logger.info("Pushing merged model to %s …", repo_id) + merged_model.push_to_hub(repo_id, token=token, safe_serialization=True) + tokenizer.push_to_hub(repo_id, token=token) + + # Upload model card + with tempfile.NamedTemporaryFile("w", suffix=".md", delete=False, encoding="utf-8") as tf: + tf.write(HF_MODEL_CARD.replace("YOUR_HF_USERNAME", repo_id.split("/")[0])) + tmp_card_path = tf.name + + api.upload_file( + path_or_fileobj=tmp_card_path, + path_in_repo="README.md", + repo_id=repo_id, + repo_type="model", + token=token, + ) + Path(tmp_card_path).unlink(missing_ok=True) + + logger.info("Done! Model published at: https://huggingface.co/%s", repo_id) + logger.info("Tag your model as free-to-use by setting the license in the repo settings.") + + +def main() -> None: + logging.basicConfig(level=logging.INFO, format="%(asctime)s | %(levelname)s | %(message)s") + args = parse_args() + merge_and_push( + adapter_path=args.adapter, + base_model_id=args.base_model, + repo_id=args.repo_id, + private=args.private, + push_dtype=args.push_dtype, + ) + + +if __name__ == "__main__": + main() diff --git a/scripts/run_pipeline.py b/scripts/run_pipeline.py new file mode 100644 index 0000000000000000000000000000000000000000..77555492f48d3a793558d80e36d3b6fa1c5eb7f9 --- /dev/null +++ b/scripts/run_pipeline.py @@ -0,0 +1,41 @@ +import argparse +import subprocess +import sys + + +STAGE_COMMANDS = { + "data": [sys.executable, "dataset_builder.py"], + "train": [sys.executable, "train.py"], + "evaluate": [sys.executable, "evaluate.py"], + "all": [ + [sys.executable, "dataset_builder.py"], + [sys.executable, "train.py"], + [sys.executable, "evaluate.py"], + ], +} + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Run WorldDisasterLM pipeline stages") + parser.add_argument("--stage", choices=["data", "train", "evaluate", "all"], default="all") + return parser.parse_args() + + +def run_command(command: list[str]) -> None: + print("Running:", " ".join(command)) + subprocess.run(command, check=True) + + +def main() -> None: + args = parse_args() + commands = STAGE_COMMANDS[args.stage] + + if args.stage == "all": + for command in commands: + run_command(command) + else: + run_command(commands) + + +if __name__ == "__main__": + main() diff --git a/scripts/train_production.py b/scripts/train_production.py new file mode 100644 index 0000000000000000000000000000000000000000..515ade2c32e64896f7d7c284008e2d742889e997 --- /dev/null +++ b/scripts/train_production.py @@ -0,0 +1,96 @@ +""" +Production training launcher with full CLI. + +Usage +----- + # Minimal (uses all defaults) + python scripts/train_production.py + + # Full options + python scripts/train_production.py \\ + --dataset data/processed/instruction_dataset.jsonl \\ + --base-model meta-llama/Llama-3.1-8B-Instruct \\ + --output checkpoints/worlddisasterlm-qlora \\ + --epochs 3 \\ + --lora-r 16 \\ + --batch-size 2 \\ + --grad-accum 8 \\ + --report-to wandb + + # Consumer GPU (RTX 4090 24 GB) + python scripts/train_production.py \\ + --batch-size 1 --grad-accum 16 --max-seq-length 1024 +""" + +from __future__ import annotations + +import argparse +import logging + +logging.basicConfig(level=logging.INFO, format="%(asctime)s | %(levelname)s | %(message)s") +logger = logging.getLogger(__name__) + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Train WorldDisasterLM with QLoRA") + + # Model / data + parser.add_argument("--base-model", default="meta-llama/Llama-3.1-8B-Instruct") + parser.add_argument("--dataset", default="data/processed/instruction_dataset.jsonl") + parser.add_argument("--output", default="checkpoints/worlddisasterlm-qlora") + parser.add_argument("--max-seq-length", type=int, default=2048) + + # Training + parser.add_argument("--epochs", type=int, default=3) + parser.add_argument("--learning-rate", type=float, default=2e-4) + parser.add_argument("--batch-size", type=int, default=2) + parser.add_argument("--grad-accum", type=int, default=8) + parser.add_argument("--warmup-ratio", type=float, default=0.03) + + # LoRA + parser.add_argument("--lora-r", type=int, default=16) + parser.add_argument("--lora-alpha", type=int, default=32) + parser.add_argument("--lora-dropout", type=float, default=0.05) + + # Hardware + parser.add_argument("--no-4bit", action="store_true", help="Disable 4-bit quantization (needs more VRAM)") + parser.add_argument("--fp16", action="store_true", help="Use fp16 instead of bf16") + + # Tracking + parser.add_argument("--report-to", choices=["mlflow", "wandb", "none"], default="none") + parser.add_argument("--seed", type=int, default=42) + + return parser.parse_args() + + +def main() -> None: + args = parse_args() + + from worlddisasterlm.training.train_qlora import QLoRAConfig, train + + config = QLoRAConfig( + base_model=args.base_model, + output_dir=args.output, + dataset_path=args.dataset, + max_seq_length=args.max_seq_length, + use_4bit=not args.no_4bit, + epochs=args.epochs, + learning_rate=args.learning_rate, + per_device_train_batch_size=args.batch_size, + gradient_accumulation_steps=args.grad_accum, + warmup_ratio=args.warmup_ratio, + lora_r=args.lora_r, + lora_alpha=args.lora_alpha, + lora_dropout=args.lora_dropout, + bf16=not args.fp16, + fp16=args.fp16, + report_to=args.report_to, + seed=args.seed, + ) + + logger.info("Effective training config: %s", config) + train(config) + + +if __name__ == "__main__": + main() diff --git a/scripts/upload_space.py b/scripts/upload_space.py new file mode 100644 index 0000000000000000000000000000000000000000..5e3943b21021e3510f529f00418296ddc3bbd1db --- /dev/null +++ b/scripts/upload_space.py @@ -0,0 +1,112 @@ +""" +Upload WorldDisasterLM-8B as a public HuggingFace Space (Gradio demo). + +Usage +----- + # Set your HF token first: + $env:HF_TOKEN = "hf_xxxxxxxxxxxxxxxxxxxx" + + # Then run: + python scripts/upload_space.py --username YOUR_HF_USERNAME + + # Optionally specify a custom space name: + python scripts/upload_space.py --username YOUR_HF_USERNAME --space-name WorldDisasterLM-8B + +Requirements +------------ + pip install huggingface_hub +""" + +from __future__ import annotations + +import argparse +import os +import sys +from pathlib import Path + +ROOT = Path(__file__).parent.parent +SPACE_DIR = ROOT / "hf_space" + + +def parse_args() -> argparse.Namespace: + p = argparse.ArgumentParser(description="Upload WorldDisasterLM-8B to HuggingFace Spaces") + p.add_argument("--username", required=True, help="Your HuggingFace username") + p.add_argument("--space-name", default="WorldDisasterLM-8B", help="Space repository name") + p.add_argument("--private", action="store_true", help="Create as private space (default: public)") + return p.parse_args() + + +def main() -> None: + args = parse_args() + + # Check token + token = os.environ.get("HF_TOKEN") + if not token: + print("ERROR: HF_TOKEN environment variable is not set.") + print(" Set it with: $env:HF_TOKEN = 'hf_xxxxxxxxxxxx'") + sys.exit(1) + + try: + from huggingface_hub import HfApi, create_repo + except ImportError: + print("ERROR: huggingface_hub not installed. Run: pip install huggingface_hub") + sys.exit(1) + + repo_id = f"{args.username}/{args.space_name}" + api = HfApi(token=token) + + print(f"\n{'='*60}") + print(f" WorldDisasterLM-8B → HuggingFace Space") + print(f" Repo : {repo_id}") + print(f" Private: {args.private}") + print(f"{'='*60}\n") + + # 1. Create the Space repo + print("Step 1/3 — Creating Space repository...") + create_repo( + repo_id=repo_id, + repo_type="space", + space_sdk="gradio", + private=args.private, + exist_ok=True, + token=token, + ) + print(f" ✓ Space created: https://huggingface.co/spaces/{repo_id}") + + # 2. Patch README.md with actual username + readme_src = SPACE_DIR / "README.md" + readme_text = readme_src.read_text(encoding="utf-8") + readme_text = readme_text.replace("YOUR_HF_USERNAME", args.username) + + import tempfile, shutil + tmp_dir = Path(tempfile.mkdtemp()) + try: + # Copy space files to temp dir with patched README + shutil.copytree(str(SPACE_DIR), str(tmp_dir / "space")) + (tmp_dir / "space" / "README.md").write_text(readme_text, encoding="utf-8") + + # 3. Upload the folder + print("Step 2/3 — Uploading files...") + api.upload_folder( + folder_path=str(tmp_dir / "space"), + repo_id=repo_id, + repo_type="space", + commit_message="Upload WorldDisasterLM-8B Space demo", + token=token, + ) + print(" ✓ Files uploaded") + finally: + shutil.rmtree(tmp_dir, ignore_errors=True) + + print("\nStep 3/3 — Verifying Space...") + space_info = api.space_info(repo_id=repo_id, token=token) + print(f" ✓ Space status: {getattr(space_info, 'runtime', {})}") + + print(f"\n{'='*60}") + print(f" DONE! Your Space is live at:") + print(f" https://huggingface.co/spaces/{repo_id}") + print(f"{'='*60}\n") + + +if __name__ == "__main__": + main() diff --git a/scripts/upload_space_urllib.py b/scripts/upload_space_urllib.py new file mode 100644 index 0000000000000000000000000000000000000000..4888b6a4b611a93d002dd9c88eff2bb41dd78143 --- /dev/null +++ b/scripts/upload_space_urllib.py @@ -0,0 +1,234 @@ +""" +Upload WorldDisasterLM-8B as a public HuggingFace Space using urllib (no httpx/requests). +Works on Python 3.14 on Windows where httpx TLS may fail. + +Usage +----- + python scripts/upload_space_urllib.py --token hf_xxx --username drdeveloper88 +""" + +from __future__ import annotations + +import argparse +import base64 +import json +import os +import shutil +import ssl +import sys +import tempfile +import urllib.request +from pathlib import Path + +ROOT = Path(__file__).parent.parent +SPACE_DIR = ROOT / "hf_space" +HF_API = "https://huggingface.co/api" + +CTX = ssl.create_default_context() + + +def api(method: str, path: str, token: str, payload: dict | None = None) -> dict: + url = f"{HF_API}{path}" + data = json.dumps(payload).encode() if payload else None + req = urllib.request.Request( + url, + data=data, + method=method, + headers={ + "Authorization": f"Bearer {token}", + "Content-Type": "application/json", + }, + ) + try: + with urllib.request.urlopen(req, context=CTX, timeout=30) as r: + return json.loads(r.read()) + except urllib.error.HTTPError as e: + body = e.read().decode() + raise RuntimeError(f"HTTP {e.code}: {body}") from e + + +def upload_file(repo_id: str, token: str, local_path: Path, repo_path: str, commit_msg: str) -> None: + """Upload a single file via the HF /api/repos endpoint (LFS-aware preupload + commit).""" + content = local_path.read_bytes() + + # Step 1: preupload (get upload URL or confirm non-LFS) + preupload_url = f"https://huggingface.co/api/spaces/{repo_id}/preupload/main" + payload = json.dumps([{"path": repo_path, "size": len(content)}]).encode() + req = urllib.request.Request( + preupload_url, + data=payload, + method="POST", + headers={ + "Authorization": f"Bearer {token}", + "Content-Type": "application/json", + }, + ) + with urllib.request.urlopen(req, context=CTX, timeout=30) as r: + preupload = json.loads(r.read()) + + files_info = preupload.get("files", [{}]) + upload_mode = files_info[0].get("uploadMode", "regular") if files_info else "regular" + + if upload_mode == "lfs": + # Upload to LFS URL + upload_url = files_info[0]["uploadUrl"] + put_req = urllib.request.Request( + upload_url, + data=content, + method="PUT", + headers={"Content-Type": "application/octet-stream"}, + ) + with urllib.request.urlopen(put_req, context=CTX, timeout=60) as r: + r.read() + oid = files_info[0]["oid"] + size = files_info[0]["size"] + lfs_content = ( + f"version https://git-lfs.github.com/spec/v1\n" + f"oid sha256:{oid}\n" + f"size {size}\n" + ).encode() + final_content = lfs_content + else: + final_content = content + + # Step 2: commit + commit_url = f"https://huggingface.co/api/spaces/{repo_id}/commit/main" + header = json.dumps({ + "summary": commit_msg, + "files": [{"path": repo_path, "encoding": "base64"}], + }) + boundary = "----HFUploadBoundary" + body_parts = [ + f"--{boundary}\r\nContent-Disposition: form-data; name=\"header\"\r\n\r\n{header}\r\n".encode(), + f"--{boundary}\r\nContent-Disposition: form-data; name=\"file\"; filename=\"{repo_path}\"\r\n\r\n".encode(), + base64.b64encode(final_content), + f"\r\n--{boundary}--\r\n".encode(), + ] + body = b"".join(body_parts) + commit_req = urllib.request.Request( + commit_url, + data=body, + method="POST", + headers={ + "Authorization": f"Bearer {token}", + "Content-Type": f"multipart/form-data; boundary={boundary}", + }, + ) + with urllib.request.urlopen(commit_req, context=CTX, timeout=60) as r: + r.read() + + +def upload_folder_simple(repo_id: str, token: str, folder: Path, readme_text: str) -> None: + """Upload all Space files using the simpler single-commit API.""" + files_payload = [] + file_contents = {} + + for f in folder.iterdir(): + if f.is_file(): + if f.name == "README.md": + content = readme_text.encode("utf-8") + else: + content = f.read_bytes() + encoded = base64.b64encode(content).decode() + files_payload.append({"path": f.name, "encoding": "base64"}) + file_contents[f.name] = encoded + + # Build multipart commit + boundary = "HFSpaceUpload42" + parts = [] + header = json.dumps({ + "summary": "Upload WorldDisasterLM-8B Space", + "files": files_payload, + }) + parts.append( + f"--{boundary}\r\nContent-Disposition: form-data; name=\"header\"\r\n\r\n{header}\r\n".encode() + ) + for item in files_payload: + fname = item["path"] + parts.append( + f"--{boundary}\r\nContent-Disposition: form-data; name=\"file\"; filename=\"{fname}\"\r\n\r\n".encode() + ) + parts.append(file_contents[fname].encode()) + parts.append(b"\r\n") + parts.append(f"--{boundary}--\r\n".encode()) + body = b"".join(parts) + + commit_url = f"https://huggingface.co/api/spaces/{repo_id}/commit/main" + req = urllib.request.Request( + commit_url, + data=body, + method="POST", + headers={ + "Authorization": f"Bearer {token}", + "Content-Type": f"multipart/form-data; boundary={boundary}", + }, + ) + try: + with urllib.request.urlopen(req, context=CTX, timeout=120) as r: + result = json.loads(r.read()) + return result + except urllib.error.HTTPError as e: + raise RuntimeError(f"Commit failed HTTP {e.code}: {e.read().decode()}") from e + + +def parse_args() -> argparse.Namespace: + p = argparse.ArgumentParser() + p.add_argument("--token", default=os.environ.get("HF_TOKEN"), help="HF token (or set HF_TOKEN env var)") + p.add_argument("--username", required=True) + p.add_argument("--space-name", default="WorldDisasterLM-8B") + p.add_argument("--private", action="store_true") + return p.parse_args() + + +def main() -> None: + args = parse_args() + token = args.token + if not token: + print("ERROR: provide --token or set HF_TOKEN"); sys.exit(1) + + repo_id = f"{args.username}/{args.space_name}" + + print(f"\n{'='*60}") + print(f" WorldDisasterLM-8B → HuggingFace Space") + print(f" Repo: {repo_id}") + print(f"{'='*60}\n") + + # Verify token + user = api("GET", "/whoami-v2", token) + print(f" Authenticated as: {user.get('name')}") + + # Step 1: Create Space + print("\nStep 1/3 — Creating Space repository...") + try: + api("POST", "/repos/create", token, { + "type": "space", + "name": args.space_name, + "sdk": "gradio", + "private": args.private, + "exist_ok": True, + }) + print(f" ✓ Space created") + except RuntimeError as e: + if "already exists" in str(e) or "409" in str(e): + print(f" ✓ Space already exists, updating") + else: + raise + + # Step 2: Patch README + readme_text = (SPACE_DIR / "README.md").read_text(encoding="utf-8") + readme_text = readme_text.replace("YOUR_HF_USERNAME", args.username) + + # Step 3: Upload files + print("Step 2/3 — Uploading files...") + result = upload_folder_simple(repo_id, token, SPACE_DIR, readme_text) + print(f" ✓ Files uploaded (commit: {result.get('commitOid', 'ok')[:12] if result.get('commitOid') else 'done'})") + + print(f"\nStep 3/3 — Done!\n") + print(f"{'='*60}") + print(f" Live at: https://huggingface.co/spaces/{repo_id}") + print(f" (Space may take 1-2 min to build)") + print(f"{'='*60}\n") + + +if __name__ == "__main__": + main() diff --git a/tests/test_api.py b/tests/test_api.py new file mode 100644 index 0000000000000000000000000000000000000000..29bc99f825cec269d9cc12445f7fb8d360f11bdd --- /dev/null +++ b/tests/test_api.py @@ -0,0 +1,59 @@ +from fastapi.testclient import TestClient + +from backend.app.main import app + +client = TestClient(app) + + +def test_health() -> None: + response = client.get("/health") + assert response.status_code == 200 + payload = response.json() + assert payload["status"] == "ok" + + +def test_chat_endpoint() -> None: + response = client.post( + "/v1/chat", + json={ + "messages": [{"role": "user", "content": "What is the first step during flood evacuation?"}], + "language": "English", + "region": "global", + }, + ) + assert response.status_code == 200 + assert "answer" in response.json() + + +def test_chat_nepali_language() -> None: + """Verify Nepali language requests return a Nepali-language answer.""" + response = client.post( + "/v1/chat", + json={ + "messages": [{"role": "user", "content": "भूकम्पको बेला के गर्ने?"}], + "language": "Nepali", + "region": "Nepal", + }, + ) + assert response.status_code == 200 + data = response.json() + assert "answer" in data + # Response should contain Devanagari characters for Nepali output + assert any("\u0900" <= ch <= "\u097F" for ch in data["answer"]), ( + "Expected Devanagari script in Nepali response" + ) + + +def test_chat_nepali_confidence_range() -> None: + """Nepali response confidence must be in valid [0, 1] range.""" + response = client.post( + "/v1/chat", + json={ + "messages": [{"role": "user", "content": "बाढीको बेला के गर्ने?"}], + "language": "Nepali", + "region": "Nepal", + }, + ) + assert response.status_code == 200 + data = response.json() + assert 0.0 <= data["confidence"] <= 1.0 diff --git a/tests/test_dataset_builder.py b/tests/test_dataset_builder.py new file mode 100644 index 0000000000000000000000000000000000000000..228d44ac5b0b05685e3f87c0eb6a4d1a99ded61d --- /dev/null +++ b/tests/test_dataset_builder.py @@ -0,0 +1,36 @@ +from worlddisasterlm.config import SUPPORTED_LANGUAGES +from worlddisasterlm.data.etl import DisasterETL +from worlddisasterlm.data.processors import build_instruction_dataset +from worlddisasterlm.data.scenario_builder import build_all_scenarios + + +def test_dataset_builder_generates_samples() -> None: + etl = DisasterETL() + records = etl.normalize(etl.deduplicate(etl.collect_records())) + samples = build_instruction_dataset(records) + assert len(samples) > 0 + assert samples[0].instruction + + +def test_nepali_in_supported_languages() -> None: + """Nepali must be present in SUPPORTED_LANGUAGES.""" + assert "Nepali" in SUPPORTED_LANGUAGES + + +def test_nepali_scenario_samples_exist() -> None: + """At least one Nepali-language training sample must be built from scenarios.""" + samples = build_all_scenarios() + nepali_samples = [s for s in samples if s.language.lower() == "nepali"] + assert len(nepali_samples) >= 3, ( + f"Expected >=3 Nepali samples, found {len(nepali_samples)}" + ) + + +def test_nepali_samples_use_devanagari() -> None: + """Nepali scenario instructions must contain Devanagari Unicode characters.""" + samples = build_all_scenarios() + nepali_samples = [s for s in samples if s.language.lower() == "nepali"] + for sample in nepali_samples: + assert any("\u0900" <= ch <= "\u097F" for ch in sample.instruction), ( + f"Nepali sample missing Devanagari: {sample.instruction!r}" + ) diff --git a/tests/test_risk_engine.py b/tests/test_risk_engine.py new file mode 100644 index 0000000000000000000000000000000000000000..e15e34859535c7c2743f88659897834ceb0dc43d --- /dev/null +++ b/tests/test_risk_engine.py @@ -0,0 +1,7 @@ +from backend.app.services.risk_engine import compute_risk + + +def test_compute_risk_range() -> None: + response = compute_risk("flood", vulnerability_index=0.7, exposure_index=0.8) + assert 0.0 <= response.risk_score <= 1.0 + assert response.risk_level in {"low", "moderate", "high", "critical"} diff --git a/train.py b/train.py new file mode 100644 index 0000000000000000000000000000000000000000..78b1b7de5390ebffa111786ba6f08f91f23cf974 --- /dev/null +++ b/train.py @@ -0,0 +1,67 @@ +"""train.py — top-level entry-point, delegates to QLoRA production pipeline. + +For full CLI options use: + python scripts/train_production.py --help +""" + +from __future__ import annotations + +import argparse +import logging + +logging.basicConfig(level=logging.INFO, format="%(asctime)s | %(levelname)s | %(message)s") +logger = logging.getLogger(__name__) + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Train WorldDisasterLM-8B (QLoRA)") + parser.add_argument("--base-model", default="meta-llama/Llama-3.1-8B-Instruct") + parser.add_argument("--dataset", default="data/processed/instruction_dataset.jsonl") + parser.add_argument("--output", default="checkpoints/worlddisasterlm-qlora") + parser.add_argument("--epochs", type=int, default=3) + parser.add_argument("--learning-rate", type=float, default=2e-4) + parser.add_argument("--batch-size", type=int, default=2) + parser.add_argument("--grad-accum", type=int, default=8) + parser.add_argument("--lora-r", type=int, default=16) + parser.add_argument("--report-to", choices=["mlflow", "wandb", "none"], default="none") + return parser.parse_args() + + +def main() -> None: + args = parse_args() + + try: + from worlddisasterlm.training.train_qlora import QLoRAConfig, train + except ImportError: + # Graceful fallback if GPU stack (torch/bitsandbytes) not installed + logger.warning( + "QLoRA dependencies not available. Using lightweight stub training. " + "Install with: pip install torch bitsandbytes peft trl" + ) + from worlddisasterlm.training.fine_tune import TrainingConfig, run_training # type: ignore[import] + run_training(TrainingConfig( + base_model=args.base_model, + dataset_path=args.dataset, + output_dir=args.output, + epochs=args.epochs, + learning_rate=args.learning_rate, + batch_size=args.batch_size, + )) + return + + config = QLoRAConfig( + base_model=args.base_model, + dataset_path=args.dataset, + output_dir=args.output, + epochs=args.epochs, + learning_rate=args.learning_rate, + per_device_train_batch_size=args.batch_size, + gradient_accumulation_steps=args.grad_accum, + lora_r=args.lora_r, + report_to=args.report_to, + ) + train(config) + + +if __name__ == "__main__": + main() diff --git a/worlddisasterlm/__init__.py b/worlddisasterlm/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..258f66ca596da41f9975113f3d68e874e2995833 --- /dev/null +++ b/worlddisasterlm/__init__.py @@ -0,0 +1,3 @@ +__all__ = [ + "config", +] diff --git a/worlddisasterlm/config.py b/worlddisasterlm/config.py new file mode 100644 index 0000000000000000000000000000000000000000..d372b5c9b223d65ce2342e712b181077daeaa403 --- /dev/null +++ b/worlddisasterlm/config.py @@ -0,0 +1,28 @@ +from dataclasses import dataclass + + +SUPPORTED_LANGUAGES = [ + "English", + "Spanish", + "French", + "Arabic", + "Hindi", + "Telugu", + "Nepali", + "Chinese", + "Japanese", + "Korean", + "Portuguese", +] + + +@dataclass +class PipelineConfig: + project_name: str = "WorldDisasterLM-8B" + base_model: str = "meta-llama/Llama-3.1-8B-Instruct" + output_dir: str = "outputs" + data_dir: str = "data" + seed: int = 42 + + +DEFAULT_CONFIG = PipelineConfig() diff --git a/worlddisasterlm/data/__init__.py b/worlddisasterlm/data/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/worlddisasterlm/data/collectors/__init__.py b/worlddisasterlm/data/collectors/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..2b9139aaf26026d72c554130eb456da46ebb6f94 --- /dev/null +++ b/worlddisasterlm/data/collectors/__init__.py @@ -0,0 +1,15 @@ +from worlddisasterlm.data.collectors.reliefweb import collect_reliefweb +from worlddisasterlm.data.collectors.usgs import collect_usgs +from worlddisasterlm.data.collectors.gdacs import collect_gdacs +from worlddisasterlm.data.collectors.noaa import collect_noaa +from worlddisasterlm.data.collectors.openfema import collect_openfema +from worlddisasterlm.data.collectors.who_rss import collect_who + +__all__ = [ + "collect_reliefweb", + "collect_usgs", + "collect_gdacs", + "collect_noaa", + "collect_openfema", + "collect_who", +] diff --git a/worlddisasterlm/data/collectors/gdacs.py b/worlddisasterlm/data/collectors/gdacs.py new file mode 100644 index 0000000000000000000000000000000000000000..f2f032bff4619cfd573a0edd344681fb863cdb03 --- /dev/null +++ b/worlddisasterlm/data/collectors/gdacs.py @@ -0,0 +1,86 @@ +""" +GDACS (Global Disaster Alert and Coordination System) collector. + +Free GeoRSS feed – no authentication required. +Feed: https://www.gdacs.org/xml/rss.xml +""" +import logging +import re +from datetime import datetime + +import feedparser + +from worlddisasterlm.data.schemas import DisasterRecord + +logger = logging.getLogger(__name__) + +GDACS_RSS_URL = "https://www.gdacs.org/xml/rss.xml" +GDACS_SCORE_RSS_URL = "https://www.gdacs.org/xml/rss_score.xml" + + +def _alert_to_severity(alert_level: str) -> str: + level = alert_level.strip().lower() + if level == "red": + return "critical" + if level == "orange": + return "high" + return "moderate" + + +def _clean_html(text: str) -> str: + clean = re.sub(r"<[^>]+>", " ", text or "") + return " ".join(clean.split())[:600] + + +def collect_gdacs(max_records: int = 2000) -> list[DisasterRecord]: + """Collect disaster events from GDACS GeoRSS (free, no auth).""" + records: list[DisasterRecord] = [] + + for url in [GDACS_RSS_URL, GDACS_SCORE_RSS_URL]: + logger.info("Parsing GDACS feed: %s", url) + try: + feed = feedparser.parse(url) + except Exception as exc: + logger.warning("Failed to parse GDACS feed %s: %s", url, exc) + continue + + if feed.bozo and feed.bozo_exception: + logger.warning("GDACS feed parse warning: %s", feed.bozo_exception) + + for entry in feed.entries: + if len(records) >= max_records: + break + + title = entry.get("title", "") + summary_raw = entry.get("summary", entry.get("description", "")) + summary = _clean_html(summary_raw) or title + + # GDACS uses gdacs: namespace tags + alert_level = ( + entry.get("gdacs_alertlevel") + or entry.get("gdacs_alertscore", "") + or "green" + ) + event_type = ( + entry.get("gdacs_eventtype") + or entry.get("gdacs_eventname", "disaster") + ) + + # Try to extract country/region + country = entry.get("gdacs_country") or entry.get("gdacs_iso3", "global") + + if not summary: + continue + + records.append( + DisasterRecord( + source="GDACS", + event_type=str(event_type).lower().strip(), + region=str(country).strip(), + summary=summary, + severity=_alert_to_severity(str(alert_level)), + ) + ) + + logger.info("GDACS collection complete: %d records", len(records)) + return records[:max_records] diff --git a/worlddisasterlm/data/collectors/noaa.py b/worlddisasterlm/data/collectors/noaa.py new file mode 100644 index 0000000000000000000000000000000000000000..57b3f513c656b7c2c60ff21a5c57e3fd93865f14 --- /dev/null +++ b/worlddisasterlm/data/collectors/noaa.py @@ -0,0 +1,113 @@ +""" +NOAA National Weather Service alerts collector. + +Free API – no authentication required. +Docs: https://www.weather.gov/documentation/services-web-api +""" +import logging +import time + +import httpx + +from worlddisasterlm.data.schemas import DisasterRecord + +logger = logging.getLogger(__name__) + +NOAA_ALERTS_API = "https://api.weather.gov/alerts" +HEADERS = {"User-Agent": "WorldDisasterLM/0.1.0 (worlddisasterlm@example.com)"} + +SEVERITY_MAP = { + "extreme": "critical", + "severe": "high", + "moderate": "moderate", + "minor": "low", + "unknown": "low", +} + +EVENT_NORMALIZE = { + "tornado warning": "tornado", + "flash flood warning": "flood", + "hurricane warning": "hurricane", + "blizzard warning": "blizzard", + "winter storm warning": "winter_storm", + "tsunami warning": "tsunami", + "earthquake warning": "earthquake", + "excessive heat warning": "heatwave", + "fire weather watch": "wildfire", + "red flag warning": "wildfire", +} + + +def collect_noaa(max_records: int = 5000) -> list[DisasterRecord]: + """Collect active and recent weather alerts from NOAA (free, no auth).""" + records: list[DisasterRecord] = [] + cursor: str | None = None + page_count = 0 + + logger.info("Collecting NOAA weather alerts (max=%d)", max_records) + + while len(records) < max_records: + params: dict[str, str | int] = {"limit": 500} + if cursor: + params["cursor"] = cursor + + try: + response = httpx.get(NOAA_ALERTS_API, headers=HEADERS, params=params, timeout=30) + response.raise_for_status() + data = response.json() + except Exception as exc: + logger.warning("NOAA request failed (page %d): %s", page_count, exc) + break + + features = data.get("features", []) + if not features: + break + + for feature in features: + if len(records) >= max_records: + break + + props = feature.get("properties", {}) + event = str(props.get("event", "weather event")).lower() + area = props.get("areaDesc", "United States") + headline = props.get("headline", "") + description = props.get("description", "") + severity_raw = str(props.get("severity", "unknown")).lower() + + summary = headline or description + if not summary: + continue + summary = " ".join(summary.split())[:600] + + normalized_event = next( + (v for k, v in EVENT_NORMALIZE.items() if k in event), + event.replace(" warning", "").replace(" watch", "").strip(), + ) + + records.append( + DisasterRecord( + source="NOAA", + event_type=normalized_event or "weather_event", + region=area[:100], + summary=summary, + severity=SEVERITY_MAP.get(severity_raw, "moderate"), + ) + ) + + # Pagination + pagination = data.get("pagination", {}) + next_url = pagination.get("next") + if next_url: + # Extract cursor from next URL query param + from urllib.parse import urlparse, parse_qs + parsed = urlparse(next_url) + qs = parse_qs(parsed.query) + cursor = qs.get("cursor", [None])[0] + else: + break + + page_count += 1 + time.sleep(0.5) + + logger.info("NOAA collection complete: %d records", len(records)) + return records diff --git a/worlddisasterlm/data/collectors/openfema.py b/worlddisasterlm/data/collectors/openfema.py new file mode 100644 index 0000000000000000000000000000000000000000..20369735eb0644bb6d32774870ce400ad2050d07 --- /dev/null +++ b/worlddisasterlm/data/collectors/openfema.py @@ -0,0 +1,111 @@ +""" +OpenFEMA disaster declarations collector. + +Free API – no authentication required. +Docs: https://www.fema.gov/about/openfema/api +""" +import logging +import time + +import httpx + +from worlddisasterlm.data.schemas import DisasterRecord + +logger = logging.getLogger(__name__) + +OPENFEMA_API = "https://www.fema.gov/api/open/v2/DisasterDeclarationsSummaries" + +INCIDENT_TYPE_MAP = { + "hurricane": "hurricane", + "flood": "flood", + "tornado": "tornado", + "earthquake": "earthquake", + "fire": "wildfire", + "winter storm": "winter_storm", + "drought": "drought", + "mudslide": "landslide", + "typhoon": "typhoon", + "tsunami": "tsunami", + "volcano": "volcano", + "chemical": "chemical", + "biological": "epidemic", +} + + +def _map_incident(raw: str) -> str: + raw_lower = raw.lower() + for key, val in INCIDENT_TYPE_MAP.items(): + if key in raw_lower: + return val + return raw_lower + + +def collect_openfema(max_records: int = 20000) -> list[DisasterRecord]: + """Collect disaster declarations from OpenFEMA (free, no auth).""" + records: list[DisasterRecord] = [] + skip = 0 + page_size = 1000 + + logger.info("Collecting OpenFEMA disaster declarations (max=%d)", max_records) + + while len(records) < max_records: + params = { + "$format": "json", + "$top": min(page_size, max_records - len(records)), + "$skip": skip, + "$orderby": "declarationDate desc", + "$select": ( + "disasterNumber,declarationTitle,incidentType," + "declarationDate,state,incidentBeginDate,incidentEndDate," + "closeoutDate,declarationType" + ), + } + + try: + response = httpx.get(OPENFEMA_API, params=params, timeout=60) + response.raise_for_status() + data = response.json() + except Exception as exc: + logger.warning("OpenFEMA request failed at skip %d: %s", skip, exc) + break + + declarations = data.get("DisasterDeclarationsSummaries", []) + if not declarations: + break + + for dec in declarations: + title = dec.get("declarationTitle", "") + incident_raw = dec.get("incidentType", "disaster") + state = dec.get("state", "US") + declaration_date = dec.get("declarationDate", "")[:10] + + incident_end = dec.get("incidentEndDate", "") + open_ended = "ongoing" if not incident_end else f"closed {incident_end[:10]}" + + summary = ( + f"Federal disaster declaration: {title}. " + f"Incident type: {incident_raw}. " + f"State/region: {state}. " + f"Declared: {declaration_date}. Status: {open_ended}." + ) + + records.append( + DisasterRecord( + source="OpenFEMA", + event_type=_map_incident(incident_raw), + region=state, + summary=summary[:600], + severity="high", + ) + ) + + skip += len(declarations) + logger.info("OpenFEMA: %d declarations collected", len(records)) + + if len(declarations) < page_size: + break + + time.sleep(0.3) + + logger.info("OpenFEMA collection complete: %d records", len(records)) + return records diff --git a/worlddisasterlm/data/collectors/reliefweb.py b/worlddisasterlm/data/collectors/reliefweb.py new file mode 100644 index 0000000000000000000000000000000000000000..91419bb0ca6d1207d0aceb6d25f18ed7f34fa088 --- /dev/null +++ b/worlddisasterlm/data/collectors/reliefweb.py @@ -0,0 +1,111 @@ +""" +ReliefWeb data collector. + +Free API – no authentication required. +Docs: https://apidoc.rwlabs.org/ +""" +import logging +import time +from typing import Any + +import httpx + +from worlddisasterlm.data.schemas import DisasterRecord + +logger = logging.getLogger(__name__) + +RELIEFWEB_API = "https://api.reliefweb.int/v1/reports" +PAGE_SIZE = 100 +REQUEST_DELAY = 0.5 # seconds between requests + + +def _severity_from_fields(fields: dict[str, Any]) -> str: + vuln_types = {e.get("name", "").lower() for e in fields.get("disaster_type", [])} + if any(t in vuln_types for t in {"earthquake", "tsunami", "nuclear", "hurricane", "cyclone"}): + return "critical" + if any(t in vuln_types for t in {"flood", "wildfire", "epidemic", "drought"}): + return "high" + return "moderate" + + +def collect_reliefweb(max_records: int = 5000) -> list[DisasterRecord]: + """Collect disaster reports from ReliefWeb (free, no auth).""" + records: list[DisasterRecord] = [] + offset = 0 + + logger.info("Collecting ReliefWeb reports (max=%d)", max_records) + + while len(records) < max_records: + batch_size = min(PAGE_SIZE, max_records - len(records)) + payload = { + "appname": "worlddisasterlm", + "limit": batch_size, + "offset": offset, + "fields": { + "include": [ + "title", + "body-html", + "primary_country.name", + "disaster_type.name", + "date.created", + "status", + ] + }, + "filter": { + "operator": "AND", + "conditions": [{"field": "status", "value": "published"}], + }, + "sort": ["date.created:desc"], + } + + try: + response = httpx.post(RELIEFWEB_API, json=payload, timeout=30) + response.raise_for_status() + data = response.json() + except Exception as exc: + logger.warning("ReliefWeb request failed at offset %d: %s", offset, exc) + break + + items = data.get("data", []) + if not items: + break + + for item in items: + fields = item.get("fields", {}) + title = fields.get("title", "") + body_html = fields.get("body-html", "") + # Strip HTML tags simply + import re + body_text = re.sub(r"<[^>]+>", " ", body_html) + body_text = " ".join(body_text.split())[:600] + + country = fields.get("primary_country", {}) + region = country.get("name", "global") if isinstance(country, dict) else "global" + + dtype_list = fields.get("disaster_type", []) + event_type = dtype_list[0].get("name", "disaster").lower() if dtype_list else "disaster" + + summary = body_text or title + if not summary: + continue + + records.append( + DisasterRecord( + source="ReliefWeb", + event_type=event_type, + region=region, + summary=summary, + severity=_severity_from_fields(fields), + ) + ) + + offset += len(items) + logger.info("ReliefWeb: collected %d / %d", len(records), max_records) + + if len(items) < batch_size: + break # last page + + time.sleep(REQUEST_DELAY) + + logger.info("ReliefWeb collection complete: %d records", len(records)) + return records diff --git a/worlddisasterlm/data/collectors/usgs.py b/worlddisasterlm/data/collectors/usgs.py new file mode 100644 index 0000000000000000000000000000000000000000..0b18e585898dc149b24cfd482d27842f5618de9b --- /dev/null +++ b/worlddisasterlm/data/collectors/usgs.py @@ -0,0 +1,95 @@ +""" +USGS Earthquake data collector. + +Free API – no authentication required. +Docs: https://earthquake.usgs.gov/fdsnws/event/1/ +""" +import logging +import time +from datetime import datetime, timedelta + +import httpx + +from worlddisasterlm.data.schemas import DisasterRecord + +logger = logging.getLogger(__name__) + +USGS_API = "https://earthquake.usgs.gov/fdsnws/event/1/query" + + +def _severity(magnitude: float) -> str: + if magnitude >= 7.5: + return "critical" + if magnitude >= 6.0: + return "high" + if magnitude >= 4.5: + return "moderate" + return "low" + + +def collect_usgs( + years_back: int = 10, + min_magnitude: float = 4.0, + max_records: int = 20000, +) -> list[DisasterRecord]: + """Collect earthquake data from USGS FDSN (free, no auth).""" + records: list[DisasterRecord] = [] + # USGS API max 20 000 per call; chunk by year to avoid that limit + end_time = datetime.utcnow() + + for year_offset in range(years_back): + if len(records) >= max_records: + break + + year_end = end_time - timedelta(days=365 * year_offset) + year_start = end_time - timedelta(days=365 * (year_offset + 1)) + + params = { + "format": "geojson", + "starttime": year_start.strftime("%Y-%m-%dT%H:%M:%S"), + "endtime": year_end.strftime("%Y-%m-%dT%H:%M:%S"), + "minmagnitude": min_magnitude, + "orderby": "time", + "limit": min(20000, max_records - len(records)), + } + + try: + response = httpx.get(USGS_API, params=params, timeout=60) + response.raise_for_status() + data = response.json() + except Exception as exc: + logger.warning("USGS request failed for year offset %d: %s", year_offset, exc) + continue + + features = data.get("features", []) + logger.info( + "USGS year -%d: %d earthquakes fetched", year_offset + 1, len(features) + ) + + for feature in features: + props = feature.get("properties", {}) + mag = props.get("mag") or 0.0 + place = props.get("place") or "Unknown location" + title = props.get("title") or f"M{mag} earthquake" + + summary = ( + f"Magnitude {mag:.1f} earthquake reported near {place}. " + f"Alert level: {props.get('alert', 'none')}. " + f"Tsunami risk: {'yes' if props.get('tsunami') else 'no'}. " + f"{title}" + ) + + records.append( + DisasterRecord( + source="USGS", + event_type="earthquake", + region=place, + summary=summary[:600], + severity=_severity(mag), + ) + ) + + time.sleep(0.3) + + logger.info("USGS collection complete: %d records", len(records)) + return records[:max_records] diff --git a/worlddisasterlm/data/collectors/who_rss.py b/worlddisasterlm/data/collectors/who_rss.py new file mode 100644 index 0000000000000000000000000000000000000000..950baea9407dca19bca8dad01dc0e571e6b02169 --- /dev/null +++ b/worlddisasterlm/data/collectors/who_rss.py @@ -0,0 +1,101 @@ +""" +WHO Disease Outbreak News (DON) RSS collector. + +Free feed – no authentication required. +Feed: https://www.who.int/feeds/entity/csr/don/en/rss.xml +""" +import logging +import re + +import feedparser + +from worlddisasterlm.data.schemas import DisasterRecord + +logger = logging.getLogger(__name__) + +WHO_DON_RSS = "https://www.who.int/feeds/entity/csr/don/en/rss.xml" + + +DISEASE_KEYWORDS = { + "ebola": "epidemic", + "cholera": "epidemic", + "dengue": "epidemic", + "malaria": "epidemic", + "covid": "pandemic", + "influenza": "epidemic", + "mpox": "epidemic", + "monkeypox": "epidemic", + "plague": "epidemic", + "meningitis": "epidemic", + "lassa": "epidemic", + "marburg": "epidemic", + "rift valley": "epidemic", + "yellow fever": "epidemic", + "polio": "epidemic", + "measles": "epidemic", + "typhoid": "epidemic", + "hepatitis": "epidemic", +} + + +def _classify_event(text: str) -> str: + lower = text.lower() + for keyword, event_type in DISEASE_KEYWORDS.items(): + if keyword in lower: + return event_type + return "public_health" + + +def _clean(text: str) -> str: + clean = re.sub(r"<[^>]+>", " ", text or "") + return " ".join(clean.split())[:600] + + +def collect_who(max_records: int = 1000) -> list[DisasterRecord]: + """Collect WHO disease outbreak news (free, no auth).""" + records: list[DisasterRecord] = [] + + logger.info("Parsing WHO Disease Outbreak News RSS feed") + + try: + feed = feedparser.parse(WHO_DON_RSS) + except Exception as exc: + logger.warning("Failed to parse WHO RSS: %s", exc) + return records + + if feed.bozo and feed.bozo_exception: + logger.warning("WHO RSS parse warning: %s", feed.bozo_exception) + + for entry in feed.entries: + if len(records) >= max_records: + break + + title = entry.get("title", "") + summary_raw = entry.get("summary", entry.get("description", "")) + summary = _clean(summary_raw) or title + + if not summary: + continue + + combined = f"{title} {summary}" + event_type = _classify_event(combined) + + # Try to extract country from title (e.g. "Ebola virus disease – Democratic Republic of the Congo") + region = "global" + if "–" in title: + region = title.split("–")[-1].strip() + elif "-" in title: + region = title.split("-")[-1].strip() + + records.append( + DisasterRecord( + source="WHO", + event_type=event_type, + region=region[:100], + summary=summary, + severity="high", + ) + ) + + logger.info("WHO collection complete: %d records", len(records)) + return records diff --git a/worlddisasterlm/data/etl.py b/worlddisasterlm/data/etl.py new file mode 100644 index 0000000000000000000000000000000000000000..98af487adf278644fabe9166a6107514f02245a8 --- /dev/null +++ b/worlddisasterlm/data/etl.py @@ -0,0 +1,55 @@ +from collections.abc import Iterable + +from worlddisasterlm.data.schemas import DisasterRecord + + +class DisasterETL: + def collect_records(self) -> list[DisasterRecord]: + # Replace with API clients and ingestion jobs for production-scale collection. + return [ + DisasterRecord( + source="ReliefWeb", + event_type="flood", + region="South Asia", + summary="Severe flooding displaced 12000 people and disrupted road access.", + severity="high", + ), + DisasterRecord( + source="WHO", + event_type="epidemic", + region="East Africa", + summary="Localized cholera outbreak with urgent water sanitation requirements.", + severity="high", + ), + DisasterRecord( + source="USGS", + event_type="earthquake", + region="Pacific Rim", + summary="Magnitude 6.8 earthquake with aftershock risk and infrastructure damage.", + severity="critical", + ), + ] + + def deduplicate(self, records: Iterable[DisasterRecord]) -> list[DisasterRecord]: + seen: set[tuple[str, str, str, str]] = set() + deduped: list[DisasterRecord] = [] + for record in records: + key = (record.source, record.event_type, record.region, record.summary) + if key not in seen: + deduped.append(record) + seen.add(key) + return deduped + + def normalize(self, records: Iterable[DisasterRecord]) -> list[DisasterRecord]: + normalized: list[DisasterRecord] = [] + for record in records: + normalized.append( + DisasterRecord( + source=record.source.strip(), + event_type=record.event_type.strip().lower(), + region=record.region.strip(), + summary=" ".join(record.summary.split()), + severity=record.severity.strip().lower(), + ) + ) + return normalized diff --git a/worlddisasterlm/data/processors.py b/worlddisasterlm/data/processors.py new file mode 100644 index 0000000000000000000000000000000000000000..275c764b5cf855044f255fce25065f6f4f0dc861 --- /dev/null +++ b/worlddisasterlm/data/processors.py @@ -0,0 +1,32 @@ +import json +from pathlib import Path + +from worlddisasterlm.data.schemas import DisasterRecord, InstructionSample +from worlddisasterlm.utils.io import ensure_dir + + +def build_instruction_dataset(records: list[DisasterRecord]) -> list[InstructionSample]: + dataset: list[InstructionSample] = [] + for record in records: + instruction = "Assess the incident and provide emergency response steps." + sample_input = ( + f"Region: {record.region}\nEvent: {record.event_type}\nSeverity: {record.severity}\n" + f"Situation: {record.summary}" + ) + output = ( + "1) Verify official alerts and incident perimeter. " + "2) Prioritize life-saving response and medical triage. " + "3) Coordinate shelter, water, food, and transport logistics. " + "4) Share multilingual updates every 30 minutes." + ) + dataset.append(InstructionSample(instruction=instruction, input=sample_input, output=output)) + return dataset + + +def save_instruction_dataset(samples: list[InstructionSample], output_path: str) -> Path: + target = Path(output_path) + ensure_dir(target.parent) + with target.open("w", encoding="utf-8") as handle: + for sample in samples: + handle.write(json.dumps(sample.__dict__, ensure_ascii=False) + "\n") + return target diff --git a/worlddisasterlm/data/qa_generator.py b/worlddisasterlm/data/qa_generator.py new file mode 100644 index 0000000000000000000000000000000000000000..5886a152e20a74d5d1e583ee56e711ebdd1d71cf --- /dev/null +++ b/worlddisasterlm/data/qa_generator.py @@ -0,0 +1,240 @@ +""" +QA pair generator. + +Converts raw DisasterRecord objects into diverse instruction-following samples. +Each record generates 8–10 QA variants to amplify the training corpus. +""" +import random + +from worlddisasterlm.data.schemas import DisasterRecord, InstructionSample + +RESPONSE_STEPS = { + "earthquake": [ + "Drop, Cover, and Hold On immediately.", + "Move away from windows, glass, and heavy furniture.", + "If outdoors, move to open space away from buildings.", + "After shaking stops, check for injuries and hazards.", + "Be prepared for aftershocks.", + "Do not use elevators; evacuate via stairways if building is unsafe.", + "Listen to emergency broadcasts for official guidance.", + ], + "flood": [ + "Move immediately to higher ground; do not wait for official evacuation order.", + "Avoid walking or driving through flood waters – 6 inches can knock you down.", + "Disconnect electrical appliances; do not touch electrical equipment if wet.", + "Follow evacuation routes designated by local emergency management.", + "Store emergency supplies including water, food, and medication for 72 hours.", + "Monitor official weather and emergency alerts for updates.", + ], + "wildfire": [ + "Evacuate immediately when ordered – do not wait.", + "Close all windows and doors to reduce smoke penetration.", + "Remove flammable items from around your home before leaving.", + "Wear N95 masks or wet cloth to protect against smoke inhalation.", + "Stay tuned to emergency broadcasts for evacuation route updates.", + ], + "hurricane": [ + "Secure or bring indoors all outdoor furniture and objects.", + "Board up windows and reinforce garage doors.", + "Prepare emergency kit: water, food, medications, documents.", + "Know your evacuation zone and route.", + "Stay indoors during the storm; the eye of the hurricane is not the all-clear.", + "After the storm, watch for downed power lines and contaminated water.", + ], + "epidemic": [ + "Report cases to local health authorities immediately.", + "Implement infection control measures: PPE, isolation protocols.", + "Coordinate with WHO, CDC, and national health agencies.", + "Establish clear case definition and surveillance system.", + "Activate contact tracing and quarantine procedures.", + "Communicate clearly with the public to prevent panic.", + ], + "pandemic": [ + "Follow national health authority guidelines.", + "Implement non-pharmaceutical interventions: masking, distancing.", + "Prioritize healthcare system capacity management.", + "Accelerate vaccine development and equitable distribution.", + "Coordinate international response through WHO frameworks.", + ], + "default": [ + "Activate emergency response plan immediately.", + "Prioritize life safety: triage injuries, evacuate if necessary.", + "Contact emergency services (fire, police, medical) as appropriate.", + "Coordinate with local emergency management authority.", + "Set up incident command structure.", + "Pre-position supplies: water, food, medical equipment, shelter.", + "Communicate regularly with affected population in plain language.", + "Document all actions for accountability and after-action review.", + ], +} + +RESOURCE_GUIDANCE = { + "earthquake": "Search and rescue teams, medical triage units, heavy machinery for debris removal, temporary shelter, water purification, emergency food supplies, structural engineers.", + "flood": "Boats and water rescue teams, pumping equipment, water purification, temporary shelters on elevated ground, food and medical supplies, sanitation units.", + "wildfire": "Aerial firefighting assets, ground crews, evacuation transport, respiratory medical support, temporary shelters, animal rescue resources.", + "hurricane": "Pre-positioned food, water and fuel, emergency shelters, power restoration crews, debris removal, search and rescue teams, mental health support.", + "epidemic": "Medical personnel with PPE, testing kits, contact tracing capacity, isolation facilities, treatment medicines, communication system.", + "default": "Emergency medical teams, shelter supplies, clean water and food, communication equipment, transport resources, coordination staff.", +} + + +def _get_response_steps(event_type: str) -> list[str]: + for key in RESPONSE_STEPS: + if key in event_type.lower(): + return RESPONSE_STEPS[key] + return RESPONSE_STEPS["default"] + + +def _get_resources(event_type: str) -> str: + for key in RESOURCE_GUIDANCE: + if key in event_type.lower(): + return RESOURCE_GUIDANCE[key] + return RESOURCE_GUIDANCE["default"] + + +def generate_qa_pairs(records: list[DisasterRecord]) -> list[InstructionSample]: + samples: list[InstructionSample] = [] + + for record in records: + steps = _get_response_steps(record.event_type) + steps_text = "\n".join(f"{i + 1}. {s}" for i, s in enumerate(steps)) + resources = _get_resources(record.event_type) + + context = ( + f"Region: {record.region}\n" + f"Hazard type: {record.event_type}\n" + f"Severity: {record.severity}\n" + f"Situation: {record.summary}" + ) + + # QA 1 – immediate response + samples.append( + InstructionSample( + instruction="What are the immediate emergency response steps for this disaster situation?", + input=context, + output=f"Immediate response priorities for a {record.severity}-severity {record.event_type} event:\n\n{steps_text}", + ) + ) + + # QA 2 – incident summary + samples.append( + InstructionSample( + instruction="Summarize this disaster event for an emergency operations center briefing.", + input=context, + output=( + f"SITUATION REPORT — {record.event_type.upper()} | {record.region}\n\n" + f"Severity: {record.severity.capitalize()}\n" + f"Source: {record.source}\n\n" + f"Summary: {record.summary}\n\n" + f"Status: Active monitoring required. Coordinate with local authorities and relevant UN agencies." + ), + ) + ) + + # QA 3 – resource planning + samples.append( + InstructionSample( + instruction="What resources and logistics are needed to respond to this disaster?", + input=context, + output=( + f"Resource requirements for a {record.event_type} event in {record.region} (severity: {record.severity}):\n\n" + f"{resources}\n\n" + f"Logistics priorities: establish forward operating base near the affected area, " + f"pre-position supplies along access routes, coordinate air and ground transport, " + f"and establish communication links with local partners." + ), + ) + ) + + # QA 4 – risk assessment + samples.append( + InstructionSample( + instruction="Perform a risk assessment for this disaster scenario.", + input=context, + output=( + f"Risk Assessment: {record.event_type.capitalize()} in {record.region}\n\n" + f"Risk Level: {record.severity.capitalize()}\n" + f"Primary hazard: {record.event_type}\n" + f"Secondary hazards: displacement, water contamination, disease outbreak, infrastructure collapse\n" + f"Vulnerable populations: elderly, children, persons with disabilities, low-income households\n\n" + f"Recommended risk reduction actions:\n" + f"1. Activate early warning dissemination for at-risk zones.\n" + f"2. Pre-position emergency supplies and first responder teams.\n" + f"3. Establish coordination hub with government and NGO partners.\n" + f"4. Issue public guidance in multiple local languages." + ), + ) + ) + + # QA 5 – humanitarian aid + samples.append( + InstructionSample( + instruction="What humanitarian aid priorities should be activated for this disaster?", + input=context, + output=( + f"Humanitarian Aid Priorities for {record.event_type} in {record.region}:\n\n" + f"1. Life-saving: Search and rescue, emergency medical care, trauma treatment.\n" + f"2. Basic needs: Emergency shelter, safe water, food assistance, sanitation.\n" + f"3. Protection: Safety monitoring for displaced persons, child protection, GBV prevention.\n" + f"4. Livelihoods: Cash transfers and livelihood support for affected households.\n" + f"5. Recovery: Debris clearance, shelter repair, economic recovery planning.\n\n" + f"Key coordination partners: OCHA, UNHCR, WFP, UNICEF, WHO, local disaster management authority." + ), + ) + ) + + # QA 6 – preparedness + samples.append( + InstructionSample( + instruction="How can communities in this region prepare for this type of disaster?", + input=context, + output=( + f"Community Preparedness for {record.event_type} in {record.region}:\n\n" + f"1. Develop and rehearse household emergency plans.\n" + f"2. Build 72-hour emergency supply kits (water, food, medication, documents).\n" + f"3. Know evacuation routes and local shelter locations.\n" + f"4. Participate in community early warning systems.\n" + f"5. Strengthen local infrastructure and building codes.\n" + f"6. Conduct regular drills with schools, workplaces, and community organizations.\n" + f"7. Ensure vulnerable populations have specific support plans." + ), + ) + ) + + # QA 7 – public communication + samples.append( + InstructionSample( + instruction="Draft an emergency public communication message for this disaster.", + input=context, + output=( + f"EMERGENCY ALERT — {record.region.upper()}\n\n" + f"A {record.severity}-severity {record.event_type} event has been reported.\n\n" + f"IMMEDIATE ACTIONS REQUIRED:\n" + f"• Follow official evacuation orders immediately.\n" + f"• Move to designated shelters or higher ground.\n" + f"• Call emergency services for life-threatening situations.\n" + f"• Do NOT spread unverified information.\n\n" + f"Stay tuned to official government and emergency management channels for updates.\n" + f"[This message should be verified and issued by the authorised emergency management authority.]" + ), + ) + ) + + # QA 8 – recovery planning + samples.append( + InstructionSample( + instruction="What are the key steps in disaster recovery planning after this event?", + input=context, + output=( + f"Recovery Planning Framework: {record.event_type} — {record.region}\n\n" + f"Phase 1 (0–72 hours): Life safety, damage assessment, displaced persons registration.\n" + f"Phase 2 (1–4 weeks): Temporary shelter provision, debris clearance, basic services restoration.\n" + f"Phase 3 (1–6 months): Infrastructure repair, economic recovery, psychosocial support.\n" + f"Phase 4 (6+ months): Long-term reconstruction, risk reduction investments, lesson-learned review.\n\n" + f"Key principles: Build Back Better, inclusion of marginalized groups, environmental sustainability, " + f"community ownership of the recovery process." + ), + ) + ) + + return samples diff --git a/worlddisasterlm/data/scenario_builder.py b/worlddisasterlm/data/scenario_builder.py new file mode 100644 index 0000000000000000000000000000000000000000..06a2b8eabdc561b6d252c6991dfa933916bde083 --- /dev/null +++ b/worlddisasterlm/data/scenario_builder.py @@ -0,0 +1,197 @@ +""" +Synthetic emergency scenario builder. + +Generates multi-turn conversations, role-play scenarios, and cross-hazard +compound disaster samples to enrich training diversity. +""" +from worlddisasterlm.data.schemas import InstructionSample + +COMPOUND_SCENARIOS = [ + { + "scenario": "An earthquake of magnitude 7.2 struck a coastal city causing building collapses, " + "followed 45 minutes later by a tsunami warning.", + "region": "Pacific coastal city", + "event_type": "compound: earthquake + tsunami", + "severity": "critical", + "response": ( + "Compound Disaster Response — Earthquake + Tsunami:\n\n" + "IMMEDIATE (0–15 min):\n" + "1. Issue tsunami warning and mandatory coastal evacuation order immediately.\n" + "2. Search and rescue operations for earthquake-trapped victims must start inland first.\n" + "3. Hospitals on high ground remain operational; coastal facilities evacuate patients.\n\n" + "SHORT-TERM (15 min – 6 hours):\n" + "4. Establish triage stations above tsunami inundation zone.\n" + "5. Account for all search and rescue teams before wave arrival.\n" + "6. Deploy helicopters for coastal cliff rescues post-wave.\n\n" + "CRITICAL NOTE: Aftershocks will compromise already-damaged structures. " + "Do not re-enter buildings until structural engineers clear them." + ), + }, + { + "scenario": "A category 4 hurricane is forecast to make landfall in 48 hours. " + "The region already has 30,000 displaced persons from last week's flooding.", + "region": "Caribbean island", + "event_type": "compound: hurricane + displaced persons", + "severity": "critical", + "response": ( + "Compound Emergency — Hurricane Landfall with Pre-existing Displacement:\n\n" + "PRE-LANDFALL (48–24 hours):\n" + "1. Mandatory evacuation of all existing displacement camps in low-lying areas.\n" + "2. Identify inland strong buildings for mass shelter; assess capacity.\n" + "3. Pre-position 72-hour food and water supplies at inland shelters.\n" + "4. Coordinate with UNHCR and local government for camp-to-shelter transfers.\n\n" + "PRE-LANDFALL (24–0 hours):\n" + "5. All personnel secured; close airport; ban sea transport.\n" + "6. Emergency services on stand-by in hardened facilities.\n\n" + "POST-LANDFALL:\n" + "7. Damage assessment sweep before re-opening any displaced camp.\n" + "8. Restore access roads before relief convoy mobilization." + ), + }, + { + "scenario": "During a wildfire evacuation, a fuel tanker overturned on the primary " + "evacuation highway, blocking egress for 15,000 evacuees.", + "region": "Southern California", + "event_type": "compound: wildfire + infrastructure failure", + "severity": "critical", + "response": ( + "Multi-Hazard Evacuation Disruption — Wildfire + Road Blockage:\n\n" + "1. Activate alternate evacuation routes immediately; publish via Emergency Alert System.\n" + "2. Dispatch hazmat team to fuel tanker — assess fire ignition risk before moving vehicle.\n" + "3. If tanker poses fire risk: keep 300m standoff; use bulldozer to open shoulder lane.\n" + "4. Coordinate law enforcement for contraflow on alternate highways.\n" + "5. Deploy CHP and National Guard for traffic management at choke points.\n" + "6. Air resources pivot to aerial evacuation for mobility-impaired stranded persons.\n" + "7. Communicate updated routes every 15 minutes via radio, app, and social media." + ), + }, +] + +MULTILINGUAL_SAMPLES = [ + { + "lang": "Nepali", + "instruction": "भूकम्पको बेला के गर्ने?", + "output": ( + "भूकम्पको बेला:\n" + "१. तुरुन्त झुक्नुहोस्, ढाक्नुहोस् र समात्नुहोस् (Drop, Cover, Hold On)।\n" + "२. झ्याल, ऐना र भारी सामानबाट टाढा रहनुहोस्।\n" + "३. बाहिर हुनुहुन्छ भने खुला ठाउँमा जानुहोस् — भवन, रुख र बिजुलीका तारबाट टाढा।\n" + "४. हल्लाउन बन्द भएपछि चोटपटक जाँच गर्नुहोस् र सावधानीसाथ बाहिर निस्कनुहोस्।\n" + "५. परकम्पको लागि तयार रहनुहोस्।\n" + "गम्भीर घाइतेको लागि तुरुन्त आपतकालीन सेवा (१०१ वा स्थानीय नम्बर) मा फोन गर्नुहोस्।" + ), + }, + { + "lang": "Nepali", + "instruction": "बाढीको बेला के गर्ने?", + "output": ( + "बाढीको बेला:\n" + "१. तुरुन्त अग्लो ठाउँमा जानुहोस् — निकासी आदेशको प्रतीक्षा नगर्नुहोस्।\n" + "२. बाढीको पानीमा नहिँड्नुहोस् वा गाडी नचलाउनुहोस् — ६ इन्च पानीले पनि गिराउन सक्छ।\n" + "३. बिजुलीका उपकरणहरू बन्द गर्नुहोस्।\n" + "४. स्थानीय अधिकारीको निकासी मार्ग पछ्याउनुहोस्।\n" + "५. आधिकारिक अलर्टको लागि रेडियो सुन्नुहोस्।\n" + "खतरामा परेमा तुरुन्त आपतकालीन सेवामा सम्पर्क गर्नुहोस्।" + ), + }, + { + "lang": "Nepali", + "instruction": "विपद् व्यवस्थापनको लागि आधारभूत तयारी के के गर्नुपर्छ?", + "output": ( + "विपद् तयारीका आधारभूत कदमहरू:\n" + "१. परिवारको आपतकालीन योजना बनाउनुहोस् र अभ्यास गर्नुहोस्।\n" + "२. ७२ घण्टाको आपतकालीन किट तयार राख्नुहोस्: पानी, खाना, औषधि, कागजात।\n" + "३. निकासी मार्ग र नजिकको आश्रयस्थलको जानकारी राख्नुहोस्।\n" + "४. समुदायको पूर्व चेतावनी प्रणालीमा सहभागी हुनुहोस्।\n" + "५. विद्यालय, कार्यस्थल र समुदायमा नियमित अभ्यास गर्नुहोस्।\n" + "राष्ट्रिय विपद् जोखिम न्यूनीकरण तथा व्यवस्थापन प्राधिकरण (NDRRMA) को निर्देशिका पालना गर्नुहोस्।" + ), + }, + { + "lang": "Spanish", + "instruction": "¿Qué debo hacer durante un terremoto?", + "output": ( + "Durante un terremoto:\n" + "1. Agáchese, cúbrase y sujétese (Drop, Cover, Hold On).\n" + "2. Aléjese de ventanas, espejos y objetos pesados.\n" + "3. Si está en el exterior, diríjase a un espacio abierto.\n" + "4. Después del sismo, verifique heridos y salga con cuidado.\n" + "5. Esté preparado para réplicas.\n" + "Llame al número de emergencias de su país para reportar heridos graves." + ), + }, + { + "lang": "French", + "instruction": "Quelles sont les étapes à suivre en cas d'inondation ?", + "output": ( + "En cas d'inondation :\n" + "1. Montez immédiatement en hauteur – ne restez pas au niveau du sol.\n" + "2. N'essayez jamais de traverser des eaux en crue, même à pied.\n" + "3. Coupez l'électricité et le gaz si c'est sans danger.\n" + "4. Suivez les consignes d'évacuation des autorités locales.\n" + "5. Appelez les secours si vous êtes en danger immédiat.\n" + "Restez à l'écoute des alertes officielles." + ), + }, + { + "lang": "Arabic", + "instruction": "ما هي الخطوات الواجب اتخاذها أثناء الإعصار؟", + "output": ( + "أثناء الإعصار:\n" + "١. ابق في الداخل بعيداً عن النوافذ والأبواب الزجاجية.\n" + "٢. اذهب إلى الطابق الأرضي أو الغرفة الداخلية دون نوافذ.\n" + "٣. لا تخرج لأن عين الإعصار تبدو هادئة لكنها مؤقتة.\n" + "٤. استمع إلى تعليمات الدفاع المدني عبر الراديو أو التليفزيون.\n" + "٥. ابتعد عن مجاري المياه والمناطق المنخفضة بعد مرور الإعصار." + ), + }, + { + "lang": "Hindi", + "instruction": "बाढ़ के दौरान क्या करें?", + "output": ( + "बाढ़ के दौरान:\n" + "1. तुरंत ऊंची जगह पर जाएं।\n" + "2. बाढ़ के पानी में न चलें—6 इंच पानी भी गिरा सकता है।\n" + "3. बिजली के उपकरण बंद करें।\n" + "4. आधिकारिक निकासी मार्ग का पालन करें।\n" + "5. आपातकालीन अलर्ट के लिए रेडियो सुनें।\n" + "खतरे में होने पर तुरंत आपातकालीन सेवा को कॉल करें।" + ), + }, +] + + +def build_compound_scenarios() -> list[InstructionSample]: + samples: list[InstructionSample] = [] + for scene in COMPOUND_SCENARIOS: + samples.append( + InstructionSample( + instruction="Provide a comprehensive emergency response plan for this compound disaster scenario.", + input=( + f"Scenario: {scene['scenario']}\n" + f"Region: {scene['region']}\n" + f"Event type: {scene['event_type']}\n" + f"Severity: {scene['severity']}" + ), + output=scene["response"], + ) + ) + return samples + + +def build_multilingual_samples() -> list[InstructionSample]: + samples: list[InstructionSample] = [] + for item in MULTILINGUAL_SAMPLES: + samples.append( + InstructionSample( + instruction=item["instruction"], + input="", + output=item["output"], + language=item["lang"], + ) + ) + return samples + + +def build_all_scenarios() -> list[InstructionSample]: + return build_compound_scenarios() + build_multilingual_samples() diff --git a/worlddisasterlm/data/schemas.py b/worlddisasterlm/data/schemas.py new file mode 100644 index 0000000000000000000000000000000000000000..dc2ecb2202bd13e89b857f1afa887e87217bb50b --- /dev/null +++ b/worlddisasterlm/data/schemas.py @@ -0,0 +1,18 @@ +from dataclasses import dataclass + + +@dataclass +class DisasterRecord: + source: str + event_type: str + region: str + summary: str + severity: str + + +@dataclass +class InstructionSample: + instruction: str + input: str + output: str + language: str = "English" diff --git a/worlddisasterlm/data/sources.py b/worlddisasterlm/data/sources.py new file mode 100644 index 0000000000000000000000000000000000000000..ac75ff21ea2862221d014646457e4c0626f2ff4f --- /dev/null +++ b/worlddisasterlm/data/sources.py @@ -0,0 +1,34 @@ +INTERNATIONAL_ORGS = [ + "UN", + "UNDRR", + "WHO", + "UNICEF", + "WFP", + "UNHCR", +] + +DISASTER_DATABASES = [ + "EM-DAT", + "ReliefWeb", + "GDACS", + "NASA Earth Data", + "NOAA", + "USGS", + "FEMA", + "World Bank Open Data", +] + +RESEARCH_SOURCES = [ + "Scientific papers", + "Government reports", + "Emergency response manuals", + "Disaster preparedness guidelines", + "Humanitarian response frameworks", +] + +REAL_TIME_SOURCES = [ + "Weather feeds", + "Satellite imagery metadata", + "Emergency alerts", + "Public safety bulletins", +] diff --git a/worlddisasterlm/evaluation/__init__.py b/worlddisasterlm/evaluation/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/worlddisasterlm/evaluation/metrics.py b/worlddisasterlm/evaluation/metrics.py new file mode 100644 index 0000000000000000000000000000000000000000..d0c48e6685f3c924b4395690cc38bebce06bd00a --- /dev/null +++ b/worlddisasterlm/evaluation/metrics.py @@ -0,0 +1,20 @@ +from dataclasses import dataclass + + +@dataclass +class EvalReport: + accuracy: float + hallucination_rate: float + safety_compliance: float + emergency_correctness: float + multilingual_score: float + + +def compute_dummy_metrics() -> EvalReport: + return EvalReport( + accuracy=0.81, + hallucination_rate=0.11, + safety_compliance=0.93, + emergency_correctness=0.86, + multilingual_score=0.78, + ) diff --git a/worlddisasterlm/evaluation/multilingual_eval.py b/worlddisasterlm/evaluation/multilingual_eval.py new file mode 100644 index 0000000000000000000000000000000000000000..83509a3392afc73b0d9a072de4492bef726ed247 --- /dev/null +++ b/worlddisasterlm/evaluation/multilingual_eval.py @@ -0,0 +1,7 @@ +from worlddisasterlm.config import SUPPORTED_LANGUAGES + + +def multilingual_coverage_score(languages_seen: list[str]) -> float: + normalized = {lang.strip().lower() for lang in languages_seen} + target = {lang.lower() for lang in SUPPORTED_LANGUAGES} + return round(len(normalized.intersection(target)) / len(target), 3) diff --git a/worlddisasterlm/evaluation/safety_eval.py b/worlddisasterlm/evaluation/safety_eval.py new file mode 100644 index 0000000000000000000000000000000000000000..203b3ebde10d7f4dc4d3f126aed7c5b2de15671a --- /dev/null +++ b/worlddisasterlm/evaluation/safety_eval.py @@ -0,0 +1,7 @@ +def safety_check(sample_output: str) -> bool: + blocked_phrases = [ + "ignore evacuation", + "do not contact authorities", + ] + lowered = sample_output.lower() + return not any(phrase in lowered for phrase in blocked_phrases) diff --git a/worlddisasterlm/optimization/__init__.py b/worlddisasterlm/optimization/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/worlddisasterlm/optimization/export_gguf.py b/worlddisasterlm/optimization/export_gguf.py new file mode 100644 index 0000000000000000000000000000000000000000..6af3b0d575af8fbc900d22de1c3cdf96f8c8af81 --- /dev/null +++ b/worlddisasterlm/optimization/export_gguf.py @@ -0,0 +1,14 @@ +from pathlib import Path + +from worlddisasterlm.utils.io import ensure_dir + + +def export_gguf(model_path: str, output_path: str = "artifacts/model.gguf") -> Path: + target = Path(output_path) + ensure_dir(target.parent) + target.write_text( + "Placeholder GGUF artifact metadata. Integrate llama.cpp conversion in production.", + encoding="utf-8", + ) + print(f"GGUF export stub created at {target} from model {model_path}") + return target diff --git a/worlddisasterlm/optimization/export_onnx.py b/worlddisasterlm/optimization/export_onnx.py new file mode 100644 index 0000000000000000000000000000000000000000..d97f97c0ef84e2945df3a29d2400a92e66be004d --- /dev/null +++ b/worlddisasterlm/optimization/export_onnx.py @@ -0,0 +1,14 @@ +from pathlib import Path + +from worlddisasterlm.utils.io import ensure_dir + + +def export_onnx(model_path: str, output_path: str = "artifacts/model.onnx") -> Path: + target = Path(output_path) + ensure_dir(target.parent) + target.write_text( + "Placeholder ONNX artifact metadata. Replace with actual transformer export logic.", + encoding="utf-8", + ) + print(f"ONNX export stub created at {target} from model {model_path}") + return target diff --git a/worlddisasterlm/optimization/tensorrt_optimize.py b/worlddisasterlm/optimization/tensorrt_optimize.py new file mode 100644 index 0000000000000000000000000000000000000000..fbb288a4e650e28122fbba35cc7a47b1cad1f6f7 --- /dev/null +++ b/worlddisasterlm/optimization/tensorrt_optimize.py @@ -0,0 +1,5 @@ +def tensorrt_optimize(onnx_path: str) -> None: + print( + "TensorRT optimization stub. Use trtexec or TensorRT Python APIs in GPU-enabled environment " + f"for ONNX model: {onnx_path}" + ) diff --git a/worlddisasterlm/training/__init__.py b/worlddisasterlm/training/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/worlddisasterlm/training/chat_format.py b/worlddisasterlm/training/chat_format.py new file mode 100644 index 0000000000000000000000000000000000000000..30c888deddd4fc2fa080b79bde62b98bde4fd209 --- /dev/null +++ b/worlddisasterlm/training/chat_format.py @@ -0,0 +1,44 @@ +""" +Llama 3.1 chat-template formatting utilities. + +Formats instruction samples into the official Llama 3.1 conversation format +consumed by the tokenizer's apply_chat_template method. +""" +DISASTER_SYSTEM_PROMPT = """You are WorldDisasterLM, an expert AI assistant for global disaster management, emergency response, humanitarian aid, and crisis intelligence. You provide accurate, actionable guidance for: +- Natural disasters (earthquakes, floods, wildfires, hurricanes, cyclones, tornadoes, tsunamis, volcanoes, avalanches, landslides, droughts, heatwaves) +- Public health emergencies (pandemics, epidemics, disease outbreaks) +- Humanitarian crises (refugee situations, food insecurity, water scarcity, conflict-related displacement) +- Industrial disasters (chemical spills, nuclear incidents, oil spills, infrastructure failures) +- Climate-related risks (extreme weather, sea-level rise, environmental degradation) + +Your responses are: +1. Accurate and grounded in established emergency management frameworks (SPHERE, IASC, ICS). +2. Actionable — prioritizing immediate life-safety steps first. +3. Appropriately cautious — always recommend professional emergency services for life-threatening situations. +4. Transparent — acknowledge uncertainty and encourage verification with official sources (UN, WHO, national authorities). +5. Inclusive — provide guidance relevant to vulnerable groups: elderly, children, persons with disabilities, low-income communities. + +Always add a verification reminder for critical operational decisions: "Verify with your local emergency management authority before taking action." """ + + +def format_as_chat_messages(instruction: str, context: str, output: str) -> list[dict[str, str]]: + """Return a list of chat messages in Llama 3.1 format.""" + user_content = instruction + if context.strip(): + user_content = f"{instruction}\n\n{context.strip()}" + + return [ + {"role": "system", "content": DISASTER_SYSTEM_PROMPT}, + {"role": "user", "content": user_content}, + {"role": "assistant", "content": output}, + ] + + +def apply_template(tokenizer, instruction: str, context: str, output: str) -> str: + """Apply the tokenizer's chat template and return a formatted string.""" + messages = format_as_chat_messages(instruction, context, output) + return tokenizer.apply_chat_template( + messages, + tokenize=False, + add_generation_prompt=False, + ) diff --git a/worlddisasterlm/training/fine_tune.py b/worlddisasterlm/training/fine_tune.py new file mode 100644 index 0000000000000000000000000000000000000000..b3f8d8baee87e9149a2f7740be64c99b56e01577 --- /dev/null +++ b/worlddisasterlm/training/fine_tune.py @@ -0,0 +1,24 @@ +from dataclasses import dataclass + + +@dataclass +class TrainingConfig: + base_model: str = "meta-llama/Llama-3.1-8B-Instruct" + dataset_path: str = "data/processed/instruction_dataset.jsonl" + output_dir: str = "checkpoints/worlddisasterlm-8b-qlora" + epochs: int = 1 + learning_rate: float = 2e-4 + batch_size: int = 2 + gradient_accumulation_steps: int = 8 + + +def run_training(config: TrainingConfig) -> None: + # This is a safe starter implementation; replace with TRL/Trainer code for full training. + print("Starting WorldDisasterLM-8B fine-tuning") + print(f"Base model: {config.base_model}") + print(f"Dataset: {config.dataset_path}") + print(f"Output: {config.output_dir}") + print( + "Configured for QLoRA/LoRA workflow. Integrate Hugging Face Trainer + PEFT + DeepSpeed/FSDP " + "according to your infrastructure." + ) diff --git a/worlddisasterlm/training/peft_utils.py b/worlddisasterlm/training/peft_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..1f0bd8428bc51d26b7aad0f2df8612f3a46eecbf --- /dev/null +++ b/worlddisasterlm/training/peft_utils.py @@ -0,0 +1,20 @@ +from dataclasses import dataclass + + +@dataclass +class PEFTConfig: + method: str = "qlora" + lora_r: int = 16 + lora_alpha: int = 32 + lora_dropout: float = 0.1 + target_modules: tuple[str, ...] = ("q_proj", "k_proj", "v_proj", "o_proj") + + +def describe_peft_stack() -> dict[str, str]: + return { + "QLoRA": "Memory-efficient low-bit adapter fine-tuning", + "LoRA": "Low-rank adaptation for fast domain tuning", + "PEFT": "Parameter-efficient framework for adapters", + "DeepSpeed": "Distributed optimization and ZeRO partitioning", + "FSDP": "Fully sharded data parallel training", + } diff --git a/worlddisasterlm/training/train_qlora.py b/worlddisasterlm/training/train_qlora.py new file mode 100644 index 0000000000000000000000000000000000000000..c75fc0eeea8cf925c53b7189a61f7a39cd20c554 --- /dev/null +++ b/worlddisasterlm/training/train_qlora.py @@ -0,0 +1,242 @@ +""" +Production QLoRA fine-tuning pipeline for WorldDisasterLM. + +Hardware requirements +--------------------- +Minimum : 1× NVIDIA A100 40 GB (or 2× RTX 3090/4090 24 GB with gradient checkpointing) +Preferred: 2–8× A100/H100 80 GB for full batch sizes +Consumer : 1× RTX 4090 24 GB → set per_device_train_batch_size=1, gradient_accumulation_steps=16 + +Prerequisites +------------- +1. Accept Meta Llama 3.1 license on Hugging Face: + https://huggingface.co/meta-llama/Llama-3.1-8B-Instruct +2. Set HF_TOKEN in your .env file: + huggingface-cli login (or export HF_TOKEN=hf_...) + +Run +--- +python scripts/train_production.py --dataset data/processed/instruction_dataset.jsonl +""" + +from __future__ import annotations + +import json +import logging +import os +from dataclasses import dataclass, field +from pathlib import Path + +from worlddisasterlm.training.chat_format import apply_template + +logger = logging.getLogger(__name__) + + +@dataclass +class QLoRAConfig: + # ── Model ───────────────────────────────────────────────────────────────── + base_model: str = "meta-llama/Llama-3.1-8B-Instruct" + output_dir: str = "checkpoints/worlddisasterlm-qlora" + + # ── Data ────────────────────────────────────────────────────────────────── + dataset_path: str = "data/processed/instruction_dataset.jsonl" + max_seq_length: int = 2048 + + # ── QLoRA / PEFT ────────────────────────────────────────────────────────── + use_4bit: bool = True + bnb_4bit_quant_type: str = "nf4" # "nf4" or "fp4" + use_double_quant: bool = True + compute_dtype: str = "bfloat16" # "bfloat16" or "float16" + + lora_r: int = 16 + lora_alpha: int = 32 + lora_dropout: float = 0.05 + lora_target_modules: list[str] = field( + default_factory=lambda: [ + "q_proj", "k_proj", "v_proj", "o_proj", + "gate_proj", "up_proj", "down_proj", + ] + ) + + # ── Optimisation ────────────────────────────────────────────────────────── + epochs: int = 3 + learning_rate: float = 2e-4 + per_device_train_batch_size: int = 2 + gradient_accumulation_steps: int = 8 + warmup_ratio: float = 0.03 + max_grad_norm: float = 0.3 + lr_scheduler_type: str = "cosine" + weight_decay: float = 0.0 + + bf16: bool = True + fp16: bool = False + gradient_checkpointing: bool = True + + # ── Logging & saving ────────────────────────────────────────────────────── + logging_steps: int = 10 + save_steps: int = 200 + save_total_limit: int = 3 + report_to: str = "none" # "mlflow" | "wandb" | "none" + + # ── Misc ────────────────────────────────────────────────────────────────── + seed: int = 42 + + +def load_jsonl(path: str) -> list[dict]: + records = [] + with open(path, encoding="utf-8") as handle: + for line in handle: + line = line.strip() + if line: + records.append(json.loads(line)) + return records + + +def build_dataset(path: str, tokenizer): + from datasets import Dataset + + raw = load_jsonl(path) + logger.info("Loaded %d instruction samples from %s", len(raw), path) + + texts = [] + for sample in raw: + try: + text = apply_template( + tokenizer, + instruction=sample.get("instruction", ""), + context=sample.get("input", ""), + output=sample.get("output", ""), + ) + texts.append(text) + except Exception as exc: + logger.warning("Failed to format sample: %s", exc) + + logger.info("Formatted %d samples with chat template", len(texts)) + return Dataset.from_dict({"text": texts}) + + +def setup_model_tokenizer(config: QLoRAConfig): + import torch + from transformers import AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig + from peft import LoraConfig, TaskType, get_peft_model, prepare_model_for_kbit_training + + # ── Tokenizer ───────────────────────────────────────────────────────────── + tokenizer = AutoTokenizer.from_pretrained( + config.base_model, + trust_remote_code=True, + padding_side="right", + ) + if tokenizer.pad_token is None: + tokenizer.pad_token = tokenizer.eos_token + tokenizer.pad_token_id = tokenizer.eos_token_id + + # ── Quantization ────────────────────────────────────────────────────────── + torch_dtype = torch.bfloat16 if config.bf16 else torch.float16 + bnb_config = None + if config.use_4bit: + bnb_config = BitsAndBytesConfig( + load_in_4bit=True, + bnb_4bit_quant_type=config.bnb_4bit_quant_type, + bnb_4bit_compute_dtype=torch_dtype, + bnb_4bit_use_double_quant=config.use_double_quant, + ) + + # ── Base model ──────────────────────────────────────────────────────────── + model = AutoModelForCausalLM.from_pretrained( + config.base_model, + quantization_config=bnb_config, + device_map="auto", + torch_dtype=torch_dtype, + trust_remote_code=True, + attn_implementation="flash_attention_2" if _flash_attn_available() else "eager", + ) + + if config.use_4bit: + model = prepare_model_for_kbit_training(model, use_gradient_checkpointing=config.gradient_checkpointing) + elif config.gradient_checkpointing: + model.gradient_checkpointing_enable() + + model.config.use_cache = False + + # ── LoRA ────────────────────────────────────────────────────────────────── + peft_config = LoraConfig( + task_type=TaskType.CAUSAL_LM, + r=config.lora_r, + lora_alpha=config.lora_alpha, + target_modules=config.lora_target_modules, + lora_dropout=config.lora_dropout, + bias="none", + inference_mode=False, + ) + model = get_peft_model(model, peft_config) + model.print_trainable_parameters() + + return model, tokenizer + + +def _flash_attn_available() -> bool: + try: + import flash_attn # noqa: F401 + return True + except ImportError: + return False + + +def train(config: QLoRAConfig) -> None: + from trl import SFTConfig, SFTTrainer + + # Seed + from worlddisasterlm.utils.seed import seed_everything + seed_everything(config.seed) + + Path(config.output_dir).mkdir(parents=True, exist_ok=True) + + # MLflow / W&B config + if config.report_to == "mlflow" and os.getenv("MLFLOW_TRACKING_URI"): + import mlflow + mlflow.set_tracking_uri(os.environ["MLFLOW_TRACKING_URI"]) + mlflow.set_experiment("worlddisasterlm-training") + + model, tokenizer = setup_model_tokenizer(config) + train_dataset = build_dataset(config.dataset_path, tokenizer) + + report_to_list = [config.report_to] if config.report_to != "none" else [] + + sft_config = SFTConfig( + output_dir=config.output_dir, + num_train_epochs=config.epochs, + per_device_train_batch_size=config.per_device_train_batch_size, + gradient_accumulation_steps=config.gradient_accumulation_steps, + gradient_checkpointing=config.gradient_checkpointing, + learning_rate=config.learning_rate, + max_grad_norm=config.max_grad_norm, + warmup_ratio=config.warmup_ratio, + lr_scheduler_type=config.lr_scheduler_type, + weight_decay=config.weight_decay, + bf16=config.bf16, + fp16=config.fp16, + logging_steps=config.logging_steps, + save_steps=config.save_steps, + save_total_limit=config.save_total_limit, + load_best_model_at_end=False, + max_seq_length=config.max_seq_length, + dataset_text_field="text", + packing=False, + report_to=report_to_list, + seed=config.seed, + ) + + trainer = SFTTrainer( + model=model, + tokenizer=tokenizer, + train_dataset=train_dataset, + args=sft_config, + ) + + logger.info("Starting QLoRA fine-tuning: %d samples, %d epochs", len(train_dataset), config.epochs) + trainer.train() + + logger.info("Saving adapter to %s", config.output_dir) + trainer.save_model(config.output_dir) + tokenizer.save_pretrained(config.output_dir) + logger.info("Training complete.") diff --git a/worlddisasterlm/utils/__init__.py b/worlddisasterlm/utils/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/worlddisasterlm/utils/io.py b/worlddisasterlm/utils/io.py new file mode 100644 index 0000000000000000000000000000000000000000..51242782289ba47768d53efda59cb2d7f48f20ed --- /dev/null +++ b/worlddisasterlm/utils/io.py @@ -0,0 +1,7 @@ +from pathlib import Path + + +def ensure_dir(path: str | Path) -> Path: + output = Path(path) + output.mkdir(parents=True, exist_ok=True) + return output diff --git a/worlddisasterlm/utils/logging.py b/worlddisasterlm/utils/logging.py new file mode 100644 index 0000000000000000000000000000000000000000..0a6fca0ee7ffe6d70c317248818e3582d0961b70 --- /dev/null +++ b/worlddisasterlm/utils/logging.py @@ -0,0 +1,8 @@ +import logging + + +def setup_logging(level: str = "INFO") -> None: + logging.basicConfig( + level=getattr(logging, level.upper(), logging.INFO), + format="%(asctime)s | %(levelname)s | %(name)s | %(message)s", + ) diff --git a/worlddisasterlm/utils/seed.py b/worlddisasterlm/utils/seed.py new file mode 100644 index 0000000000000000000000000000000000000000..716b6a64c3f93ba88d8f868848ec2351bd389727 --- /dev/null +++ b/worlddisasterlm/utils/seed.py @@ -0,0 +1,18 @@ +import random + +import numpy as np + + +try: + import torch +except Exception: # pragma: no cover + torch = None + + +def seed_everything(seed: int = 42) -> None: + random.seed(seed) + np.random.seed(seed) + if torch is not None: + torch.manual_seed(seed) + if torch.cuda.is_available(): + torch.cuda.manual_seed_all(seed)