Sync: correct language list (en/ne/es/fr/ar/hi/te/zh/ja/ko/pt), updated README from project, full source code sync

README.md

@@ -32,11 +32,11 @@ model-index:
 
 # WorldDisasterLM-8B
 
-> **Open-source AI foundation model for global disaster intelligence, emergency response, and humanitarian aid — supporting 11 languages including Nepali (नेपाली).**
+> **Open-source AI foundation model for global disaster intelligence, emergency response, and humanitarian aid — supporting 11 languages including Nepali.**
 
 [![Python 3.11+](https://img.shields.io/badge/python-3.11+-blue.svg)](https://python.org)
 [![License](https://img.shields.io/badge/license-Llama3-green.svg)](https://ai.meta.com/llama/license/)
-[![HuggingFace](https://img.shields.io/badge/🤗-WorldDisasterLM--8B-yellow)](https://huggingface.co)
+[![HuggingFace](https://img.shields.io/badge/🤗-WorldDisasterLM--8B-yellow)](https://huggingface.co/drdeveloper88/WorldDisasterLM-8B)
 [![Tests](https://img.shields.io/badge/tests-9%20passed-brightgreen.svg)]()
 
 ---
@@ -93,7 +93,7 @@ Training data is collected **live** from six free public APIs: ReliefWeb, USGS E
 ### 1. Clone and install
 
 ```bash
-git clone <your-repo-url> worlddisasterllm
+git clone https://huggingface.co/drdeveloper88/WorldDisasterLM-8B worlddisasterllm
 cd worlddisasterllm
 python -m venv .venv
 

worlddisasterlm/data/collectors/__init__.py

@@ -1,15 +0,0 @@
-from worlddisasterlm.data.collectors.reliefweb import collect_reliefweb
-from worlddisasterlm.data.collectors.usgs import collect_usgs
-from worlddisasterlm.data.collectors.gdacs import collect_gdacs
-from worlddisasterlm.data.collectors.noaa import collect_noaa
-from worlddisasterlm.data.collectors.openfema import collect_openfema
-from worlddisasterlm.data.collectors.who_rss import collect_who
-
-__all__ = [
-    "collect_reliefweb",
-    "collect_usgs",
-    "collect_gdacs",
-    "collect_noaa",
-    "collect_openfema",
-    "collect_who",
-]

worlddisasterlm/data/collectors/gdacs.py

@@ -1,86 +0,0 @@
-"""
-GDACS (Global Disaster Alert and Coordination System) collector.
-
-Free GeoRSS feed – no authentication required.
-Feed: https://www.gdacs.org/xml/rss.xml
-"""
-import logging
-import re
-from datetime import datetime
-
-import feedparser
-
-from worlddisasterlm.data.schemas import DisasterRecord
-
-logger = logging.getLogger(__name__)
-
-GDACS_RSS_URL = "https://www.gdacs.org/xml/rss.xml"
-GDACS_SCORE_RSS_URL = "https://www.gdacs.org/xml/rss_score.xml"
-
-
-def _alert_to_severity(alert_level: str) -> str:
-    level = alert_level.strip().lower()
-    if level == "red":
-        return "critical"
-    if level == "orange":
-        return "high"
-    return "moderate"
-
-
-def _clean_html(text: str) -> str:
-    clean = re.sub(r"<[^>]+>", " ", text or "")
-    return " ".join(clean.split())[:600]
-
-
-def collect_gdacs(max_records: int = 2000) -> list[DisasterRecord]:
-    """Collect disaster events from GDACS GeoRSS (free, no auth)."""
-    records: list[DisasterRecord] = []
-
-    for url in [GDACS_RSS_URL, GDACS_SCORE_RSS_URL]:
-        logger.info("Parsing GDACS feed: %s", url)
-        try:
-            feed = feedparser.parse(url)
-        except Exception as exc:
-            logger.warning("Failed to parse GDACS feed %s: %s", url, exc)
-            continue
-
-        if feed.bozo and feed.bozo_exception:
-            logger.warning("GDACS feed parse warning: %s", feed.bozo_exception)
-
-        for entry in feed.entries:
-            if len(records) >= max_records:
-                break
-
-            title = entry.get("title", "")
-            summary_raw = entry.get("summary", entry.get("description", ""))
-            summary = _clean_html(summary_raw) or title
-
-            # GDACS uses gdacs: namespace tags
-            alert_level = (
-                entry.get("gdacs_alertlevel")
-                or entry.get("gdacs_alertscore", "")
-                or "green"
-            )
-            event_type = (
-                entry.get("gdacs_eventtype")
-                or entry.get("gdacs_eventname", "disaster")
-            )
-
-            # Try to extract country/region
-            country = entry.get("gdacs_country") or entry.get("gdacs_iso3", "global")
-
-            if not summary:
-                continue
-
-            records.append(
-                DisasterRecord(
-                    source="GDACS",
-                    event_type=str(event_type).lower().strip(),
-                    region=str(country).strip(),
-                    summary=summary,
-                    severity=_alert_to_severity(str(alert_level)),
-                )
-            )
-
-    logger.info("GDACS collection complete: %d records", len(records))
-    return records[:max_records]

worlddisasterlm/data/collectors/noaa.py

@@ -1,113 +0,0 @@
-"""
-NOAA National Weather Service alerts collector.
-
-Free API – no authentication required.
-Docs: https://www.weather.gov/documentation/services-web-api
-"""
-import logging
-import time
-
-import httpx
-
-from worlddisasterlm.data.schemas import DisasterRecord
-
-logger = logging.getLogger(__name__)
-
-NOAA_ALERTS_API = "https://api.weather.gov/alerts"
-HEADERS = {"User-Agent": "WorldDisasterLM/0.1.0 (worlddisasterlm@example.com)"}
-
-SEVERITY_MAP = {
-    "extreme": "critical",
-    "severe": "high",
-    "moderate": "moderate",
-    "minor": "low",
-    "unknown": "low",
-}
-
-EVENT_NORMALIZE = {
-    "tornado warning": "tornado",
-    "flash flood warning": "flood",
-    "hurricane warning": "hurricane",
-    "blizzard warning": "blizzard",
-    "winter storm warning": "winter_storm",
-    "tsunami warning": "tsunami",
-    "earthquake warning": "earthquake",
-    "excessive heat warning": "heatwave",
-    "fire weather watch": "wildfire",
-    "red flag warning": "wildfire",
-}
-
-
-def collect_noaa(max_records: int = 5000) -> list[DisasterRecord]:
-    """Collect active and recent weather alerts from NOAA (free, no auth)."""
-    records: list[DisasterRecord] = []
-    cursor: str | None = None
-    page_count = 0
-
-    logger.info("Collecting NOAA weather alerts (max=%d)", max_records)
-
-    while len(records) < max_records:
-        params: dict[str, str | int] = {"limit": 500}
-        if cursor:
-            params["cursor"] = cursor
-
-        try:
-            response = httpx.get(NOAA_ALERTS_API, headers=HEADERS, params=params, timeout=30)
-            response.raise_for_status()
-            data = response.json()
-        except Exception as exc:
-            logger.warning("NOAA request failed (page %d): %s", page_count, exc)
-            break
-
-        features = data.get("features", [])
-        if not features:
-            break
-
-        for feature in features:
-            if len(records) >= max_records:
-                break
-
-            props = feature.get("properties", {})
-            event = str(props.get("event", "weather event")).lower()
-            area = props.get("areaDesc", "United States")
-            headline = props.get("headline", "")
-            description = props.get("description", "")
-            severity_raw = str(props.get("severity", "unknown")).lower()
-
-            summary = headline or description
-            if not summary:
-                continue
-            summary = " ".join(summary.split())[:600]
-
-            normalized_event = next(
-                (v for k, v in EVENT_NORMALIZE.items() if k in event),
-                event.replace(" warning", "").replace(" watch", "").strip(),
-            )
-
-            records.append(
-                DisasterRecord(
-                    source="NOAA",
-                    event_type=normalized_event or "weather_event",
-                    region=area[:100],
-                    summary=summary,
-                    severity=SEVERITY_MAP.get(severity_raw, "moderate"),
-                )
-            )
-
-        # Pagination
-        pagination = data.get("pagination", {})
-        next_url = pagination.get("next")
-        if next_url:
-            # Extract cursor from next URL query param
-            from urllib.parse import urlparse, parse_qs
-            parsed = urlparse(next_url)
-            qs = parse_qs(parsed.query)
-            cursor = qs.get("cursor", [None])[0]
-        else:
-            break
-
-        page_count += 1
-        time.sleep(0.5)
-
-    logger.info("NOAA collection complete: %d records", len(records))
-    return records

worlddisasterlm/data/collectors/openfema.py

@@ -1,111 +0,0 @@
-"""
-OpenFEMA disaster declarations collector.
-
-Free API – no authentication required.
-Docs: https://www.fema.gov/about/openfema/api
-"""
-import logging
-import time
-
-import httpx
-
-from worlddisasterlm.data.schemas import DisasterRecord
-
-logger = logging.getLogger(__name__)
-
-OPENFEMA_API = "https://www.fema.gov/api/open/v2/DisasterDeclarationsSummaries"
-
-INCIDENT_TYPE_MAP = {
-    "hurricane": "hurricane",
-    "flood": "flood",
-    "tornado": "tornado",
-    "earthquake": "earthquake",
-    "fire": "wildfire",
-    "winter storm": "winter_storm",
-    "drought": "drought",
-    "mudslide": "landslide",
-    "typhoon": "typhoon",
-    "tsunami": "tsunami",
-    "volcano": "volcano",
-    "chemical": "chemical",
-    "biological": "epidemic",
-}
-
-
-def _map_incident(raw: str) -> str:
-    raw_lower = raw.lower()
-    for key, val in INCIDENT_TYPE_MAP.items():
-        if key in raw_lower:
-            return val
-    return raw_lower
-
-
-def collect_openfema(max_records: int = 20000) -> list[DisasterRecord]:
-    """Collect disaster declarations from OpenFEMA (free, no auth)."""
-    records: list[DisasterRecord] = []
-    skip = 0
-    page_size = 1000
-
-    logger.info("Collecting OpenFEMA disaster declarations (max=%d)", max_records)
-
-    while len(records) < max_records:
-        params = {
-            "$format": "json",
-            "$top": min(page_size, max_records - len(records)),
-            "$skip": skip,
-            "$orderby": "declarationDate desc",
-            "$select": (
-                "disasterNumber,declarationTitle,incidentType,"
-                "declarationDate,state,incidentBeginDate,incidentEndDate,"
-                "closeoutDate,declarationType"
-            ),
-        }
-
-        try:
-            response = httpx.get(OPENFEMA_API, params=params, timeout=60)
-            response.raise_for_status()
-            data = response.json()
-        except Exception as exc:
-            logger.warning("OpenFEMA request failed at skip %d: %s", skip, exc)
-            break
-
-        declarations = data.get("DisasterDeclarationsSummaries", [])
-        if not declarations:
-            break
-
-        for dec in declarations:
-            title = dec.get("declarationTitle", "")
-            incident_raw = dec.get("incidentType", "disaster")
-            state = dec.get("state", "US")
-            declaration_date = dec.get("declarationDate", "")[:10]
-
-            incident_end = dec.get("incidentEndDate", "")
-            open_ended = "ongoing" if not incident_end else f"closed {incident_end[:10]}"
-
-            summary = (
-                f"Federal disaster declaration: {title}. "
-                f"Incident type: {incident_raw}. "
-                f"State/region: {state}. "
-                f"Declared: {declaration_date}. Status: {open_ended}."
-            )
-
-            records.append(
-                DisasterRecord(
-                    source="OpenFEMA",
-                    event_type=_map_incident(incident_raw),
-                    region=state,
-                    summary=summary[:600],
-                    severity="high",
-                )
-            )
-
-        skip += len(declarations)
-        logger.info("OpenFEMA: %d declarations collected", len(records))
-
-        if len(declarations) < page_size:
-            break
-
-        time.sleep(0.3)
-
-    logger.info("OpenFEMA collection complete: %d records", len(records))
-    return records

worlddisasterlm/data/collectors/reliefweb.py

@@ -1,111 +0,0 @@
-"""
-ReliefWeb data collector.
-
-Free API – no authentication required.
-Docs: https://apidoc.rwlabs.org/
-"""
-import logging
-import time
-from typing import Any
-
-import httpx
-
-from worlddisasterlm.data.schemas import DisasterRecord
-
-logger = logging.getLogger(__name__)
-
-RELIEFWEB_API = "https://api.reliefweb.int/v1/reports"
-PAGE_SIZE = 100
-REQUEST_DELAY = 0.5  # seconds between requests
-
-
-def _severity_from_fields(fields: dict[str, Any]) -> str:
-    vuln_types = {e.get("name", "").lower() for e in fields.get("disaster_type", [])}
-    if any(t in vuln_types for t in {"earthquake", "tsunami", "nuclear", "hurricane", "cyclone"}):
-        return "critical"
-    if any(t in vuln_types for t in {"flood", "wildfire", "epidemic", "drought"}):
-        return "high"
-    return "moderate"
-
-
-def collect_reliefweb(max_records: int = 5000) -> list[DisasterRecord]:
-    """Collect disaster reports from ReliefWeb (free, no auth)."""
-    records: list[DisasterRecord] = []
-    offset = 0
-
-    logger.info("Collecting ReliefWeb reports (max=%d)", max_records)
-
-    while len(records) < max_records:
-        batch_size = min(PAGE_SIZE, max_records - len(records))
-        payload = {
-            "appname": "worlddisasterlm",
-            "limit": batch_size,
-            "offset": offset,
-            "fields": {
-                "include": [
-                    "title",
-                    "body-html",
-                    "primary_country.name",
-                    "disaster_type.name",
-                    "date.created",
-                    "status",
-                ]
-            },
-            "filter": {
-                "operator": "AND",
-                "conditions": [{"field": "status", "value": "published"}],
-            },
-            "sort": ["date.created:desc"],
-        }
-
-        try:
-            response = httpx.post(RELIEFWEB_API, json=payload, timeout=30)
-            response.raise_for_status()
-            data = response.json()
-        except Exception as exc:
-            logger.warning("ReliefWeb request failed at offset %d: %s", offset, exc)
-            break
-
-        items = data.get("data", [])
-        if not items:
-            break
-
-        for item in items:
-            fields = item.get("fields", {})
-            title = fields.get("title", "")
-            body_html = fields.get("body-html", "")
-            # Strip HTML tags simply
-            import re
-            body_text = re.sub(r"<[^>]+>", " ", body_html)
-            body_text = " ".join(body_text.split())[:600]
-
-            country = fields.get("primary_country", {})
-            region = country.get("name", "global") if isinstance(country, dict) else "global"
-
-            dtype_list = fields.get("disaster_type", [])
-            event_type = dtype_list[0].get("name", "disaster").lower() if dtype_list else "disaster"
-
-            summary = body_text or title
-            if not summary:
-                continue
-
-            records.append(
-                DisasterRecord(
-                    source="ReliefWeb",
-                    event_type=event_type,
-                    region=region,
-                    summary=summary,
-                    severity=_severity_from_fields(fields),
-                )
-            )
-
-        offset += len(items)
-        logger.info("ReliefWeb: collected %d / %d", len(records), max_records)
-
-        if len(items) < batch_size:
-            break  # last page
-
-        time.sleep(REQUEST_DELAY)
-
-    logger.info("ReliefWeb collection complete: %d records", len(records))
-    return records

worlddisasterlm/data/collectors/usgs.py

@@ -1,95 +0,0 @@
-"""
-USGS Earthquake data collector.
-
-Free API – no authentication required.
-Docs: https://earthquake.usgs.gov/fdsnws/event/1/
-"""
-import logging
-import time
-from datetime import datetime, timedelta
-
-import httpx
-
-from worlddisasterlm.data.schemas import DisasterRecord
-
-logger = logging.getLogger(__name__)
-
-USGS_API = "https://earthquake.usgs.gov/fdsnws/event/1/query"
-
-
-def _severity(magnitude: float) -> str:
-    if magnitude >= 7.5:
-        return "critical"
-    if magnitude >= 6.0:
-        return "high"
-    if magnitude >= 4.5:
-        return "moderate"
-    return "low"
-
-
-def collect_usgs(
-    years_back: int = 10,
-    min_magnitude: float = 4.0,
-    max_records: int = 20000,
-) -> list[DisasterRecord]:
-    """Collect earthquake data from USGS FDSN (free, no auth)."""
-    records: list[DisasterRecord] = []
-    # USGS API max 20 000 per call; chunk by year to avoid that limit
-    end_time = datetime.utcnow()
-
-    for year_offset in range(years_back):
-        if len(records) >= max_records:
-            break
-
-        year_end = end_time - timedelta(days=365 * year_offset)
-        year_start = end_time - timedelta(days=365 * (year_offset + 1))
-
-        params = {
-            "format": "geojson",
-            "starttime": year_start.strftime("%Y-%m-%dT%H:%M:%S"),
-            "endtime": year_end.strftime("%Y-%m-%dT%H:%M:%S"),
-            "minmagnitude": min_magnitude,
-            "orderby": "time",
-            "limit": min(20000, max_records - len(records)),
-        }
-
-        try:
-            response = httpx.get(USGS_API, params=params, timeout=60)
-            response.raise_for_status()
-            data = response.json()
-        except Exception as exc:
-            logger.warning("USGS request failed for year offset %d: %s", year_offset, exc)
-            continue
-
-        features = data.get("features", [])
-        logger.info(
-            "USGS year -%d: %d earthquakes fetched", year_offset + 1, len(features)
-        )
-
-        for feature in features:
-            props = feature.get("properties", {})
-            mag = props.get("mag") or 0.0
-            place = props.get("place") or "Unknown location"
-            title = props.get("title") or f"M{mag} earthquake"
-
-            summary = (
-                f"Magnitude {mag:.1f} earthquake reported near {place}. "
-                f"Alert level: {props.get('alert', 'none')}. "
-                f"Tsunami risk: {'yes' if props.get('tsunami') else 'no'}. "
-                f"{title}"
-            )
-
-            records.append(
-                DisasterRecord(
-                    source="USGS",
-                    event_type="earthquake",
-                    region=place,
-                    summary=summary[:600],
-                    severity=_severity(mag),
-                )
-            )
-
-        time.sleep(0.3)
-
-    logger.info("USGS collection complete: %d records", len(records))
-    return records[:max_records]

worlddisasterlm/data/collectors/who_rss.py

@@ -1,101 +0,0 @@
-"""
-WHO Disease Outbreak News (DON) RSS collector.
-
-Free feed – no authentication required.
-Feed: https://www.who.int/feeds/entity/csr/don/en/rss.xml
-"""
-import logging
-import re
-
-import feedparser
-
-from worlddisasterlm.data.schemas import DisasterRecord
-
-logger = logging.getLogger(__name__)
-
-WHO_DON_RSS = "https://www.who.int/feeds/entity/csr/don/en/rss.xml"
-
-
-DISEASE_KEYWORDS = {
-    "ebola": "epidemic",
-    "cholera": "epidemic",
-    "dengue": "epidemic",
-    "malaria": "epidemic",
-    "covid": "pandemic",
-    "influenza": "epidemic",
-    "mpox": "epidemic",
-    "monkeypox": "epidemic",
-    "plague": "epidemic",
-    "meningitis": "epidemic",
-    "lassa": "epidemic",
-    "marburg": "epidemic",
-    "rift valley": "epidemic",
-    "yellow fever": "epidemic",
-    "polio": "epidemic",
-    "measles": "epidemic",
-    "typhoid": "epidemic",
-    "hepatitis": "epidemic",
-}
-
-
-def _classify_event(text: str) -> str:
-    lower = text.lower()
-    for keyword, event_type in DISEASE_KEYWORDS.items():
-        if keyword in lower:
-            return event_type
-    return "public_health"
-
-
-def _clean(text: str) -> str:
-    clean = re.sub(r"<[^>]+>", " ", text or "")
-    return " ".join(clean.split())[:600]
-
-
-def collect_who(max_records: int = 1000) -> list[DisasterRecord]:
-    """Collect WHO disease outbreak news (free, no auth)."""
-    records: list[DisasterRecord] = []
-
-    logger.info("Parsing WHO Disease Outbreak News RSS feed")
-
-    try:
-        feed = feedparser.parse(WHO_DON_RSS)
-    except Exception as exc:
-        logger.warning("Failed to parse WHO RSS: %s", exc)
-        return records
-
-    if feed.bozo and feed.bozo_exception:
-        logger.warning("WHO RSS parse warning: %s", feed.bozo_exception)
-
-    for entry in feed.entries:
-        if len(records) >= max_records:
-            break
-
-        title = entry.get("title", "")
-        summary_raw = entry.get("summary", entry.get("description", ""))
-        summary = _clean(summary_raw) or title
-
-        if not summary:
-            continue
-
-        combined = f"{title} {summary}"
-        event_type = _classify_event(combined)
-
-        # Try to extract country from title (e.g. "Ebola virus disease – Democratic Republic of the Congo")
-        region = "global"
-        if "–" in title:
-            region = title.split("–")[-1].strip()
-        elif "-" in title:
-            region = title.split("-")[-1].strip()
-
-        records.append(
-            DisasterRecord(
-                source="WHO",
-                event_type=event_type,
-                region=region[:100],
-                summary=summary,
-                severity="high",
-            )
-        )
-
-    logger.info("WHO collection complete: %d records", len(records))
-    return records

worlddisasterlm/data/etl.py

@@ -1,55 +0,0 @@
-from collections.abc import Iterable
-
-from worlddisasterlm.data.schemas import DisasterRecord
-
-
-class DisasterETL:
-    def collect_records(self) -> list[DisasterRecord]:
-        # Replace with API clients and ingestion jobs for production-scale collection.
-        return [
-            DisasterRecord(
-                source="ReliefWeb",
-                event_type="flood",
-                region="South Asia",
-                summary="Severe flooding displaced 12000 people and disrupted road access.",
-                severity="high",
-            ),
-            DisasterRecord(
-                source="WHO",
-                event_type="epidemic",
-                region="East Africa",
-                summary="Localized cholera outbreak with urgent water sanitation requirements.",
-                severity="high",
-            ),
-            DisasterRecord(
-                source="USGS",
-                event_type="earthquake",
-                region="Pacific Rim",
-                summary="Magnitude 6.8 earthquake with aftershock risk and infrastructure damage.",
-                severity="critical",
-            ),
-        ]
-
-    def deduplicate(self, records: Iterable[DisasterRecord]) -> list[DisasterRecord]:
-        seen: set[tuple[str, str, str, str]] = set()
-        deduped: list[DisasterRecord] = []
-        for record in records:
-            key = (record.source, record.event_type, record.region, record.summary)
-            if key not in seen:
-                deduped.append(record)
-                seen.add(key)
-        return deduped
-
-    def normalize(self, records: Iterable[DisasterRecord]) -> list[DisasterRecord]:
-        normalized: list[DisasterRecord] = []
-        for record in records:
-            normalized.append(
-                DisasterRecord(
-                    source=record.source.strip(),
-                    event_type=record.event_type.strip().lower(),
-                    region=record.region.strip(),
-                    summary=" ".join(record.summary.split()),
-                    severity=record.severity.strip().lower(),
-                )
-            )
-        return normalized

worlddisasterlm/data/processors.py

@@ -1,32 +0,0 @@
-import json
-from pathlib import Path
-
-from worlddisasterlm.data.schemas import DisasterRecord, InstructionSample
-from worlddisasterlm.utils.io import ensure_dir
-
-
-def build_instruction_dataset(records: list[DisasterRecord]) -> list[InstructionSample]:
-    dataset: list[InstructionSample] = []
-    for record in records:
-        instruction = "Assess the incident and provide emergency response steps."
-        sample_input = (
-            f"Region: {record.region}\nEvent: {record.event_type}\nSeverity: {record.severity}\n"
-            f"Situation: {record.summary}"
-        )
-        output = (
-            "1) Verify official alerts and incident perimeter. "
-            "2) Prioritize life-saving response and medical triage. "
-            "3) Coordinate shelter, water, food, and transport logistics. "
-            "4) Share multilingual updates every 30 minutes."
-        )
-        dataset.append(InstructionSample(instruction=instruction, input=sample_input, output=output))
-    return dataset
-
-
-def save_instruction_dataset(samples: list[InstructionSample], output_path: str) -> Path:
-    target = Path(output_path)
-    ensure_dir(target.parent)
-    with target.open("w", encoding="utf-8") as handle:
-        for sample in samples:
-            handle.write(json.dumps(sample.__dict__, ensure_ascii=False) + "\n")
-    return target

worlddisasterlm/data/qa_generator.py

@@ -1,240 +0,0 @@
-"""
-QA pair generator.
-
-Converts raw DisasterRecord objects into diverse instruction-following samples.
-Each record generates 8–10 QA variants to amplify the training corpus.
-"""
-import random
-
-from worlddisasterlm.data.schemas import DisasterRecord, InstructionSample
-
-RESPONSE_STEPS = {
-    "earthquake": [
-        "Drop, Cover, and Hold On immediately.",
-        "Move away from windows, glass, and heavy furniture.",
-        "If outdoors, move to open space away from buildings.",
-        "After shaking stops, check for injuries and hazards.",
-        "Be prepared for aftershocks.",
-        "Do not use elevators; evacuate via stairways if building is unsafe.",
-        "Listen to emergency broadcasts for official guidance.",
-    ],
-    "flood": [
-        "Move immediately to higher ground; do not wait for official evacuation order.",
-        "Avoid walking or driving through flood waters – 6 inches can knock you down.",
-        "Disconnect electrical appliances; do not touch electrical equipment if wet.",
-        "Follow evacuation routes designated by local emergency management.",
-        "Store emergency supplies including water, food, and medication for 72 hours.",
-        "Monitor official weather and emergency alerts for updates.",
-    ],
-    "wildfire": [
-        "Evacuate immediately when ordered – do not wait.",
-        "Close all windows and doors to reduce smoke penetration.",
-        "Remove flammable items from around your home before leaving.",
-        "Wear N95 masks or wet cloth to protect against smoke inhalation.",
-        "Stay tuned to emergency broadcasts for evacuation route updates.",
-    ],
-    "hurricane": [
-        "Secure or bring indoors all outdoor furniture and objects.",
-        "Board up windows and reinforce garage doors.",
-        "Prepare emergency kit: water, food, medications, documents.",
-        "Know your evacuation zone and route.",
-        "Stay indoors during the storm; the eye of the hurricane is not the all-clear.",
-        "After the storm, watch for downed power lines and contaminated water.",
-    ],
-    "epidemic": [
-        "Report cases to local health authorities immediately.",
-        "Implement infection control measures: PPE, isolation protocols.",
-        "Coordinate with WHO, CDC, and national health agencies.",
-        "Establish clear case definition and surveillance system.",
-        "Activate contact tracing and quarantine procedures.",
-        "Communicate clearly with the public to prevent panic.",
-    ],
-    "pandemic": [
-        "Follow national health authority guidelines.",
-        "Implement non-pharmaceutical interventions: masking, distancing.",
-        "Prioritize healthcare system capacity management.",
-        "Accelerate vaccine development and equitable distribution.",
-        "Coordinate international response through WHO frameworks.",
-    ],
-    "default": [
-        "Activate emergency response plan immediately.",
-        "Prioritize life safety: triage injuries, evacuate if necessary.",
-        "Contact emergency services (fire, police, medical) as appropriate.",
-        "Coordinate with local emergency management authority.",
-        "Set up incident command structure.",
-        "Pre-position supplies: water, food, medical equipment, shelter.",
-        "Communicate regularly with affected population in plain language.",
-        "Document all actions for accountability and after-action review.",
-    ],
-}
-
-RESOURCE_GUIDANCE = {
-    "earthquake": "Search and rescue teams, medical triage units, heavy machinery for debris removal, temporary shelter, water purification, emergency food supplies, structural engineers.",
-    "flood": "Boats and water rescue teams, pumping equipment, water purification, temporary shelters on elevated ground, food and medical supplies, sanitation units.",
-    "wildfire": "Aerial firefighting assets, ground crews, evacuation transport, respiratory medical support, temporary shelters, animal rescue resources.",
-    "hurricane": "Pre-positioned food, water and fuel, emergency shelters, power restoration crews, debris removal, search and rescue teams, mental health support.",
-    "epidemic": "Medical personnel with PPE, testing kits, contact tracing capacity, isolation facilities, treatment medicines, communication system.",
-    "default": "Emergency medical teams, shelter supplies, clean water and food, communication equipment, transport resources, coordination staff.",
-}
-
-
-def _get_response_steps(event_type: str) -> list[str]:
-    for key in RESPONSE_STEPS:
-        if key in event_type.lower():
-            return RESPONSE_STEPS[key]
-    return RESPONSE_STEPS["default"]
-
-
-def _get_resources(event_type: str) -> str:
-    for key in RESOURCE_GUIDANCE:
-        if key in event_type.lower():
-            return RESOURCE_GUIDANCE[key]
-    return RESOURCE_GUIDANCE["default"]
-
-
-def generate_qa_pairs(records: list[DisasterRecord]) -> list[InstructionSample]:
-    samples: list[InstructionSample] = []
-
-    for record in records:
-        steps = _get_response_steps(record.event_type)
-        steps_text = "\n".join(f"{i + 1}. {s}" for i, s in enumerate(steps))
-        resources = _get_resources(record.event_type)
-
-        context = (
-            f"Region: {record.region}\n"
-            f"Hazard type: {record.event_type}\n"
-            f"Severity: {record.severity}\n"
-            f"Situation: {record.summary}"
-        )
-
-        # QA 1 – immediate response
-        samples.append(
-            InstructionSample(
-                instruction="What are the immediate emergency response steps for this disaster situation?",
-                input=context,
-                output=f"Immediate response priorities for a {record.severity}-severity {record.event_type} event:\n\n{steps_text}",
-            )
-        )
-
-        # QA 2 – incident summary
-        samples.append(
-            InstructionSample(
-                instruction="Summarize this disaster event for an emergency operations center briefing.",
-                input=context,
-                output=(
-                    f"SITUATION REPORT — {record.event_type.upper()} | {record.region}\n\n"
-                    f"Severity: {record.severity.capitalize()}\n"
-                    f"Source: {record.source}\n\n"
-                    f"Summary: {record.summary}\n\n"
-                    f"Status: Active monitoring required. Coordinate with local authorities and relevant UN agencies."
-                ),
-            )
-        )
-
-        # QA 3 – resource planning
-        samples.append(
-            InstructionSample(
-                instruction="What resources and logistics are needed to respond to this disaster?",
-                input=context,
-                output=(
-                    f"Resource requirements for a {record.event_type} event in {record.region} (severity: {record.severity}):\n\n"
-                    f"{resources}\n\n"
-                    f"Logistics priorities: establish forward operating base near the affected area, "
-                    f"pre-position supplies along access routes, coordinate air and ground transport, "
-                    f"and establish communication links with local partners."
-                ),
-            )
-        )
-
-        # QA 4 – risk assessment
-        samples.append(
-            InstructionSample(
-                instruction="Perform a risk assessment for this disaster scenario.",
-                input=context,
-                output=(
-                    f"Risk Assessment: {record.event_type.capitalize()} in {record.region}\n\n"
-                    f"Risk Level: {record.severity.capitalize()}\n"
-                    f"Primary hazard: {record.event_type}\n"
-                    f"Secondary hazards: displacement, water contamination, disease outbreak, infrastructure collapse\n"
-                    f"Vulnerable populations: elderly, children, persons with disabilities, low-income households\n\n"
-                    f"Recommended risk reduction actions:\n"
-                    f"1. Activate early warning dissemination for at-risk zones.\n"
-                    f"2. Pre-position emergency supplies and first responder teams.\n"
-                    f"3. Establish coordination hub with government and NGO partners.\n"
-                    f"4. Issue public guidance in multiple local languages."
-                ),
-            )
-        )
-
-        # QA 5 – humanitarian aid
-        samples.append(
-            InstructionSample(
-                instruction="What humanitarian aid priorities should be activated for this disaster?",
-                input=context,
-                output=(
-                    f"Humanitarian Aid Priorities for {record.event_type} in {record.region}:\n\n"
-                    f"1. Life-saving: Search and rescue, emergency medical care, trauma treatment.\n"
-                    f"2. Basic needs: Emergency shelter, safe water, food assistance, sanitation.\n"
-                    f"3. Protection: Safety monitoring for displaced persons, child protection, GBV prevention.\n"
-                    f"4. Livelihoods: Cash transfers and livelihood support for affected households.\n"
-                    f"5. Recovery: Debris clearance, shelter repair, economic recovery planning.\n\n"
-                    f"Key coordination partners: OCHA, UNHCR, WFP, UNICEF, WHO, local disaster management authority."
-                ),
-            )
-        )
-
-        # QA 6 – preparedness
-        samples.append(
-            InstructionSample(
-                instruction="How can communities in this region prepare for this type of disaster?",
-                input=context,
-                output=(
-                    f"Community Preparedness for {record.event_type} in {record.region}:\n\n"
-                    f"1. Develop and rehearse household emergency plans.\n"
-                    f"2. Build 72-hour emergency supply kits (water, food, medication, documents).\n"
-                    f"3. Know evacuation routes and local shelter locations.\n"
-                    f"4. Participate in community early warning systems.\n"
-                    f"5. Strengthen local infrastructure and building codes.\n"
-                    f"6. Conduct regular drills with schools, workplaces, and community organizations.\n"
-                    f"7. Ensure vulnerable populations have specific support plans."
-                ),
-            )
-        )
-
-        # QA 7 – public communication
-        samples.append(
-            InstructionSample(
-                instruction="Draft an emergency public communication message for this disaster.",
-                input=context,
-                output=(
-                    f"EMERGENCY ALERT — {record.region.upper()}\n\n"
-                    f"A {record.severity}-severity {record.event_type} event has been reported.\n\n"
-                    f"IMMEDIATE ACTIONS REQUIRED:\n"
-                    f"• Follow official evacuation orders immediately.\n"
-                    f"• Move to designated shelters or higher ground.\n"
-                    f"• Call emergency services for life-threatening situations.\n"
-                    f"• Do NOT spread unverified information.\n\n"
-                    f"Stay tuned to official government and emergency management channels for updates.\n"
-                    f"[This message should be verified and issued by the authorised emergency management authority.]"
-                ),
-            )
-        )
-
-        # QA 8 – recovery planning
-        samples.append(
-            InstructionSample(
-                instruction="What are the key steps in disaster recovery planning after this event?",
-                input=context,
-                output=(
-                    f"Recovery Planning Framework: {record.event_type} — {record.region}\n\n"
-                    f"Phase 1 (0–72 hours): Life safety, damage assessment, displaced persons registration.\n"
-                    f"Phase 2 (1–4 weeks): Temporary shelter provision, debris clearance, basic services restoration.\n"
-                    f"Phase 3 (1–6 months): Infrastructure repair, economic recovery, psychosocial support.\n"
-                    f"Phase 4 (6+ months): Long-term reconstruction, risk reduction investments, lesson-learned review.\n\n"
-                    f"Key principles: Build Back Better, inclusion of marginalized groups, environmental sustainability, "
-                    f"community ownership of the recovery process."
-                ),
-            )
-        )
-
-    return samples

worlddisasterlm/data/scenario_builder.py

@@ -1,197 +0,0 @@
-"""
-Synthetic emergency scenario builder.
-
-Generates multi-turn conversations, role-play scenarios, and cross-hazard
-compound disaster samples to enrich training diversity.
-"""
-from worlddisasterlm.data.schemas import InstructionSample
-
-COMPOUND_SCENARIOS = [
-    {
-        "scenario": "An earthquake of magnitude 7.2 struck a coastal city causing building collapses, "
-                    "followed 45 minutes later by a tsunami warning.",
-        "region": "Pacific coastal city",
-        "event_type": "compound: earthquake + tsunami",
-        "severity": "critical",
-        "response": (
-            "Compound Disaster Response — Earthquake + Tsunami:\n\n"
-            "IMMEDIATE (0–15 min):\n"
-            "1. Issue tsunami warning and mandatory coastal evacuation order immediately.\n"
-            "2. Search and rescue operations for earthquake-trapped victims must start inland first.\n"
-            "3. Hospitals on high ground remain operational; coastal facilities evacuate patients.\n\n"
-            "SHORT-TERM (15 min – 6 hours):\n"
-            "4. Establish triage stations above tsunami inundation zone.\n"
-            "5. Account for all search and rescue teams before wave arrival.\n"
-            "6. Deploy helicopters for coastal cliff rescues post-wave.\n\n"
-            "CRITICAL NOTE: Aftershocks will compromise already-damaged structures. "
-            "Do not re-enter buildings until structural engineers clear them."
-        ),
-    },
-    {
-        "scenario": "A category 4 hurricane is forecast to make landfall in 48 hours. "
-                    "The region already has 30,000 displaced persons from last week's flooding.",
-        "region": "Caribbean island",
-        "event_type": "compound: hurricane + displaced persons",
-        "severity": "critical",
-        "response": (
-            "Compound Emergency — Hurricane Landfall with Pre-existing Displacement:\n\n"
-            "PRE-LANDFALL (48–24 hours):\n"
-            "1. Mandatory evacuation of all existing displacement camps in low-lying areas.\n"
-            "2. Identify inland strong buildings for mass shelter; assess capacity.\n"
-            "3. Pre-position 72-hour food and water supplies at inland shelters.\n"
-            "4. Coordinate with UNHCR and local government for camp-to-shelter transfers.\n\n"
-            "PRE-LANDFALL (24–0 hours):\n"
-            "5. All personnel secured; close airport; ban sea transport.\n"
-            "6. Emergency services on stand-by in hardened facilities.\n\n"
-            "POST-LANDFALL:\n"
-            "7. Damage assessment sweep before re-opening any displaced camp.\n"
-            "8. Restore access roads before relief convoy mobilization."
-        ),
-    },
-    {
-        "scenario": "During a wildfire evacuation, a fuel tanker overturned on the primary "
-                    "evacuation highway, blocking egress for 15,000 evacuees.",
-        "region": "Southern California",
-        "event_type": "compound: wildfire + infrastructure failure",
-        "severity": "critical",
-        "response": (
-            "Multi-Hazard Evacuation Disruption — Wildfire + Road Blockage:\n\n"
-            "1. Activate alternate evacuation routes immediately; publish via Emergency Alert System.\n"
-            "2. Dispatch hazmat team to fuel tanker — assess fire ignition risk before moving vehicle.\n"
-            "3. If tanker poses fire risk: keep 300m standoff; use bulldozer to open shoulder lane.\n"
-            "4. Coordinate law enforcement for contraflow on alternate highways.\n"
-            "5. Deploy CHP and National Guard for traffic management at choke points.\n"
-            "6. Air resources pivot to aerial evacuation for mobility-impaired stranded persons.\n"
-            "7. Communicate updated routes every 15 minutes via radio, app, and social media."
-        ),
-    },
-]
-
-MULTILINGUAL_SAMPLES = [
-    {
-        "lang": "Nepali",
-        "instruction": "भूकम्पको बेला के गर्ने?",
-        "output": (
-            "भूकम्पको बेला:\n"
-            "१. तुरुन्त झुक्नुहोस्, ढाक्नुहोस् र समात्नुहोस् (Drop, Cover, Hold On)।\n"
-            "२. झ्याल, ऐना र भारी सामानबाट टाढा रहनुहोस्।\n"
-            "३. बाहिर हुनुहुन्छ भने खुला ठाउँमा जानुहोस् — भवन, रुख र बिजुलीका तारबाट टाढा।\n"
-            "४. हल्लाउन बन्द भएपछि चोटपटक जाँच गर्नुहोस् र सावधानीसाथ बाहिर निस्कनुहोस्।\n"
-            "५. परकम्पको लागि तयार रहनुहोस्।\n"
-            "गम्भीर घाइतेको लागि तुरुन्त आपतकालीन सेवा (१०१ वा स्थानीय नम्बर) मा फोन गर्नुहोस्।"
-        ),
-    },
-    {
-        "lang": "Nepali",
-        "instruction": "बाढीको बेला के गर्ने?",
-        "output": (
-            "बाढीको बेला:\n"
-            "१. तुरुन्त अग्लो ठाउँमा जानुहोस् — निकासी आदेशको प्रतीक्षा नगर्नुहोस्।\n"
-            "२. बाढीको पानीमा नहिँड्नुहोस् वा गाडी नचलाउनुहोस् — ६ इन्च पानीले पनि गिराउन सक्छ।\n"
-            "३. बिजुलीका उपकरणहरू बन्द गर्नुहोस्।\n"
-            "४. स्थानीय अधिकारीको निकासी मार्ग पछ्याउनुहोस्।\n"
-            "५. आधिकारिक अलर्टको लागि रेडियो सुन्नुहोस्।\n"
-            "खतरामा परेमा तुरुन्त आपतकालीन सेवामा सम्पर्क गर्नुहोस्।"
-        ),
-    },
-    {
-        "lang": "Nepali",
-        "instruction": "विपद् व्यवस्थापनको लागि आधारभूत तयारी के के गर्नुपर्छ?",
-        "output": (
-            "विपद् तयारीका आधारभूत कदमहरू:\n"
-            "१. परिवारको आपतकालीन योजना बनाउनुहोस् र अभ्यास गर्नुहोस्।\n"
-            "२. ७२ घण्टाको आपतकालीन किट तयार राख्नुहोस्: पानी, खाना, औषधि, कागजात।\n"
-            "३. निकासी मार्ग र नजिकको आश्रयस्थलको जानकारी राख्नुहोस्।\n"
-            "४. समुदायको पूर्व चेतावनी प्रणालीमा सहभागी हुनुहोस्।\n"
-            "५. विद्यालय, कार्यस्थल र समुदायमा नियमित अभ्यास गर्नुहोस्।\n"
-            "राष्ट्रिय विपद् जोखिम न्यूनीकरण तथा व्यवस्थापन प्राधिकरण (NDRRMA) को निर्देशिका पालना गर्नुहोस्।"
-        ),
-    },
-    {
-        "lang": "Spanish",
-        "instruction": "¿Qué debo hacer durante un terremoto?",
-        "output": (
-            "Durante un terremoto:\n"
-            "1. Agáchese, cúbrase y sujétese (Drop, Cover, Hold On).\n"
-            "2. Aléjese de ventanas, espejos y objetos pesados.\n"
-            "3. Si está en el exterior, diríjase a un espacio abierto.\n"
-            "4. Después del sismo, verifique heridos y salga con cuidado.\n"
-            "5. Esté preparado para réplicas.\n"
-            "Llame al número de emergencias de su país para reportar heridos graves."
-        ),
-    },
-    {
-        "lang": "French",
-        "instruction": "Quelles sont les étapes à suivre en cas d'inondation ?",
-        "output": (
-            "En cas d'inondation :\n"
-            "1. Montez immédiatement en hauteur – ne restez pas au niveau du sol.\n"
-            "2. N'essayez jamais de traverser des eaux en crue, même à pied.\n"
-            "3. Coupez l'électricité et le gaz si c'est sans danger.\n"
-            "4. Suivez les consignes d'évacuation des autorités locales.\n"
-            "5. Appelez les secours si vous êtes en danger immédiat.\n"
-            "Restez à l'écoute des alertes officielles."
-        ),
-    },
-    {
-        "lang": "Arabic",
-        "instruction": "ما هي الخطوات الواجب اتخاذها أثناء الإعصار؟",
-        "output": (
-            "أثناء الإعصار:\n"
-            "١. ابق في الداخل بعيداً عن النوافذ والأبواب الزجاجية.\n"
-            "٢. اذهب إلى الطابق الأرضي أو الغرفة الداخلية دون نوافذ.\n"
-            "٣. لا تخرج لأن عين الإعصار تبدو هادئة لكنها مؤقتة.\n"
-            "٤. استمع إلى تعليمات الدفاع المدني عبر الراديو أو التليفزيون.\n"
-            "٥. ابتعد عن مجاري المياه والمناطق المنخفضة بعد مرور الإعصار."
-        ),
-    },
-    {
-        "lang": "Hindi",
-        "instruction": "बाढ़ के दौरान क्या करें?",
-        "output": (
-            "बाढ़ ���े दौरान:\n"
-            "1. तुरंत ऊंची जगह पर जाएं।\n"
-            "2. बाढ़ के पानी में न चलें—6 इंच पानी भी गिरा सकता है।\n"
-            "3. बिजली के उपकरण बंद करें।\n"
-            "4. आधिकारिक निकासी मार्ग का पालन करें।\n"
-            "5. आपातकालीन अलर्ट के लिए रेडियो सुनें।\n"
-            "खतरे में होने पर तुरंत आपातकालीन सेवा को कॉल करें।"
-        ),
-    },
-]
-
-
-def build_compound_scenarios() -> list[InstructionSample]:
-    samples: list[InstructionSample] = []
-    for scene in COMPOUND_SCENARIOS:
-        samples.append(
-            InstructionSample(
-                instruction="Provide a comprehensive emergency response plan for this compound disaster scenario.",
-                input=(
-                    f"Scenario: {scene['scenario']}\n"
-                    f"Region: {scene['region']}\n"
-                    f"Event type: {scene['event_type']}\n"
-                    f"Severity: {scene['severity']}"
-                ),
-                output=scene["response"],
-            )
-        )
-    return samples
-
-
-def build_multilingual_samples() -> list[InstructionSample]:
-    samples: list[InstructionSample] = []
-    for item in MULTILINGUAL_SAMPLES:
-        samples.append(
-            InstructionSample(
-                instruction=item["instruction"],
-                input="",
-                output=item["output"],
-                language=item["lang"],
-            )
-        )
-    return samples
-
-
-def build_all_scenarios() -> list[InstructionSample]:
-    return build_compound_scenarios() + build_multilingual_samples()

worlddisasterlm/data/schemas.py

@@ -1,18 +0,0 @@
-from dataclasses import dataclass
-
-
-@dataclass
-class DisasterRecord:
-    source: str
-    event_type: str
-    region: str
-    summary: str
-    severity: str
-
-
-@dataclass
-class InstructionSample:
-    instruction: str
-    input: str
-    output: str
-    language: str = "English"

worlddisasterlm/data/sources.py

@@ -1,34 +0,0 @@
-INTERNATIONAL_ORGS = [
-    "UN",
-    "UNDRR",
-    "WHO",
-    "UNICEF",
-    "WFP",
-    "UNHCR",
-]
-
-DISASTER_DATABASES = [
-    "EM-DAT",
-    "ReliefWeb",
-    "GDACS",
-    "NASA Earth Data",
-    "NOAA",
-    "USGS",
-    "FEMA",
-    "World Bank Open Data",
-]
-
-RESEARCH_SOURCES = [
-    "Scientific papers",
-    "Government reports",
-    "Emergency response manuals",
-    "Disaster preparedness guidelines",
-    "Humanitarian response frameworks",
-]
-
-REAL_TIME_SOURCES = [
-    "Weather feeds",
-    "Satellite imagery metadata",
-    "Emergency alerts",
-    "Public safety bulletins",
-]