scripts/build_tech_docs.py

"""Generate the AgroSense Technical Guide and Standard Operating Procedure (SOP),
each as Word (.docx) and PDF, into docs/.

    pip install python-docx fpdf2
    python scripts/build_tech_docs.py
"""
from __future__ import annotations

import sys
from pathlib import Path

sys.path.insert(0, str(Path(__file__).resolve().parent))
from docgen import build_both  # noqa: E402

DOCS_DIR = Path(__file__).resolve().parent.parent / "docs"

# --------------------------------------------------------------------------- #
# Technical Guide
# --------------------------------------------------------------------------- #
TECH_TITLE = "AgroSense - Technical Guide"
TECH_SUBTITLE = "Architecture, modules, configuration and deployment  -  v0.1 (POC)"

TECH_SECTIONS = [
    ("1. System Overview", [
        ("p", "AgroSense is a FastAPI application that also serves a single-page web "
              "app. At its core is a Retrieval-Augmented Generation (RAG) engine over "
              "an agriculture knowledge base, surrounded by data-source modules "
              "(weather, satellite, market, hazards, etc.) and service modules "
              "(plant vision, telemedicine, live consultation, doctor registry, "
              "knowledge-base admin)."),
        ("h2", "Layers"),
        ("ul", ["Data layer - static knowledge base plus live sources (Open-Meteo, "
                "NASA, data.gov.in, Yahoo Finance, Google News).",
                "Knowledge layer - documents embedded into a vector store.",
                "Cognitive layer - retrieval + re-ranking + grounded generation, plus "
                "decision-fusion advisories and plant vision/telemedicine.",
                "Interaction layer - the web app (served by FastAPI), Streamlit UI, CLI.",
                "Engagement layer - multilingual, live consultation, notifications."]),
        ("h2", "RAG pipeline"),
        ("p", "query -> preprocess/translate -> embed -> vector search (top-k) -> "
              "CARO-style metadata re-rank + relevance gate -> grounded extractive "
              "generation with citations -> optional translation back to the user "
              "language."),
    ]),
    ("2. Technology Stack", [
        ("ul", ["Language/runtime: Python 3.10+ (validated on 3.14).",
                "Core: FastAPI + Uvicorn, Pydantic, NumPy.",
                "Web app: HTML + CSS + vanilla JS (no build step), served at /ui.",
                "Optional accelerators: faiss-cpu (vector search), sentence-transformers "
                "(embeddings) - pure-Python fallbacks exist when absent.",
                "Optional: TensorFlow (plant-vision CNNs), Earth Engine API (NDVI), "
                "Pillow (image heuristic), deep-translator/argostranslate (i18n), "
                "python-docx + fpdf2 (these documents).",
                "Tests: a self-contained runner (python tests/test_rag.py) plus a "
                "jsdom SPA smoke test (node web/smoke.mjs)."]),
    ]),
    ("3. Repository Layout", [
        ("ul", ["agrosense/ - the library (engine + all modules).",
                "agrosense/rag.py - RAGEngine, the single orchestration entry point.",
                "agrosense/{embeddings,vector_store,retriever,generator}.py - RAG core.",
                "agrosense/{weather,satellite,environment,planetary,hazards}.py - data.",
                "agrosense/{fusion,crop_profiles}.py - decision-fusion advisories.",
                "agrosense/{vision,telemedicine,ipm}.py - plant clinic.",
                "agrosense/{consultation,doctors,notifications}.py - live doctor.",
                "agrosense/{clubs,traditional}.py - farmer clubs + traditional/Panchang.",
                "agrosense/{subsidies,finance,land_records,trading}.py - schemes, loans, "
                "land records, produce marketplace.",
                "agrosense/{prices,commodities,news,radio,calendars,translation}.py - misc.",
                "agrosense/{kb_admin,evaluation}.py - admin + evaluation.",
                "api/main.py - FastAPI app and all endpoints.",
                "web/ - the single-page app. ui/ - Streamlit UI. cli.py - CLI.",
                "data/ - knowledge_base.json, agri_experts.json, subsidies.json, "
                "finance.json, land_records.json, eval_set.json, etc.",
                "scripts/ - evaluate, train_plant_models, build_manual, build_tech_docs, "
                "docgen (shared Word/PDF renderer), build_docs (regenerate all).",
                "tests/ - test_rag.py (logic), test_ui_render.py (Streamlit render)."]),
    ]),
    ("4. The RAG Engine", [
        ("p", "RAGEngine (agrosense/rag.py) builds the embedder, generator, documents, "
              "vector store and retriever on init, and owns every feature method. "
              "Embeddings default to an offline hashing embedder; the vector store "
              "defaults to a NumPy cosine store; both upgrade automatically when the "
              "optional libraries are installed."),
        ("ul", ["Retrieval: vector search returns top-k; a CARO-style pass re-weights "
                "by metadata overlap (crop, soil, region) and lexical overlap.",
                "Relevance gate: a candidate must share a content term with the query "
                "(matched against field values, not labels) - off-domain queries hit a "
                "safe fallback instead of fabricating advice.",
                "Generation: an extractive generator composes the answer ONLY from "
                "retrieved KB fields and attaches numbered citations (low hallucination "
                "by construction). An OpenAI/Azure seam exists for a real LLM.",
                "reload_kb() rebuilds documents + store + retriever after admin edits."]),
    ]),
    ("5. Knowledge Base", [
        ("p", "data/knowledge_base.json is a list of entries (crop, soil_type, "
              "rainfall_mm, recommended_fertilizer, disease_prevention, pest_management, "
              "source, ...). agrosense/kb_admin.py provides a validated CRUD store "
              "(KBStore) used by the admin endpoints; saving rewrites the file and the "
              "engine rebuilds the index so changes are live."),
    ]),
    ("6. Modules and Data Sources", [
        ("ul", ["Keyless: weather/environment (Open-Meteo), satellite imagery + "
                "agroclimate (NASA GIBS/POWER), hazards (NASA EONET), planetary "
                "(local ephemeris), news (Google News), commodities (Yahoo Finance), "
                "internet radio (Radio Browser).",
                "Keyed: prices + groundwater (data.gov.in), fires (NASA FIRMS), "
                "field NDVI (Earth Engine).",
                "Curated knowledge bases (no live feed exists): government schemes "
                "(subsidies.json), agri-finance products (finance.json), state land-record "
                "systems (land_records.json). The land-record guide is rendered to Word/PDF "
                "on the fly via scripts/docgen.py.",
                "User-generated runtime stores (auto-seeded / gitignored): farmer clubs "
                "(clubs.json), produce listings (market_listings.json), loan enquiries "
                "(finance_applications.json), doctor registry (plant_doctors.json).",
                "Each client degrades gracefully (returns None/empty) when offline or "
                "unconfigured; parsing/geometry logic is pure and unit-tested, network "
                "calls are mock-tested."]),
    ]),
    ("7. API Reference", [
        ("ul", ["Advisor: POST /query.",
                "Location: GET /weather /satellite /environment /planetary /hazards "
                "/advisories.",
                "Market/info: GET /prices /commodities /news /radio /datetime /languages.",
                "Trading: GET/POST /market/listings, GET /market/listings/{id}, "
                "POST /market/listings/{id}/inquire, POST /market/listings/{id}/close.",
                "Plant clinic: POST /vision/classify, POST /telemedicine.",
                "Doctors: GET /experts, GET /doctors/{id}, POST /doctors/{id}/rate, "
                "POST /doctors/apply.",
                "Consults: POST /consult/request, GET/POST /consult/{id}[/message], "
                "GET /notifications.",
                "Clubs: GET/POST /clubs, GET /clubs/{id}, POST /clubs/{id}/join|post.",
                "Traditional: GET /traditional.",
                "Schemes: GET /subsidies[/{id}], GET /subsidies/updates.",
                "Finance: GET /finance[/{id}], POST /finance/{id}/apply.",
                "Land records: GET /land-records, GET /land-records/guide.{pdf,docx}.",
                "Admin (X-Admin-Token): /admin/kb (CRUD), /admin/doctors, "
                "/admin/doctors/{id}/verify, /admin/subsidies/{id}/update, "
                "/admin/finance/applications.",
                "Docs: GET /downloads/{user-manual,technical-guide,sop}.{pdf,docx}.",
                "App: GET / -> /ui/ ; interactive API docs at /docs."]),
    ]),
    ("8. Configuration (environment variables)", [
        ("ul", ["AGROSENSE_EMBEDDING_BACKEND - auto | sentence-transformers | hashing.",
                "AGROSENSE_GENERATION_BACKEND - extractive | openai.",
                "AGROSENSE_ADMIN_TOKEN - admin gate (change the default 'admin').",
                "AGROSENSE_DATAGOV_API_KEY - prices + groundwater.",
                "AGROSENSE_GROUNDWATER_RESOURCE - CGWB resource id.",
                "AGROSENSE_FIRMS_MAP_KEY - active fires.",
                "AGROSENSE_EE_PROJECT / _EE_SERVICE_ACCOUNT / _EE_KEY_FILE - NDVI.",
                "AGROSENSE_VISION_{DISEASE,PLANT,PEST}_MODEL / _LABELS - vision models.",
                "AGROSENSE_TRANSLATION_BACKEND - auto | argos | deep | none.",
                "AGROSENSE_NOTIFY_WEBHOOK / SMTP vars - expert notifications.",
                "AGROSENSE_VIDEO_BASE - Jitsi base for live video.",
                "AGROSENSE_KB_PATH - knowledge base path override."]),
    ]),
    ("9. Extending AgroSense", [
        ("ul", ["Add KB knowledge: use the Admin tab or POST /admin/kb (or edit the "
                "JSON and call reload).",
                "Add a data source: create a module with a pure parser + a client that "
                "returns None on failure; add an engine method and an endpoint; unit-"
                "test the parser and mock the fetch.",
                "Add a vision model: train with scripts/train_plant_models.py and set "
                "the model env vars; inference activates automatically.",
                "Add an expert specialization: extend the tag map in doctors.py and the "
                "routing map in consultation.py."]),
    ]),
    ("10. Testing and Quality", [
        ("ul", ["Logic suite: python tests/test_rag.py (offline; set "
                "AGROSENSE_EMBEDDING_BACKEND=hashing for determinism).",
                "Evaluation harness: python scripts/evaluate.py - context relevance, "
                "faithfulness, hallucination rate, latency vs targets.",
                "SPA smoke: node web/smoke.mjs (jsdom; mocks fetch; asserts no runtime "
                "errors and that sections render).",
                "Streamlit render: python tests/test_ui_render.py."]),
    ]),
    ("11. Deployment", [
        ("ol", ["Provision a host with Python 3.10+; pip install -r requirements.txt.",
                "Set required environment variables (at least change AGROSENSE_ADMIN_TOKEN).",
                "Run: uvicorn api.main:app --host 0.0.0.0 --port 8000 (add --workers N "
                "behind a process manager; put a reverse proxy / TLS in front).",
                "Persist data/ (knowledge_base.json, plant_doctors.json) on durable "
                "storage; back it up regularly.",
                "Generate docs once (python scripts/build_manual.py ; "
                "python scripts/build_tech_docs.py)."]),
        ("p", "Note: engine state (the consultation/notification stores and the doctor "
              "registry) is in-process and per-worker; for multi-worker or multi-host "
              "deployments move that state to a shared database."),
    ]),
    ("12. Security", [
        ("ul", ["Admin endpoints use a shared-secret token (X-Admin-Token) - suitable "
                "for a single-admin POC; use real auth (accounts, roles, audit) in "
                "production.",
                "CORS is open by default - restrict allowed origins for a real deployment.",
                "Onboarding and ratings are unauthenticated - add accounts and rate "
                "limiting to prevent spam/abuse.",
                "Keep API keys in environment variables / a secrets manager, never in "
                "code. Validate and size-limit uploaded images."]),
    ]),
]

# --------------------------------------------------------------------------- #
# Standard Operating Procedure
# --------------------------------------------------------------------------- #
SOP_TITLE = "AgroSense - Standard Operating Procedure (SOP)"
SOP_SUBTITLE = "Operational runbook for deploying, running and administering AgroSense"

SOP_SECTIONS = [
    ("1. Purpose and Scope", [
        ("p", "This SOP defines the standard procedures to deploy, operate, administer "
              "and recover the AgroSense service. It applies to administrators and "
              "operators running the FastAPI service and the web app."),
    ]),
    ("2. Roles and Responsibilities", [
        ("ul", ["Operator - starts/stops the service, monitors health, applies config, "
                "performs backups and incident response.",
                "Administrator - manages the knowledge base and verifies plant doctors "
                "(holds the admin token).",
                "Plant doctor - onboards, gets verified, handles live consultations.",
                "Farmer (end user) - uses the advisor, plant clinic and consultations."]),
    ]),
    ("3. Pre-requisites", [
        ("ol", ["Host with Python 3.10+ and network egress for live features.",
                "pip install -r requirements.txt.",
                "Environment variables set (see the Technical Guide). At minimum set a "
                "strong AGROSENSE_ADMIN_TOKEN.",
                "data/ directory present and writable (knowledge base + doctor registry)."]),
    ]),
    ("4. Start, Stop, Restart", [
        ("ol", ["Start: uvicorn api.main:app --host 0.0.0.0 --port 8000 "
                "(use a process manager such as systemd/pm2 in production).",
                "Verify: open http://HOST:8000/ui/ and GET http://HOST:8000/health.",
                "Stop: stop the uvicorn process (Ctrl+C or the process manager).",
                "Restart: stop then start; the engine reloads the knowledge base and "
                "rebuilds the index at boot."]),
    ]),
    ("5. Health Checks and Monitoring", [
        ("ul", ["GET /health returns status ok and the knowledge-base document count.",
                "GET / should redirect (307) to /ui/; /ui/ should return the web app.",
                "Watch the uvicorn logs for errors; a feature returning 'unavailable' "
                "indicates a missing key or no network, not a service fault.",
                "Latency target for the advisor is under 3 seconds; the offline core is "
                "typically sub-second."]),
    ]),
    ("6. Configuration Management", [
        ("ol", ["Set configuration via environment variables before starting the "
                "service (do not hard-code secrets).",
                "To change a key/setting: update the environment, then restart the "
                "service.",
                "Record changes (who/when/what) in your change log."]),
    ]),
    ("7. Knowledge Base Management", [
        ("ol", ["Back up data/knowledge_base.json before bulk changes.",
                "Open the web app Admin tab, enter the admin token, click Unlock.",
                "Add an entry (crop and source are required), or Edit/Delete an existing "
                "one. Alternatively use POST/PUT/DELETE /admin/kb with the X-Admin-Token "
                "header.",
                "Saving rebuilds the live index automatically - verify by asking the "
                "advisor a question that should hit the new entry."]),
    ]),
    ("8. Plant Doctor Onboarding and Verification", [
        ("ol", ["A doctor applies via the Plant clinic tab's live-doctor section (or "
                "POST /doctors/apply).",
                "Admin opens the Admin tab -> Plant doctor verification, and reviews the "
                "application: name, specialization, region, languages, contact, "
                "credentials and the ICAR/registration number.",
                "Check the credentials and registration number against your records / "
                "the relevant registry (manual step).",
                "Click Verify to approve (the doctor enters the directory and can take "
                "consults) or Reject. Verified doctors are notified on new consults via "
                "the configured channels."]),
    ]),
    ("9. Handling Live Consultations", [
        ("ul", ["A consult request is routed to a matching available verified doctor; a "
                "video room link and chat thread are created and the doctor is notified "
                "(in-app log always; webhook/email if configured).",
                "Operators ensure the video backend (Jitsi) is reachable; for production "
                "use a self-hosted Jitsi via AGROSENSE_VIDEO_BASE."]),
    ]),
    ("10. Incident Response", [
        ("ul", ["Service down: check the process and logs; restart; confirm /health.",
                "Web app blank: confirm the server is up and open /ui/ (root redirects).",
                "A live feature is 'unavailable': verify the relevant key and network "
                "egress; the core advisor keeps working offline.",
                "Expired/invalid key: rotate the key in the environment and restart.",
                "Bad knowledge entry causing wrong answers: edit/delete it in the Admin "
                "tab (index rebuilds immediately) or restore the KB backup.",
                "Abusive doctor application or rating: reject the doctor / remove the "
                "entry; consider enabling authentication and rate limiting."]),
    ]),
    ("11. Backup and Recovery", [
        ("ol", ["Regularly back up the committed content: data/knowledge_base.json, "
                "subsidies.json, finance.json and land_records.json.",
                "Also back up the runtime state: plant_doctors.json (doctor registry + "
                "ratings), clubs.json (clubs + posts), market_listings.json (trading) and "
                "finance_applications.json (loan enquiries).",
                "To recover: stop the service, restore the JSON files from backup, start "
                "the service; the index rebuilds at boot.",
                "Keep agri_experts.json under version control; the runtime files above are "
                "auto-seeded (or created) if missing."]),
    ]),
    ("12. Routine Maintenance", [
        ("ul", ["Review and update the knowledge base as guidance changes.",
                "Keep the curated content current: scheme details in subsidies.json, "
                "loan products in finance.json, and state portals in land_records.json; "
                "post scheme announcements from the Admin tab as they are released.",
                "Review the loan enquiries lodged in the Finance tab "
                "(GET /admin/finance/applications) and follow up.",
                "Rotate the admin token periodically.",
                "Review doctor ratings/reviews and trading listings for abuse.",
                "Update dependencies and re-run the test suite + SPA smoke before "
                "deploying.",
                "Regenerate all documents (python scripts/build_docs.py) after "
                "feature changes."]),
    ]),
    ("13. Change Management", [
        ("ol", ["Make changes in a non-production environment first.",
                "Run python tests/test_rag.py and node web/smoke.mjs; run "
                "scripts/evaluate.py to confirm quality targets.",
                "Deploy during a low-traffic window; verify /health and key flows.",
                "If a problem appears, roll back to the previous version and restore "
                "backed-up data."]),
    ]),
    ("14. Compliance and Disclaimer", [
        ("p", "AgroSense provides decision support, not professional or medical "
              "authority. Operators and doctors must ensure recommendations comply with "
              "local agricultural regulations. Always confirm product names, dosages and "
              "pre-harvest intervals before advising chemical use."),
    ]),
]


def main() -> int:
    build_both(TECH_TITLE, TECH_SUBTITLE, TECH_SECTIONS, DOCS_DIR,
               "AgroSense_Technical_Guide")
    build_both(SOP_TITLE, SOP_SUBTITLE, SOP_SECTIONS, DOCS_DIR, "AgroSense_SOP")
    return 0


if __name__ == "__main__":
    raise SystemExit(main())