"""Generate the AgroSense Technical Guide and Standard Operating Procedure (SOP), each as Word (.docx) and PDF, into docs/. pip install python-docx fpdf2 python scripts/build_tech_docs.py """ from __future__ import annotations import sys from pathlib import Path sys.path.insert(0, str(Path(__file__).resolve().parent)) from docgen import build_both # noqa: E402 DOCS_DIR = Path(__file__).resolve().parent.parent / "docs" # --------------------------------------------------------------------------- # # Technical Guide # --------------------------------------------------------------------------- # TECH_TITLE = "AgroSense - Technical Guide" TECH_SUBTITLE = "Architecture, modules, configuration and deployment - v0.1 (POC)" TECH_SECTIONS = [ ("1. System Overview", [ ("p", "AgroSense is a FastAPI application that also serves a single-page web " "app. At its core is a Retrieval-Augmented Generation (RAG) engine over " "an agriculture knowledge base, surrounded by data-source modules " "(weather, satellite, market, hazards, etc.) and service modules " "(plant vision, telemedicine, live consultation, doctor registry, " "knowledge-base admin)."), ("h2", "Layers"), ("ul", ["Data layer - static knowledge base plus live sources (Open-Meteo, " "NASA, data.gov.in, Yahoo Finance, Google News).", "Knowledge layer - documents embedded into a vector store.", "Cognitive layer - retrieval + re-ranking + grounded generation, plus " "decision-fusion advisories and plant vision/telemedicine.", "Interaction layer - the web app (served by FastAPI), Streamlit UI, CLI.", "Engagement layer - multilingual, live consultation, notifications."]), ("h2", "RAG pipeline"), ("p", "query -> preprocess/translate -> embed -> vector search (top-k) -> " "CARO-style metadata re-rank + relevance gate -> grounded extractive " "generation with citations -> optional translation back to the user " "language."), ]), ("2. Technology Stack", [ ("ul", ["Language/runtime: Python 3.10+ (validated on 3.14).", "Core: FastAPI + Uvicorn, Pydantic, NumPy.", "Web app: HTML + CSS + vanilla JS (no build step), served at /ui.", "Optional accelerators: faiss-cpu (vector search), sentence-transformers " "(embeddings) - pure-Python fallbacks exist when absent.", "Optional: TensorFlow (plant-vision CNNs), Earth Engine API (NDVI), " "Pillow (image heuristic), deep-translator/argostranslate (i18n), " "python-docx + fpdf2 (these documents).", "Tests: a self-contained runner (python tests/test_rag.py) plus a " "jsdom SPA smoke test (node web/smoke.mjs)."]), ]), ("3. Repository Layout", [ ("ul", ["agrosense/ - the library (engine + all modules).", "agrosense/rag.py - RAGEngine, the single orchestration entry point.", "agrosense/{embeddings,vector_store,retriever,generator}.py - RAG core.", "agrosense/{weather,satellite,environment,planetary,hazards}.py - data.", "agrosense/{fusion,crop_profiles}.py - decision-fusion advisories.", "agrosense/{vision,telemedicine,ipm}.py - plant clinic.", "agrosense/{consultation,doctors,notifications}.py - live doctor.", "agrosense/{clubs,traditional}.py - farmer clubs + traditional/Panchang.", "agrosense/{subsidies,finance,land_records,trading}.py - schemes, loans, " "land records, produce marketplace.", "agrosense/{prices,commodities,news,radio,calendars,translation}.py - misc.", "agrosense/{kb_admin,evaluation}.py - admin + evaluation.", "api/main.py - FastAPI app and all endpoints.", "web/ - the single-page app. ui/ - Streamlit UI. cli.py - CLI.", "data/ - knowledge_base.json, agri_experts.json, subsidies.json, " "finance.json, land_records.json, eval_set.json, etc.", "scripts/ - evaluate, train_plant_models, build_manual, build_tech_docs, " "docgen (shared Word/PDF renderer), build_docs (regenerate all).", "tests/ - test_rag.py (logic), test_ui_render.py (Streamlit render)."]), ]), ("4. The RAG Engine", [ ("p", "RAGEngine (agrosense/rag.py) builds the embedder, generator, documents, " "vector store and retriever on init, and owns every feature method. " "Embeddings default to an offline hashing embedder; the vector store " "defaults to a NumPy cosine store; both upgrade automatically when the " "optional libraries are installed."), ("ul", ["Retrieval: vector search returns top-k; a CARO-style pass re-weights " "by metadata overlap (crop, soil, region) and lexical overlap.", "Relevance gate: a candidate must share a content term with the query " "(matched against field values, not labels) - off-domain queries hit a " "safe fallback instead of fabricating advice.", "Generation: an extractive generator composes the answer ONLY from " "retrieved KB fields and attaches numbered citations (low hallucination " "by construction). An OpenAI/Azure seam exists for a real LLM.", "reload_kb() rebuilds documents + store + retriever after admin edits."]), ]), ("5. Knowledge Base", [ ("p", "data/knowledge_base.json is a list of entries (crop, soil_type, " "rainfall_mm, recommended_fertilizer, disease_prevention, pest_management, " "source, ...). agrosense/kb_admin.py provides a validated CRUD store " "(KBStore) used by the admin endpoints; saving rewrites the file and the " "engine rebuilds the index so changes are live."), ]), ("6. Modules and Data Sources", [ ("ul", ["Keyless: weather/environment (Open-Meteo), satellite imagery + " "agroclimate (NASA GIBS/POWER), hazards (NASA EONET), planetary " "(local ephemeris), news (Google News), commodities (Yahoo Finance), " "internet radio (Radio Browser).", "Keyed: prices + groundwater (data.gov.in), fires (NASA FIRMS), " "field NDVI (Earth Engine).", "Curated knowledge bases (no live feed exists): government schemes " "(subsidies.json), agri-finance products (finance.json), state land-record " "systems (land_records.json). The land-record guide is rendered to Word/PDF " "on the fly via scripts/docgen.py.", "User-generated runtime stores (auto-seeded / gitignored): farmer clubs " "(clubs.json), produce listings (market_listings.json), loan enquiries " "(finance_applications.json), doctor registry (plant_doctors.json).", "Each client degrades gracefully (returns None/empty) when offline or " "unconfigured; parsing/geometry logic is pure and unit-tested, network " "calls are mock-tested."]), ]), ("7. API Reference", [ ("ul", ["Advisor: POST /query.", "Location: GET /weather /satellite /environment /planetary /hazards " "/advisories.", "Market/info: GET /prices /commodities /news /radio /datetime /languages.", "Trading: GET/POST /market/listings, GET /market/listings/{id}, " "POST /market/listings/{id}/inquire, POST /market/listings/{id}/close.", "Plant clinic: POST /vision/classify, POST /telemedicine.", "Doctors: GET /experts, GET /doctors/{id}, POST /doctors/{id}/rate, " "POST /doctors/apply.", "Consults: POST /consult/request, GET/POST /consult/{id}[/message], " "GET /notifications.", "Clubs: GET/POST /clubs, GET /clubs/{id}, POST /clubs/{id}/join|post.", "Traditional: GET /traditional.", "Schemes: GET /subsidies[/{id}], GET /subsidies/updates.", "Finance: GET /finance[/{id}], POST /finance/{id}/apply.", "Land records: GET /land-records, GET /land-records/guide.{pdf,docx}.", "Admin (X-Admin-Token): /admin/kb (CRUD), /admin/doctors, " "/admin/doctors/{id}/verify, /admin/subsidies/{id}/update, " "/admin/finance/applications.", "Docs: GET /downloads/{user-manual,technical-guide,sop}.{pdf,docx}.", "App: GET / -> /ui/ ; interactive API docs at /docs."]), ]), ("8. Configuration (environment variables)", [ ("ul", ["AGROSENSE_EMBEDDING_BACKEND - auto | sentence-transformers | hashing.", "AGROSENSE_GENERATION_BACKEND - extractive | openai.", "AGROSENSE_ADMIN_TOKEN - admin gate (change the default 'admin').", "AGROSENSE_DATAGOV_API_KEY - prices + groundwater.", "AGROSENSE_GROUNDWATER_RESOURCE - CGWB resource id.", "AGROSENSE_FIRMS_MAP_KEY - active fires.", "AGROSENSE_EE_PROJECT / _EE_SERVICE_ACCOUNT / _EE_KEY_FILE - NDVI.", "AGROSENSE_VISION_{DISEASE,PLANT,PEST}_MODEL / _LABELS - vision models.", "AGROSENSE_TRANSLATION_BACKEND - auto | argos | deep | none.", "AGROSENSE_NOTIFY_WEBHOOK / SMTP vars - expert notifications.", "AGROSENSE_VIDEO_BASE - Jitsi base for live video.", "AGROSENSE_KB_PATH - knowledge base path override."]), ]), ("9. Extending AgroSense", [ ("ul", ["Add KB knowledge: use the Admin tab or POST /admin/kb (or edit the " "JSON and call reload).", "Add a data source: create a module with a pure parser + a client that " "returns None on failure; add an engine method and an endpoint; unit-" "test the parser and mock the fetch.", "Add a vision model: train with scripts/train_plant_models.py and set " "the model env vars; inference activates automatically.", "Add an expert specialization: extend the tag map in doctors.py and the " "routing map in consultation.py."]), ]), ("10. Testing and Quality", [ ("ul", ["Logic suite: python tests/test_rag.py (offline; set " "AGROSENSE_EMBEDDING_BACKEND=hashing for determinism).", "Evaluation harness: python scripts/evaluate.py - context relevance, " "faithfulness, hallucination rate, latency vs targets.", "SPA smoke: node web/smoke.mjs (jsdom; mocks fetch; asserts no runtime " "errors and that sections render).", "Streamlit render: python tests/test_ui_render.py."]), ]), ("11. Deployment", [ ("ol", ["Provision a host with Python 3.10+; pip install -r requirements.txt.", "Set required environment variables (at least change AGROSENSE_ADMIN_TOKEN).", "Run: uvicorn api.main:app --host 0.0.0.0 --port 8000 (add --workers N " "behind a process manager; put a reverse proxy / TLS in front).", "Persist data/ (knowledge_base.json, plant_doctors.json) on durable " "storage; back it up regularly.", "Generate docs once (python scripts/build_manual.py ; " "python scripts/build_tech_docs.py)."]), ("p", "Note: engine state (the consultation/notification stores and the doctor " "registry) is in-process and per-worker; for multi-worker or multi-host " "deployments move that state to a shared database."), ]), ("12. Security", [ ("ul", ["Admin endpoints use a shared-secret token (X-Admin-Token) - suitable " "for a single-admin POC; use real auth (accounts, roles, audit) in " "production.", "CORS is open by default - restrict allowed origins for a real deployment.", "Onboarding and ratings are unauthenticated - add accounts and rate " "limiting to prevent spam/abuse.", "Keep API keys in environment variables / a secrets manager, never in " "code. Validate and size-limit uploaded images."]), ]), ] # --------------------------------------------------------------------------- # # Standard Operating Procedure # --------------------------------------------------------------------------- # SOP_TITLE = "AgroSense - Standard Operating Procedure (SOP)" SOP_SUBTITLE = "Operational runbook for deploying, running and administering AgroSense" SOP_SECTIONS = [ ("1. Purpose and Scope", [ ("p", "This SOP defines the standard procedures to deploy, operate, administer " "and recover the AgroSense service. It applies to administrators and " "operators running the FastAPI service and the web app."), ]), ("2. Roles and Responsibilities", [ ("ul", ["Operator - starts/stops the service, monitors health, applies config, " "performs backups and incident response.", "Administrator - manages the knowledge base and verifies plant doctors " "(holds the admin token).", "Plant doctor - onboards, gets verified, handles live consultations.", "Farmer (end user) - uses the advisor, plant clinic and consultations."]), ]), ("3. Pre-requisites", [ ("ol", ["Host with Python 3.10+ and network egress for live features.", "pip install -r requirements.txt.", "Environment variables set (see the Technical Guide). At minimum set a " "strong AGROSENSE_ADMIN_TOKEN.", "data/ directory present and writable (knowledge base + doctor registry)."]), ]), ("4. Start, Stop, Restart", [ ("ol", ["Start: uvicorn api.main:app --host 0.0.0.0 --port 8000 " "(use a process manager such as systemd/pm2 in production).", "Verify: open http://HOST:8000/ui/ and GET http://HOST:8000/health.", "Stop: stop the uvicorn process (Ctrl+C or the process manager).", "Restart: stop then start; the engine reloads the knowledge base and " "rebuilds the index at boot."]), ]), ("5. Health Checks and Monitoring", [ ("ul", ["GET /health returns status ok and the knowledge-base document count.", "GET / should redirect (307) to /ui/; /ui/ should return the web app.", "Watch the uvicorn logs for errors; a feature returning 'unavailable' " "indicates a missing key or no network, not a service fault.", "Latency target for the advisor is under 3 seconds; the offline core is " "typically sub-second."]), ]), ("6. Configuration Management", [ ("ol", ["Set configuration via environment variables before starting the " "service (do not hard-code secrets).", "To change a key/setting: update the environment, then restart the " "service.", "Record changes (who/when/what) in your change log."]), ]), ("7. Knowledge Base Management", [ ("ol", ["Back up data/knowledge_base.json before bulk changes.", "Open the web app Admin tab, enter the admin token, click Unlock.", "Add an entry (crop and source are required), or Edit/Delete an existing " "one. Alternatively use POST/PUT/DELETE /admin/kb with the X-Admin-Token " "header.", "Saving rebuilds the live index automatically - verify by asking the " "advisor a question that should hit the new entry."]), ]), ("8. Plant Doctor Onboarding and Verification", [ ("ol", ["A doctor applies via the Plant clinic tab's live-doctor section (or " "POST /doctors/apply).", "Admin opens the Admin tab -> Plant doctor verification, and reviews the " "application: name, specialization, region, languages, contact, " "credentials and the ICAR/registration number.", "Check the credentials and registration number against your records / " "the relevant registry (manual step).", "Click Verify to approve (the doctor enters the directory and can take " "consults) or Reject. Verified doctors are notified on new consults via " "the configured channels."]), ]), ("9. Handling Live Consultations", [ ("ul", ["A consult request is routed to a matching available verified doctor; a " "video room link and chat thread are created and the doctor is notified " "(in-app log always; webhook/email if configured).", "Operators ensure the video backend (Jitsi) is reachable; for production " "use a self-hosted Jitsi via AGROSENSE_VIDEO_BASE."]), ]), ("10. Incident Response", [ ("ul", ["Service down: check the process and logs; restart; confirm /health.", "Web app blank: confirm the server is up and open /ui/ (root redirects).", "A live feature is 'unavailable': verify the relevant key and network " "egress; the core advisor keeps working offline.", "Expired/invalid key: rotate the key in the environment and restart.", "Bad knowledge entry causing wrong answers: edit/delete it in the Admin " "tab (index rebuilds immediately) or restore the KB backup.", "Abusive doctor application or rating: reject the doctor / remove the " "entry; consider enabling authentication and rate limiting."]), ]), ("11. Backup and Recovery", [ ("ol", ["Regularly back up the committed content: data/knowledge_base.json, " "subsidies.json, finance.json and land_records.json.", "Also back up the runtime state: plant_doctors.json (doctor registry + " "ratings), clubs.json (clubs + posts), market_listings.json (trading) and " "finance_applications.json (loan enquiries).", "To recover: stop the service, restore the JSON files from backup, start " "the service; the index rebuilds at boot.", "Keep agri_experts.json under version control; the runtime files above are " "auto-seeded (or created) if missing."]), ]), ("12. Routine Maintenance", [ ("ul", ["Review and update the knowledge base as guidance changes.", "Keep the curated content current: scheme details in subsidies.json, " "loan products in finance.json, and state portals in land_records.json; " "post scheme announcements from the Admin tab as they are released.", "Review the loan enquiries lodged in the Finance tab " "(GET /admin/finance/applications) and follow up.", "Rotate the admin token periodically.", "Review doctor ratings/reviews and trading listings for abuse.", "Update dependencies and re-run the test suite + SPA smoke before " "deploying.", "Regenerate all documents (python scripts/build_docs.py) after " "feature changes."]), ]), ("13. Change Management", [ ("ol", ["Make changes in a non-production environment first.", "Run python tests/test_rag.py and node web/smoke.mjs; run " "scripts/evaluate.py to confirm quality targets.", "Deploy during a low-traffic window; verify /health and key flows.", "If a problem appears, roll back to the previous version and restore " "backed-up data."]), ]), ("14. Compliance and Disclaimer", [ ("p", "AgroSense provides decision support, not professional or medical " "authority. Operators and doctors must ensure recommendations comply with " "local agricultural regulations. Always confirm product names, dosages and " "pre-harvest intervals before advising chemical use."), ]), ] def main() -> int: build_both(TECH_TITLE, TECH_SUBTITLE, TECH_SECTIONS, DOCS_DIR, "AgroSense_Technical_Guide") build_both(SOP_TITLE, SOP_SUBTITLE, SOP_SECTIONS, DOCS_DIR, "AgroSense_SOP") return 0 if __name__ == "__main__": raise SystemExit(main())