Spaces:
Sleeping
Sleeping
| """Generate the AgroSense Technical Guide and Standard Operating Procedure (SOP), | |
| each as Word (.docx) and PDF, into docs/. | |
| pip install python-docx fpdf2 | |
| python scripts/build_tech_docs.py | |
| """ | |
| from __future__ import annotations | |
| import sys | |
| from pathlib import Path | |
| sys.path.insert(0, str(Path(__file__).resolve().parent)) | |
| from docgen import build_both # noqa: E402 | |
| DOCS_DIR = Path(__file__).resolve().parent.parent / "docs" | |
| # --------------------------------------------------------------------------- # | |
| # Technical Guide | |
| # --------------------------------------------------------------------------- # | |
| TECH_TITLE = "AgroSense - Technical Guide" | |
| TECH_SUBTITLE = "Architecture, modules, configuration and deployment - v0.1 (POC)" | |
| TECH_SECTIONS = [ | |
| ("1. System Overview", [ | |
| ("p", "AgroSense is a FastAPI application that also serves a single-page web " | |
| "app. At its core is a Retrieval-Augmented Generation (RAG) engine over " | |
| "an agriculture knowledge base, surrounded by data-source modules " | |
| "(weather, satellite, market, hazards, etc.) and service modules " | |
| "(plant vision, telemedicine, live consultation, doctor registry, " | |
| "knowledge-base admin)."), | |
| ("h2", "Layers"), | |
| ("ul", ["Data layer - static knowledge base plus live sources (Open-Meteo, " | |
| "NASA, data.gov.in, Yahoo Finance, Google News).", | |
| "Knowledge layer - documents embedded into a vector store.", | |
| "Cognitive layer - retrieval + re-ranking + grounded generation, plus " | |
| "decision-fusion advisories and plant vision/telemedicine.", | |
| "Interaction layer - the web app (served by FastAPI), Streamlit UI, CLI.", | |
| "Engagement layer - multilingual, live consultation, notifications."]), | |
| ("h2", "RAG pipeline"), | |
| ("p", "query -> preprocess/translate -> embed -> vector search (top-k) -> " | |
| "CARO-style metadata re-rank + relevance gate -> grounded extractive " | |
| "generation with citations -> optional translation back to the user " | |
| "language."), | |
| ]), | |
| ("2. Technology Stack", [ | |
| ("ul", ["Language/runtime: Python 3.10+ (validated on 3.14).", | |
| "Core: FastAPI + Uvicorn, Pydantic, NumPy.", | |
| "Web app: HTML + CSS + vanilla JS (no build step), served at /ui.", | |
| "Optional accelerators: faiss-cpu (vector search), sentence-transformers " | |
| "(embeddings) - pure-Python fallbacks exist when absent.", | |
| "Optional: TensorFlow (plant-vision CNNs), Earth Engine API (NDVI), " | |
| "Pillow (image heuristic), deep-translator/argostranslate (i18n), " | |
| "python-docx + fpdf2 (these documents).", | |
| "Tests: a self-contained runner (python tests/test_rag.py) plus a " | |
| "jsdom SPA smoke test (node web/smoke.mjs)."]), | |
| ]), | |
| ("3. Repository Layout", [ | |
| ("ul", ["agrosense/ - the library (engine + all modules).", | |
| "agrosense/rag.py - RAGEngine, the single orchestration entry point.", | |
| "agrosense/{embeddings,vector_store,retriever,generator}.py - RAG core.", | |
| "agrosense/{weather,satellite,environment,planetary,hazards}.py - data.", | |
| "agrosense/{fusion,crop_profiles}.py - decision-fusion advisories.", | |
| "agrosense/{vision,telemedicine,ipm}.py - plant clinic.", | |
| "agrosense/{consultation,doctors,notifications}.py - live doctor.", | |
| "agrosense/{clubs,traditional}.py - farmer clubs + traditional/Panchang.", | |
| "agrosense/{subsidies,finance,land_records,trading}.py - schemes, loans, " | |
| "land records, produce marketplace.", | |
| "agrosense/{prices,commodities,news,radio,calendars,translation}.py - misc.", | |
| "agrosense/{kb_admin,evaluation}.py - admin + evaluation.", | |
| "api/main.py - FastAPI app and all endpoints.", | |
| "web/ - the single-page app. ui/ - Streamlit UI. cli.py - CLI.", | |
| "data/ - knowledge_base.json, agri_experts.json, subsidies.json, " | |
| "finance.json, land_records.json, eval_set.json, etc.", | |
| "scripts/ - evaluate, train_plant_models, build_manual, build_tech_docs, " | |
| "docgen (shared Word/PDF renderer), build_docs (regenerate all).", | |
| "tests/ - test_rag.py (logic), test_ui_render.py (Streamlit render)."]), | |
| ]), | |
| ("4. The RAG Engine", [ | |
| ("p", "RAGEngine (agrosense/rag.py) builds the embedder, generator, documents, " | |
| "vector store and retriever on init, and owns every feature method. " | |
| "Embeddings default to an offline hashing embedder; the vector store " | |
| "defaults to a NumPy cosine store; both upgrade automatically when the " | |
| "optional libraries are installed."), | |
| ("ul", ["Retrieval: vector search returns top-k; a CARO-style pass re-weights " | |
| "by metadata overlap (crop, soil, region) and lexical overlap.", | |
| "Relevance gate: a candidate must share a content term with the query " | |
| "(matched against field values, not labels) - off-domain queries hit a " | |
| "safe fallback instead of fabricating advice.", | |
| "Generation: an extractive generator composes the answer ONLY from " | |
| "retrieved KB fields and attaches numbered citations (low hallucination " | |
| "by construction). An OpenAI/Azure seam exists for a real LLM.", | |
| "reload_kb() rebuilds documents + store + retriever after admin edits."]), | |
| ]), | |
| ("5. Knowledge Base", [ | |
| ("p", "data/knowledge_base.json is a list of entries (crop, soil_type, " | |
| "rainfall_mm, recommended_fertilizer, disease_prevention, pest_management, " | |
| "source, ...). agrosense/kb_admin.py provides a validated CRUD store " | |
| "(KBStore) used by the admin endpoints; saving rewrites the file and the " | |
| "engine rebuilds the index so changes are live."), | |
| ]), | |
| ("6. Modules and Data Sources", [ | |
| ("ul", ["Keyless: weather/environment (Open-Meteo), satellite imagery + " | |
| "agroclimate (NASA GIBS/POWER), hazards (NASA EONET), planetary " | |
| "(local ephemeris), news (Google News), commodities (Yahoo Finance), " | |
| "internet radio (Radio Browser).", | |
| "Keyed: prices + groundwater (data.gov.in), fires (NASA FIRMS), " | |
| "field NDVI (Earth Engine).", | |
| "Curated knowledge bases (no live feed exists): government schemes " | |
| "(subsidies.json), agri-finance products (finance.json), state land-record " | |
| "systems (land_records.json). The land-record guide is rendered to Word/PDF " | |
| "on the fly via scripts/docgen.py.", | |
| "User-generated runtime stores (auto-seeded / gitignored): farmer clubs " | |
| "(clubs.json), produce listings (market_listings.json), loan enquiries " | |
| "(finance_applications.json), doctor registry (plant_doctors.json).", | |
| "Each client degrades gracefully (returns None/empty) when offline or " | |
| "unconfigured; parsing/geometry logic is pure and unit-tested, network " | |
| "calls are mock-tested."]), | |
| ]), | |
| ("7. API Reference", [ | |
| ("ul", ["Advisor: POST /query.", | |
| "Location: GET /weather /satellite /environment /planetary /hazards " | |
| "/advisories.", | |
| "Market/info: GET /prices /commodities /news /radio /datetime /languages.", | |
| "Trading: GET/POST /market/listings, GET /market/listings/{id}, " | |
| "POST /market/listings/{id}/inquire, POST /market/listings/{id}/close.", | |
| "Plant clinic: POST /vision/classify, POST /telemedicine.", | |
| "Doctors: GET /experts, GET /doctors/{id}, POST /doctors/{id}/rate, " | |
| "POST /doctors/apply.", | |
| "Consults: POST /consult/request, GET/POST /consult/{id}[/message], " | |
| "GET /notifications.", | |
| "Clubs: GET/POST /clubs, GET /clubs/{id}, POST /clubs/{id}/join|post.", | |
| "Traditional: GET /traditional.", | |
| "Schemes: GET /subsidies[/{id}], GET /subsidies/updates.", | |
| "Finance: GET /finance[/{id}], POST /finance/{id}/apply.", | |
| "Land records: GET /land-records, GET /land-records/guide.{pdf,docx}.", | |
| "Admin (X-Admin-Token): /admin/kb (CRUD), /admin/doctors, " | |
| "/admin/doctors/{id}/verify, /admin/subsidies/{id}/update, " | |
| "/admin/finance/applications.", | |
| "Docs: GET /downloads/{user-manual,technical-guide,sop}.{pdf,docx}.", | |
| "App: GET / -> /ui/ ; interactive API docs at /docs."]), | |
| ]), | |
| ("8. Configuration (environment variables)", [ | |
| ("ul", ["AGROSENSE_EMBEDDING_BACKEND - auto | sentence-transformers | hashing.", | |
| "AGROSENSE_GENERATION_BACKEND - extractive | openai.", | |
| "AGROSENSE_ADMIN_TOKEN - admin gate (change the default 'admin').", | |
| "AGROSENSE_DATAGOV_API_KEY - prices + groundwater.", | |
| "AGROSENSE_GROUNDWATER_RESOURCE - CGWB resource id.", | |
| "AGROSENSE_FIRMS_MAP_KEY - active fires.", | |
| "AGROSENSE_EE_PROJECT / _EE_SERVICE_ACCOUNT / _EE_KEY_FILE - NDVI.", | |
| "AGROSENSE_VISION_{DISEASE,PLANT,PEST}_MODEL / _LABELS - vision models.", | |
| "AGROSENSE_TRANSLATION_BACKEND - auto | argos | deep | none.", | |
| "AGROSENSE_NOTIFY_WEBHOOK / SMTP vars - expert notifications.", | |
| "AGROSENSE_VIDEO_BASE - Jitsi base for live video.", | |
| "AGROSENSE_KB_PATH - knowledge base path override."]), | |
| ]), | |
| ("9. Extending AgroSense", [ | |
| ("ul", ["Add KB knowledge: use the Admin tab or POST /admin/kb (or edit the " | |
| "JSON and call reload).", | |
| "Add a data source: create a module with a pure parser + a client that " | |
| "returns None on failure; add an engine method and an endpoint; unit-" | |
| "test the parser and mock the fetch.", | |
| "Add a vision model: train with scripts/train_plant_models.py and set " | |
| "the model env vars; inference activates automatically.", | |
| "Add an expert specialization: extend the tag map in doctors.py and the " | |
| "routing map in consultation.py."]), | |
| ]), | |
| ("10. Testing and Quality", [ | |
| ("ul", ["Logic suite: python tests/test_rag.py (offline; set " | |
| "AGROSENSE_EMBEDDING_BACKEND=hashing for determinism).", | |
| "Evaluation harness: python scripts/evaluate.py - context relevance, " | |
| "faithfulness, hallucination rate, latency vs targets.", | |
| "SPA smoke: node web/smoke.mjs (jsdom; mocks fetch; asserts no runtime " | |
| "errors and that sections render).", | |
| "Streamlit render: python tests/test_ui_render.py."]), | |
| ]), | |
| ("11. Deployment", [ | |
| ("ol", ["Provision a host with Python 3.10+; pip install -r requirements.txt.", | |
| "Set required environment variables (at least change AGROSENSE_ADMIN_TOKEN).", | |
| "Run: uvicorn api.main:app --host 0.0.0.0 --port 8000 (add --workers N " | |
| "behind a process manager; put a reverse proxy / TLS in front).", | |
| "Persist data/ (knowledge_base.json, plant_doctors.json) on durable " | |
| "storage; back it up regularly.", | |
| "Generate docs once (python scripts/build_manual.py ; " | |
| "python scripts/build_tech_docs.py)."]), | |
| ("p", "Note: engine state (the consultation/notification stores and the doctor " | |
| "registry) is in-process and per-worker; for multi-worker or multi-host " | |
| "deployments move that state to a shared database."), | |
| ]), | |
| ("12. Security", [ | |
| ("ul", ["Admin endpoints use a shared-secret token (X-Admin-Token) - suitable " | |
| "for a single-admin POC; use real auth (accounts, roles, audit) in " | |
| "production.", | |
| "CORS is open by default - restrict allowed origins for a real deployment.", | |
| "Onboarding and ratings are unauthenticated - add accounts and rate " | |
| "limiting to prevent spam/abuse.", | |
| "Keep API keys in environment variables / a secrets manager, never in " | |
| "code. Validate and size-limit uploaded images."]), | |
| ]), | |
| ] | |
| # --------------------------------------------------------------------------- # | |
| # Standard Operating Procedure | |
| # --------------------------------------------------------------------------- # | |
| SOP_TITLE = "AgroSense - Standard Operating Procedure (SOP)" | |
| SOP_SUBTITLE = "Operational runbook for deploying, running and administering AgroSense" | |
| SOP_SECTIONS = [ | |
| ("1. Purpose and Scope", [ | |
| ("p", "This SOP defines the standard procedures to deploy, operate, administer " | |
| "and recover the AgroSense service. It applies to administrators and " | |
| "operators running the FastAPI service and the web app."), | |
| ]), | |
| ("2. Roles and Responsibilities", [ | |
| ("ul", ["Operator - starts/stops the service, monitors health, applies config, " | |
| "performs backups and incident response.", | |
| "Administrator - manages the knowledge base and verifies plant doctors " | |
| "(holds the admin token).", | |
| "Plant doctor - onboards, gets verified, handles live consultations.", | |
| "Farmer (end user) - uses the advisor, plant clinic and consultations."]), | |
| ]), | |
| ("3. Pre-requisites", [ | |
| ("ol", ["Host with Python 3.10+ and network egress for live features.", | |
| "pip install -r requirements.txt.", | |
| "Environment variables set (see the Technical Guide). At minimum set a " | |
| "strong AGROSENSE_ADMIN_TOKEN.", | |
| "data/ directory present and writable (knowledge base + doctor registry)."]), | |
| ]), | |
| ("4. Start, Stop, Restart", [ | |
| ("ol", ["Start: uvicorn api.main:app --host 0.0.0.0 --port 8000 " | |
| "(use a process manager such as systemd/pm2 in production).", | |
| "Verify: open http://HOST:8000/ui/ and GET http://HOST:8000/health.", | |
| "Stop: stop the uvicorn process (Ctrl+C or the process manager).", | |
| "Restart: stop then start; the engine reloads the knowledge base and " | |
| "rebuilds the index at boot."]), | |
| ]), | |
| ("5. Health Checks and Monitoring", [ | |
| ("ul", ["GET /health returns status ok and the knowledge-base document count.", | |
| "GET / should redirect (307) to /ui/; /ui/ should return the web app.", | |
| "Watch the uvicorn logs for errors; a feature returning 'unavailable' " | |
| "indicates a missing key or no network, not a service fault.", | |
| "Latency target for the advisor is under 3 seconds; the offline core is " | |
| "typically sub-second."]), | |
| ]), | |
| ("6. Configuration Management", [ | |
| ("ol", ["Set configuration via environment variables before starting the " | |
| "service (do not hard-code secrets).", | |
| "To change a key/setting: update the environment, then restart the " | |
| "service.", | |
| "Record changes (who/when/what) in your change log."]), | |
| ]), | |
| ("7. Knowledge Base Management", [ | |
| ("ol", ["Back up data/knowledge_base.json before bulk changes.", | |
| "Open the web app Admin tab, enter the admin token, click Unlock.", | |
| "Add an entry (crop and source are required), or Edit/Delete an existing " | |
| "one. Alternatively use POST/PUT/DELETE /admin/kb with the X-Admin-Token " | |
| "header.", | |
| "Saving rebuilds the live index automatically - verify by asking the " | |
| "advisor a question that should hit the new entry."]), | |
| ]), | |
| ("8. Plant Doctor Onboarding and Verification", [ | |
| ("ol", ["A doctor applies via the Plant clinic tab's live-doctor section (or " | |
| "POST /doctors/apply).", | |
| "Admin opens the Admin tab -> Plant doctor verification, and reviews the " | |
| "application: name, specialization, region, languages, contact, " | |
| "credentials and the ICAR/registration number.", | |
| "Check the credentials and registration number against your records / " | |
| "the relevant registry (manual step).", | |
| "Click Verify to approve (the doctor enters the directory and can take " | |
| "consults) or Reject. Verified doctors are notified on new consults via " | |
| "the configured channels."]), | |
| ]), | |
| ("9. Handling Live Consultations", [ | |
| ("ul", ["A consult request is routed to a matching available verified doctor; a " | |
| "video room link and chat thread are created and the doctor is notified " | |
| "(in-app log always; webhook/email if configured).", | |
| "Operators ensure the video backend (Jitsi) is reachable; for production " | |
| "use a self-hosted Jitsi via AGROSENSE_VIDEO_BASE."]), | |
| ]), | |
| ("10. Incident Response", [ | |
| ("ul", ["Service down: check the process and logs; restart; confirm /health.", | |
| "Web app blank: confirm the server is up and open /ui/ (root redirects).", | |
| "A live feature is 'unavailable': verify the relevant key and network " | |
| "egress; the core advisor keeps working offline.", | |
| "Expired/invalid key: rotate the key in the environment and restart.", | |
| "Bad knowledge entry causing wrong answers: edit/delete it in the Admin " | |
| "tab (index rebuilds immediately) or restore the KB backup.", | |
| "Abusive doctor application or rating: reject the doctor / remove the " | |
| "entry; consider enabling authentication and rate limiting."]), | |
| ]), | |
| ("11. Backup and Recovery", [ | |
| ("ol", ["Regularly back up the committed content: data/knowledge_base.json, " | |
| "subsidies.json, finance.json and land_records.json.", | |
| "Also back up the runtime state: plant_doctors.json (doctor registry + " | |
| "ratings), clubs.json (clubs + posts), market_listings.json (trading) and " | |
| "finance_applications.json (loan enquiries).", | |
| "To recover: stop the service, restore the JSON files from backup, start " | |
| "the service; the index rebuilds at boot.", | |
| "Keep agri_experts.json under version control; the runtime files above are " | |
| "auto-seeded (or created) if missing."]), | |
| ]), | |
| ("12. Routine Maintenance", [ | |
| ("ul", ["Review and update the knowledge base as guidance changes.", | |
| "Keep the curated content current: scheme details in subsidies.json, " | |
| "loan products in finance.json, and state portals in land_records.json; " | |
| "post scheme announcements from the Admin tab as they are released.", | |
| "Review the loan enquiries lodged in the Finance tab " | |
| "(GET /admin/finance/applications) and follow up.", | |
| "Rotate the admin token periodically.", | |
| "Review doctor ratings/reviews and trading listings for abuse.", | |
| "Update dependencies and re-run the test suite + SPA smoke before " | |
| "deploying.", | |
| "Regenerate all documents (python scripts/build_docs.py) after " | |
| "feature changes."]), | |
| ]), | |
| ("13. Change Management", [ | |
| ("ol", ["Make changes in a non-production environment first.", | |
| "Run python tests/test_rag.py and node web/smoke.mjs; run " | |
| "scripts/evaluate.py to confirm quality targets.", | |
| "Deploy during a low-traffic window; verify /health and key flows.", | |
| "If a problem appears, roll back to the previous version and restore " | |
| "backed-up data."]), | |
| ]), | |
| ("14. Compliance and Disclaimer", [ | |
| ("p", "AgroSense provides decision support, not professional or medical " | |
| "authority. Operators and doctors must ensure recommendations comply with " | |
| "local agricultural regulations. Always confirm product names, dosages and " | |
| "pre-harvest intervals before advising chemical use."), | |
| ]), | |
| ] | |
| def main() -> int: | |
| build_both(TECH_TITLE, TECH_SUBTITLE, TECH_SECTIONS, DOCS_DIR, | |
| "AgroSense_Technical_Guide") | |
| build_both(SOP_TITLE, SOP_SUBTITLE, SOP_SECTIONS, DOCS_DIR, "AgroSense_SOP") | |
| return 0 | |
| if __name__ == "__main__": | |
| raise SystemExit(main()) | |