sync processor from GitLab 6d86455: 版本 1.8.6 —— systemd 单元入库(僵尸挂载自愈 + 看门狗)

processor/app/version.py

@@ -33,4 +33,9 @@
 # 1.8.5 顺带删掉 renderGroupedWorkspaceLegacy(187 行死代码,全仓库零调用,
 #       里面还留着一份同样的 × 按钮)。它不渲染,删它纯粹是防止将来有人
 #       复活它把 × 带回来 —— 用户可见行为与 1.8.4 完全相同
-__version__ = "1.8.5"
+# 1.8.6 部署加固:systemd 单元入库(scripts/systemd/host-units/,含安装脚本与
+#       说明)。修「换网络/休眠后整套服务静默挂掉」:挂载加僵尸态自愈
+#       (启动前 fusermount3 -u -z)、API/worker 改 Requires+After 挂载、
+#       Restart 一律 always;另加每分钟看门狗自动重挂/重启。纯运维配置,
+#       无代码行为与用户可见变化
+__version__ = "1.8.6"

processor/scripts/systemd/host-units/README.md

@@ -0,0 +1,47 @@
+# 生产机 systemd 用户单元(实际运行配置)
+
+本目录是生产机 `192.168.110.58` 上**实际在跑**的那套 systemd 用户单元,入库是为了
+换机/重装时可直接复用。与上级目录的 `*.service.in`(deploy.py 的模板,另一套命名
+`egodata-backend` / `egodata-workers`)并存,互不影响 —— 当前运行的是本目录这套。
+
+## 包含什么
+
+| 文件 | 作用 |
+|---|---|
+| `egodata-storage.service` | sshfs 挂载 NAS 数据(`~/.cache/egodata-data`)。含**僵尸挂载自愈**:启动前先 `fusermount3 -u -z` 清理残留,再重挂 |
+| `egodata-api.service` | uvicorn 后端 + 前端网页(同一进程,0.0.0.0:8000)。`Requires`+`After` 挂载,挂载没就绪不会启动 |
+| `egodata-worker.service` | 异步处理 worker(领工作流任务) |
+| `egodata-watchdog.sh` + `.service` + `.timer` | 每分钟巡检:挂载掉了重挂、API 不健康重启。启动中(activating)跳过、连续 2 次不健康才动手,避开 API 冷启约 20s 的误判 |
+| `install.sh` | 一键安装到 `~/.config/systemd/user/` + `~/.local/bin/` |
+
+## 为什么这样配(踩过的坑)
+
+1. **换网络/休眠后服务全挂**:sshfs 断线 → 挂载单元失败 → API 启动时扫描数据目录撞上
+   死挂载崩溃 → systemd 因"正常退出码"不重启。已用 `Restart=always` + `Requires=` 修正。
+2. **僵尸挂载**(`Transport endpoint is not connected`):断线后挂载点残留,`mkdir`/`sshfs`
+   全失败,重启重试 73 次也挂不上。已用 `ExecStartPre=fusermount3 -u -z` 自愈。
+3. **API 冷启约 20s**:看门狗若在这期间判为宕机就会反复重启,故加 `activating` 跳过 +
+   连续 2 次阈值。
+
+## 安装
+
+```bash
+bash install.sh
+```
+
+依赖:
+- `sshfs`(`processor/.tools/sshfs/usr/bin/sshfs`,单元里写死此路径)
+- NAS 的 SSH 私钥 `~/.ssh/id_rsa`
+- `~/.config/egodata-worker-env`(含 `EGODATA_WORKER_API_KEY`,**不入库**,install.sh 会
+  生成占位,取值见 `processor/.env`)
+
+## 常用操作
+
+```bash
+systemctl --user status egodata-api          # 状态
+journalctl --user -u egodata-api -f          # 实时日志
+systemctl --user restart egodata-api         # 重启后端(前端同一个进程)
+systemctl --user list-timers egodata-watchdog.timer   # 看门狗巡检计划
+```
+
+> 注意:前端与后端是**同一个服务**(uvicorn 同时提供 API 和网页模板),没有独立的前端进程。

processor/scripts/systemd/host-units/egodata-api.service

@@ -0,0 +1,18 @@
+[Unit]
+Description=EgoData API (uvicorn, port 8000)
+After=network-online.target egodata-storage.service
+# 数据在 NAS 上:挂载没就绪就别启动(否则启动扫描会撞上死挂载直接崩)
+Requires=egodata-storage.service
+Wants=network-online.target
+StartLimitIntervalSec=0
+
+[Service]
+Type=simple
+WorkingDirectory=/home/stouching/Desktop/Date-V/processor
+ExecStart=/home/stouching/Desktop/Date-V/processor/.venv-linux/bin/python -m uvicorn app.main:app --host 0.0.0.0 --port 8000
+Restart=always
+RestartSec=5
+Environment=PYTHONUNBUFFERED=1
+
+[Install]
+WantedBy=default.target

processor/scripts/systemd/host-units/egodata-storage.service

@@ -0,0 +1,22 @@
+[Unit]
+Description=EgoData remote data storage mount
+After=network-online.target
+Wants=network-online.target
+Before=egodata-backend.service egodata-api.service egodata-worker.service
+# 挂载是整条链路的地基:网络/换网/休眠导致断线时不要放弃重试,一直试到挂上为止
+StartLimitIntervalSec=0
+
+[Service]
+Type=forking
+# 断线后挂载点可能残留成"僵尸挂载"(Transport endpoint is not connected):
+# 此时 mkdir/stat 都会失败,重试再多次也挂不上。先强制(惰性)卸载清掉残留,
+# 失败用 "-" 前缀忽略(本来就没挂载时 fusermount 会返回非零)。
+ExecStartPre=-/usr/bin/fusermount3 -u -z /home/stouching/.cache/egodata-data
+ExecStartPre=/usr/bin/mkdir -p /home/stouching/.cache/egodata-data
+ExecStart="/home/stouching/Desktop/Date-V/processor/.tools/sshfs/usr/bin/sshfs" "Stouch@192.168.110.41:/vol1/docker/odoo-dev/Data Acquisition/data" /home/stouching/.cache/egodata-data -o IdentityFile=/home/stouching/.ssh/id_rsa -o StrictHostKeyChecking=no -o reconnect -o ServerAliveInterval=15 -o ServerAliveCountMax=3 -o idmap=user -o cache=yes -o cache_timeout=10
+ExecStop=/usr/bin/fusermount3 -u -z /home/stouching/.cache/egodata-data
+Restart=always
+RestartSec=10
+
+[Install]
+WantedBy=default.target

processor/scripts/systemd/host-units/egodata-watchdog.service

@@ -0,0 +1,7 @@
+[Unit]
+Description=EgoData watchdog (storage mount + API health)
+
+[Service]
+Type=oneshot
+ExecStart=/home/stouching/.local/bin/egodata-watchdog.sh
+Environment=XDG_RUNTIME_DIR=/run/user/1000

processor/scripts/systemd/host-units/egodata-watchdog.sh

@@ -0,0 +1,47 @@
+#!/bin/bash
+# EgoData 看门狗:挂载掉了重挂,API 掉了重启。
+# 由 egodata-watchdog.timer 每分钟调用一次;只做本地检查(不读 NAS 数据),
+# 开销可忽略。systemctl --user 在用户单元里可用,输出进 journal。
+set -u
+
+MOUNT=/home/stouching/.cache/egodata-data
+API_URL=http://127.0.0.1:8000/health
+STATE_FILE=/run/user/1000/egodata-watchdog.fails
+FAIL_THRESHOLD=2      # 连续 2 次不健康才动手,避开"启动中"的误判
+
+# 1) 挂载在不在(mountpoint 查内核挂载表,零网络开销)。
+#    断线可能留下"僵尸挂载"(Transport endpoint is not connected):
+#    mountpoint 判为未挂载,但残留条目会让 mkdir/sshfs 全部失败 ——
+#    先强制卸载清掉,再重启挂载单元(单元里也有同样的 ExecStartPre 兜底)。
+if ! mountpoint -q "$MOUNT"; then
+    echo "storage 未挂载,清理残留并重启 egodata-storage"
+    fusermount3 -u -z "$MOUNT" 2>/dev/null
+    systemctl --user reset-failed egodata-storage.service 2>/dev/null
+    systemctl --user restart egodata-storage.service
+    sleep 5
+fi
+
+# 2) API 健不健康。启动中(activating)直接放行 —— API 冷启要约 20s,
+#    正好落在两次巡检之间会被误判成宕机并反复重启。
+state=$(systemctl --user is-active egodata-api.service 2>/dev/null)
+if [ "$state" = "activating" ] || [ "$state" = "deactivating" ]; then
+    echo "API 正在启动/停止中($state),跳过本次巡检"
+    exit 0
+fi
+
+code=$(curl -s -o /dev/null -w '%{http_code}' --max-time 10 "$API_URL" 2>/dev/null)
+if [ "$code" = "200" ]; then
+    rm -f "$STATE_FILE"
+    exit 0
+fi
+
+fails=$(( $(cat "$STATE_FILE" 2>/dev/null || echo 0) + 1 ))
+echo "$fails" > "$STATE_FILE"
+if [ "$fails" -lt "$FAIL_THRESHOLD" ]; then
+    echo "API health=$code(连续第 $fails 次),再观察一轮"
+    exit 0
+fi
+
+echo "API health=$code(连续 $fails 次),重启 egodata-api"
+rm -f "$STATE_FILE"
+systemctl --user restart egodata-api.service

processor/scripts/systemd/host-units/egodata-watchdog.timer

@@ -0,0 +1,10 @@
+[Unit]
+Description=Run EgoData watchdog every minute
+
+[Timer]
+OnBootSec=3min
+OnUnitActiveSec=1min
+AccuracySec=10s
+
+[Install]
+WantedBy=timers.target

processor/scripts/systemd/host-units/egodata-worker.service

@@ -0,0 +1,17 @@
+[Unit]
+Description=EgoData processing worker
+After=network-online.target egodata-api.service egodata-storage.service
+Requires=egodata-storage.service
+StartLimitIntervalSec=0
+
+[Service]
+Type=simple
+WorkingDirectory=/home/stouching/Desktop/Date-V/processor
+EnvironmentFile=%h/.config/egodata-worker-env
+ExecStart=/home/stouching/Desktop/Date-V/processor/.venv-linux/bin/python -m worker
+Restart=always
+RestartSec=5
+Environment=PYTHONUNBUFFERED=1
+
+[Install]
+WantedBy=default.target

processor/scripts/systemd/host-units/install.sh

@@ -0,0 +1,40 @@
+#!/usr/bin/env bash
+# 安装本目录下的 systemd 用户单元(生产机实际配置)。
+#
+# 覆盖内容:NAS 挂载(egodata-storage)、API(egodata-api)、worker(egodata-worker)、
+# 看门狗(每分钟巡检挂载+API 并自动恢复)。
+#
+# 用法:bash install.sh
+# 卸载:systemctl --user disable --now egodata-storage egodata-api egodata-worker egodata-watchdog.timer
+set -euo pipefail
+
+HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+UNIT_DIR="${HOME}/.config/systemd/user"
+BIN_DIR="${HOME}/.local/bin"
+
+mkdir -p "${UNIT_DIR}" "${BIN_DIR}"
+
+install -m 644 "${HERE}"/egodata-storage.service \
+               "${HERE}"/egodata-api.service \
+               "${HERE}"/egodata-worker.service \
+               "${HERE}"/egodata-watchdog.service \
+               "${HERE}"/egodata-watchdog.timer "${UNIT_DIR}/"
+install -m 755 "${HERE}/egodata-watchdog.sh" "${BIN_DIR}/egodata-watchdog.sh"
+
+# worker 的 API key 文件(含密钥,故意不入库):不存在时建占位,需人工填。
+ENV_FILE="${HOME}/.config/egodata-worker-env"
+if [ ! -f "${ENV_FILE}" ]; then
+    umask 077
+    echo "EGODATA_WORKER_API_KEY=change-me" > "${ENV_FILE}"
+    chmod 600 "${ENV_FILE}"
+    echo "⚠️  已生成 ${ENV_FILE} 占位,请填入真实 WORKER_API_KEY(取值见 processor/.env)"
+fi
+
+systemctl --user daemon-reload
+systemctl --user enable --now egodata-storage.service \
+                            egodata-api.service \
+                            egodata-worker.service \
+                            egodata-watchdog.timer
+
+echo "✅ 安装完成。状态:"
+systemctl --user is-active egodata-storage egodata-api egodata-worker egodata-watchdog.timer