Upload folder using huggingface_hub
Browse filesThis view is limited to 50 files because it contains too many changes. See raw diff
- .dockerignore +28 -0
- .gitattributes +36 -0
- .gitignore +29 -0
- .hfignore +14 -0
- Agent.md +99 -0
- CHANGELOG.md +19 -0
- Dockerfile +34 -0
- README.md +62 -0
- config/__init__.py +0 -0
- config/examples/detector-only.yaml +23 -0
- config/examples/disabled.yaml +17 -0
- config/examples/hybrid.yaml +84 -0
- config/examples/laya-nvidia-hybrid.yaml +47 -0
- config/examples/laya-only.yaml +30 -0
- config/examples/laya-pending.yaml +22 -0
- config/examples/laya-vlm-fallback.yaml +42 -0
- config/examples/nvidia-laya-sequential.yaml +30 -0
- config/examples/vlm-only.yaml +41 -0
- config/schema.py +141 -0
- deploy/backup-repository/inactive-workflows/backup-12h.yml +26 -0
- deploy/backup-repository/inactive-workflows/qualify-private-staging.yml +25 -0
- deploy/hf-staging/README.md +67 -0
- diagnostics/README.md +179 -0
- diagnostics/qualify_github_release.py +218 -0
- diagnostics/tests/test_release_probe.py +95 -0
- docs/backup-restore.md +167 -0
- docs/credential-qualification.md +106 -0
- docs/deployment-hf.md +118 -0
- docs/evidence/backend-tests.txt +9 -0
- docs/evidence/browser-administration.txt +14 -0
- docs/evidence/browser-regression.txt +29 -0
- docs/evidence/credential-qualification.json +47 -0
- docs/evidence/dedicated-token-api.json +28 -0
- docs/evidence/github-auth-probe.json +79 -0
- docs/evidence/github-backup-results.json +55 -0
- docs/evidence/github-recovery-live.json +132 -0
- docs/evidence/github-recovery-prior-blocked.json +115 -0
- docs/evidence/github-upload-client-comparison.json +32 -0
- docs/evidence/hf-space-observation.json +17 -0
- docs/evidence/live-qualification-backend-tests.txt +9 -0
- docs/evidence/private-staging-live.json +8 -0
- docs/evidence/standalone-probes/github-release-qualification-20261017T223752Z-a13651aa74504b7ca9a9a2f1f83e0100.json +44 -0
- docs/github-backup-migration.md +294 -0
- docs/original-research.md +129 -0
- docs/providers.md +53 -0
- docs/v2-evidence/backend-tests.txt +9 -0
- docs/v2-evidence/browser-tests.txt +33 -0
- docs/v2-evidence/build.txt +120 -0
- docs/v2-evidence/container.json +22 -0
- docs/v2-evidence/laya-live.json +70 -0
.dockerignore
ADDED
|
@@ -0,0 +1,28 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
.git
|
| 2 |
+
.venv
|
| 3 |
+
**/__pycache__
|
| 4 |
+
.pytest_cache
|
| 5 |
+
web/node_modules
|
| 6 |
+
web/.next
|
| 7 |
+
serve
|
| 8 |
+
sams-videos
|
| 9 |
+
data
|
| 10 |
+
.env*
|
| 11 |
+
*.pem
|
| 12 |
+
*.log
|
| 13 |
+
|
| 14 |
+
web/test-results
|
| 15 |
+
web/playwright-report
|
| 16 |
+
|
| 17 |
+
web/tests
|
| 18 |
+
web/playwright.config.ts
|
| 19 |
+
# Local credentials must stay out of every build stage, including nested web files.
|
| 20 |
+
**/.env*
|
| 21 |
+
**/*.pem
|
| 22 |
+
**/*.age
|
| 23 |
+
**/*.agekey
|
| 24 |
+
**/*.key
|
| 25 |
+
**/rclone.conf
|
| 26 |
+
backups
|
| 27 |
+
snapshots
|
| 28 |
+
docs
|
.gitattributes
CHANGED
|
@@ -35,3 +35,39 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
|
|
| 35 |
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
| 36 |
or-logo1.png filter=lfs diff=lfs merge=lfs -text
|
| 37 |
lambda.png filter=lfs diff=lfs merge=lfs -text
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 35 |
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
| 36 |
or-logo1.png filter=lfs diff=lfs merge=lfs -text
|
| 37 |
lambda.png filter=lfs diff=lfs merge=lfs -text
|
| 38 |
+
docs/v2-evidence/main/architecture-desktop-linux.png filter=lfs diff=lfs merge=lfs -text
|
| 39 |
+
docs/v2-evidence/main/architecture-laptop-linux.png filter=lfs diff=lfs merge=lfs -text
|
| 40 |
+
docs/v2-evidence/main/architecture-mobile-linux.png filter=lfs diff=lfs merge=lfs -text
|
| 41 |
+
docs/v2-evidence/main/architecture-tablet-linux.png filter=lfs diff=lfs merge=lfs -text
|
| 42 |
+
docs/v2-evidence/main/experiments-desktop-linux.png filter=lfs diff=lfs merge=lfs -text
|
| 43 |
+
docs/v2-evidence/main/experiments-laptop-linux.png filter=lfs diff=lfs merge=lfs -text
|
| 44 |
+
docs/v2-evidence/main/experiments-mobile-linux.png filter=lfs diff=lfs merge=lfs -text
|
| 45 |
+
docs/v2-evidence/main/experiments-tablet-linux.png filter=lfs diff=lfs merge=lfs -text
|
| 46 |
+
docs/v2-evidence/main/library-desktop-linux.png filter=lfs diff=lfs merge=lfs -text
|
| 47 |
+
docs/v2-evidence/main/library-laptop-linux.png filter=lfs diff=lfs merge=lfs -text
|
| 48 |
+
docs/v2-evidence/main/library-mobile-linux.png filter=lfs diff=lfs merge=lfs -text
|
| 49 |
+
docs/v2-evidence/main/library-tablet-linux.png filter=lfs diff=lfs merge=lfs -text
|
| 50 |
+
docs/v2-evidence/main/search-desktop-linux.png filter=lfs diff=lfs merge=lfs -text
|
| 51 |
+
docs/v2-evidence/main/search-laptop-linux.png filter=lfs diff=lfs merge=lfs -text
|
| 52 |
+
docs/v2-evidence/main/search-mobile-linux.png filter=lfs diff=lfs merge=lfs -text
|
| 53 |
+
docs/v2-evidence/main/search-tablet-linux.png filter=lfs diff=lfs merge=lfs -text
|
| 54 |
+
web/tests/parity.spec.ts-snapshots/architecture-desktop-linux.png filter=lfs diff=lfs merge=lfs -text
|
| 55 |
+
web/tests/parity.spec.ts-snapshots/architecture-laptop-linux.png filter=lfs diff=lfs merge=lfs -text
|
| 56 |
+
web/tests/parity.spec.ts-snapshots/architecture-mobile-linux.png filter=lfs diff=lfs merge=lfs -text
|
| 57 |
+
web/tests/parity.spec.ts-snapshots/architecture-tablet-linux.png filter=lfs diff=lfs merge=lfs -text
|
| 58 |
+
web/tests/parity.spec.ts-snapshots/experiments-desktop-linux.png filter=lfs diff=lfs merge=lfs -text
|
| 59 |
+
web/tests/parity.spec.ts-snapshots/experiments-laptop-linux.png filter=lfs diff=lfs merge=lfs -text
|
| 60 |
+
web/tests/parity.spec.ts-snapshots/experiments-mobile-linux.png filter=lfs diff=lfs merge=lfs -text
|
| 61 |
+
web/tests/parity.spec.ts-snapshots/experiments-tablet-linux.png filter=lfs diff=lfs merge=lfs -text
|
| 62 |
+
web/tests/parity.spec.ts-snapshots/library-desktop-linux.png filter=lfs diff=lfs merge=lfs -text
|
| 63 |
+
web/tests/parity.spec.ts-snapshots/library-laptop-linux.png filter=lfs diff=lfs merge=lfs -text
|
| 64 |
+
web/tests/parity.spec.ts-snapshots/library-mobile-linux.png filter=lfs diff=lfs merge=lfs -text
|
| 65 |
+
web/tests/parity.spec.ts-snapshots/library-tablet-linux.png filter=lfs diff=lfs merge=lfs -text
|
| 66 |
+
web/tests/parity.spec.ts-snapshots/search-desktop-linux.png filter=lfs diff=lfs merge=lfs -text
|
| 67 |
+
web/tests/parity.spec.ts-snapshots/search-laptop-linux.png filter=lfs diff=lfs merge=lfs -text
|
| 68 |
+
web/tests/parity.spec.ts-snapshots/search-mobile-linux.png filter=lfs diff=lfs merge=lfs -text
|
| 69 |
+
web/tests/parity.spec.ts-snapshots/search-tablet-linux.png filter=lfs diff=lfs merge=lfs -text
|
| 70 |
+
web/tests/studio.spec.ts-snapshots/live-desktop-linux.png filter=lfs diff=lfs merge=lfs -text
|
| 71 |
+
web/tests/studio.spec.ts-snapshots/live-mobile-linux.png filter=lfs diff=lfs merge=lfs -text
|
| 72 |
+
web/tests/studio.spec.ts-snapshots/models-desktop-linux.png filter=lfs diff=lfs merge=lfs -text
|
| 73 |
+
web/tests/studio.spec.ts-snapshots/models-mobile-linux.png filter=lfs diff=lfs merge=lfs -text
|
.gitignore
ADDED
|
@@ -0,0 +1,29 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
.venv/
|
| 2 |
+
serve/.venv/
|
| 3 |
+
serve/logs/
|
| 4 |
+
__pycache__/
|
| 5 |
+
data/
|
| 6 |
+
sams-videos/
|
| 7 |
+
.research/
|
| 8 |
+
web/node_modules/
|
| 9 |
+
web/.next/
|
| 10 |
+
*.log
|
| 11 |
+
|
| 12 |
+
# OS / editor
|
| 13 |
+
.DS_Store
|
| 14 |
+
.idea/
|
| 15 |
+
.vscode/
|
| 16 |
+
|
| 17 |
+
# secrets / local config
|
| 18 |
+
.env
|
| 19 |
+
.env.*
|
| 20 |
+
*.pem
|
| 21 |
+
web/test-results/
|
| 22 |
+
web/playwright-report/
|
| 23 |
+
test-results/
|
| 24 |
+
*.age
|
| 25 |
+
*.agekey
|
| 26 |
+
*.key
|
| 27 |
+
rclone.conf
|
| 28 |
+
/backups/
|
| 29 |
+
snapshots/
|
.hfignore
ADDED
|
@@ -0,0 +1,14 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
.git
|
| 2 |
+
data
|
| 3 |
+
serve
|
| 4 |
+
sams-videos
|
| 5 |
+
**/.env*
|
| 6 |
+
**/*.pem
|
| 7 |
+
**/*.key
|
| 8 |
+
**/*.agekey
|
| 9 |
+
**/*.age
|
| 10 |
+
**/node_modules
|
| 11 |
+
**/.next
|
| 12 |
+
**/__pycache__
|
| 13 |
+
web/test-results
|
| 14 |
+
web/playwright-report
|
Agent.md
ADDED
|
@@ -0,0 +1,99 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
## 1. Deployment Configuration
|
| 2 |
+
|
| 3 |
+
### Target Space
|
| 4 |
+
- **Profile:** `Leon4gr45`
|
| 5 |
+
- **Space:** `openweb`
|
| 6 |
+
- **Full Identifier:** `Leon4gr45/openweb`
|
| 7 |
+
- **Frontend Port:** `7860` (mandatory for all Hugging Face Spaces)
|
| 8 |
+
|
| 9 |
+
### Deployment Method
|
| 10 |
+
Choose the correct SDK based on the app type based on the codebase language:
|
| 11 |
+
|
| 12 |
+
- **Gradio SDK** — for Gradio applications
|
| 13 |
+
- **Streamlit SDK** — for Streamlit applications
|
| 14 |
+
- **Docker SDK** — for all other applications (recommended default for flexibility)
|
| 15 |
+
|
| 16 |
+
### HF Token
|
| 17 |
+
- The environment variable **`HF_TOKEN` will always be provided at execution time**.
|
| 18 |
+
- Never hardcode the token. Always read it from the environment.
|
| 19 |
+
- All monitoring and log‑streaming commands rely on `HF_TOKEN`.
|
| 20 |
+
|
| 21 |
+
### Required Files
|
| 22 |
+
- `Dockerfile` (or `app.py` for Gradio/Streamlit SDKs)
|
| 23 |
+
- `README.md` with Hugging Face YAML frontmatter:
|
| 24 |
+
```yaml
|
| 25 |
+
---
|
| 26 |
+
title: <APP NAME>
|
| 27 |
+
sdk: docker | gradio | streamlit
|
| 28 |
+
app_port: 7860
|
| 29 |
+
---
|
| 30 |
+
```
|
| 31 |
+
- `.hfignore` to exclude unnecessary files
|
| 32 |
+
- This `Agent.md` file (must be committed before deployment)
|
| 33 |
+
|
| 34 |
+
---
|
| 35 |
+
|
| 36 |
+
## 2. API Exposure and Documentation
|
| 37 |
+
|
| 38 |
+
### Mandatory Endpoints
|
| 39 |
+
Every deployment **must** expose:
|
| 40 |
+
|
| 41 |
+
- **`/health`**
|
| 42 |
+
- Returns HTTP 200 when the app is ready.
|
| 43 |
+
- Required for Hugging Face to transition the Space from *starting* → *running*.
|
| 44 |
+
|
| 45 |
+
- **`/api-docs`**
|
| 46 |
+
- Documents **all** available API endpoints.
|
| 47 |
+
- Must be reachable at:
|
| 48 |
+
`https://HF_PROFILE-openweb.hf.space/api-docs`
|
| 49 |
+
|
| 50 |
+
### Functional Endpoints
|
| 51 |
+
Document each endpoint here. For every endpoint, include:
|
| 52 |
+
|
| 53 |
+
- **Method:** GET/POST/PUT/DELETE
|
| 54 |
+
- **Path:** `/predict`, `/generate`, `/upload`, etc.
|
| 55 |
+
- **Purpose:** What the endpoint does
|
| 56 |
+
- **Request Example:** JSON or query parameters
|
| 57 |
+
- **Response Example:** JSON schema or example payload
|
| 58 |
+
|
| 59 |
+
Example format:
|
| 60 |
+
|
| 61 |
+
```
|
| 62 |
+
### /predict
|
| 63 |
+
- Method: POST
|
| 64 |
+
- Purpose: Run model inference
|
| 65 |
+
- Request:
|
| 66 |
+
{
|
| 67 |
+
"text": "hello world"
|
| 68 |
+
}
|
| 69 |
+
- Response:
|
| 70 |
+
{
|
| 71 |
+
"prediction": "…"
|
| 72 |
+
}
|
| 73 |
+
```
|
| 74 |
+
|
| 75 |
+
All endpoints listed here **must** appear in `/api-docs`.
|
| 76 |
+
|
| 77 |
+
---
|
| 78 |
+
|
| 79 |
+
## 3. Deployment Workflow
|
| 80 |
+
|
| 81 |
+
Precondition: Use the huggingface hub cli hf to check that the space is empty of files nd delete any which are still in there and not belonging to the project to be uploaded
|
| 82 |
+
|
| 83 |
+
### Standard Deployment Command
|
| 84 |
+
After any code change, run:
|
| 85 |
+
|
| 86 |
+
```bash
|
| 87 |
+
hf upload Leon4gr45/openweb --repo-type=space
|
| 88 |
+
---
|
| 89 |
+
|
| 90 |
+
Scan build and run logs # Get build logs (SSE) curl -N
|
| 91 |
+
-H "Authorization: Bearer HF_TOKEN"
|
| 92 |
+
"https://huggingface.co/api/spaces/Leon4gr45/openweb/logs/build"
|
| 93 |
+
|
| 94 |
+
Get run logs (SSE) once the build logs succeed
|
| 95 |
+
curl -N
|
| 96 |
+
-H "Authorization: Bearer HF_TOKEN"
|
| 97 |
+
"https://huggingface.co/api/spaces/Leon4gr45/openweb/logs/run"
|
| 98 |
+
|
| 99 |
+
after 300 seconds to see if the deployment has been successful, and if not, fix the errors of deployment, and redeploy and monitor in a cycle until the space is running and reacts to the api endpoints you created.
|
CHANGELOG.md
ADDED
|
@@ -0,0 +1,19 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Changes
|
| 2 |
+
|
| 3 |
+
## Video RAG · Visual Mentor v2 (2026-10-17, development)
|
| 4 |
+
|
| 5 |
+
- New main-based branch; original Library landing, cards, Search/Ask, charts,
|
| 6 |
+
Architecture and timestamped clip workflows preserved. Four-size baseline and
|
| 7 |
+
current screenshot tests, immutable synthetic sample corpus and playable media.
|
| 8 |
+
- Selective remote backend/security migration; protected private library/media,
|
| 9 |
+
user/admin roles, strict disabled-by-default YAML profiles and placeholder rejection.
|
| 10 |
+
- Persistent bounded upload/CPU-FFmpeg queue, explicit missing-provider stages,
|
| 11 |
+
remote transcription and model-isolated atomic text-index revisions.
|
| 12 |
+
- Native Live dual-camera workspace, consent/voice/stale guards, local snapshots,
|
| 13 |
+
freeze/fullscreen, timeline inspection and isolated session route overrides.
|
| 14 |
+
- Models provider cards, route builder/matrix, YAML draft diff/import/export,
|
| 15 |
+
validation gating, CAS apply/rollback and synthetic execution trace.
|
| 16 |
+
- Real age encryption, consistent SQLite snapshots, independent media checksums,
|
| 17 |
+
isolated restore, conservative retention and approval-gated private workflow templates.
|
| 18 |
+
- Private HF Docker package on one port; no inference infrastructure modifications.
|
| 19 |
+
Production deployment, durability and backup activation remain unqualified gates.
|
Dockerfile
ADDED
|
@@ -0,0 +1,34 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
FROM node:24-bookworm-slim@sha256:a5f4366e834695d5f363b920c104953a535d3a9864d1bdf74566a0ebe01daee5 AS web
|
| 2 |
+
WORKDIR /build/web
|
| 3 |
+
ENV NEXT_TELEMETRY_DISABLED=1
|
| 4 |
+
COPY web/package*.json ./
|
| 5 |
+
RUN --mount=type=secret,id=build_ca if [ -f /run/secrets/build_ca ]; then export NODE_EXTRA_CA_CERTS=/run/secrets/build_ca; fi; npm ci --no-audit --no-fund
|
| 6 |
+
COPY web/ ./
|
| 7 |
+
RUN npm run build
|
| 8 |
+
|
| 9 |
+
# Assemble application assets without repeatedly copying the FFmpeg runtime layer.
|
| 10 |
+
FROM scratch AS bundle
|
| 11 |
+
COPY vrag /app/vrag
|
| 12 |
+
COPY config /app/config
|
| 13 |
+
COPY scripts/cloud-start.py /app/scripts/cloud-start.py
|
| 14 |
+
COPY --from=web /build/web/.next/standalone /app/web/.next/standalone
|
| 15 |
+
COPY --from=web /build/web/.next/static /app/web/.next/standalone/.next/static
|
| 16 |
+
COPY --from=web /build/web/public /app/web/.next/standalone/public
|
| 17 |
+
|
| 18 |
+
FROM python:3.12-slim-bookworm@sha256:34386ef0cb081344d7ec1c103ba398e6e9f64e9ab3a1509accc92a4e24a07258 AS runtime
|
| 19 |
+
WORKDIR /app
|
| 20 |
+
COPY requirements-runtime.txt ./
|
| 21 |
+
RUN --mount=type=secret,id=build_ca \
|
| 22 |
+
export http_proxy="$HTTP_PROXY" https_proxy="$HTTPS_PROXY"; \
|
| 23 |
+
if [ -f /run/secrets/build_ca ]; then export PIP_CERT=/run/secrets/build_ca; fi; \
|
| 24 |
+
apt-get update && apt-get install -y --no-install-recommends ca-certificates libatomic1 ffmpeg age && \
|
| 25 |
+
pip install --no-cache-dir -r requirements-runtime.txt && \
|
| 26 |
+
rm -rf /var/lib/apt/lists/* && \
|
| 27 |
+
useradd -m -u 1000 mentor && mkdir -p /app/data/videos /app/work && chown -R mentor:mentor /app/data /app/work
|
| 28 |
+
COPY --from=web /usr/local/bin/node /usr/local/bin/node
|
| 29 |
+
COPY --from=bundle --chown=1000:1000 /app /app
|
| 30 |
+
USER mentor
|
| 31 |
+
ENV PYTHONUNBUFFERED=1 NEXT_TELEMETRY_DISABLED=1 VRAG_DATA=/app/work/data VM_STATE_DIR=/app/work/state
|
| 32 |
+
EXPOSE 7860
|
| 33 |
+
HEALTHCHECK --interval=30s --timeout=5s --start-period=30s CMD python -c "import urllib.request; urllib.request.urlopen('http://127.0.0.1:7860/api/health', timeout=4)"
|
| 34 |
+
CMD ["python", "scripts/cloud-start.py"]
|
README.md
ADDED
|
@@ -0,0 +1,62 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
---
|
| 2 |
+
title: Video RAG · Visual Mentor
|
| 3 |
+
sdk: docker
|
| 4 |
+
app_port: 7860
|
| 5 |
+
---
|
| 6 |
+
|
| 7 |
+
# Video RAG · Visual Mentor v2
|
| 8 |
+
|
| 9 |
+
The original Library, Search, Ask, Experiments, Architecture and clip-player UI,
|
| 10 |
+
with remote inference, a dual-camera Live workspace and Models & Routing studio.
|
| 11 |
+
The landing page is Library. Without a connected private-library token it shows
|
| 12 |
+
an immutable synthetic sample corpus, keyword retrieval and labeled evidence
|
| 13 |
+
extracts. No model inference or benchmark quality is implied by samples.
|
| 14 |
+
|
| 15 |
+
Run a CPU-only development environment (Python 3.12+, Node 24+, FFmpeg):
|
| 16 |
+
|
| 17 |
+
```
|
| 18 |
+
python -m venv .venv
|
| 19 |
+
.venv/bin/pip install -r requirements-runtime.txt
|
| 20 |
+
npm --prefix web ci
|
| 21 |
+
PYTHON=.venv/bin/python bash scripts/dev.sh
|
| 22 |
+
```
|
| 23 |
+
|
| 24 |
+
Open port 7860. FastAPI stays on loopback 8000. Configure independent
|
| 25 |
+
VM_USER_TOKEN/VM_ADMIN_TOKEN (at least 24 random characters), VM_ALLOWED_ORIGINS,
|
| 26 |
+
VRAG_DATA and VM_STATE_DIR in environment settings; never commit values.
|
| 27 |
+
No local inference services are started. For a production container:
|
| 28 |
+
|
| 29 |
+
```
|
| 30 |
+
docker build -t video-rag-mentor:v2 .
|
| 31 |
+
```
|
| 32 |
+
|
| 33 |
+
See [HF staging/deployment steps](docs/deployment-hf.md),
|
| 34 |
+
[v2 qualification and remaining gates](docs/v2-release-readiness.md),
|
| 35 |
+
[private backup/restore](docs/backup-restore.md), and
|
| 36 |
+
[porting decisions](docs/v2-porting-plan.md). Private Space deployment, backup
|
| 37 |
+
repository creation, credentials and activation remain approval-gated.
|
| 38 |
+
The base Space disk is ephemeral; local files alone do not establish durability.
|
| 39 |
+
|
| 40 |
+
Browser uploads are capped at 64 MiB/30 minutes, with one processing worker and
|
| 41 |
+
two simultaneous uploads. Jobs prepare CPU H.264 proxies, keyframes and timestamped
|
| 42 |
+
segments, transcribe via remote Whisper-compatible API and build isolated remote
|
| 43 |
+
text indexes. Missing providers become explicit blocked jobs with retry/manual
|
| 44 |
+
transcript support. Models/routing use strict schema version 1, not the illustrative
|
| 45 |
+
version-2 YAML in the product specification. All example routes start disabled.
|
| 46 |
+
|
| 47 |
+
Large multimodal inference stays remote. Text adapters cannot encode images,
|
| 48 |
+
video or audio into a shared vector space. Historical MRL/RRF indexes and charts
|
| 49 |
+
remain viewable where compatible; [original research documentation](docs/original-research.md)
|
| 50 |
+
is retained separately and is not this cloud runtime's setup procedure.
|
| 51 |
+
|
| 52 |
+
Validation:
|
| 53 |
+
|
| 54 |
+
```
|
| 55 |
+
python -m pytest -q
|
| 56 |
+
npm --prefix web run lint
|
| 57 |
+
npm --prefix web run build
|
| 58 |
+
cd web && CHROMIUM_PATH=/usr/bin/chromium npx playwright test
|
| 59 |
+
```
|
| 60 |
+
|
| 61 |
+
Install `age` to run actual encrypted backup tests. Test tooling requires pytest,
|
| 62 |
+
pytest-asyncio and Playwright; Python dev dependencies are in the uv dev group.
|
config/__init__.py
ADDED
|
File without changes
|
config/examples/detector-only.yaml
ADDED
|
@@ -0,0 +1,23 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version: 1
|
| 2 |
+
profile: detector-only
|
| 3 |
+
providers:
|
| 4 |
+
detector:
|
| 5 |
+
adapter: detection-http
|
| 6 |
+
base_url_env: NVIDIA_DETECTION_BASE_URL
|
| 7 |
+
api_key_secret: NVIDIA_DETECTION_API_KEY
|
| 8 |
+
model: replace-with-verified-gateway-model
|
| 9 |
+
capabilities:
|
| 10 |
+
- object_detection
|
| 11 |
+
- bounding_boxes
|
| 12 |
+
trust: approved_sensitive
|
| 13 |
+
features:
|
| 14 |
+
live.camera1.detection:
|
| 15 |
+
strategy: single
|
| 16 |
+
providers:
|
| 17 |
+
- detector
|
| 18 |
+
enabled: false
|
| 19 |
+
live.camera2.detection:
|
| 20 |
+
strategy: single
|
| 21 |
+
providers:
|
| 22 |
+
- detector
|
| 23 |
+
enabled: false
|
config/examples/disabled.yaml
ADDED
|
@@ -0,0 +1,17 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version: 1
|
| 2 |
+
profile: local-preview
|
| 3 |
+
providers: {}
|
| 4 |
+
features: {}
|
| 5 |
+
cameras:
|
| 6 |
+
camera1:
|
| 7 |
+
analysis_fps: 1
|
| 8 |
+
width: 640
|
| 9 |
+
height: 480
|
| 10 |
+
camera2:
|
| 11 |
+
analysis_fps: 1
|
| 12 |
+
width: 640
|
| 13 |
+
height: 480
|
| 14 |
+
privacy:
|
| 15 |
+
record_video: false
|
| 16 |
+
upload_only_on_explicit_start: true
|
| 17 |
+
require_approved_destination_for_camera: true
|
config/examples/hybrid.yaml
ADDED
|
@@ -0,0 +1,84 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version: 1
|
| 2 |
+
profile: hybrid
|
| 3 |
+
providers:
|
| 4 |
+
vlm:
|
| 5 |
+
adapter: openai-chat-vision
|
| 6 |
+
base_url_env: VLM_BASE_URL
|
| 7 |
+
api_key_secret: VLM_API_KEY
|
| 8 |
+
model: replace-with-verified-model-id
|
| 9 |
+
capabilities:
|
| 10 |
+
- image_understanding
|
| 11 |
+
- reasoning
|
| 12 |
+
- visual_decision
|
| 13 |
+
trust: approved_sensitive
|
| 14 |
+
timeout_ms: 10000
|
| 15 |
+
concurrency: 2
|
| 16 |
+
detector:
|
| 17 |
+
adapter: detection-http
|
| 18 |
+
base_url_env: NVIDIA_DETECTION_BASE_URL
|
| 19 |
+
api_key_secret: NVIDIA_DETECTION_API_KEY
|
| 20 |
+
model: replace-with-verified-gateway-model
|
| 21 |
+
capabilities:
|
| 22 |
+
- object_detection
|
| 23 |
+
- bounding_boxes
|
| 24 |
+
trust: approved_sensitive
|
| 25 |
+
embeddings:
|
| 26 |
+
adapter: openai-embeddings
|
| 27 |
+
base_url_env: EMBEDDINGS_BASE_URL
|
| 28 |
+
api_key_secret: EMBEDDINGS_API_KEY
|
| 29 |
+
model: replace-with-embedding-model
|
| 30 |
+
capabilities:
|
| 31 |
+
- text_embeddings
|
| 32 |
+
trust: approved_sensitive
|
| 33 |
+
whisper:
|
| 34 |
+
adapter: openai-transcription
|
| 35 |
+
base_url_env: WHISPER_BASE_URL
|
| 36 |
+
api_key_secret: WHISPER_API_KEY
|
| 37 |
+
model: replace-with-whisper-model
|
| 38 |
+
capabilities:
|
| 39 |
+
- transcription
|
| 40 |
+
trust: approved_sensitive
|
| 41 |
+
features:
|
| 42 |
+
live.camera1.scene:
|
| 43 |
+
strategy: single
|
| 44 |
+
providers:
|
| 45 |
+
- vlm
|
| 46 |
+
enabled: false
|
| 47 |
+
live.camera2.scene:
|
| 48 |
+
strategy: single
|
| 49 |
+
providers:
|
| 50 |
+
- vlm
|
| 51 |
+
enabled: false
|
| 52 |
+
live.visual_decision:
|
| 53 |
+
strategy: single
|
| 54 |
+
providers:
|
| 55 |
+
- vlm
|
| 56 |
+
enabled: false
|
| 57 |
+
rag.reasoning:
|
| 58 |
+
strategy: single
|
| 59 |
+
providers:
|
| 60 |
+
- vlm
|
| 61 |
+
enabled: false
|
| 62 |
+
mentor.reasoning:
|
| 63 |
+
strategy: single
|
| 64 |
+
providers:
|
| 65 |
+
- vlm
|
| 66 |
+
enabled: false
|
| 67 |
+
live.camera2.detection:
|
| 68 |
+
strategy: parallel
|
| 69 |
+
providers:
|
| 70 |
+
- detector
|
| 71 |
+
- vlm
|
| 72 |
+
result_policy: all
|
| 73 |
+
fusion: spatial_semantic
|
| 74 |
+
enabled: false
|
| 75 |
+
rag.text_embeddings:
|
| 76 |
+
strategy: single
|
| 77 |
+
providers:
|
| 78 |
+
- embeddings
|
| 79 |
+
enabled: false
|
| 80 |
+
audio.transcription:
|
| 81 |
+
strategy: single
|
| 82 |
+
providers:
|
| 83 |
+
- whisper
|
| 84 |
+
enabled: false
|
config/examples/laya-nvidia-hybrid.yaml
ADDED
|
@@ -0,0 +1,47 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version: 1
|
| 2 |
+
profile: laya-nvidia-hybrid
|
| 3 |
+
providers:
|
| 4 |
+
laya:
|
| 5 |
+
label: Needle Router Laya Vision
|
| 6 |
+
adapter: laya-vision
|
| 7 |
+
base_url_env: LAYA_BASE_URL
|
| 8 |
+
model: thaitea/laya-vision
|
| 9 |
+
capabilities:
|
| 10 |
+
- image_understanding
|
| 11 |
+
- visual_decision
|
| 12 |
+
trust: approved_sensitive
|
| 13 |
+
timeout_ms: 60000
|
| 14 |
+
concurrency: 1
|
| 15 |
+
detector:
|
| 16 |
+
adapter: detection-http
|
| 17 |
+
base_url_env: NVIDIA_DETECTION_BASE_URL
|
| 18 |
+
api_key_secret: NVIDIA_DETECTION_API_KEY
|
| 19 |
+
model: replace-with-verified-gateway-model
|
| 20 |
+
capabilities:
|
| 21 |
+
- object_detection
|
| 22 |
+
- bounding_boxes
|
| 23 |
+
trust: approved_sensitive
|
| 24 |
+
features:
|
| 25 |
+
live.camera1.scene:
|
| 26 |
+
strategy: single
|
| 27 |
+
providers:
|
| 28 |
+
- laya
|
| 29 |
+
enabled: false
|
| 30 |
+
live.camera2.scene:
|
| 31 |
+
strategy: single
|
| 32 |
+
providers:
|
| 33 |
+
- laya
|
| 34 |
+
enabled: false
|
| 35 |
+
live.visual_decision:
|
| 36 |
+
strategy: single
|
| 37 |
+
providers:
|
| 38 |
+
- laya
|
| 39 |
+
enabled: false
|
| 40 |
+
live.camera2.detection:
|
| 41 |
+
strategy: parallel
|
| 42 |
+
providers:
|
| 43 |
+
- laya
|
| 44 |
+
- detector
|
| 45 |
+
result_policy: all
|
| 46 |
+
fusion: spatial_semantic
|
| 47 |
+
enabled: false
|
config/examples/laya-only.yaml
ADDED
|
@@ -0,0 +1,30 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version: 1
|
| 2 |
+
profile: laya-only
|
| 3 |
+
providers:
|
| 4 |
+
laya:
|
| 5 |
+
label: Needle Router Laya Vision
|
| 6 |
+
adapter: laya-vision
|
| 7 |
+
base_url_env: LAYA_BASE_URL
|
| 8 |
+
model: thaitea/laya-vision
|
| 9 |
+
capabilities:
|
| 10 |
+
- image_understanding
|
| 11 |
+
- visual_decision
|
| 12 |
+
trust: approved_sensitive
|
| 13 |
+
timeout_ms: 60000
|
| 14 |
+
concurrency: 1
|
| 15 |
+
features:
|
| 16 |
+
live.camera1.scene:
|
| 17 |
+
strategy: single
|
| 18 |
+
providers:
|
| 19 |
+
- laya
|
| 20 |
+
enabled: false
|
| 21 |
+
live.camera2.scene:
|
| 22 |
+
strategy: single
|
| 23 |
+
providers:
|
| 24 |
+
- laya
|
| 25 |
+
enabled: false
|
| 26 |
+
live.visual_decision:
|
| 27 |
+
strategy: single
|
| 28 |
+
providers:
|
| 29 |
+
- laya
|
| 30 |
+
enabled: false
|
config/examples/laya-pending.yaml
ADDED
|
@@ -0,0 +1,22 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version: 1
|
| 2 |
+
profile: laya-pending-protocol
|
| 3 |
+
providers:
|
| 4 |
+
laya:
|
| 5 |
+
label: Needle router / Laya
|
| 6 |
+
adapter: laya-vision
|
| 7 |
+
base_url_env: LAYA_BASE_URL
|
| 8 |
+
api_key_secret: LAYA_API_KEY
|
| 9 |
+
model: pending-v1-info-verification
|
| 10 |
+
capabilities: []
|
| 11 |
+
trust: approved_sensitive
|
| 12 |
+
features:
|
| 13 |
+
live.camera1.scene:
|
| 14 |
+
strategy: single
|
| 15 |
+
providers:
|
| 16 |
+
- laya
|
| 17 |
+
enabled: false
|
| 18 |
+
live.camera2.scene:
|
| 19 |
+
strategy: single
|
| 20 |
+
providers:
|
| 21 |
+
- laya
|
| 22 |
+
enabled: false
|
config/examples/laya-vlm-fallback.yaml
ADDED
|
@@ -0,0 +1,42 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version: 1
|
| 2 |
+
profile: laya-vlm-fallback
|
| 3 |
+
providers:
|
| 4 |
+
laya:
|
| 5 |
+
label: Needle Router Laya Vision
|
| 6 |
+
adapter: laya-vision
|
| 7 |
+
base_url_env: LAYA_BASE_URL
|
| 8 |
+
model: thaitea/laya-vision
|
| 9 |
+
capabilities:
|
| 10 |
+
- image_understanding
|
| 11 |
+
- visual_decision
|
| 12 |
+
trust: approved_sensitive
|
| 13 |
+
timeout_ms: 60000
|
| 14 |
+
concurrency: 1
|
| 15 |
+
detector:
|
| 16 |
+
adapter: detection-http
|
| 17 |
+
base_url_env: NVIDIA_DETECTION_BASE_URL
|
| 18 |
+
api_key_secret: NVIDIA_DETECTION_API_KEY
|
| 19 |
+
model: replace-with-verified-gateway-model
|
| 20 |
+
capabilities:
|
| 21 |
+
- object_detection
|
| 22 |
+
- bounding_boxes
|
| 23 |
+
trust: approved_sensitive
|
| 24 |
+
vlm:
|
| 25 |
+
adapter: openai-chat-vision
|
| 26 |
+
base_url_env: VLM_BASE_URL
|
| 27 |
+
api_key_secret: VLM_API_KEY
|
| 28 |
+
model: replace-with-verified-model-id
|
| 29 |
+
capabilities:
|
| 30 |
+
- image_understanding
|
| 31 |
+
- reasoning
|
| 32 |
+
- visual_decision
|
| 33 |
+
trust: approved_sensitive
|
| 34 |
+
timeout_ms: 10000
|
| 35 |
+
concurrency: 2
|
| 36 |
+
features:
|
| 37 |
+
live.camera1.scene:
|
| 38 |
+
strategy: fallback
|
| 39 |
+
providers:
|
| 40 |
+
- laya
|
| 41 |
+
- vlm
|
| 42 |
+
enabled: false
|
config/examples/nvidia-laya-sequential.yaml
ADDED
|
@@ -0,0 +1,30 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version: 1
|
| 2 |
+
profile: nvidia-laya-sequential
|
| 3 |
+
providers:
|
| 4 |
+
laya:
|
| 5 |
+
label: Needle Router Laya Vision
|
| 6 |
+
adapter: laya-vision
|
| 7 |
+
base_url_env: LAYA_BASE_URL
|
| 8 |
+
model: thaitea/laya-vision
|
| 9 |
+
capabilities:
|
| 10 |
+
- image_understanding
|
| 11 |
+
- visual_decision
|
| 12 |
+
trust: approved_sensitive
|
| 13 |
+
timeout_ms: 60000
|
| 14 |
+
concurrency: 1
|
| 15 |
+
detector:
|
| 16 |
+
adapter: detection-http
|
| 17 |
+
base_url_env: NVIDIA_DETECTION_BASE_URL
|
| 18 |
+
api_key_secret: NVIDIA_DETECTION_API_KEY
|
| 19 |
+
model: replace-with-verified-gateway-model
|
| 20 |
+
capabilities:
|
| 21 |
+
- object_detection
|
| 22 |
+
- bounding_boxes
|
| 23 |
+
trust: approved_sensitive
|
| 24 |
+
features:
|
| 25 |
+
live.camera1.scene:
|
| 26 |
+
strategy: sequential
|
| 27 |
+
providers:
|
| 28 |
+
- detector
|
| 29 |
+
- laya
|
| 30 |
+
enabled: false
|
config/examples/vlm-only.yaml
ADDED
|
@@ -0,0 +1,41 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version: 1
|
| 2 |
+
profile: vlm-only
|
| 3 |
+
providers:
|
| 4 |
+
vlm:
|
| 5 |
+
adapter: openai-chat-vision
|
| 6 |
+
base_url_env: VLM_BASE_URL
|
| 7 |
+
api_key_secret: VLM_API_KEY
|
| 8 |
+
model: replace-with-verified-model-id
|
| 9 |
+
capabilities:
|
| 10 |
+
- image_understanding
|
| 11 |
+
- reasoning
|
| 12 |
+
- visual_decision
|
| 13 |
+
trust: approved_sensitive
|
| 14 |
+
timeout_ms: 10000
|
| 15 |
+
concurrency: 2
|
| 16 |
+
features:
|
| 17 |
+
live.camera1.scene:
|
| 18 |
+
strategy: single
|
| 19 |
+
providers:
|
| 20 |
+
- vlm
|
| 21 |
+
enabled: false
|
| 22 |
+
live.camera2.scene:
|
| 23 |
+
strategy: single
|
| 24 |
+
providers:
|
| 25 |
+
- vlm
|
| 26 |
+
enabled: false
|
| 27 |
+
live.visual_decision:
|
| 28 |
+
strategy: single
|
| 29 |
+
providers:
|
| 30 |
+
- vlm
|
| 31 |
+
enabled: false
|
| 32 |
+
rag.reasoning:
|
| 33 |
+
strategy: single
|
| 34 |
+
providers:
|
| 35 |
+
- vlm
|
| 36 |
+
enabled: false
|
| 37 |
+
mentor.reasoning:
|
| 38 |
+
strategy: single
|
| 39 |
+
providers:
|
| 40 |
+
- vlm
|
| 41 |
+
enabled: false
|
config/schema.py
ADDED
|
@@ -0,0 +1,141 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Strict, bounded configuration. Credentials are references, never YAML values."""
|
| 2 |
+
from typing import Literal
|
| 3 |
+
import re
|
| 4 |
+
import yaml
|
| 5 |
+
from pydantic import BaseModel, ConfigDict, Field, model_validator
|
| 6 |
+
|
| 7 |
+
Capability = Literal['image_understanding', 'visual_decision', 'object_detection', 'bounding_boxes', 'object_tracking', 'reasoning', 'text_embeddings', 'multimodal_embeddings', 'transcription']
|
| 8 |
+
|
| 9 |
+
class Strict(BaseModel):
|
| 10 |
+
model_config = ConfigDict(extra='forbid', frozen=True)
|
| 11 |
+
|
| 12 |
+
class Provider(Strict):
|
| 13 |
+
label: str = Field(default='', max_length=100)
|
| 14 |
+
adapter: Literal['openai-chat-vision', 'openai-embeddings', 'openai-transcription', 'detection-http', 'laya-vision']
|
| 15 |
+
base_url_env: str = Field(pattern=r'^[A-Z][A-Z0-9_]{0,79}$')
|
| 16 |
+
api_key_secret: str | None = Field(default=None, pattern=r'^[A-Z][A-Z0-9_]{0,79}$')
|
| 17 |
+
model: str = Field(min_length=1, max_length=200)
|
| 18 |
+
capabilities: list[Capability] = Field(max_length=10)
|
| 19 |
+
trust: Literal['approved_sensitive', 'public_only'] = 'public_only'
|
| 20 |
+
timeout_ms: int = Field(default=10000, ge=100, le=60000)
|
| 21 |
+
concurrency: int = Field(default=2, ge=1, le=8)
|
| 22 |
+
@model_validator(mode='after')
|
| 23 |
+
def supported(self):
|
| 24 |
+
if self.api_key_secret and (self.api_key_secret.startswith(('VM_', 'HF_', 'HUGGING_FACE_', 'GITHUB_', 'GH_', 'AWS_', 'RCLONE_', 'AGE_', 'OIC_', 'AZURE_', 'GOOGLE_')) or self.api_key_secret in ('HTTP_PROXY','HTTPS_PROXY','ALL_PROXY')):
|
| 25 |
+
raise ValueError('provider credentials cannot reference application, infrastructure or recovery secrets')
|
| 26 |
+
supported = {
|
| 27 |
+
'openai-chat-vision': {'image_understanding', 'reasoning', 'visual_decision'},
|
| 28 |
+
'openai-embeddings': {'text_embeddings'},
|
| 29 |
+
'openai-transcription': {'transcription'},
|
| 30 |
+
'detection-http': {'object_detection', 'bounding_boxes', 'object_tracking'},
|
| 31 |
+
# Needle Router imageBase64 + typed noul answers; not generative chat.
|
| 32 |
+
'laya-vision': {'image_understanding', 'visual_decision'},
|
| 33 |
+
}
|
| 34 |
+
if not set(self.capabilities) <= supported[self.adapter]:
|
| 35 |
+
raise ValueError('capability is not supported by this adapter; use a verified provider protocol')
|
| 36 |
+
if self.adapter == 'laya-vision' and self.capabilities and set(self.capabilities) != {'image_understanding','visual_decision'}:
|
| 37 |
+
raise ValueError('Laya image scoring requires image_understanding and visual_decision')
|
| 38 |
+
if self.adapter == 'detection-http' and set(self.capabilities) & {'object_detection','object_tracking'} and 'bounding_boxes' not in self.capabilities:
|
| 39 |
+
raise ValueError('detector/tracker contract requires bounding_boxes')
|
| 40 |
+
return self
|
| 41 |
+
|
| 42 |
+
class Condition(Strict):
|
| 43 |
+
field: Literal['camera_id', 'has_detections']
|
| 44 |
+
equals: str | bool
|
| 45 |
+
provider: str
|
| 46 |
+
@model_validator(mode='after')
|
| 47 |
+
def predicate_type(self):
|
| 48 |
+
if self.field == 'camera_id' and (not isinstance(self.equals, str) or self.equals not in ('camera1','camera2')):
|
| 49 |
+
raise ValueError('camera condition requires a logical camera slot')
|
| 50 |
+
if self.field == 'has_detections' and not isinstance(self.equals, bool):
|
| 51 |
+
raise ValueError('has_detections requires a boolean')
|
| 52 |
+
return self
|
| 53 |
+
|
| 54 |
+
class Route(Strict):
|
| 55 |
+
strategy: Literal['single', 'fallback', 'parallel', 'sequential', 'conditional'] = 'single'
|
| 56 |
+
providers: list[str] = Field(min_length=1, max_length=8)
|
| 57 |
+
enabled: bool = True
|
| 58 |
+
result_policy: Literal['all', 'first_valid', 'quorum'] = 'all'
|
| 59 |
+
quorum: int = Field(default=1, ge=1, le=8)
|
| 60 |
+
fusion: Literal['spatial_semantic'] | None = None
|
| 61 |
+
fallback_on: list[Literal['timeout', 'provider_error', 'invalid_output']] = Field(default_factory=lambda: ['timeout', 'provider_error', 'invalid_output'])
|
| 62 |
+
conditions: list[Condition] = Field(default_factory=list, max_length=8)
|
| 63 |
+
@model_validator(mode='after')
|
| 64 |
+
def valid(self):
|
| 65 |
+
if len(set(self.providers)) != len(self.providers):
|
| 66 |
+
raise ValueError('duplicate providers')
|
| 67 |
+
if self.strategy != 'parallel' and self.result_policy != 'all':
|
| 68 |
+
raise ValueError('result_policy is only supported for parallel routing')
|
| 69 |
+
if self.strategy == 'single' and len(self.providers) != 1:
|
| 70 |
+
raise ValueError('single requires exactly one provider')
|
| 71 |
+
if self.quorum > len(self.providers):
|
| 72 |
+
raise ValueError('quorum exceeds provider count')
|
| 73 |
+
if self.strategy == 'conditional' and not self.conditions:
|
| 74 |
+
raise ValueError('conditional requires deterministic conditions')
|
| 75 |
+
if any(c.provider not in self.providers for c in self.conditions):
|
| 76 |
+
raise ValueError('condition provider must belong to route')
|
| 77 |
+
return self
|
| 78 |
+
|
| 79 |
+
class Camera(Strict):
|
| 80 |
+
analysis_fps: float = Field(default=1, gt=0, le=2)
|
| 81 |
+
width: int = Field(default=640, ge=160, le=1920)
|
| 82 |
+
height: int = Field(default=480, ge=120, le=1080)
|
| 83 |
+
|
| 84 |
+
class Privacy(Strict):
|
| 85 |
+
record_video: Literal[False] = False
|
| 86 |
+
upload_only_on_explicit_start: Literal[True] = True
|
| 87 |
+
require_approved_destination_for_camera: Literal[True] = True
|
| 88 |
+
|
| 89 |
+
class Configuration(Strict):
|
| 90 |
+
version: Literal[1] = 1
|
| 91 |
+
profile: str = Field(default='unconfigured', min_length=1, max_length=100)
|
| 92 |
+
providers: dict[str, Provider] = Field(default_factory=dict, max_length=32)
|
| 93 |
+
features: dict[str, Route] = Field(default_factory=dict, max_length=64)
|
| 94 |
+
cameras: dict[Literal['camera1', 'camera2'], Camera] = Field(default_factory=lambda: {'camera1': Camera(), 'camera2': Camera()})
|
| 95 |
+
privacy: Privacy = Field(default_factory=Privacy)
|
| 96 |
+
@model_validator(mode='after')
|
| 97 |
+
def references(self):
|
| 98 |
+
for key in self.providers:
|
| 99 |
+
if not re.fullmatch(r'[a-zA-Z0-9_-]{1,64}', key):
|
| 100 |
+
raise ValueError('invalid provider identifier')
|
| 101 |
+
required = {'rag.text_embeddings': 'text_embeddings', 'audio.transcription': 'transcription', 'rag.reasoning': 'reasoning', 'mentor.reasoning': 'reasoning'}
|
| 102 |
+
for feature, route in self.features.items():
|
| 103 |
+
if any(p not in self.providers for p in route.providers):
|
| 104 |
+
raise ValueError(f'{feature}: unknown provider')
|
| 105 |
+
if route.enabled:
|
| 106 |
+
if any(self.providers[p].model.lower().startswith(('replace-with','verified_','your-')) for p in route.providers):
|
| 107 |
+
raise ValueError('replace placeholder model identifiers before enabling routes')
|
| 108 |
+
cap = required.get(feature)
|
| 109 |
+
caps = {c for p in route.providers for c in self.providers[p].capabilities}
|
| 110 |
+
if cap and any(cap not in self.providers[p].capabilities for p in route.providers):
|
| 111 |
+
raise ValueError(f'{feature}: missing {cap} capability')
|
| 112 |
+
if feature == 'rag.text_embeddings' and route.strategy != 'single':
|
| 113 |
+
raise ValueError('embedding indexes require one explicit provider identity')
|
| 114 |
+
if feature.endswith('.tracking') and 'object_tracking' not in caps:
|
| 115 |
+
raise ValueError('tracking requires a real tracker')
|
| 116 |
+
if feature.endswith('.detection') and 'bounding_boxes' not in caps:
|
| 117 |
+
raise ValueError('detection requires a spatial detector')
|
| 118 |
+
if feature.startswith('live.') and any(self.providers[p].adapter in ('openai-chat-vision','laya-vision') and 'image_understanding' not in self.providers[p].capabilities for p in route.providers):
|
| 119 |
+
raise ValueError('live semantic providers must accept images')
|
| 120 |
+
if feature.startswith('live.') and any(not set(self.providers[p].capabilities) & {'image_understanding', 'visual_decision', 'object_detection', 'object_tracking'} for p in route.providers):
|
| 121 |
+
raise ValueError('live route requires visual capability')
|
| 122 |
+
return self
|
| 123 |
+
|
| 124 |
+
def parse_yaml(text: str) -> Configuration:
|
| 125 |
+
if len(text.encode()) > 65536:
|
| 126 |
+
raise ValueError('configuration exceeds 64 KiB')
|
| 127 |
+
# Reject aliases/anchors entirely rather than expanding attacker-controlled graphs.
|
| 128 |
+
if any(isinstance(t, (yaml.tokens.AliasToken, yaml.tokens.AnchorToken)) for t in yaml.scan(text)):
|
| 129 |
+
raise ValueError('YAML aliases and anchors are disabled')
|
| 130 |
+
class UniqueLoader(yaml.SafeLoader):
|
| 131 |
+
pass
|
| 132 |
+
def mapping(loader, node):
|
| 133 |
+
result = {}
|
| 134 |
+
for k, v in node.value:
|
| 135 |
+
key = loader.construct_object(k)
|
| 136 |
+
if key in result:
|
| 137 |
+
raise ValueError('duplicate YAML key')
|
| 138 |
+
result[key] = loader.construct_object(v)
|
| 139 |
+
return result
|
| 140 |
+
UniqueLoader.add_constructor(yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG, mapping)
|
| 141 |
+
return Configuration.model_validate(yaml.load(text, Loader=UniqueLoader))
|
deploy/backup-repository/inactive-workflows/backup-12h.yml
ADDED
|
@@ -0,0 +1,26 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
name: Coordinate private Space GitHub checkpoint
|
| 2 |
+
on:
|
| 3 |
+
schedule:
|
| 4 |
+
- cron: '17 0,12 * * *'
|
| 5 |
+
workflow_dispatch:
|
| 6 |
+
permissions:
|
| 7 |
+
contents: read
|
| 8 |
+
concurrency:
|
| 9 |
+
group: backup-production
|
| 10 |
+
cancel-in-progress: false
|
| 11 |
+
jobs:
|
| 12 |
+
checkpoint:
|
| 13 |
+
if: vars.BACKUPS_APPROVED == 'true' && vars.COORDINATOR_VALIDATED == 'true'
|
| 14 |
+
environment: private-backup
|
| 15 |
+
runs-on: ubuntu-24.04
|
| 16 |
+
timeout-minutes: 35
|
| 17 |
+
steps:
|
| 18 |
+
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683
|
| 19 |
+
- run: python -m pip install httpx==0.28.1
|
| 20 |
+
- name: Verify private Space reachability and completed checkpoint
|
| 21 |
+
env:
|
| 22 |
+
VM_SPACE_ID: ${{ vars.VM_SPACE_ID }}
|
| 23 |
+
VM_SPACE_URL: ${{ vars.VM_SPACE_URL }}
|
| 24 |
+
HF_SPACE_TOKEN: ${{ secrets.HF_SPACE_TOKEN }}
|
| 25 |
+
VM_BACKUP_TRIGGER_TOKEN: ${{ secrets.VM_BACKUP_TRIGGER_TOKEN }}
|
| 26 |
+
run: python scripts/coordinate_backup.py
|
deploy/backup-repository/inactive-workflows/qualify-private-staging.yml
ADDED
|
@@ -0,0 +1,25 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
name: Qualify private staging backup manually
|
| 2 |
+
on:
|
| 3 |
+
workflow_dispatch:
|
| 4 |
+
permissions:
|
| 5 |
+
contents: read
|
| 6 |
+
concurrency:
|
| 7 |
+
group: backup-staging-qualification
|
| 8 |
+
cancel-in-progress: false
|
| 9 |
+
jobs:
|
| 10 |
+
qualify:
|
| 11 |
+
runs-on: ubuntu-24.04
|
| 12 |
+
environment: private-backup-staging
|
| 13 |
+
timeout-minutes: 35
|
| 14 |
+
steps:
|
| 15 |
+
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683
|
| 16 |
+
- run: python -m pip install httpx==0.28.1
|
| 17 |
+
- name: Verify private access and a new checkpoint
|
| 18 |
+
env:
|
| 19 |
+
VM_SPACE_ID: ${{ vars.STAGING_SPACE_ID }}
|
| 20 |
+
VM_SPACE_URL: ${{ vars.STAGING_SPACE_URL }}
|
| 21 |
+
HF_SPACE_TOKEN: ${{ secrets.STAGING_HF_TOKEN }}
|
| 22 |
+
VM_BACKUP_TRIGGER_TOKEN: ${{ secrets.STAGING_BACKUP_TRIGGER_TOKEN }}
|
| 23 |
+
run: >-
|
| 24 |
+
python scripts/qualify_private_space.py
|
| 25 |
+
--space "$VM_SPACE_ID" --url "$VM_SPACE_URL" --trigger
|
deploy/hf-staging/README.md
ADDED
|
@@ -0,0 +1,67 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Private staging qualification package (not deployed)
|
| 2 |
+
|
| 3 |
+
Proposed target: **Leon4gr45/openweb-v2-staging**, private Docker Space, CPU Basic.
|
| 4 |
+
Production `Leon4gr45/openweb` is excluded. A live anonymous HF metadata probe on
|
| 5 |
+
17 October 2026 reported production `private=false`, Docker/CPU Basic/RUNNING and
|
| 6 |
+
no deployed backup-status route (404). This package never changes that Space.
|
| 7 |
+
|
| 8 |
+
Upload only the application allowlist from docs/deployment-hf.md, using the reviewed
|
| 9 |
+
feature-branch commit and an HF identity authorized to write this staging repository.
|
| 10 |
+
A credential restricted to an existing staging repository is preferred: create the
|
| 11 |
+
private staging repository separately first, then grant write only to that target.
|
| 12 |
+
Creation itself requires repository-creation rights for the Leon4gr45 namespace.
|
| 13 |
+
No deployment command has been executed and no HF credential is currently present.
|
| 14 |
+
|
| 15 |
+
HF staging Variables:
|
| 16 |
+
|
| 17 |
+
```
|
| 18 |
+
VM_BACKUP_ENABLED=true
|
| 19 |
+
VM_BACKUP_REPOSITORY=JsonLord/visual-mentor-backups
|
| 20 |
+
VM_BACKUP_NAMESPACE=staging-qualification
|
| 21 |
+
VM_RECOVERY_ROOT=/app/work
|
| 22 |
+
VRAG_DATA=/app/work/data
|
| 23 |
+
VM_STATE_DIR=/app/work/state
|
| 24 |
+
VM_ALLOWED_ORIGINS=https://leon4gr45-openweb-v2-staging.hf.space
|
| 25 |
+
VM_BACKUP_RECIPIENT=<public age recipient>
|
| 26 |
+
```
|
| 27 |
+
|
| 28 |
+
Secrets: independent VM_USER_TOKEN / VM_ADMIN_TOKEN (>=24 characters),
|
| 29 |
+
VM_BACKUP_TRIGGER_TOKEN (>=32 characters), VM_BACKUP_GITHUB_TOKEN (backup repository
|
| 30 |
+
Contents read/write, Metadata read, API + upload-host access), VM_BACKUP_IDENTITY
|
| 31 |
+
(age private recovery identity, direct HF Secret; do not proxy this value).
|
| 32 |
+
Keep the independent operator recovery key copy. Provider settings remain disabled
|
| 33 |
+
unless an explicitly authorized synthetic-provider test needs them. Source and
|
| 34 |
+
media/index fixtures must come from a separate disposable test namespace.
|
| 35 |
+
|
| 36 |
+
For a reviewed first initialization of this *new staging namespace only*, temporarily
|
| 37 |
+
set VM_BACKUP_ALLOW_EMPTY=true; remove it after the first verified checkpoint.
|
| 38 |
+
Never use this override to hide a failed restore of a previously initialized root.
|
| 39 |
+
Use synthetic video ingestion or the offline fixture exporter with writers stopped.
|
| 40 |
+
|
| 41 |
+
After the deployment is ready, execute the read-only private boundary probe:
|
| 42 |
+
|
| 43 |
+
```
|
| 44 |
+
python scripts/qualify_private_space.py \
|
| 45 |
+
--space Leon4gr45/openweb-v2-staging \
|
| 46 |
+
--url https://leon4gr45-openweb-v2-staging.hf.space
|
| 47 |
+
```
|
| 48 |
+
|
| 49 |
+
It requires HF_SPACE_TOKEN (read access to this private staging repo, plus direct
|
| 50 |
+
private app route access) and VM_BACKUP_TRIGGER_TOKEN from secure settings.
|
| 51 |
+
Add `--trigger` only when ready for an actual new checkpoint; it uses the bounded
|
| 52 |
+
coordinator. It records anonymous denial, authorized status, rejection of a wrong
|
| 53 |
+
trigger token, initial runtime stage, and checkpoint outcome. An already running
|
| 54 |
+
Space cannot establish a sleep/wake claim. Do not call restart/resume/hardware APIs;
|
| 55 |
+
wait for a naturally sleeping staging instance and test the authenticated route.
|
| 56 |
+
|
| 57 |
+
The backup repository's `inactive-workflows/qualify-private-staging.yml` is a
|
| 58 |
+
**manual-only** test workflow, with separate `private-backup-staging` environment
|
| 59 |
+
and STAGING_* variables/secrets. No production timer is activated. Once staging
|
| 60 |
+
credentials and routing work locally, review and install only this manual workflow,
|
| 61 |
+
dispatch from an actual runner, then record its run URL/checkpoint. The 12-hour
|
| 62 |
+
production workflow stays in inactive-workflows with approval guards unset.
|
| 63 |
+
|
| 64 |
+
Final drill: delete only this staging test working root, start its clean container,
|
| 65 |
+
and verify restored Library/Search/media/config/index from GitHub + protected key.
|
| 66 |
+
Record initial/final file hashes, SQLite integrity/logical rows, Range response and
|
| 67 |
+
actual media decoding. Production and pre-existing backups/releases remain untouched.
|
diagnostics/README.md
ADDED
|
@@ -0,0 +1,179 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Standalone GitHub encrypted-release probe
|
| 2 |
+
|
| 3 |
+
Copy **only `qualify_github_release.py`** to your trusted laptop/server if desired.
|
| 4 |
+
It needs Python 3.10+ and the official `age` / `age-keygen` executables on PATH.
|
| 5 |
+
No Visual Mentor installation, pip packages, GitHub CLI or application models are
|
| 6 |
+
required. It works with Linux and native Windows Python; Windows execution has not
|
| 7 |
+
been tested in this managed Linux environment.
|
| 8 |
+
|
| 9 |
+
Inject `VM_BACKUP_GITHUB_TOKEN` using your trusted machine's secure secret manager.
|
| 10 |
+
The script reads **only this environment variable** for credentials; it never reads
|
| 11 |
+
GH_TOKEN, credential files or command-line token arguments. Use the same permitted
|
| 12 |
+
credential as in the managed test, not a copied proxy placeholder. Limit it to the
|
| 13 |
+
private JsonLord/visual-mentor-backups repository, Contents read/write and Metadata
|
| 14 |
+
read. Do not paste credentials into command history, reports or chat.
|
| 15 |
+
|
| 16 |
+
The script keeps the machine's approved proxy/CA/TLS settings. Execute it on the
|
| 17 |
+
independent trusted machine normally; do not disable this managed environment's
|
| 18 |
+
proxy, firewall or certificate checks. Optional `--age` and `--age-keygen` arguments
|
| 19 |
+
accept executable paths when those tools are not on PATH.
|
| 20 |
+
|
| 21 |
+
## Linux / Debian: standalone upload test
|
| 22 |
+
|
| 23 |
+
Install the local prerequisites on your trusted Debian/Ubuntu server if needed:
|
| 24 |
+
|
| 25 |
+
```sh
|
| 26 |
+
sudo apt-get update
|
| 27 |
+
sudo apt-get install --no-install-recommends python3 age ca-certificates
|
| 28 |
+
```
|
| 29 |
+
|
| 30 |
+
With the credential already injected securely, run from the downloaded script's
|
| 31 |
+
folder (or use its absolute path):
|
| 32 |
+
|
| 33 |
+
```sh
|
| 34 |
+
python3 qualify_github_release.py --output standalone-upload.json
|
| 35 |
+
```
|
| 36 |
+
|
| 37 |
+
From the repository checkout the equivalent command is:
|
| 38 |
+
|
| 39 |
+
```sh
|
| 40 |
+
python3 diagnostics/qualify_github_release.py --output standalone-upload.json
|
| 41 |
+
```
|
| 42 |
+
|
| 43 |
+
Exit 0 means the new encrypted asset's SHA-256 was verified by downloading it.
|
| 44 |
+
Exit 2 means NO-GO; stop and inspect the sanitized evidence. No request is retried.
|
| 45 |
+
Without `--output`, a unique `github-release-qualification-<UTC timestamp>-<UUID>.json` is written in
|
| 46 |
+
the current directory. `--output /path/to/new.json` selects a new file explicitly;
|
| 47 |
+
an existing file is refused before any network call. Missing tools/credentials
|
| 48 |
+
also fail before creating a release.
|
| 49 |
+
|
| 50 |
+
## Windows: standalone upload test
|
| 51 |
+
|
| 52 |
+
Use Python 3.10+ (`py -3`) and PowerShell 7. Install the official pinned age tools
|
| 53 |
+
if they are not already on PATH:
|
| 54 |
+
|
| 55 |
+
```powershell
|
| 56 |
+
$tools = Join-Path $env:LOCALAPPDATA 'VisualMentorDiagnosticTools'
|
| 57 |
+
New-Item -ItemType Directory -Force -Path $tools | Out-Null
|
| 58 |
+
$zip = Join-Path $tools 'age-v1.2.1-windows-amd64.zip'
|
| 59 |
+
Invoke-WebRequest -Uri 'https://github.com/FiloSottile/age/releases/download/v1.2.1/age-v1.2.1-windows-amd64.zip' -OutFile $zip
|
| 60 |
+
Expand-Archive -Path $zip -DestinationPath $tools -Force
|
| 61 |
+
$env:Path = "$(Join-Path $tools 'age');$env:Path"
|
| 62 |
+
Get-Command age.exe, age-keygen.exe
|
| 63 |
+
```
|
| 64 |
+
|
| 65 |
+
These commands are for x64 Windows; use the matching official age release for
|
| 66 |
+
other architectures. They contain no credential values. With the dedicated token
|
| 67 |
+
already injected securely into this PowerShell session, run:
|
| 68 |
+
|
| 69 |
+
```powershell
|
| 70 |
+
py -3 .\qualify_github_release.py --output .\standalone-upload.json
|
| 71 |
+
if ($LASTEXITCODE -ne 0) { throw 'Upload qualification failed; inspect the new JSON evidence.' }
|
| 72 |
+
```
|
| 73 |
+
|
| 74 |
+
From the application checkout use `py -3 .\diagnostics\qualify_github_release.py --output .\standalone-upload.json`.
|
| 75 |
+
Do not add a token argument. Paths with spaces work with `--age` / `--age-keygen`:
|
| 76 |
+
|
| 77 |
+
```powershell
|
| 78 |
+
py -3 .\qualify_github_release.py --age "$tools\age\age.exe" --age-keygen "$tools\age\age-keygen.exe"
|
| 79 |
+
```
|
| 80 |
+
|
| 81 |
+
## What the probe does
|
| 82 |
+
|
| 83 |
+
1. Generates a small synthetic payload and a disposable age identity in a temporary
|
| 84 |
+
directory, encrypts the payload, and verifies a local decryption roundtrip.
|
| 85 |
+
2. Confirms the repository is private. Explicit denial of reported push permission
|
| 86 |
+
stops the run; successful creation of the new test release proves an actual
|
| 87 |
+
Contents-write operation (no repository file or checkpoint is edited).
|
| 88 |
+
3. Creates exactly one uniquely named `vm-credential-test-*` prerelease.
|
| 89 |
+
4. Validates and uses GitHub's returned upload_url for exactly one encrypted asset.
|
| 90 |
+
5. Downloads the asset by API ID and compares its ciphertext SHA-256 and byte count.
|
| 91 |
+
Signed release redirects are allowlisted and receive **no Authorization header**.
|
| 92 |
+
6. Saves method, host, status, GitHub request ID, release/asset IDs and hash results.
|
| 93 |
+
Tokens, plaintext payloads, recovery identities, response bodies and signed URLs
|
| 94 |
+
are never printed or written to evidence. The temporary identity is destroyed.
|
| 95 |
+
|
| 96 |
+
Any HTTP rejection stops immediately, including 401/403/404. The unique release
|
| 97 |
+
and any successful synthetic upload remain for inspection; nothing is deleted.
|
| 98 |
+
Existing releases/checkpoints and all workflow settings remain untouched. The
|
| 99 |
+
probe does not dispatch workflows or deploy anything. Its upload GO alone does
|
| 100 |
+
not establish a complete recovery or private-HF staging GO.
|
| 101 |
+
|
| 102 |
+
## Only after upload GO: complete destructive GitHub recovery
|
| 103 |
+
|
| 104 |
+
The full existing drill requires the application checkout, Python 3.12, uv, FFmpeg,
|
| 105 |
+
age/age-keygen and the same dedicated credential. It does not require HF access.
|
| 106 |
+
Only the disposable local fixture is destroyed. All previous remote backups remain.
|
| 107 |
+
The commands below reuse the preceding `standalone-upload.json`; they do not create
|
| 108 |
+
another standalone probe release. If you selected a different output filename,
|
| 109 |
+
substitute that filename in the evidence check.
|
| 110 |
+
Its query embedding is a deterministic synthetic stub; index retrieval is real.
|
| 111 |
+
|
| 112 |
+
**Linux**, from the application checkout:
|
| 113 |
+
|
| 114 |
+
```sh
|
| 115 |
+
set -e
|
| 116 |
+
if [ -z "${VM_BACKUP_GITHUB_TOKEN:-}" ]; then
|
| 117 |
+
echo 'Dedicated credential required; refusing integration fallback' >&2
|
| 118 |
+
exit 2
|
| 119 |
+
fi
|
| 120 |
+
python3 -c 'import json; r=json.load(open("standalone-upload.json")); assert r["status"] == "passed" and r["asset_readback_verified"] and r["credential_source"] == "VM_BACKUP_GITHUB_TOKEN"'
|
| 121 |
+
# set -e stops unless the preceding standalone report proves upload + readback.
|
| 122 |
+
archive=$(mktemp -d "$PWD/docs/evidence/trusted-recovery-XXXXXXXX")
|
| 123 |
+
if [ -f docs/evidence/github-recovery-live.json ]; then
|
| 124 |
+
cp docs/evidence/github-recovery-live.json "$archive/before.json"
|
| 125 |
+
fi
|
| 126 |
+
set +e
|
| 127 |
+
uv run --frozen --group dev --python 3.12 python tests/live_github_recovery.py
|
| 128 |
+
result=$?
|
| 129 |
+
set -e
|
| 130 |
+
if [ -f docs/evidence/github-recovery-live.json ]; then
|
| 131 |
+
cp docs/evidence/github-recovery-live.json "$archive/after.json"
|
| 132 |
+
fi
|
| 133 |
+
if [ "$result" -ne 0 ]; then
|
| 134 |
+
echo "NO-GO: recovery failed; evidence saved under $archive" >&2
|
| 135 |
+
exit "$result"
|
| 136 |
+
fi
|
| 137 |
+
echo "GitHub recovery GO; evidence saved under $archive"
|
| 138 |
+
```
|
| 139 |
+
|
| 140 |
+
**Windows PowerShell 7**, from the application checkout with uv, FFmpeg and age on
|
| 141 |
+
PATH and Python 3.12 installed:
|
| 142 |
+
|
| 143 |
+
```powershell
|
| 144 |
+
if (-not $env:VM_BACKUP_GITHUB_TOKEN) { throw 'Dedicated credential required; refusing integration fallback.' }
|
| 145 |
+
py -3 -c "import json; r=json.load(open('standalone-upload.json')); assert r['status'] == 'passed' and r['asset_readback_verified'] and r['credential_source'] == 'VM_BACKUP_GITHUB_TOKEN'"
|
| 146 |
+
if ($LASTEXITCODE -ne 0) { throw 'NO-GO: upload/readback report missing or failed; do not run recovery.' }
|
| 147 |
+
$archive = Join-Path (Get-Location) ('docs\evidence\trusted-recovery-' + [guid]::NewGuid().ToString('N'))
|
| 148 |
+
New-Item -ItemType Directory -Path $archive | Out-Null
|
| 149 |
+
$evidence = '.\docs\evidence\github-recovery-live.json'
|
| 150 |
+
if (Test-Path $evidence) { Copy-Item $evidence (Join-Path $archive 'before.json') }
|
| 151 |
+
uv run --frozen --group dev --python 3.12 python tests\live_github_recovery.py
|
| 152 |
+
$result = $LASTEXITCODE
|
| 153 |
+
if (Test-Path $evidence) { Copy-Item $evidence (Join-Path $archive 'after.json') }
|
| 154 |
+
if ($result -ne 0) { throw "NO-GO: recovery failed; evidence saved under $archive" }
|
| 155 |
+
Write-Host "GitHub recovery GO; evidence saved under $archive"
|
| 156 |
+
```
|
| 157 |
+
|
| 158 |
+
The harness creates another disposable namespace, commits two verified checkpoints,
|
| 159 |
+
checks unchanged incremental dedup, deletes source and staging, recovers using
|
| 160 |
+
GitHub plus the protected test identity, and checks every restored file hash,
|
| 161 |
+
SQLite integrity/logical rows, fresh-process Library/Search/config/index, authorized
|
| 162 |
+
media Range, anonymous media denial and FFmpeg decoding. It never deletes remote
|
| 163 |
+
releases or activates workflows. Archive copies preserve pre-existing local evidence
|
| 164 |
+
before the existing harness writes its fixed report filename.
|
| 165 |
+
|
| 166 |
+
Return only the new standalone JSON and the archived `after.json` for review.
|
| 167 |
+
Outside success with managed failure implicates host-specific credential/proxy
|
| 168 |
+
handling; matching failures require repository/permission/policy investigation.
|
| 169 |
+
Neither result alone proves whether a proxy strips versus replaces Authorization.
|
| 170 |
+
|
| 171 |
+
## Recorded managed result
|
| 172 |
+
|
| 173 |
+
The first standalone run reproduced API write success and upload **404**; full
|
| 174 |
+
recovery was therefore **not run**. See the immutable unique JSON under
|
| 175 |
+
`docs/evidence/standalone-probes/` and the qualification report. No outside-machine
|
| 176 |
+
or Windows live run is claimed. Four stdlib unittest cases validate the mocked
|
| 177 |
+
success/security/failure paths (`python -m unittest discover -s diagnostics/tests -v`).
|
| 178 |
+
Production workflows stay inactive. First private staging deployment remains NO-GO
|
| 179 |
+
until complete live recovery and the remaining private-HF checks are established.
|
diagnostics/qualify_github_release.py
ADDED
|
@@ -0,0 +1,218 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env python3
|
| 2 |
+
"""Standalone release probe: Python stdlib + age/age-keygen; no app imports.
|
| 3 |
+
|
| 4 |
+
Only VM_BACKUP_GITHUB_TOKEN is read for authentication. Existing releases and
|
| 5 |
+
checkpoints are never changed. Each run creates at most one new test release.
|
| 6 |
+
"""
|
| 7 |
+
import argparse
|
| 8 |
+
from datetime import datetime, timezone
|
| 9 |
+
import hashlib
|
| 10 |
+
import json
|
| 11 |
+
import os
|
| 12 |
+
from pathlib import Path
|
| 13 |
+
import re
|
| 14 |
+
import shutil
|
| 15 |
+
import ssl
|
| 16 |
+
import subprocess
|
| 17 |
+
import tempfile
|
| 18 |
+
from urllib.error import HTTPError, URLError
|
| 19 |
+
from urllib.parse import urlencode, urlsplit
|
| 20 |
+
from urllib.request import HTTPRedirectHandler, HTTPSHandler, Request, build_opener
|
| 21 |
+
import uuid
|
| 22 |
+
|
| 23 |
+
LIMIT = 1024 * 1024
|
| 24 |
+
|
| 25 |
+
|
| 26 |
+
class Failure(Exception):
|
| 27 |
+
pass
|
| 28 |
+
|
| 29 |
+
|
| 30 |
+
class NoRedirect(HTTPRedirectHandler):
|
| 31 |
+
def redirect_request(self, req, fp, code, msg, headers, newurl):
|
| 32 |
+
return None
|
| 33 |
+
|
| 34 |
+
|
| 35 |
+
class Probe:
|
| 36 |
+
def __init__(self, repo, token, report, save):
|
| 37 |
+
self.repo, self.token, self.report, self.save = repo, token, report, save
|
| 38 |
+
self.base = 'https://api.github.com/repos/' + repo
|
| 39 |
+
# Preserve the machine's approved proxy settings and CA trust.
|
| 40 |
+
context = ssl.create_default_context(cafile=os.environ.get('SSL_CERT_FILE') or None)
|
| 41 |
+
self.opener = build_opener(NoRedirect(), HTTPSHandler(context=context))
|
| 42 |
+
|
| 43 |
+
def request(self, method, url, data=None, authenticated=True, redirect=False):
|
| 44 |
+
parsed = urlsplit(url)
|
| 45 |
+
hosts = ('api.github.com', 'uploads.github.com') if authenticated else (
|
| 46 |
+
'release-assets.githubusercontent.com', 'objects.githubusercontent.com')
|
| 47 |
+
if (parsed.scheme != 'https' or parsed.hostname not in hosts
|
| 48 |
+
or parsed.username or parsed.password or parsed.port not in (None, 443)
|
| 49 |
+
or parsed.fragment):
|
| 50 |
+
raise Failure('destination_denied')
|
| 51 |
+
headers = {'Accept': 'application/vnd.github+json', 'User-Agent': 'vm-release-credential-probe',
|
| 52 |
+
'X-GitHub-Api-Version': '2022-11-28'}
|
| 53 |
+
if authenticated:
|
| 54 |
+
headers['Authorization'] = 'Bearer ' + self.token
|
| 55 |
+
if isinstance(data, dict):
|
| 56 |
+
data = json.dumps(data).encode()
|
| 57 |
+
headers['Content-Type'] = 'application/json'
|
| 58 |
+
elif data is not None:
|
| 59 |
+
headers['Content-Type'] = 'application/octet-stream'
|
| 60 |
+
if '/releases/assets/' in parsed.path or not authenticated:
|
| 61 |
+
headers['Accept'] = 'application/octet-stream'
|
| 62 |
+
try:
|
| 63 |
+
response = self.opener.open(Request(url, data=data, headers=headers, method=method), timeout=30)
|
| 64 |
+
except HTTPError as error:
|
| 65 |
+
response = error # Record response metadata only; never print its URL/body.
|
| 66 |
+
except (URLError, OSError):
|
| 67 |
+
raise Failure('network_or_tls_failure') from None
|
| 68 |
+
with response:
|
| 69 |
+
status = response.code
|
| 70 |
+
event = {'host': parsed.hostname, 'method': method, 'status': status,
|
| 71 |
+
'request_id': response.headers.get('X-GitHub-Request-Id'),
|
| 72 |
+
'authenticated': authenticated,
|
| 73 |
+
'accepted_permissions': response.headers.get('X-Accepted-GitHub-Permissions')}
|
| 74 |
+
self.report['http_events'].append(event)
|
| 75 |
+
self.save()
|
| 76 |
+
if redirect and status in (301, 302, 303, 307, 308):
|
| 77 |
+
return status, b'', response.headers.get('Location', '')
|
| 78 |
+
if not 200 <= status < 300:
|
| 79 |
+
# No retries, including 401/403/404: preserve the first rejection.
|
| 80 |
+
raise Failure('authentication_rejected' if status == 401 else 'http_rejected_' + str(status))
|
| 81 |
+
body = response.read(LIMIT + 1)
|
| 82 |
+
if len(body) > LIMIT:
|
| 83 |
+
raise Failure('response_too_large')
|
| 84 |
+
return status, body, None
|
| 85 |
+
|
| 86 |
+
def json_request(self, method, url, data=None):
|
| 87 |
+
status, body, _ = self.request(method, url, data)
|
| 88 |
+
try:
|
| 89 |
+
return status, json.loads(body)
|
| 90 |
+
except (ValueError, UnicodeError):
|
| 91 |
+
raise Failure('invalid_json_response') from None
|
| 92 |
+
|
| 93 |
+
def run(self, ciphertext, run_id):
|
| 94 |
+
self.report.update(ciphertext_bytes=len(ciphertext),
|
| 95 |
+
expected_sha256=hashlib.sha256(ciphertext).hexdigest())
|
| 96 |
+
self.save()
|
| 97 |
+
_, repo = self.json_request('GET', self.base)
|
| 98 |
+
if repo.get('private') is not True:
|
| 99 |
+
raise Failure('repository_not_private')
|
| 100 |
+
self.report['private_repository_verified'] = True
|
| 101 |
+
push = repo.get('permissions', {}).get('push')
|
| 102 |
+
self.report['repository_push_permission_reported'] = push
|
| 103 |
+
if push is False:
|
| 104 |
+
raise Failure('repository_write_permission_denied')
|
| 105 |
+
tag = 'vm-credential-test-' + run_id
|
| 106 |
+
status, release = self.json_request('POST', self.base + '/releases', {
|
| 107 |
+
'tag_name': tag, 'name': tag, 'prerelease': True, 'draft': False,
|
| 108 |
+
'body': 'Disposable synthetic encrypted credential probe. No application data.'})
|
| 109 |
+
if status != 201:
|
| 110 |
+
raise Failure('release_creation_not_confirmed')
|
| 111 |
+
release_id = release.get('id')
|
| 112 |
+
if not isinstance(release_id, int) or isinstance(release_id, bool) or release_id <= 0:
|
| 113 |
+
raise Failure('invalid_release_id')
|
| 114 |
+
self.report.update(release_id=release_id, release_tag=tag, contents_write_verified=True)
|
| 115 |
+
self.save() # Successful release creation is an actual Contents-write operation.
|
| 116 |
+
upload = release.get('upload_url', '').split('{', 1)[0]
|
| 117 |
+
parsed = urlsplit(upload)
|
| 118 |
+
expected = '/repos/' + self.repo + '/releases/' + str(release_id) + '/assets'
|
| 119 |
+
if (parsed.scheme != 'https' or parsed.hostname != 'uploads.github.com'
|
| 120 |
+
or parsed.path != expected or parsed.query or parsed.fragment
|
| 121 |
+
or parsed.username or parsed.password or parsed.port not in (None, 443)):
|
| 122 |
+
raise Failure('upload_url_denied')
|
| 123 |
+
status, asset = self.json_request('POST', upload + '?' + urlencode({'name': 'synthetic.age'}), ciphertext)
|
| 124 |
+
if status != 201:
|
| 125 |
+
raise Failure('asset_creation_not_confirmed')
|
| 126 |
+
asset_id = asset.get('id')
|
| 127 |
+
if not isinstance(asset_id, int) or isinstance(asset_id, bool) or asset_id <= 0:
|
| 128 |
+
raise Failure('invalid_asset_id')
|
| 129 |
+
self.report['asset_id'] = asset_id
|
| 130 |
+
self.save()
|
| 131 |
+
status, downloaded, location = self.request('GET', self.base + '/releases/assets/' + str(asset_id), redirect=True)
|
| 132 |
+
if location is not None:
|
| 133 |
+
_, downloaded, _ = self.request('GET', location, authenticated=False)
|
| 134 |
+
expected_hash = hashlib.sha256(ciphertext).hexdigest()
|
| 135 |
+
actual_hash = hashlib.sha256(downloaded).hexdigest()
|
| 136 |
+
self.report.update(ciphertext_bytes=len(ciphertext), expected_sha256=expected_hash,
|
| 137 |
+
downloaded_sha256=actual_hash)
|
| 138 |
+
if actual_hash != expected_hash or len(downloaded) != len(ciphertext):
|
| 139 |
+
raise Failure('asset_readback_mismatch')
|
| 140 |
+
self.report.update(asset_readback_verified=True, upload_decision='GO',
|
| 141 |
+
staging_decision='NO-GO: complete recovery and private HF qualification still required',
|
| 142 |
+
status='passed')
|
| 143 |
+
|
| 144 |
+
|
| 145 |
+
def encrypt_fixture(age, keygen):
|
| 146 |
+
with tempfile.TemporaryDirectory(prefix='vm-release-probe-') as scratch:
|
| 147 |
+
root = Path(scratch)
|
| 148 |
+
identity, clear, encrypted = root / 'identity.agekey', root / 'synthetic.txt', root / 'synthetic.age'
|
| 149 |
+
def command(args):
|
| 150 |
+
try:
|
| 151 |
+
result = subprocess.run(args, capture_output=True, timeout=30)
|
| 152 |
+
except (OSError, subprocess.TimeoutExpired):
|
| 153 |
+
raise Failure('age_tool_failed') from None
|
| 154 |
+
if result.returncode:
|
| 155 |
+
raise Failure('age_tool_failed')
|
| 156 |
+
return result.stdout
|
| 157 |
+
command([keygen, '-o', str(identity)])
|
| 158 |
+
identity.chmod(0o600)
|
| 159 |
+
recipient = command([keygen, '-y', str(identity)]).decode().strip()
|
| 160 |
+
payload = b'Visual Mentor synthetic release credential qualification; no user data.\n'
|
| 161 |
+
clear.write_bytes(payload)
|
| 162 |
+
command([age, '-r', recipient, '-o', str(encrypted), str(clear)])
|
| 163 |
+
if command([age, '-d', '-i', str(identity), str(encrypted)]) != payload:
|
| 164 |
+
raise Failure('local_age_roundtrip_failed')
|
| 165 |
+
ciphertext = encrypted.read_bytes()
|
| 166 |
+
if not ciphertext.startswith(b'age-encryption.org/v1') or len(ciphertext) >= LIMIT:
|
| 167 |
+
raise Failure('invalid_encrypted_fixture')
|
| 168 |
+
return ciphertext # The disposable test key is destroyed on exit.
|
| 169 |
+
|
| 170 |
+
|
| 171 |
+
def main():
|
| 172 |
+
parser = argparse.ArgumentParser(description=__doc__)
|
| 173 |
+
parser.add_argument('--repo', default='JsonLord/visual-mentor-backups')
|
| 174 |
+
parser.add_argument('--output', type=Path, help='New evidence file; existing files are refused')
|
| 175 |
+
parser.add_argument('--age', default='age', help='age executable path')
|
| 176 |
+
parser.add_argument('--age-keygen', default='age-keygen', help='age-keygen executable path')
|
| 177 |
+
args = parser.parse_args()
|
| 178 |
+
if not re.fullmatch(r'[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+', args.repo):
|
| 179 |
+
parser.error('invalid owner/repository')
|
| 180 |
+
run_id = datetime.now(timezone.utc).strftime('%Y%m%dT%H%M%SZ') + '-' + uuid.uuid4().hex
|
| 181 |
+
output = args.output or Path('github-release-qualification-' + run_id + '.json')
|
| 182 |
+
report = {'run_id': run_id, 'repository': args.repo, 'credential_source': 'VM_BACKUP_GITHUB_TOKEN',
|
| 183 |
+
'status': 'started', 'upload_decision': 'NO-GO', 'staging_decision': 'NO-GO',
|
| 184 |
+
'contents_write_verified': False, 'asset_readback_verified': False, 'http_events': [],
|
| 185 |
+
'recovery_run': False, 'production_or_workflows_changed': False}
|
| 186 |
+
# Exclusive creation protects all previous evidence. Only this run's new file is updated.
|
| 187 |
+
try:
|
| 188 |
+
with output.open('x', encoding='utf-8') as stream:
|
| 189 |
+
stream.write(json.dumps(report, indent=2) + '\n')
|
| 190 |
+
output.chmod(0o600)
|
| 191 |
+
except OSError:
|
| 192 |
+
print('NO-GO: evidence path unavailable or already exists; no requests made.')
|
| 193 |
+
return 2
|
| 194 |
+
def save():
|
| 195 |
+
output.write_text(json.dumps(report, indent=2) + '\n', encoding='utf-8')
|
| 196 |
+
try:
|
| 197 |
+
token = os.environ.get('VM_BACKUP_GITHUB_TOKEN')
|
| 198 |
+
if not token:
|
| 199 |
+
raise Failure('dedicated_credential_missing')
|
| 200 |
+
age, keygen = shutil.which(args.age), shutil.which(args.age_keygen)
|
| 201 |
+
if not age or not keygen:
|
| 202 |
+
raise Failure('install_age_and_age_keygen')
|
| 203 |
+
ciphertext = encrypt_fixture(age, keygen)
|
| 204 |
+
report['local_age_roundtrip_verified'] = True
|
| 205 |
+
Probe(args.repo, token, report, save).run(ciphertext, run_id)
|
| 206 |
+
except Failure as error:
|
| 207 |
+
report.update(status='failed', failure_code=str(error))
|
| 208 |
+
except Exception:
|
| 209 |
+
report.update(status='failed', failure_code='unexpected_local_failure')
|
| 210 |
+
save()
|
| 211 |
+
print(json.dumps({'status': report['status'], 'upload_decision': report['upload_decision'],
|
| 212 |
+
'evidence': str(output), 'failure_code': report.get('failure_code'),
|
| 213 |
+
'asset_id': report.get('asset_id')}))
|
| 214 |
+
return 0 if report['status'] == 'passed' else 2
|
| 215 |
+
|
| 216 |
+
|
| 217 |
+
if __name__ == '__main__':
|
| 218 |
+
raise SystemExit(main())
|
diagnostics/tests/test_release_probe.py
ADDED
|
@@ -0,0 +1,95 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import hashlib
|
| 2 |
+
import importlib.util
|
| 3 |
+
import io
|
| 4 |
+
import json
|
| 5 |
+
from pathlib import Path
|
| 6 |
+
import tempfile
|
| 7 |
+
import unittest
|
| 8 |
+
from urllib.error import HTTPError
|
| 9 |
+
from unittest.mock import patch
|
| 10 |
+
|
| 11 |
+
spec = importlib.util.spec_from_file_location('probe', Path(__file__).parents[1] / 'qualify_github_release.py')
|
| 12 |
+
probe = importlib.util.module_from_spec(spec)
|
| 13 |
+
spec.loader.exec_module(probe)
|
| 14 |
+
|
| 15 |
+
|
| 16 |
+
class Response(io.BytesIO):
|
| 17 |
+
def __init__(self, code, body=b'', headers=None):
|
| 18 |
+
super().__init__(body)
|
| 19 |
+
self.code, self.headers = code, headers or {'X-GitHub-Request-Id': 'TEST:SAFE'}
|
| 20 |
+
|
| 21 |
+
|
| 22 |
+
class ProbeTests(unittest.TestCase):
|
| 23 |
+
def make(self):
|
| 24 |
+
report = {'http_events': []}
|
| 25 |
+
return probe.Probe('test/private', 'secret-never-log', report, lambda: None), report
|
| 26 |
+
|
| 27 |
+
def test_roundtrip_and_stripped_signed_download(self):
|
| 28 |
+
client, report = self.make()
|
| 29 |
+
ciphertext = b'age-encryption.org/v1\nsynthetic mock ciphertext'
|
| 30 |
+
requests = []
|
| 31 |
+
class Opener:
|
| 32 |
+
def open(self, req, timeout):
|
| 33 |
+
requests.append(req)
|
| 34 |
+
url, method = req.full_url, req.get_method()
|
| 35 |
+
if url == client.base:
|
| 36 |
+
return Response(200, json.dumps({'private': True, 'permissions': {'push': True}}).encode())
|
| 37 |
+
if url.endswith('/releases'):
|
| 38 |
+
return Response(201, json.dumps({'id': 9, 'upload_url': 'https://uploads.github.com/repos/test/private/releases/9/assets{?name,label}'}).encode())
|
| 39 |
+
if 'uploads.github.com' in url:
|
| 40 |
+
self.uploaded = req.data
|
| 41 |
+
return Response(201, b'{"id":19}')
|
| 42 |
+
if '/releases/assets/19' in url:
|
| 43 |
+
return Response(302, headers={'Location': 'https://release-assets.githubusercontent.com/object?signature=not-recorded'})
|
| 44 |
+
assert req.get_header('Authorization') is None
|
| 45 |
+
return Response(200, self.uploaded)
|
| 46 |
+
client.opener = Opener()
|
| 47 |
+
client.run(ciphertext, 'unique')
|
| 48 |
+
self.assertTrue(report['contents_write_verified'])
|
| 49 |
+
self.assertTrue(report['asset_readback_verified'])
|
| 50 |
+
self.assertEqual(report['downloaded_sha256'], hashlib.sha256(ciphertext).hexdigest())
|
| 51 |
+
self.assertEqual(report['asset_id'], 19)
|
| 52 |
+
self.assertEqual(len(requests), 5)
|
| 53 |
+
self.assertNotIn('secret-never-log', json.dumps(report))
|
| 54 |
+
self.assertNotIn('signature', json.dumps(report))
|
| 55 |
+
self.assertEqual(sum(r.get_method() == 'POST' and r.full_url.endswith('/releases') for r in requests), 1)
|
| 56 |
+
|
| 57 |
+
def test_permanent_failure_stops_without_retry(self):
|
| 58 |
+
for status in (401, 403, 404, 422):
|
| 59 |
+
client, report = self.make()
|
| 60 |
+
calls = []
|
| 61 |
+
class Opener:
|
| 62 |
+
def open(self, req, timeout):
|
| 63 |
+
calls.append(req)
|
| 64 |
+
raise HTTPError(req.full_url, status, 'redacted', {'X-GitHub-Request-Id': 'REJECT'}, io.BytesIO(b'never-record-this-body'))
|
| 65 |
+
client.opener = Opener()
|
| 66 |
+
with self.assertRaises(probe.Failure):
|
| 67 |
+
client.request('POST', 'https://uploads.github.com/endpoint', b'encrypted')
|
| 68 |
+
self.assertEqual(len(calls), 1)
|
| 69 |
+
self.assertEqual(report['http_events'][0]['status'], status)
|
| 70 |
+
self.assertNotIn('never-record', json.dumps(report))
|
| 71 |
+
|
| 72 |
+
def test_untrusted_destination_rejected_before_request(self):
|
| 73 |
+
client, _ = self.make()
|
| 74 |
+
class Forbidden:
|
| 75 |
+
def open(self, *args, **kwargs):
|
| 76 |
+
raise AssertionError('must not send')
|
| 77 |
+
client.opener = Forbidden()
|
| 78 |
+
for url in ('http://api.github.com/private', 'https://evil.example/private', 'https://token@api.github.com/private'):
|
| 79 |
+
with self.assertRaises(probe.Failure):
|
| 80 |
+
client.request('GET', url)
|
| 81 |
+
|
| 82 |
+
def test_missing_dedicated_token_does_not_use_integration_token(self):
|
| 83 |
+
with tempfile.TemporaryDirectory() as root:
|
| 84 |
+
output = Path(root) / 'new.json'
|
| 85 |
+
with patch.dict(probe.os.environ, {'GH_TOKEN': 'never-use'}, clear=True), patch('sys.argv', ['probe', '--output', str(output)]), patch('builtins.print'):
|
| 86 |
+
self.assertEqual(probe.main(), 2)
|
| 87 |
+
self.assertEqual(json.loads(output.read_text())['failure_code'], 'dedicated_credential_missing')
|
| 88 |
+
original = output.read_bytes()
|
| 89 |
+
with patch('sys.argv', ['probe', '--output', str(output)]), patch('builtins.print'):
|
| 90 |
+
self.assertEqual(probe.main(), 2)
|
| 91 |
+
self.assertEqual(output.read_bytes(), original)
|
| 92 |
+
|
| 93 |
+
|
| 94 |
+
if __name__ == '__main__':
|
| 95 |
+
unittest.main()
|
docs/backup-restore.md
ADDED
|
@@ -0,0 +1,167 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# GitHub-native encrypted backup and recovery
|
| 2 |
+
|
| 3 |
+
The active filesystem is HF CPU Basic ephemeral disk. The private
|
| 4 |
+
`JsonLord/visual-mentor-backups` repository is the sole persistent recovery store.
|
| 5 |
+
No object store, rclone remote, HF paid disk, or independent media copy is required.
|
| 6 |
+
This implementation is prepared for review; production scheduling and HF deployment
|
| 7 |
+
remain disabled. See `github-backup-migration.md` for actual test evidence and gates.
|
| 8 |
+
|
| 9 |
+
## Recovery protocol (format 2)
|
| 10 |
+
|
| 11 |
+
The Space pauses ingestion/upload and configuration writes using one asyncio lock,
|
| 12 |
+
then copies its complete recognized data tree and config history into a private
|
| 13 |
+
snapshot directory. SQLite uses the online backup API, integrity check and VACUUM.
|
| 14 |
+
The copied index includes CURRENT and all versioned revisions, manifests, vectors
|
| 15 |
+
and segments. Coverage validates published videos and referenced index media.
|
| 16 |
+
All original videos, proxies, thumbnails, segment frames, transcript files, index
|
| 17 |
+
revisions, experiments/evaluation files, saved answer history, queue DB and configuration history travel
|
| 18 |
+
with the checkpoint. Unknown data/config files, symlinks, unfinished temporary
|
| 19 |
+
files, missing required media, incompatible index layout and capacity violations
|
| 20 |
+
fail the checkpoint. No success is reported for an omitted required file.
|
| 21 |
+
|
| 22 |
+
Uploads happen after releasing the write lock. Files are split into 16 MiB chunks,
|
| 23 |
+
addressed by plaintext SHA-256 **inside the encrypted manifest**. Identical chunks
|
| 24 |
+
in the same checkpoint and its immediate predecessor reuse existing assets. A
|
| 25 |
+
chunk absent from that predecessor may upload again even if an older checkpoint
|
| 26 |
+
contains it; this is incremental deduplication, not global garbage collection.
|
| 27 |
+
New chunks are age-encrypted before upload. GitHub release asset names are hashes
|
| 28 |
+
of ciphertext. Decryptor/recipient compatibility is checked before publication.
|
| 29 |
+
Existing objects and new uploads are read back and hash-verified. A manifest
|
| 30 |
+
contains file paths, hashes, chunk references, index revision/model space and
|
| 31 |
+
coverage. It is itself encrypted and uploaded as a private release asset.
|
| 32 |
+
|
| 33 |
+
Only after every referenced object verifies does a unique GitHub Contents commit
|
| 34 |
+
publish `checkpoints/<namespace>/<year>/<timestamp>-<uuid>.json`. That small record
|
| 35 |
+
contains format, creation time, encrypted manifest asset ID/size/hash; it contains
|
| 36 |
+
no media, transcript, config, plaintext hashes or key. GitHub also observes repo,
|
| 37 |
+
release names, object sizes, timing and access. These operational metadata cannot
|
| 38 |
+
be hidden from the storage provider. No plaintext user payload is transmitted.
|
| 39 |
+
The repository record is the atomic commit point. Partial releases are uncommitted
|
| 40 |
+
orphans, not restore points. Existing records/assets are never overwritten or
|
| 41 |
+
pruned. Transient calls retry at most three times; immutable names reconcile
|
| 42 |
+
uncertain responses. Snapshot attempts also retry three times, with 90-second
|
| 43 |
+
lock acquisition bounds. Concurrent triggers coalesce in the one-worker queue.
|
| 44 |
+
Only one application process/replica is supported. Do not run external writers or
|
| 45 |
+
CLI ingestion alongside the serving process; use offline CLI after stopping it.
|
| 46 |
+
|
| 47 |
+
## Space settings (after deployment approval)
|
| 48 |
+
|
| 49 |
+
HF Variables:
|
| 50 |
+
|
| 51 |
+
```
|
| 52 |
+
VM_BACKUP_ENABLED=true
|
| 53 |
+
VM_BACKUP_REPOSITORY=JsonLord/visual-mentor-backups
|
| 54 |
+
VM_BACKUP_NAMESPACE=production
|
| 55 |
+
VM_RECOVERY_ROOT=/app/work
|
| 56 |
+
VRAG_DATA=/app/work/data
|
| 57 |
+
VM_STATE_DIR=/app/work/state
|
| 58 |
+
VM_BACKUP_RECIPIENT=age1<public-recipient>
|
| 59 |
+
```
|
| 60 |
+
|
| 61 |
+
HF **Secrets**, entered securely in settings:
|
| 62 |
+
|
| 63 |
+
- `VM_BACKUP_GITHUB_TOKEN`: fine-grained identity limited to the private backup
|
| 64 |
+
repository, Contents read/write and release asset access. Test both
|
| 65 |
+
`api.github.com` and `uploads.github.com`. It requires neither repository
|
| 66 |
+
administration nor Actions permission. Private downloads may redirect to signed
|
| 67 |
+
GitHub release URLs; the client never forwards its bearer token to that host.
|
| 68 |
+
- `VM_BACKUP_IDENTITY`: age private identity text matching the public recipient.
|
| 69 |
+
It is materialized with mode 0600 into a temporary file outside the working tree
|
| 70 |
+
before startup; the raw variable is removed from inherited child environment.
|
| 71 |
+
Alternatively provide a securely provisioned `VM_BACKUP_IDENTITY_FILE` outside
|
| 72 |
+
`/app/work`. Protect an independent operator copy of this key: GitHub data alone
|
| 73 |
+
cannot recover without it. This key is intentionally never backed up to GitHub.
|
| 74 |
+
- `VM_BACKUP_TRIGGER_TOKEN`: independent random >=32-character token. It authorizes
|
| 75 |
+
backup enqueue/status only, through `X-VM-Backup-Token`. It grants no config,
|
| 76 |
+
inference, media or isolated-recovery-test privileges. `VM_ADMIN_TOKEN` retains
|
| 77 |
+
admin access and is never given to the coordinator.
|
| 78 |
+
|
| 79 |
+
Keep provider credentials and app role tokens as before. They are excluded from
|
| 80 |
+
checkpoints and must be re-provisioned for a recreated Space. Configuration retains
|
| 81 |
+
only secret/endpoint environment-variable references; the environment values are
|
| 82 |
+
not recovered. If using a custom index path, it must equal the covered data/index.
|
| 83 |
+
|
| 84 |
+
The supervisor runs recovery **before** importing ConfigStore, index or workers.
|
| 85 |
+
On an empty working root it selects the newest committed record, fetches and
|
| 86 |
+
decrypts only GitHub-hosted objects, checks every byte/hash, SQLite, config and
|
| 87 |
+
index coverage, and renames a sibling staging directory into place atomically.
|
| 88 |
+
Failure aborts startup. It never merges into or overwrites nonempty state and never
|
| 89 |
+
silently falls back from a corrupt latest checkpoint. To choose an earlier point,
|
| 90 |
+
use `python scripts/restore_state.py --destination /new/isolated-root --checkpoint
|
| 91 |
+
<checkpoint>` with the same protected environment, then review before promotion.
|
| 92 |
+
|
| 93 |
+
On a brand-new namespace with no checkpoint, startup refuses by default. A reviewed
|
| 94 |
+
first initialization may temporarily set `VM_BACKUP_ALLOW_EMPTY=true`; remove it
|
| 95 |
+
immediately after the first verified checkpoint. A namespace with prior data must
|
| 96 |
+
never use this escape hatch to conceal a recovery failure.
|
| 97 |
+
|
| 98 |
+
## Triggers, administration and coordinator
|
| 99 |
+
|
| 100 |
+
Backup requests occur on startup, accepted media upload, completed ingestion,
|
| 101 |
+
applied/rolled-back configuration, authenticated manual trigger and the resident
|
| 102 |
+
12-hour timer. The timer continues only while the Space process is running. It
|
| 103 |
+
cannot wake a sleeping Space and is **not** an independently durable scheduler.
|
| 104 |
+
The queue reports failure codes, attempt time, pending encrypted uploads, latest
|
| 105 |
+
successful checkpoint, inventory bytes including historical/orphan assets, restore
|
| 106 |
+
points, and last isolated recovery result. `/backups` is the administration panel;
|
| 107 |
+
server routes require administrator or the deliberately limited coordinator token.
|
| 108 |
+
The panel does not claim enqueue acceptance is backup success.
|
| 109 |
+
|
| 110 |
+
`deploy/backup-repository/inactive-workflows/backup-12h.yml` runs at 00:17 and 12:17
|
| 111 |
+
UTC (02:17/14:17 Berlin summer time; 01:17/13:17 Berlin winter time). It is outside
|
| 112 |
+
`.github/workflows` and cannot run. It also requires both `BACKUPS_APPROVED` and
|
| 113 |
+
`COORDINATOR_VALIDATED` to equal `true`, plus the protected `private-backup`
|
| 114 |
+
environment. No workflow is activated by source upload.
|
| 115 |
+
|
| 116 |
+
The coordinator verifies Space privacy using the HF API, sends the scoped HF
|
| 117 |
+
bearer token through the private Space access boundary, and sends the independent
|
| 118 |
+
app trigger token in a separate header. It refuses redirects and unexpected URL
|
| 119 |
+
origins. It probes up to ten nominal minutes for a sleeping instance, then triggers
|
| 120 |
+
and waits for a **new completed checkpoint**, not just HTTP acceptance. Network
|
| 121 |
+
calls have timeouts and attempts are bounded; Actions has a 35-minute outer timeout.
|
| 122 |
+
A failed private route/wake/authentication test is a deployment blocker. Do not
|
| 123 |
+
replace it with a public Space, arbitrary proxy, app admin token or unauthenticated
|
| 124 |
+
export. The resident timer is a tested local fallback while awake only; it is not
|
| 125 |
+
claimed as a validated sleeping-Space alternative.
|
| 126 |
+
|
| 127 |
+
## Limits, retention and recovery point objective
|
| 128 |
+
|
| 129 |
+
Current application upload limit is 64 MiB/video, 30 minutes, 10k indexed segments.
|
| 130 |
+
Backup plaintext capacity is 20 GiB / 100k files. Capture needs free scratch space
|
| 131 |
+
at least the source size plus 64 MiB; source+snapshot and restore staging compete
|
| 132 |
+
with HF's 50 GB ephemeral allowance. Actual usable capacity is lower after image,
|
| 133 |
+
processing scratch and previous index revisions. Budget below 20 GiB, monitor
|
| 134 |
+
usage and fail explicitly at capacity. Objects are 16 MiB plus age overhead,
|
| 135 |
+
well below GitHub's 2 GiB per-asset limit. Uploads shard at 800 assets per release
|
| 136 |
+
to stay below the 1000-assets/release limit. GitHub release policy, API quotas,
|
| 137 |
+
Acceptable Use and private-account storage entitlements still need operator review;
|
| 138 |
+
release assets must not be assumed an unlimited general-purpose storage service.
|
| 139 |
+
Checkpoint metadata is sharded by namespace/year and refuses a 1000-entry listing;
|
| 140 |
+
rotate to a reviewed namespace before reaching that limit (12h plus event triggers
|
| 141 |
+
can reach it sooner than a year). No destructive retention is automated.
|
| 142 |
+
|
| 143 |
+
Unchanged chunks cost no new payload assets, but dedup readback downloads referenced
|
| 144 |
+
ciphertext to ensure it still exists. Checkpoints add an encrypted manifest and a
|
| 145 |
+
small repository commit. A 64 MiB original plus 20 MiB proxy, 1 MiB images/transcript
|
| 146 |
+
and a 3 MiB index starts around 88 MiB; compressed media does not shrink under age.
|
| 147 |
+
At 100 similarly sized videos, media is roughly 8.5 GiB plus accumulated indexes.
|
| 148 |
+
A 10k x 768 float32 text matrix is 30.72 MB (29.30 MiB) per revision. Five views are
|
| 149 |
+
153.6 MB (146.48 MiB) per revision. Twenty retained single-view revisions add about
|
| 150 |
+
586 MiB, excluding JSON. Metadata-only edits upload only changed chunks and a new
|
| 151 |
+
manifest; changed full matrix files can upload most chunks again.
|
| 152 |
+
|
| 153 |
+
Ephemeral disk losses can destroy uncheckpointed uploads. Event triggers reduce
|
| 154 |
+
but do not eliminate that window; backups are asynchronous and a sleeping Space
|
| 155 |
+
cannot expose lost disk to Actions. GitHub downtime, token expiry, key loss, disk
|
| 156 |
+
capacity and quotas can prevent recovery. Require a successful staging drill and
|
| 157 |
+
real sleep/wake run before asserting a 12-hour recovery objective.
|
| 158 |
+
|
| 159 |
+
## Migration and activation
|
| 160 |
+
|
| 161 |
+
Retain format-1 archives unchanged. They used external asset coverage and are not
|
| 162 |
+
self-contained format-2 checkpoints. Import their verified full restored data into
|
| 163 |
+
an isolated format-2 root while writers are stopped, then create and independently
|
| 164 |
+
recover a GitHub-native checkpoint before retiring external copies. Never delete
|
| 165 |
+
old assets as part of this refactor. Old helper functions remain only for format-1
|
| 166 |
+
compatibility tests; the exporter and active external-source templates are retired.
|
| 167 |
+
Production activation checklist is in `github-backup-migration.md`.
|
docs/credential-qualification.md
ADDED
|
@@ -0,0 +1,106 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Dedicated credential qualification: NO-GO (18 October 2026)
|
| 2 |
+
|
| 3 |
+
The dedicated secure binding is now present and reports ready. Every qualification
|
| 4 |
+
request in this run used **VM_BACKUP_GITHUB_TOKEN**. The integration-managed GH_TOKEN
|
| 5 |
+
was not substituted. No backup architecture or application code was changed.
|
| 6 |
+
|
| 7 |
+
## Actual live results
|
| 8 |
+
|
| 9 |
+
| Check | Result |
|
| 10 |
+
| --- | --- |
|
| 11 |
+
| Private backup repository read, api.github.com | 200; private=true, read/write reported |
|
| 12 |
+
| New disposable release creation, api.github.com | 201 |
|
| 13 |
+
| First small age-encrypted asset POST, uploads.github.com | **404**, one attempt |
|
| 14 |
+
| Uploaded assets / committed checkpoints | **0 / 0**, verified by read-only queries |
|
| 15 |
+
| Destructive recovery and incremental checkpoint | Blocked by upload failure |
|
| 16 |
+
| Trusted-machine comparison | Unavailable: no independent trusted machine attached |
|
| 17 |
+
| Secure HF credential | Absent; private staging remains unqualified |
|
| 18 |
+
|
| 19 |
+
Upload request ID: `FE93:319F8B:36F88A:4FB435:6AD3F6B8`.
|
| 20 |
+
Disposable namespace: `test-recovery-20261017-745d0278` (the harness uses UTC host
|
| 21 |
+
clock naming; this qualification report uses the client Berlin date).
|
| 22 |
+
The new release is uncommitted and empty; no pre-existing release or checkpoint
|
| 23 |
+
was altered or deleted. No recurring workflow or production setting changed.
|
| 24 |
+
|
| 25 |
+
The client used GitHub's returned upload_url, validated its destination and
|
| 26 |
+
attached the dedicated authorization header. A GitHub response/request ID proves
|
| 27 |
+
an HTTPS service response, not which credential the proxy ultimately forwarded.
|
| 28 |
+
The earlier missing-binding problem is resolved. The former integration-token 401
|
| 29 |
+
is not reused as evidence about this token. The new 404 may conceal a private
|
| 30 |
+
resource/permission rejection or reflect credential propagation differences
|
| 31 |
+
between hosts. **Scope versus injection/proxy attribution remains undetermined.**
|
| 32 |
+
API release creation proves a successful write through that route; it does not
|
| 33 |
+
prove that the same upstream identity was used on the upload route.
|
| 34 |
+
|
| 35 |
+
Evidence:
|
| 36 |
+
|
| 37 |
+
- [Dedicated API probe](evidence/dedicated-token-api.json)
|
| 38 |
+
- [Current live upload/recovery attempt](evidence/github-recovery-live.json)
|
| 39 |
+
- [Qualification decision](evidence/credential-qualification.json)
|
| 40 |
+
- [Prior blocked attempt retained](evidence/github-recovery-prior-blocked.json)
|
| 41 |
+
|
| 42 |
+
## Exact next step: trusted-machine comparison
|
| 43 |
+
|
| 44 |
+
Use an independently trusted machine outside this managed proxy, provision the
|
| 45 |
+
**same permitted dedicated credential** there from its secure store, and check out
|
| 46 |
+
application branch feature/video-rag-mentor-v2. Do not export a proxy placeholder,
|
| 47 |
+
copy integration-managed credentials, paste credentials into chat, disable TLS,
|
| 48 |
+
or bypass this managed environment's network controls.
|
| 49 |
+
|
| 50 |
+
With Python 3.12+, uv, FFmpeg and age/age-keygen available, run from the checkout:
|
| 51 |
+
|
| 52 |
+
```sh
|
| 53 |
+
if [ -z "${VM_BACKUP_GITHUB_TOKEN:-}" ]; then
|
| 54 |
+
echo 'Dedicated backup credential required' >&2
|
| 55 |
+
exit 2
|
| 56 |
+
fi
|
| 57 |
+
uv run --frozen --group dev python tests/live_github_recovery.py
|
| 58 |
+
```
|
| 59 |
+
|
| 60 |
+
The guard prevents the harness's legacy GH_TOKEN fallback. The drill creates a new
|
| 61 |
+
disposable namespace and never deletes existing remote releases. Successful upload
|
| 62 |
+
continues to readback, committed checkpoints, unchanged incremental dedup and full
|
| 63 |
+
recovery after destroying local source/staging state, with fresh-process Library,
|
| 64 |
+
Search, config/index and media checks. Compare sanitized evidence JSON only.
|
| 65 |
+
|
| 66 |
+
- Outside succeeds while managed fails: managed credential propagation or proxy
|
| 67 |
+
behavior is implicated; header stripping versus rewriting still needs trusted
|
| 68 |
+
proxy observability. Do not weaken or work around its security controls.
|
| 69 |
+
- Outside also fails: review the credential's resource owner, selected private
|
| 70 |
+
repository JsonLord/visual-mentor-backups, Contents read/write, Metadata read,
|
| 71 |
+
expiry and any applicable organization authorization. Matching failures alone
|
| 72 |
+
do not conclusively distinguish every GitHub policy/resource issue.
|
| 73 |
+
|
| 74 |
+
Private staging package remains prepared at deploy/hf-staging/README.md. Secure HF
|
| 75 |
+
credentials are still absent, and the existing public Space remains unchanged.
|
| 76 |
+
**NO-GO for first private staging deployment** until a dedicated-token asset upload
|
| 77 |
+
and complete GitHub-only recovery succeed and private staging access is qualified.
|
| 78 |
+
|
| 79 |
+
|
| 80 |
+
## Standalone trusted-machine diagnostic prepared (18 October 2026)
|
| 81 |
+
|
| 82 |
+
The independent standard-library probe is [qualify_github_release.py](../diagnostics/qualify_github_release.py).
|
| 83 |
+
It needs only Python and age/age-keygen; it imports no application code, reads only
|
| 84 |
+
VM_BACKUP_GITHUB_TOKEN, creates one unique test release, uses the returned validated
|
| 85 |
+
upload_url, and verifies encrypted asset SHA-256 by readback. Permanent HTTP errors
|
| 86 |
+
stop immediately without retries. Its new evidence filename is unique and existing
|
| 87 |
+
files are refused. All prior evidence above has been preserved.
|
| 88 |
+
|
| 89 |
+
[Exact Linux and Windows instructions](../diagnostics/README.md) include a gated
|
| 90 |
+
full-recovery invocation after upload GO and archival of the existing recovery
|
| 91 |
+
report before the unchanged harness runs. Windows execution and an independent
|
| 92 |
+
trusted machine are not available in this managed Linux session.
|
| 93 |
+
|
| 94 |
+
Four standalone stdlib unit tests passed: mocked successful readback/hash, removal
|
| 95 |
+
of authorization on signed redirects, no fallback to GH_TOKEN, evidence-overwrite
|
| 96 |
+
refusal, destination validation and permanent-rejection stopping. The managed live
|
| 97 |
+
probe again created a new synthetic test release successfully, but its first asset
|
| 98 |
+
upload returned 404. No asset ID/readback or complete recovery success is claimed.
|
| 99 |
+
The full destructive test was not run because its upload prerequisite did not pass.
|
| 100 |
+
|
| 101 |
+
New evidence: [standalone probe JSON](evidence/standalone-probes/github-release-qualification-20261017T223752Z-a13651aa74504b7ca9a9a2f1f83e0100.json).
|
| 102 |
+
**Managed upload: NO-GO; full GitHub recovery: not run; first private staging: NO-GO.**
|
| 103 |
+
No backup code was refactored, deployment performed, workflow activated, or existing
|
| 104 |
+
release/checkpoint modified or deleted. The next action is to run this standalone
|
| 105 |
+
script on the trusted laptop/Debian server with the same securely provisioned token
|
| 106 |
+
and return its sanitized JSON, never the token or a proxy placeholder.
|
docs/deployment-hf.md
ADDED
|
@@ -0,0 +1,118 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Visual Mentor v2: private Hugging Face staging and rollback
|
| 2 |
+
|
| 3 |
+
**Prepared only. No Space has been created, uploaded or modified.** Current
|
| 4 |
+
production target `Leon4gr45/openweb` is protected by explicit approval and a staging
|
| 5 |
+
UI review. No DGX/Needle/inference infrastructure change is required.
|
| 6 |
+
|
| 7 |
+
## Local package and evidence
|
| 8 |
+
|
| 9 |
+
Docker runs Node/Next.js and one FastAPI worker, supervised by `scripts/cloud-start.py`.
|
| 10 |
+
Only port **7860** is exposed. FastAPI binds loopback 8000. One authoritative
|
| 11 |
+
`web/next.config.ts` preserves `/api` and `/media`, with Range seeking and bounded
|
| 12 |
+
65MB proxy buffering for the application's 64MiB upload cap. Node/Python bases are
|
| 13 |
+
pinned. No torch, transformers, Whisper weights, CUDA runtime or GPU device is
|
| 14 |
+
required. FFmpeg/age are CPU media/encryption tools, not inference models.
|
| 15 |
+
|
| 16 |
+
Build in this managed environment (session CA is mounted only during install):
|
| 17 |
+
|
| 18 |
+
```
|
| 19 |
+
BUILDX_CONFIG=/tmp/v2-buildx docker --host=unix:///var/run/docker.sock build \
|
| 20 |
+
--secret id=build_ca,src=/etc/ssl/certs/ca-certificates.crt \
|
| 21 |
+
-t video-rag-mentor:v2-review .
|
| 22 |
+
python scripts/qualify-release.py --image video-rag-mentor:v2-review
|
| 23 |
+
VM_RUNTIME_IMAGE=video-rag-mentor:v2-review python scripts/measure-runtime.py
|
| 24 |
+
```
|
| 25 |
+
|
| 26 |
+
These checks publish no host ports, run with network=none and a 2-vCPU/16GiB limit,
|
| 27 |
+
and remove their own containers/volumes. Qualification checks application bearer
|
| 28 |
+
roles, origin rejection, configuration CAS/rollback and volume recreation. A Docker
|
| 29 |
+
volume in a test is not evidence that HF supplies durable storage.
|
| 30 |
+
|
| 31 |
+
## Configuration before staging
|
| 32 |
+
|
| 33 |
+
Use HF Settings / Secrets (do not put values in YAML or browser bundles):
|
| 34 |
+
|
| 35 |
+
- `VM_USER_TOKEN`, `VM_ADMIN_TOKEN`: separate random tokens, >=24 characters.
|
| 36 |
+
The user token opens private library/capture operations; admin token opens config
|
| 37 |
+
and backup status only. This is a shared-role app, not per-person accounts/SSO.
|
| 38 |
+
- `VM_ALLOWED_ORIGINS=https://<staging-app-host>`: exact external origin.
|
| 39 |
+
- `VM_ALLOWED_HOSTS`: exact approved inference hostnames only.
|
| 40 |
+
- `VRAG_DATA=/app/work/data`, `VM_STATE_DIR=/app/work/state`; default index under
|
| 41 |
+
`/app/work/data/index`. Configure GitHub-native backup and protected age identity
|
| 42 |
+
as documented in backup-restore.md; the working disk remains ephemeral.
|
| 43 |
+
- `LAYA_BASE_URL` is the router root; OpenAI-compatible VLM/embedding/Whisper
|
| 44 |
+
base URLs generally include `/v1`. Credential environment references are the
|
| 45 |
+
exact `api_key_secret` fields in the reviewed profile. Do not assume all endpoints
|
| 46 |
+
share a protocol or that NVIDIA's public API implements this `/detect` gateway.
|
| 47 |
+
- Backup settings, protected trigger and empty-state startup recovery: see
|
| 48 |
+
[backup-restore.md](backup-restore.md). No plaintext export is exposed.
|
| 49 |
+
- Decide whether answer-history persistence (`VRAG_SAVE_ANSWERS`) is permitted.
|
| 50 |
+
Persisted answer history is included in encrypted GitHub checkpoints; review its
|
| 51 |
+
privacy scope and opt out of local history if required.
|
| 52 |
+
|
| 53 |
+
All profile examples are disabled until model IDs, endpoints, capabilities,
|
| 54 |
+
privacy trust and credentials are explicitly verified. Active placeholder IDs are
|
| 55 |
+
rejected. Load/import YAML, review form/diff, validate, then apply using the current
|
| 56 |
+
revision. Credentials remain server-only references. Transcripts/keyframes uploaded
|
| 57 |
+
through Import may be sent to the selected authorized providers.
|
| 58 |
+
|
| 59 |
+
## Mandatory approval and external gates
|
| 60 |
+
|
| 61 |
+
1. Approve a **private staging** Space (proposed `Leon4gr45/openweb-v2-staging`),
|
| 62 |
+
GitHub backup storage budget and recovery integration. Obtain a fine-grained HF
|
| 63 |
+
identity limited to intended repos. The current environment has no configured HF
|
| 64 |
+
identity; openweb source, visibility, settings and screenshots were not inspected.
|
| 65 |
+
2. Before touching production, snapshot the existing Space Git revision, settings
|
| 66 |
+
inventory (secret names only), private access policy and all data. Verify restore.
|
| 67 |
+
3. Qualify GitHub-native encrypted release assets and repository checkpoints as the
|
| 68 |
+
sole recovery store. HF CPU Basic has 50GB ephemeral disk; no persistent mount
|
| 69 |
+
or external object store is required. Complete the live GitHub destructive
|
| 70 |
+
recovery drill and private Actions sleep/wake test in backup-restore.md.
|
| 71 |
+
4. Review `docs/v2-evidence/main/` against current Playwright screenshots. Confirm
|
| 72 |
+
original workflows remain recognizable at desktop/laptop/tablet/mobile sizes.
|
| 73 |
+
5. With explicit staging approval, create a private Docker Space and upload the
|
| 74 |
+
reviewed source, using the official SDK and environment-provisioned HF_TOKEN:
|
| 75 |
+
|
| 76 |
+
```python
|
| 77 |
+
import os
|
| 78 |
+
from huggingface_hub import HfApi
|
| 79 |
+
api = HfApi(token=os.environ['HF_TOKEN'])
|
| 80 |
+
repo = 'Leon4gr45/openweb-v2-staging' # approved staging name
|
| 81 |
+
api.create_repo(repo_id=repo, repo_type='space', space_sdk='docker', private=True)
|
| 82 |
+
api.upload_folder(
|
| 83 |
+
repo_id=repo, repo_type='space', folder_path='.',
|
| 84 |
+
allow_patterns=['README.md', 'Dockerfile', 'requirements-runtime.txt',
|
| 85 |
+
'vrag/**', 'config/**', 'scripts/cloud-start.py', 'web/**'],
|
| 86 |
+
ignore_patterns=['**/.env*', '**/*.pem', '**/*.key', '**/*.agekey', '**/*.age',
|
| 87 |
+
'**/__pycache__/**', '**/node_modules/**', '**/.next/**',
|
| 88 |
+
'web/tests/**', 'web/test-results/**', 'web/playwright-report/**'],
|
| 89 |
+
commit_message='Stage reviewed Video RAG Visual Mentor v2',
|
| 90 |
+
)
|
| 91 |
+
```
|
| 92 |
+
|
| 93 |
+
6. Configure secrets/variables in staging, select CPU Basic (2 vCPU/16GB), inspect
|
| 94 |
+
logs and health. Verify actual Space privacy by anonymous page/API/media requests;
|
| 95 |
+
verify logged-in access with app user/admin roles and no access cross-over.
|
| 96 |
+
Exercise `/`, `/search`, `/ask`, `/experiments`, `/architecture`, detail/clip Range
|
| 97 |
+
seeking, `/live` and `/models`. Public sample assets are synthetic but still sit
|
| 98 |
+
behind the private HF page/access boundary.
|
| 99 |
+
7. From a physical laptop browser over HTTPS, test laptop + USB simultaneous
|
| 100 |
+
selection, permission denial/contention, disconnect, hidden-tab cleanup and
|
| 101 |
+
stale results. Verify each real provider and costs using approved media only.
|
| 102 |
+
Run real upload→Whisper→segments→embeddings→Search/Ask and verify citations/playback.
|
| 103 |
+
8. Recreate staging and restore all media/index/configuration from GitHub alone
|
| 104 |
+
with the protected age key. Confirm a real recent checkpoint survives Space
|
| 105 |
+
recreation. Mock transport tests alone do not pass this gate.
|
| 106 |
+
9. Present actual staging screenshots, provider test table, storage/backup/restore
|
| 107 |
+
history and performance; obtain explicit approval before changing openweb.
|
| 108 |
+
|
| 109 |
+
## Controlled production switch / rollback
|
| 110 |
+
|
| 111 |
+
Save the existing production Space commit and tested GitHub checkpoint IDs before
|
| 112 |
+
upload. Obtain UI and deployment approval. Deploy the reviewed v2 source using the
|
| 113 |
+
same procedure targeted at openweb; do not merge the deployment PR wholesale.
|
| 114 |
+
Keep previous image/source and compatible snapshot available. On failure, pause
|
| 115 |
+
writes, restore the previous Space revision and point at an isolated verified
|
| 116 |
+
compatible state snapshot. Do not overwrite current data by automated restore or
|
| 117 |
+
run the legacy `serve/` scripts. Recheck private access, API/media and clips after
|
| 118 |
+
rollback. Production promotion and rollback execution are operator actions.
|
docs/evidence/backend-tests.txt
ADDED
|
@@ -0,0 +1,9 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
........................................................................ [ 66%]
|
| 2 |
+
..................................... [100%]
|
| 3 |
+
=============================== warnings summary ===============================
|
| 4 |
+
../visual-mentor-venv/lib/python3.12/site-packages/fastapi/testclient.py:1
|
| 5 |
+
/workspace/visual-mentor-venv/lib/python3.12/site-packages/fastapi/testclient.py:1: StarletteDeprecationWarning: Using `httpx` with `starlette.testclient` is deprecated; install `httpx2` instead.
|
| 6 |
+
from starlette.testclient import TestClient as TestClient # noqa
|
| 7 |
+
|
| 8 |
+
-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html
|
| 9 |
+
109 passed, 1 warning in 9.64s
|
docs/evidence/browser-administration.txt
ADDED
|
@@ -0,0 +1,14 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
|
| 2 |
+
Running 2 tests using 1 worker
|
| 3 |
+
|
| 4 |
+
(node:65537) Warning: The 'NO_COLOR' env is ignored due to the 'FORCE_COLOR' env being set.
|
| 5 |
+
(Use `node --trace-warnings ...` to show where the warning was created)
|
| 6 |
+
✓ 1 tests/backups.spec.ts:3:5 › backup administration distinguishes acceptance from verified completion (2.0s)
|
| 7 |
+
✓ 2 tests/backups.spec.ts:21:5 › backup administration handles denied access (2.8s)
|
| 8 |
+
|
| 9 |
+
2 passed (8.6s)
|
| 10 |
+
npm notice
|
| 11 |
+
npm notice New major version of npm available! 11.9.0 -> 12.2.0
|
| 12 |
+
npm notice Changelog: https://github.com/npm/cli/releases/tag/v12.2.0
|
| 13 |
+
npm notice To update run: npm install -g npm@12.2.0
|
| 14 |
+
npm notice
|
docs/evidence/browser-regression.txt
ADDED
|
@@ -0,0 +1,29 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
|
| 2 |
+
Running 17 tests using 1 worker
|
| 3 |
+
|
| 4 |
+
(node:63329) Warning: The 'NO_COLOR' env is ignored due to the 'FORCE_COLOR' env being set.
|
| 5 |
+
(Use `node --trace-warnings ...` to show where the warning was created)
|
| 6 |
+
✓ 1 tests/live.spec.ts:2:5 › dual simulated previews require explicit transfer and stop on navigation (4.5s)
|
| 7 |
+
✓ 2 tests/live.spec.ts:63:5 › models studio validates and atomically switches a profile (1.4s)
|
| 8 |
+
✓ 3 tests/parity.spec.ts:10:7 › original UI desktop (8.3s)
|
| 9 |
+
✓ 4 tests/parity.spec.ts:10:7 › original UI laptop (6.7s)
|
| 10 |
+
✓ 5 tests/parity.spec.ts:10:7 › original UI tablet (9.9s)
|
| 11 |
+
✓ 6 tests/parity.spec.ts:10:7 › original UI mobile (9.5s)
|
| 12 |
+
✓ 7 tests/parity.spec.ts:48:5 › clip and citation interactions (1.8s)
|
| 13 |
+
✓ 8 tests/release.spec.ts:11:5 › permission denial shows an error without uploading (878ms)
|
| 14 |
+
✓ 9 tests/release.spec.ts:19:7 › rejects frame mismatch and permits a later matching observation (1.2s)
|
| 15 |
+
✓ 10 tests/release.spec.ts:19:7 › rejects config mismatch and permits a later matching observation (1.2s)
|
| 16 |
+
✓ 11 tests/release.spec.ts:19:7 › rejects dimensions mismatch and permits a later matching observation (1.2s)
|
| 17 |
+
✓ 12 tests/release.spec.ts:19:7 › rejects box mismatch and permits a later matching observation (1.2s)
|
| 18 |
+
✓ 13 tests/release.spec.ts:46:5 › stopping a camera discards its delayed result (1.2s)
|
| 19 |
+
✓ 14 tests/release.spec.ts:59:5 › a pending camera permission grant after navigation stops the returned tracks (3.1s)
|
| 20 |
+
✓ 15 tests/release.spec.ts:72:5 › device contention is reported and track end stops the preview (1.0s)
|
| 21 |
+
✓ 16 tests/release.spec.ts:84:5 › hiding the page stops microphone capture without uploading audio (1.1s)
|
| 22 |
+
✓ 17 tests/release.spec.ts:98:5 › recorder errors release microphone tracks and discard audio (1.0s)
|
| 23 |
+
|
| 24 |
+
17 passed (1.0m)
|
| 25 |
+
npm notice
|
| 26 |
+
npm notice New major version of npm available! 11.9.0 -> 12.2.0
|
| 27 |
+
npm notice Changelog: https://github.com/npm/cli/releases/tag/v12.2.0
|
| 28 |
+
npm notice To update run: npm install -g npm@12.2.0
|
| 29 |
+
npm notice
|
docs/evidence/credential-qualification.json
ADDED
|
@@ -0,0 +1,47 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"observed_at": "2026-10-17T22:10:28.212207+02:00",
|
| 3 |
+
"application_commit": "c229b819f5f68236781876ad3d7865cd501df16e",
|
| 4 |
+
"backup_repository_commit": "17fbad6c0bd00d36258f2376d2e8b9547e3727d2",
|
| 5 |
+
"credential_source_required": "VM_BACKUP_GITHUB_TOKEN",
|
| 6 |
+
"dedicated_variable_present": true,
|
| 7 |
+
"saved_credential_binding_present": true,
|
| 8 |
+
"declared_credential_destinations": [
|
| 9 |
+
"api.github.com",
|
| 10 |
+
"uploads.github.com"
|
| 11 |
+
],
|
| 12 |
+
"integration_credential_substituted": false,
|
| 13 |
+
"api_permission_test": "private repository GET 200; disposable release POST 201",
|
| 14 |
+
"encrypted_release_asset_test": "uploads.github.com POST 404; one attempt",
|
| 15 |
+
"trusted_machine_comparison": "not_run_no_attached_trusted_machine",
|
| 16 |
+
"destructive_live_recovery": "not_run_upload_rejected_before_commit",
|
| 17 |
+
"attribution": {
|
| 18 |
+
"established_blocker": "encrypted release asset upload rejected with HTTP 404",
|
| 19 |
+
"credential_scope": "undetermined across upload destination",
|
| 20 |
+
"credential_injection": "undetermined",
|
| 21 |
+
"network_proxy_behavior": "HTTPS response received with GitHub request ID; credential forwarding not observable"
|
| 22 |
+
},
|
| 23 |
+
"hf_credential_present": false,
|
| 24 |
+
"private_staging_preparation": "existing_deploy/hf-staging_package_available; authenticated_qualification_blocked",
|
| 25 |
+
"staging_deployment_decision": "NO-GO",
|
| 26 |
+
"recurring_workflows_activated": false,
|
| 27 |
+
"production_changed": false,
|
| 28 |
+
"releases_created_modified_or_deleted": false,
|
| 29 |
+
"architecture_changed": false,
|
| 30 |
+
"latest_recheck": {
|
| 31 |
+
"client_date": "2026-10-18",
|
| 32 |
+
"timezone": "Europe/Berlin",
|
| 33 |
+
"dedicated_variable_present": true,
|
| 34 |
+
"saved_credential_binding_present": true,
|
| 35 |
+
"integration_credential_substituted": false,
|
| 36 |
+
"api_status": 200,
|
| 37 |
+
"release_creation_status": 201,
|
| 38 |
+
"asset_upload_status": 404,
|
| 39 |
+
"upload_attempts": 1,
|
| 40 |
+
"uploaded_assets": 0,
|
| 41 |
+
"committed_checkpoints": 0,
|
| 42 |
+
"request_id": "FE93:319F8B:36F88A:4FB435:6AD3F6B8"
|
| 43 |
+
},
|
| 44 |
+
"checked_on": "2026-10-18",
|
| 45 |
+
"credential_readiness": "ready",
|
| 46 |
+
"current_live_evidence": "github-recovery-live.json"
|
| 47 |
+
}
|
docs/evidence/dedicated-token-api.json
ADDED
|
@@ -0,0 +1,28 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"credential_source": "VM_BACKUP_GITHUB_TOKEN",
|
| 3 |
+
"http": {
|
| 4 |
+
"method": "GET",
|
| 5 |
+
"host": "api.github.com",
|
| 6 |
+
"status": 200,
|
| 7 |
+
"headers": {
|
| 8 |
+
"x-github-request-id": "80E9:137167:4D2D1:F8EC7:6AD3F69C",
|
| 9 |
+
"x-github-api-version-selected": "2022-11-28",
|
| 10 |
+
"x-ratelimit-limit": "15000",
|
| 11 |
+
"x-ratelimit-remaining": "14976",
|
| 12 |
+
"x-ratelimit-resource": "core",
|
| 13 |
+
"x-accepted-github-permissions": "metadata=read",
|
| 14 |
+
"x-oauth-scopes": "",
|
| 15 |
+
"content-type": "application/json; charset=utf-8",
|
| 16 |
+
"server": "envoy"
|
| 17 |
+
},
|
| 18 |
+
"client_authorization_attached": true
|
| 19 |
+
},
|
| 20 |
+
"private": true,
|
| 21 |
+
"permissions": {
|
| 22 |
+
"admin": true,
|
| 23 |
+
"maintain": true,
|
| 24 |
+
"push": true,
|
| 25 |
+
"triage": true,
|
| 26 |
+
"pull": true
|
| 27 |
+
}
|
| 28 |
+
}
|
docs/evidence/github-auth-probe.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"observations": [
|
| 3 |
+
{
|
| 4 |
+
"method": "GET",
|
| 5 |
+
"host": "api.github.com",
|
| 6 |
+
"path": "/user",
|
| 7 |
+
"credential_source": "GH_TOKEN",
|
| 8 |
+
"client_authorization_attached": true,
|
| 9 |
+
"status": 200,
|
| 10 |
+
"headers": {
|
| 11 |
+
"x-github-request-id": "9690:32E8B7:DBB396:2C7BB5A:6AD3D3BD",
|
| 12 |
+
"x-github-api-version-selected": "2022-11-28",
|
| 13 |
+
"x-ratelimit-limit": "15000",
|
| 14 |
+
"x-ratelimit-remaining": "14923",
|
| 15 |
+
"x-ratelimit-resource": "core",
|
| 16 |
+
"x-accepted-github-permissions": "allows_permissionless_access=true",
|
| 17 |
+
"content-type": "application/json; charset=utf-8",
|
| 18 |
+
"server": "envoy"
|
| 19 |
+
}
|
| 20 |
+
},
|
| 21 |
+
{
|
| 22 |
+
"method": "GET",
|
| 23 |
+
"host": "api.github.com",
|
| 24 |
+
"path": "/repos/JsonLord/visual-mentor-backups",
|
| 25 |
+
"credential_source": "GH_TOKEN",
|
| 26 |
+
"client_authorization_attached": true,
|
| 27 |
+
"status": 200,
|
| 28 |
+
"headers": {
|
| 29 |
+
"x-github-request-id": "3DF2:319386:45059D:E020B0:6AD3D3BE",
|
| 30 |
+
"x-github-api-version-selected": "2022-11-28",
|
| 31 |
+
"x-ratelimit-limit": "15000",
|
| 32 |
+
"x-ratelimit-remaining": "14922",
|
| 33 |
+
"x-ratelimit-resource": "core",
|
| 34 |
+
"x-accepted-github-permissions": "metadata=read",
|
| 35 |
+
"content-type": "application/json; charset=utf-8",
|
| 36 |
+
"server": "envoy"
|
| 37 |
+
}
|
| 38 |
+
},
|
| 39 |
+
{
|
| 40 |
+
"method": "GET",
|
| 41 |
+
"host": "uploads.github.com",
|
| 42 |
+
"path": "/",
|
| 43 |
+
"credential_source": "GH_TOKEN",
|
| 44 |
+
"client_authorization_attached": true,
|
| 45 |
+
"status": 302,
|
| 46 |
+
"headers": {
|
| 47 |
+
"x-github-request-id": "2E62:2F9DB5:A7FA8:C8D62:6AD3D3BE",
|
| 48 |
+
"server": "envoy"
|
| 49 |
+
}
|
| 50 |
+
},
|
| 51 |
+
{
|
| 52 |
+
"method": "GET",
|
| 53 |
+
"host": "huggingface.co",
|
| 54 |
+
"path": "/api/spaces/Leon4gr45/openweb",
|
| 55 |
+
"credential_source": "anonymous",
|
| 56 |
+
"client_authorization_attached": false,
|
| 57 |
+
"status": 200,
|
| 58 |
+
"headers": {
|
| 59 |
+
"content-type": "application/json; charset=utf-8",
|
| 60 |
+
"server": "envoy",
|
| 61 |
+
"via": "1.1 eb382ff675fc4a4551773e2e1657c302.cloudfront.net (CloudFront)"
|
| 62 |
+
}
|
| 63 |
+
},
|
| 64 |
+
{
|
| 65 |
+
"method": "GET",
|
| 66 |
+
"host": "leon4gr45-openweb.hf.space",
|
| 67 |
+
"path": "/api/admin/backup-status",
|
| 68 |
+
"credential_source": "anonymous",
|
| 69 |
+
"client_authorization_attached": false,
|
| 70 |
+
"status": 404,
|
| 71 |
+
"headers": {
|
| 72 |
+
"content-type": "application/json",
|
| 73 |
+
"server": "envoy"
|
| 74 |
+
},
|
| 75 |
+
"error_code": "remote_error"
|
| 76 |
+
}
|
| 77 |
+
],
|
| 78 |
+
"proxy_interior_authorization": "not observable; no credential extraction or interception performed"
|
| 79 |
+
}
|
docs/evidence/github-backup-results.json
ADDED
|
@@ -0,0 +1,55 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"date": "2026-10-17",
|
| 3 |
+
"backend": {
|
| 4 |
+
"passed": 109,
|
| 5 |
+
"encryption": "real age CLI",
|
| 6 |
+
"sqlite": "real online snapshots",
|
| 7 |
+
"media": "real FFmpeg-generated video",
|
| 8 |
+
"github_transport": "mocked for successful recovery tests"
|
| 9 |
+
},
|
| 10 |
+
"browser": {
|
| 11 |
+
"existing_parity_camera_routing_passed": 17,
|
| 12 |
+
"backup_administration_passed": 2,
|
| 13 |
+
"providers_and_hardware": "mocked or simulated"
|
| 14 |
+
},
|
| 15 |
+
"frontend": {
|
| 16 |
+
"production_build": "passed",
|
| 17 |
+
"lint_errors": 0,
|
| 18 |
+
"existing_lint_warnings": 1
|
| 19 |
+
},
|
| 20 |
+
"live_github": {
|
| 21 |
+
"private_repository_verified": true,
|
| 22 |
+
"release_creation": "passed",
|
| 23 |
+
"release_tag": "vm-test-recovery-20261017-deae94ff-20261017T173832-f8cd66db742745159168b741ea1a751e",
|
| 24 |
+
"asset_upload": "blocked: HTTP 401 Bad credentials from uploads.github.com",
|
| 25 |
+
"committed_recovery_checkpoints": 0,
|
| 26 |
+
"complete_github_only_recovery": "NOT VERIFIED",
|
| 27 |
+
"existing_backups_overwritten": false
|
| 28 |
+
},
|
| 29 |
+
"private_space": {
|
| 30 |
+
"identity": "Leon4gr45/openweb",
|
| 31 |
+
"origin": "https://leon4gr45-openweb.hf.space",
|
| 32 |
+
"probe": "blocked by network proxy policy",
|
| 33 |
+
"hf_test_credential_present": false,
|
| 34 |
+
"actions_private_reachability": "NOT VERIFIED",
|
| 35 |
+
"sleep_wake": "NOT VERIFIED",
|
| 36 |
+
"production_scheduling_activated": false,
|
| 37 |
+
"deployment_modified": false
|
| 38 |
+
},
|
| 39 |
+
"alternative": {
|
| 40 |
+
"resident_timer": "mock-clock branch and bounded retry/health tests passed",
|
| 41 |
+
"sleeping_space_wake": "not provided or claimed"
|
| 42 |
+
},
|
| 43 |
+
"live_qualification_followup": {
|
| 44 |
+
"backend_passed": 116,
|
| 45 |
+
"backup_tests_passed": 20,
|
| 46 |
+
"upload_url": "GitHub-returned template; validated exact destination",
|
| 47 |
+
"upload_authentication": "401 once; no retry",
|
| 48 |
+
"upload_clients": "HTTPX Bearer, HTTPX token, native gh all 401 using GH_TOKEN",
|
| 49 |
+
"proxy_header_forwarding": "not observable; cannot establish strip versus rewrite",
|
| 50 |
+
"github_recovery": "blocked; zero assets and no committed checkpoint",
|
| 51 |
+
"production_space_private": false,
|
| 52 |
+
"private_staging": "prepared, not deployed; missing HF credential",
|
| 53 |
+
"production_scheduling_activated": false
|
| 54 |
+
}
|
| 55 |
+
}
|
docs/evidence/github-recovery-live.json
ADDED
|
@@ -0,0 +1,132 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"live": true,
|
| 3 |
+
"status": "blocked",
|
| 4 |
+
"namespace": "test-recovery-20261017-745d0278",
|
| 5 |
+
"repository": "JsonLord/visual-mentor-backups",
|
| 6 |
+
"credential_source": "VM_BACKUP_GITHUB_TOKEN",
|
| 7 |
+
"http_events": [
|
| 8 |
+
{
|
| 9 |
+
"method": "GET",
|
| 10 |
+
"host": "api.github.com",
|
| 11 |
+
"status": 200,
|
| 12 |
+
"headers": {
|
| 13 |
+
"x-github-request-id": "B7AE:25708C:84BF5C:1B16D75:6AD3F6B6",
|
| 14 |
+
"x-github-api-version-selected": "2022-11-28",
|
| 15 |
+
"x-ratelimit-limit": "15000",
|
| 16 |
+
"x-ratelimit-remaining": "14958",
|
| 17 |
+
"x-ratelimit-resource": "core",
|
| 18 |
+
"x-accepted-github-permissions": "metadata=read",
|
| 19 |
+
"x-oauth-scopes": "",
|
| 20 |
+
"content-type": "application/json; charset=utf-8",
|
| 21 |
+
"server": "envoy"
|
| 22 |
+
},
|
| 23 |
+
"client_authorization_attached": true
|
| 24 |
+
},
|
| 25 |
+
{
|
| 26 |
+
"method": "GET",
|
| 27 |
+
"host": "api.github.com",
|
| 28 |
+
"status": 404,
|
| 29 |
+
"headers": {
|
| 30 |
+
"x-github-request-id": "41CF:1D584D:F1627F:3129345:6AD3F6B6",
|
| 31 |
+
"x-github-api-version-selected": "2022-11-28",
|
| 32 |
+
"x-ratelimit-limit": "15000",
|
| 33 |
+
"x-ratelimit-remaining": "14957",
|
| 34 |
+
"x-ratelimit-resource": "core",
|
| 35 |
+
"x-accepted-github-permissions": "contents=read",
|
| 36 |
+
"x-oauth-scopes": "",
|
| 37 |
+
"content-type": "application/json; charset=utf-8",
|
| 38 |
+
"server": "envoy"
|
| 39 |
+
},
|
| 40 |
+
"client_authorization_attached": true
|
| 41 |
+
},
|
| 42 |
+
{
|
| 43 |
+
"method": "GET",
|
| 44 |
+
"host": "api.github.com",
|
| 45 |
+
"status": 404,
|
| 46 |
+
"headers": {
|
| 47 |
+
"x-github-request-id": "8F54:18C49:4101B:D3AE7:6AD3F6B6",
|
| 48 |
+
"x-github-api-version-selected": "2022-11-28",
|
| 49 |
+
"x-ratelimit-limit": "15000",
|
| 50 |
+
"x-ratelimit-remaining": "14956",
|
| 51 |
+
"x-ratelimit-resource": "core",
|
| 52 |
+
"x-accepted-github-permissions": "contents=read",
|
| 53 |
+
"x-oauth-scopes": "",
|
| 54 |
+
"content-type": "application/json; charset=utf-8",
|
| 55 |
+
"server": "envoy"
|
| 56 |
+
},
|
| 57 |
+
"client_authorization_attached": true
|
| 58 |
+
},
|
| 59 |
+
{
|
| 60 |
+
"method": "POST",
|
| 61 |
+
"host": "api.github.com",
|
| 62 |
+
"status": 201,
|
| 63 |
+
"headers": {
|
| 64 |
+
"x-github-request-id": "5146:2519A0:67DC3:153C3C:6AD3F6B7",
|
| 65 |
+
"x-github-api-version-selected": "2022-11-28",
|
| 66 |
+
"x-ratelimit-limit": "15000",
|
| 67 |
+
"x-ratelimit-remaining": "14989",
|
| 68 |
+
"x-ratelimit-resource": "core",
|
| 69 |
+
"x-accepted-github-permissions": "contents=write; contents=write,workflows=write",
|
| 70 |
+
"x-oauth-scopes": "",
|
| 71 |
+
"content-type": "application/json; charset=utf-8",
|
| 72 |
+
"server": "envoy"
|
| 73 |
+
},
|
| 74 |
+
"client_authorization_attached": true
|
| 75 |
+
},
|
| 76 |
+
{
|
| 77 |
+
"method": "GET",
|
| 78 |
+
"host": "api.github.com",
|
| 79 |
+
"status": 200,
|
| 80 |
+
"headers": {
|
| 81 |
+
"x-github-request-id": "41CF:1D584D:F16D27:312B69F:6AD3F6B6",
|
| 82 |
+
"x-github-api-version-selected": "2022-11-28",
|
| 83 |
+
"x-ratelimit-limit": "15000",
|
| 84 |
+
"x-ratelimit-remaining": "14988",
|
| 85 |
+
"x-ratelimit-resource": "core",
|
| 86 |
+
"x-accepted-github-permissions": "contents=read",
|
| 87 |
+
"x-oauth-scopes": "",
|
| 88 |
+
"content-type": "application/json; charset=utf-8",
|
| 89 |
+
"server": "envoy"
|
| 90 |
+
},
|
| 91 |
+
"client_authorization_attached": true
|
| 92 |
+
},
|
| 93 |
+
{
|
| 94 |
+
"method": "POST",
|
| 95 |
+
"host": "uploads.github.com",
|
| 96 |
+
"status": 404,
|
| 97 |
+
"headers": {
|
| 98 |
+
"x-github-request-id": "FE93:319F8B:36F88A:4FB435:6AD3F6B8",
|
| 99 |
+
"content-type": "application/json; charset=utf-8",
|
| 100 |
+
"server": "envoy"
|
| 101 |
+
},
|
| 102 |
+
"client_authorization_attached": true
|
| 103 |
+
}
|
| 104 |
+
],
|
| 105 |
+
"fixture_file_hashes": {
|
| 106 |
+
"state/config-history.json": "fc54a50cefc2d5955a43999a8f287dea13f432a53b379e3620986984a4862d29",
|
| 107 |
+
"data/jobs.sqlite": "61ea188727defa04c534eb336ee7781ea36583ff97205361f8b182bee574469c",
|
| 108 |
+
"data/index/CURRENT": "3ba3f5f43b92602683c19aee62a20342b084dd5971ddd33808d81a328879a547",
|
| 109 |
+
"data/answers/history.jsonl": "24286a82bed379831e51587c4928b22fac236936ae7bdbfaf4af4b65d33cc96c",
|
| 110 |
+
"data/videos/v1/video.mp4": "bc616d8ada2b297f447e74ba212cc84f05a03345070d4145a62c0bfe3b5cb848",
|
| 111 |
+
"data/videos/v1/transcript.json": "4a8fb2f5a7707c7e0ae59b3f649c684e186511e32308bed8c608562014894d22",
|
| 112 |
+
"data/videos/v1/thumb.jpg": "8a2404e720bb32c6430c4772e8d2a866a87662b1969395927c1aeef2b79b7566",
|
| 113 |
+
"data/videos/v1/segments.json": "016ce6672b25eacc268d418b6d0a161a7fa2a7396cfd3543e53d1938a40a4809",
|
| 114 |
+
"data/videos/v1/original": "bc616d8ada2b297f447e74ba212cc84f05a03345070d4145a62c0bfe3b5cb848",
|
| 115 |
+
"data/videos/v1/meta.json": "ec593249ce82f4928a210ae42105d0678c7993847ab52468a37e28263bae6d16",
|
| 116 |
+
"data/videos/v1/segments/000.jpg": "8a2404e720bb32c6430c4772e8d2a866a87662b1969395927c1aeef2b79b7566",
|
| 117 |
+
"data/index/revisions/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/text.npy": "770eebe4eb273d2e066dcf9e6085d73c43192b2a4d6a0bd191759e6745bdf507",
|
| 118 |
+
"data/index/revisions/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/segments.json": "016ce6672b25eacc268d418b6d0a161a7fa2a7396cfd3543e53d1938a40a4809",
|
| 119 |
+
"data/index/revisions/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/manifest.json": "d380ed784acc1640825dba8528743125483af0f32d1cf327970edcc03b582b70"
|
| 120 |
+
},
|
| 121 |
+
"failure_code": "github_request_rejected",
|
| 122 |
+
"status_code": 404,
|
| 123 |
+
"failure_host": "uploads.github.com",
|
| 124 |
+
"request_id": "FE93:319F8B:36F88A:4FB435:6AD3F6B8",
|
| 125 |
+
"committed_checkpoints": 0,
|
| 126 |
+
"first_checkpoint": null,
|
| 127 |
+
"recovery_verified": false,
|
| 128 |
+
"uncommitted_release_tag": "vm-test-recovery-20261017-745d0278-20261017T222910307870749-6b71d153db4944d590521b43e3cb4eb1",
|
| 129 |
+
"release_id": 416107443,
|
| 130 |
+
"asset_count": 0,
|
| 131 |
+
"namespace_checkpoint_exists": false
|
| 132 |
+
}
|
docs/evidence/github-recovery-prior-blocked.json
ADDED
|
@@ -0,0 +1,115 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"live": true,
|
| 3 |
+
"status": "blocked",
|
| 4 |
+
"namespace": "test-recovery-20261017-937e26ae",
|
| 5 |
+
"repository": "JsonLord/visual-mentor-backups",
|
| 6 |
+
"credential_source": "GH_TOKEN",
|
| 7 |
+
"http_events": [
|
| 8 |
+
{
|
| 9 |
+
"method": "GET",
|
| 10 |
+
"host": "api.github.com",
|
| 11 |
+
"status": 200,
|
| 12 |
+
"headers": {
|
| 13 |
+
"x-github-request-id": "8E2E:1BB071:2286C:711C7:6AD3D411",
|
| 14 |
+
"x-github-api-version-selected": "2022-11-28",
|
| 15 |
+
"x-ratelimit-limit": "15000",
|
| 16 |
+
"x-ratelimit-remaining": "14921",
|
| 17 |
+
"x-ratelimit-resource": "core",
|
| 18 |
+
"x-accepted-github-permissions": "metadata=read",
|
| 19 |
+
"x-oauth-scopes": "",
|
| 20 |
+
"content-type": "application/json; charset=utf-8",
|
| 21 |
+
"server": "envoy"
|
| 22 |
+
},
|
| 23 |
+
"client_authorization_attached": true
|
| 24 |
+
},
|
| 25 |
+
{
|
| 26 |
+
"method": "GET",
|
| 27 |
+
"host": "api.github.com",
|
| 28 |
+
"status": 404,
|
| 29 |
+
"headers": {
|
| 30 |
+
"x-github-request-id": "77B1:3FD457:FABDAE:341D926:6AD3D411",
|
| 31 |
+
"x-github-api-version-selected": "2022-11-28",
|
| 32 |
+
"x-ratelimit-limit": "15000",
|
| 33 |
+
"x-ratelimit-remaining": "14920",
|
| 34 |
+
"x-ratelimit-resource": "core",
|
| 35 |
+
"x-accepted-github-permissions": "contents=read",
|
| 36 |
+
"x-oauth-scopes": "",
|
| 37 |
+
"content-type": "application/json; charset=utf-8",
|
| 38 |
+
"server": "envoy"
|
| 39 |
+
},
|
| 40 |
+
"client_authorization_attached": true
|
| 41 |
+
},
|
| 42 |
+
{
|
| 43 |
+
"method": "GET",
|
| 44 |
+
"host": "api.github.com",
|
| 45 |
+
"status": 404,
|
| 46 |
+
"headers": {
|
| 47 |
+
"x-github-request-id": "9E38:15CAE4:251FB:79F1E:6AD3D412",
|
| 48 |
+
"x-github-api-version-selected": "2022-11-28",
|
| 49 |
+
"x-ratelimit-limit": "15000",
|
| 50 |
+
"x-ratelimit-remaining": "14919",
|
| 51 |
+
"x-ratelimit-resource": "core",
|
| 52 |
+
"x-accepted-github-permissions": "contents=read",
|
| 53 |
+
"x-oauth-scopes": "",
|
| 54 |
+
"content-type": "application/json; charset=utf-8",
|
| 55 |
+
"server": "envoy"
|
| 56 |
+
},
|
| 57 |
+
"client_authorization_attached": true
|
| 58 |
+
},
|
| 59 |
+
{
|
| 60 |
+
"method": "POST",
|
| 61 |
+
"host": "api.github.com",
|
| 62 |
+
"status": 201,
|
| 63 |
+
"headers": {
|
| 64 |
+
"x-github-request-id": "F605:363BB9:DDC6D8:2CF6533:6AD3D412",
|
| 65 |
+
"x-github-api-version-selected": "2022-11-28",
|
| 66 |
+
"x-ratelimit-limit": "15000",
|
| 67 |
+
"x-ratelimit-remaining": "14999",
|
| 68 |
+
"x-ratelimit-resource": "core",
|
| 69 |
+
"x-accepted-github-permissions": "contents=write; contents=write,workflows=write",
|
| 70 |
+
"x-oauth-scopes": "",
|
| 71 |
+
"content-type": "application/json; charset=utf-8",
|
| 72 |
+
"server": "envoy"
|
| 73 |
+
},
|
| 74 |
+
"client_authorization_attached": true
|
| 75 |
+
},
|
| 76 |
+
{
|
| 77 |
+
"method": "GET",
|
| 78 |
+
"host": "api.github.com",
|
| 79 |
+
"status": 200,
|
| 80 |
+
"headers": {
|
| 81 |
+
"x-github-request-id": "77B1:3FD457:FAC77F:341F981:6AD3D412",
|
| 82 |
+
"x-github-api-version-selected": "2022-11-28",
|
| 83 |
+
"x-ratelimit-limit": "15000",
|
| 84 |
+
"x-ratelimit-remaining": "14998",
|
| 85 |
+
"x-ratelimit-resource": "core",
|
| 86 |
+
"x-accepted-github-permissions": "contents=read",
|
| 87 |
+
"x-oauth-scopes": "",
|
| 88 |
+
"content-type": "application/json; charset=utf-8",
|
| 89 |
+
"server": "envoy"
|
| 90 |
+
},
|
| 91 |
+
"client_authorization_attached": true
|
| 92 |
+
},
|
| 93 |
+
{
|
| 94 |
+
"method": "POST",
|
| 95 |
+
"host": "uploads.github.com",
|
| 96 |
+
"status": 401,
|
| 97 |
+
"headers": {
|
| 98 |
+
"x-github-request-id": "FD3D:10E5C1:CD4E9:E9D17:6AD3D414",
|
| 99 |
+
"content-type": "application/json; charset=utf-8",
|
| 100 |
+
"server": "envoy"
|
| 101 |
+
},
|
| 102 |
+
"client_authorization_attached": true
|
| 103 |
+
}
|
| 104 |
+
],
|
| 105 |
+
"failure_code": "github_authentication_failed",
|
| 106 |
+
"status_code": 401,
|
| 107 |
+
"failure_host": "uploads.github.com",
|
| 108 |
+
"request_id": "FD3D:10E5C1:CD4E9:E9D17:6AD3D414",
|
| 109 |
+
"committed_checkpoints": 0,
|
| 110 |
+
"recovery_verified": false,
|
| 111 |
+
"uncommitted_release_tag": "vm-test-recovery-20261017-937e26ae-20261017T200121863305766-33b55696676543a781ab99b712d7bf15",
|
| 112 |
+
"release_id": 416029153,
|
| 113 |
+
"asset_count": 0,
|
| 114 |
+
"namespace_checkpoint_lookup_status": 404
|
| 115 |
+
}
|
docs/evidence/github-upload-client-comparison.json
ADDED
|
@@ -0,0 +1,32 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"observations": [
|
| 3 |
+
{
|
| 4 |
+
"method": "POST",
|
| 5 |
+
"host": "uploads.github.com",
|
| 6 |
+
"status": 401,
|
| 7 |
+
"headers": {
|
| 8 |
+
"x-github-request-id": "4CD7:31E81D:C5069:E70BE:6AD3D43C",
|
| 9 |
+
"content-type": "application/json; charset=utf-8",
|
| 10 |
+
"server": "envoy"
|
| 11 |
+
},
|
| 12 |
+
"client_authorization_attached": true,
|
| 13 |
+
"client": "httpx",
|
| 14 |
+
"auth_scheme": "token",
|
| 15 |
+
"credential_source": "GH_TOKEN"
|
| 16 |
+
},
|
| 17 |
+
{
|
| 18 |
+
"client": "gh",
|
| 19 |
+
"host": "uploads.github.com",
|
| 20 |
+
"method": "POST",
|
| 21 |
+
"status": 401,
|
| 22 |
+
"exit_code": 1,
|
| 23 |
+
"headers": {
|
| 24 |
+
"content-type": "application/json; charset=utf-8",
|
| 25 |
+
"server": "envoy",
|
| 26 |
+
"x-github-request-id": "ADAA:355B8B:2A3FE2:3F6FD8:6AD3D43D"
|
| 27 |
+
},
|
| 28 |
+
"credential_source": "GH_TOKEN"
|
| 29 |
+
}
|
| 30 |
+
],
|
| 31 |
+
"conclusion": "Client-side header attachment confirmed for HTTPX; proxy-side header contents are not observable. Current host-scoped credential propagation remains unverified."
|
| 32 |
+
}
|
docs/evidence/hf-space-observation.json
ADDED
|
@@ -0,0 +1,17 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"production_space": "Leon4gr45/openweb",
|
| 3 |
+
"metadata_status": 200,
|
| 4 |
+
"private": false,
|
| 5 |
+
"sdk": "docker",
|
| 6 |
+
"sha": "5a12f98129417859ebddae6d7e74ca4a8d3dd5e4",
|
| 7 |
+
"runtime_stage": "RUNNING",
|
| 8 |
+
"hardware": {
|
| 9 |
+
"current": "cpu-basic",
|
| 10 |
+
"requested": "cpu-basic"
|
| 11 |
+
},
|
| 12 |
+
"production_modified": false,
|
| 13 |
+
"staging_target": "Leon4gr45/openweb-v2-staging",
|
| 14 |
+
"anonymous_staging_metadata_status": 401,
|
| 15 |
+
"staging_exists": "unverified without authorized HF access",
|
| 16 |
+
"staging_deployed": false
|
| 17 |
+
}
|
docs/evidence/live-qualification-backend-tests.txt
ADDED
|
@@ -0,0 +1,9 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
........................................................................ [ 62%]
|
| 2 |
+
............................................ [100%]
|
| 3 |
+
=============================== warnings summary ===============================
|
| 4 |
+
../visual-mentor-venv/lib/python3.12/site-packages/fastapi/testclient.py:1
|
| 5 |
+
/workspace/visual-mentor-venv/lib/python3.12/site-packages/fastapi/testclient.py:1: StarletteDeprecationWarning: Using `httpx` with `starlette.testclient` is deprecated; install `httpx2` instead.
|
| 6 |
+
from starlette.testclient import TestClient as TestClient # noqa
|
| 7 |
+
|
| 8 |
+
-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html
|
| 9 |
+
116 passed, 1 warning in 9.43s
|
docs/evidence/private-staging-live.json
ADDED
|
@@ -0,0 +1,8 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"status": "blocked",
|
| 3 |
+
"private_boundary_verified": false,
|
| 4 |
+
"checkpoint_verified": false,
|
| 5 |
+
"sleep_wake_verified": false,
|
| 6 |
+
"failure_code": "missing_secure_staging_credentials",
|
| 7 |
+
"http_events": []
|
| 8 |
+
}
|
docs/evidence/standalone-probes/github-release-qualification-20261017T223752Z-a13651aa74504b7ca9a9a2f1f83e0100.json
ADDED
|
@@ -0,0 +1,44 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"run_id": "20261017T223752Z-a13651aa74504b7ca9a9a2f1f83e0100",
|
| 3 |
+
"repository": "JsonLord/visual-mentor-backups",
|
| 4 |
+
"credential_source": "VM_BACKUP_GITHUB_TOKEN",
|
| 5 |
+
"status": "failed",
|
| 6 |
+
"upload_decision": "NO-GO",
|
| 7 |
+
"staging_decision": "NO-GO",
|
| 8 |
+
"contents_write_verified": true,
|
| 9 |
+
"asset_readback_verified": false,
|
| 10 |
+
"http_events": [
|
| 11 |
+
{
|
| 12 |
+
"host": "api.github.com",
|
| 13 |
+
"method": "GET",
|
| 14 |
+
"status": 200,
|
| 15 |
+
"request_id": "B7D9:353E69:DAF87:2CA245:6AD3F8C0",
|
| 16 |
+
"authenticated": true,
|
| 17 |
+
"accepted_permissions": "metadata=read"
|
| 18 |
+
},
|
| 19 |
+
{
|
| 20 |
+
"host": "api.github.com",
|
| 21 |
+
"method": "POST",
|
| 22 |
+
"status": 201,
|
| 23 |
+
"request_id": "E502:35277:C81D0:28AAFC:6AD3F8C0",
|
| 24 |
+
"authenticated": true,
|
| 25 |
+
"accepted_permissions": "contents=write; contents=write,workflows=write"
|
| 26 |
+
},
|
| 27 |
+
{
|
| 28 |
+
"host": "uploads.github.com",
|
| 29 |
+
"method": "POST",
|
| 30 |
+
"status": 404,
|
| 31 |
+
"request_id": "8FF6:3E5B5B:35838A:4E7443:6AD3F8C1",
|
| 32 |
+
"authenticated": true,
|
| 33 |
+
"accepted_permissions": null
|
| 34 |
+
}
|
| 35 |
+
],
|
| 36 |
+
"recovery_run": false,
|
| 37 |
+
"production_or_workflows_changed": false,
|
| 38 |
+
"local_age_roundtrip_verified": true,
|
| 39 |
+
"private_repository_verified": true,
|
| 40 |
+
"repository_push_permission_reported": true,
|
| 41 |
+
"release_id": 416110789,
|
| 42 |
+
"release_tag": "vm-credential-test-20261017T223752Z-a13651aa74504b7ca9a9a2f1f83e0100",
|
| 43 |
+
"failure_code": "http_rejected_404"
|
| 44 |
+
}
|
docs/github-backup-migration.md
ADDED
|
@@ -0,0 +1,294 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# GitHub-native backup migration qualification — 17 October 2026
|
| 2 |
+
|
| 3 |
+
**Implementation prepared; activation is blocked.** No production Space deployment,
|
| 4 |
+
secret provisioning, scheduling activation, existing-backup overwrite or DGX service
|
| 5 |
+
change was performed. The source changes are on `feature/video-rag-mentor-v2`; backup
|
| 6 |
+
coordinator source is on private backup `main`, outside `.github/workflows`.
|
| 7 |
+
|
| 8 |
+
## Architectural change
|
| 9 |
+
|
| 10 |
+
The old design required an immutable external store snapshot plus separately stored
|
| 11 |
+
media. The new runtime operates on HF CPU Basic ephemeral files, creates consistent
|
| 12 |
+
SQLite/config/index/media snapshots, encrypts content-addressed chunks with age,
|
| 13 |
+
and uses private GitHub release assets plus atomic repository checkpoint records
|
| 14 |
+
as its sole persistent recovery source. No S3, R2, rclone or HF persistent disk is
|
| 15 |
+
required. The original Library/Search/Ask/Experiments/Architecture/detail/live/model
|
| 16 |
+
routes remain; a backup admin panel is added at `/backups`.
|
| 17 |
+
|
| 18 |
+
The backup repository now coordinates the Space instead of fetching external data.
|
| 19 |
+
The Space holds a scoped GitHub writer and age recovery key; Actions holds only a
|
| 20 |
+
scoped HF access token and limited backup-trigger token. This avoids giving the
|
| 21 |
+
coordinator configuration/media/inference administrator privileges. Encryption is
|
| 22 |
+
performed inside the Space before GitHub transmission, including the recovery
|
| 23 |
+
manifest. Only operational ciphertext references/timing/size metadata is public to
|
| 24 |
+
the private repository and GitHub itself.
|
| 25 |
+
|
| 26 |
+
The activation runbook and estimates are in [backup-restore.md](backup-restore.md).
|
| 27 |
+
Format-1 snapshots are retained unchanged and never automatically reinterpreted.
|
| 28 |
+
Existing data must first be independently restored and imported into a format-2
|
| 29 |
+
working root. No automated pruning or history rewriting is included.
|
| 30 |
+
|
| 31 |
+
## Evidence and its limits
|
| 32 |
+
|
| 33 |
+
Machine-readable summary: [github-backup-results.json](evidence/github-backup-results.json).
|
| 34 |
+
|
| 35 |
+
| Check | Result | Evidence boundary |
|
| 36 |
+
| --- | --- | --- |
|
| 37 |
+
| Backend regression suite | 109 passed | Real CPU media, SQLite and age; provider calls mocked |
|
| 38 |
+
| New backup tests | 13 passed (included above) | Successful GitHub storage transport mocked |
|
| 39 |
+
| Destructive complete-library recovery | Passed locally | Generated video, original/proxy/frame/thumbnail, transcript, segments, SQLite row, saved answer history, config history, versioned vector index; delete source + capture stage, recover only encrypted mock remote assets + key, compare all file hashes |
|
| 40 |
+
| Fresh application startup | Passed locally | Separate Python interpreter loads recovered ConfigStore and index and serves protected library endpoint; no stale in-memory state |
|
| 41 |
+
| Incremental backup | Passed locally | Identical original/proxy chunks share objects; unchanged second checkpoint uploads manifest only |
|
| 42 |
+
| Safety/failure behavior | Passed locally | Partial upload publishes no record, corrupt/missing objects block restore, wrong key publishes nothing, invalid index/unknown state fail, existing roots are not overwritten; mocked documented HTTP endpoints and cross-origin credential stripping pass |
|
| 43 |
+
| Protected administration | Passed locally | Missing/wrong role denied, coordinator token scoped to enqueue/status, browser-origin coordinator rejected |
|
| 44 |
+
| Timer and retries | Passed simulated clock | Exercises 12-hour timeout branch, three snapshot attempts, status and zero pending uploads; does not simulate a platform sleep/wake |
|
| 45 |
+
| Frontend build/lint | Passed | One existing unused-variable warning in Architecture; zero lint errors |
|
| 46 |
+
| Browser parity/camera/routing | 17 passed | Screenshot comparisons and mock API/simulated devices, not physical camera/provider integrations |
|
| 47 |
+
| Backup administration browser | 2 passed | Mock status and trigger endpoints; displays actual reported status and distinguishes acceptance from completion |
|
| 48 |
+
| GitHub private metadata | Passed live | Repository private; release creation succeeds with existing identity |
|
| 49 |
+
| GitHub encrypted asset upload | **Blocked live** | `uploads.github.com` responds 401 Bad credentials; no recovery record committed |
|
| 50 |
+
| Private HF reachability | **Blocked live** | Both HF metadata and `leon4gr45-openweb.hf.space` rejected by current network proxy policy; HF test credential absent |
|
| 51 |
+
| Actions wake/completion | **Not run / not established** | No deployed trigger on staging, no validated private access route, no activated workflow |
|
| 52 |
+
|
| 53 |
+
The unsuccessful live attempt created one empty, uniquely named test release:
|
| 54 |
+
`vm-test-recovery-20261017-deae94ff-20261017T173832-f8cd66db742745159168b741ea1a751e`.
|
| 55 |
+
It is not a checkpoint. No original user data or private credentials were uploaded.
|
| 56 |
+
The attempted ciphertext upload and a synthetic header-only diagnostic both failed
|
| 57 |
+
with 401. No production namespace was written and old assets were not deleted.
|
| 58 |
+
|
| 59 |
+
The environment draft now preserves existing domains and adds API/upload/signed
|
| 60 |
+
release hosts plus `huggingface.co` and the exact supplied Space origin. It also
|
| 61 |
+
records scoped `VM_BACKUP_GITHUB_TOKEN` and optional `HF_SPACE_TOKEN` requirements.
|
| 62 |
+
Saving this draft did not apply or publish it, provision credentials, or change HF
|
| 63 |
+
settings. Review/save and publish the environment configuration to apply access.
|
| 64 |
+
Do not paste tokens into chat. GitHub API access alone does not establish upload
|
| 65 |
+
host credentials; both routes must be tested using the same authorized identity.
|
| 66 |
+
|
| 67 |
+
## Scheduling decision
|
| 68 |
+
|
| 69 |
+
The inactive Actions template has 00:17/12:17 UTC schedule and manual dispatch,
|
| 70 |
+
concurrency and outer timeout. It requires explicit approval plus validated private
|
| 71 |
+
routing; it checks HF metadata privacy, uses HF bearer authentication and separate
|
| 72 |
+
app trigger header, then polls for a newly completed checkpoint. YAML existence
|
| 73 |
+
and a 202-style enqueue response are never reported as schedule success.
|
| 74 |
+
|
| 75 |
+
The local resident timer and important-state-change triggers are implemented and
|
| 76 |
+
covered with mock-clock/local tests. They run only while the Space is awake. They
|
| 77 |
+
are useful as a local fallback but cannot meet a sleeping-Space wake guarantee.
|
| 78 |
+
No secure alternative wake path is validated in this environment. Reliable
|
| 79 |
+
12-hour scheduling remains blocked pending a real private staging sleep/wake test.
|
| 80 |
+
The implementation does not create a public endpoint, restart the Space, change
|
| 81 |
+
hardware, buy persistent storage or expose additional ports to evade that blocker.
|
| 82 |
+
|
| 83 |
+
## Capacity estimate
|
| 84 |
+
|
| 85 |
+
Example 64 MiB original + 20 MiB proxy + 1 MiB transcript/images + 3 MiB index:
|
| 86 |
+
approximately 88 MiB initial encrypted storage. One hundred similar videos:
|
| 87 |
+
roughly 8.5 GiB media plus index revisions/manifests. A 10k x 768 float32 view is
|
| 88 |
+
29.30 MiB; five views are 146.48 MiB/revision. Twenty single-view revisions add
|
| 89 |
+
about 586 MiB. Age adds small framing overhead and does not compress media.
|
| 90 |
+
A configuration-only change uploads changed chunks and a small encrypted manifest;
|
| 91 |
+
unchanged chunks are reused but read back for coverage. Older removed/reintroduced
|
| 92 |
+
chunks may upload again because the dedup catalog is the preceding manifest.
|
| 93 |
+
|
| 94 |
+
Enforce 20 GiB / 100k-file plaintext ceiling and source-size + 64 MiB free scratch
|
| 95 |
+
preflight. HF's 50 GB includes app, active files, processing and backup/restore
|
| 96 |
+
staging. Keep operational usage below the theoretical limit. Check GitHub account
|
| 97 |
+
quotas and release-asset acceptable use before treating the estimate as approval.
|
| 98 |
+
GitHub listings fail closed at capacity instead of silently dropping files; rotate
|
| 99 |
+
reviewed namespaces before a year shard reaches 1000 records. No deletion is automatic.
|
| 100 |
+
|
| 101 |
+
## Activation checklist — requires separate production approval
|
| 102 |
+
|
| 103 |
+
1. Review the two commits and storage/key policy; keep all existing backup data.
|
| 104 |
+
2. Apply the saved environment network settings. Provision a fine-grained private
|
| 105 |
+
repository token in secure settings with Contents read/write, authenticated on
|
| 106 |
+
**both** API and uploads hosts. Provision read-only HF access for the supplied
|
| 107 |
+
private Space. Do not provide a broad app administrator token to Actions.
|
| 108 |
+
3. Rerun `AGE_BIN=/path/to/age python tests/live_github_recovery.py`. It creates a
|
| 109 |
+
fresh `test-recovery-*` namespace, backs up real generated video/index/config,
|
| 110 |
+
checks unchanged dedup, destroys source + snapshot stage, starts a fresh Python
|
| 111 |
+
application instance and recovers using GitHub + protected ephemeral test key.
|
| 112 |
+
Require `docs/evidence/github-recovery-live.json` with all hashes matching.
|
| 113 |
+
The protected test key is deleted at drill end; test assets are disposable and
|
| 114 |
+
must never be mistaken for production recovery points.
|
| 115 |
+
4. Obtain staging deployment approval. Configure the age identity as an HF Secret,
|
| 116 |
+
keep an independent protected operator recovery copy, configure roots exactly as
|
| 117 |
+
documented, and import reviewed existing data with writers stopped. First-start
|
| 118 |
+
empty authorization is temporary and is forbidden for a failed recovery.
|
| 119 |
+
5. Deploy only to the approved private staging Space. Verify anonymous denial,
|
| 120 |
+
correct HF identity access, user/admin separation and scoped coordinator access.
|
| 121 |
+
6. Run the coordinator manually against staging with approved secrets. Let staging
|
| 122 |
+
sleep, then repeat from an actual GitHub Actions runner. Require verified private
|
| 123 |
+
access, wake, accepted trigger **and a new committed checkpoint**. Record run URL,
|
| 124 |
+
timestamps and restore-point ID. Do not claim these tests from local mock results.
|
| 125 |
+
7. Destroy staging state and start its clean container. Verify complete library,
|
| 126 |
+
media/Range, index queries, YAML history and health from GitHub + age key only.
|
| 127 |
+
Run an isolated admin recovery test and capture its result in `/backups`.
|
| 128 |
+
8. Exercise ingestion-complete and config-apply/rollback triggers, failed uploads,
|
| 129 |
+
exhausted retries, revoked credentials, capacity failure and stale health.
|
| 130 |
+
Check the existing inference/camera/routing/UI flows with unchanged providers.
|
| 131 |
+
9. After explicit approval, move the reviewed inactive coordinator into
|
| 132 |
+
`.github/workflows`, provision the protected environment and set
|
| 133 |
+
`COORDINATOR_VALIDATED=true` and `BACKUPS_APPROVED=true`. First observe manual
|
| 134 |
+
completion, then actual scheduled runs. No such activation was performed here.
|
| 135 |
+
10. Obtain a distinct production deployment/scheduling approval before applying any
|
| 136 |
+
changes to `Leon4gr45/openweb`. Retain a tested previous source and recovery
|
| 137 |
+
checkpoint; rollback into an isolated root, never overwrite active data.
|
| 138 |
+
|
| 139 |
+
|
| 140 |
+
## Subsequent live qualification (17 October 2026)
|
| 141 |
+
|
| 142 |
+
This section supersedes the earlier network-policy observations above. Fixes are
|
| 143 |
+
based on application `30dbccac29e8056ccffff98f45247515f32edaad` and backup tooling
|
| 144 |
+
`96e6391b303e82ca821937c57da5e03776f3eedc`. No production changes or workflow activation
|
| 145 |
+
were made. Existing checkpoints/releases were neither deleted nor overwritten.
|
| 146 |
+
|
| 147 |
+
### Phase 1: host-specific 401 diagnosis
|
| 148 |
+
|
| 149 |
+
Using the **same GH_TOKEN source**, GitHub user/private repository reads return
|
| 150 |
+
200 and new disposable release creation returns 201 on `api.github.com`. The
|
| 151 |
+
uploader now consumes the returned `upload_url`, removes its URI-template suffix,
|
| 152 |
+
and validates HTTPS, exact uploads host, expected repository/release path, no
|
| 153 |
+
userinfo/query/fragment and no redirects before attaching Authorization.
|
| 154 |
+
|
| 155 |
+
The resulting POST to `uploads.github.com` returns **401**, request ID
|
| 156 |
+
`FD3D:10E5C1:CD4E9:E9D17:6AD3D414`. A separate supported `token` authorization-scheme
|
| 157 |
+
probe returns 401 (`4CD7:31E81D:C5069:E70BE:6AD3D43C`); the native gh CLI using the
|
| 158 |
+
same credential source also returns 401 (`ADAA:355B8B:2A3FE2:3F6FD8:6AD3D43D`). The
|
| 159 |
+
upload-host root's 302 is not an authentication test and is not counted as success.
|
| 160 |
+
|
| 161 |
+
HTTPX confirms Authorization is attached before the outbound proxy. API responses
|
| 162 |
+
include GitHub request IDs; the server response header is `envoy`. There is no
|
| 163 |
+
available trusted proxy trace that shows the forwarded Authorization value. These
|
| 164 |
+
observations localize the rejection to authentication at the upload destination;
|
| 165 |
+
they do **not** establish whether the proxy stripped, rewrote, or failed to inject
|
| 166 |
+
a credential. No proxy-held credentials were extracted, and no direct connection,
|
| 167 |
+
alternate API route or TLS bypass was attempted. Both HTTPX and gh fail, so this
|
| 168 |
+
is not demonstrated to be an HTTPX-only request-format issue. The declared scoped
|
| 169 |
+
VM_BACKUP_GITHUB_TOKEN binding still has no saved credential, and its variable is
|
| 170 |
+
absent in the running environment. A valid credential on both destinations remains
|
| 171 |
+
the external prerequisite; changing upload_url alone did not fix the 401.
|
| 172 |
+
|
| 173 |
+
401 and other permanent 4xx requests now raise sanitized GitHubPermanentError,
|
| 174 |
+
without blind upload or outer checkpoint retries. The health report distinguishes
|
| 175 |
+
`github_authentication_failed` from a generic retry exhaustion. Transient upload
|
| 176 |
+
failures still reconcile the immutable asset name before retrying; a test verifies
|
| 177 |
+
that an uncertain 503 with an actually created asset does not produce a duplicate.
|
| 178 |
+
Sanitized diagnostics record method/host/status/GitHub request ID and allowlisted
|
| 179 |
+
response headers, never Authorization values, request bodies, response payloads or
|
| 180 |
+
signed Location URLs. Tests cover destination authorization and credential stripping
|
| 181 |
+
on signed downloads.
|
| 182 |
+
|
| 183 |
+
Evidence:
|
| 184 |
+
|
| 185 |
+
- [Host probes](evidence/github-auth-probe.json)
|
| 186 |
+
- [HTTPX / gh comparison](evidence/github-upload-client-comparison.json)
|
| 187 |
+
- [Latest live drill](evidence/github-recovery-live.json)
|
| 188 |
+
|
| 189 |
+
### Phase 2: attempted real recovery
|
| 190 |
+
|
| 191 |
+
Namespace `test-recovery-20261017-937e26ae` contains a newly created, uncommitted
|
| 192 |
+
release only. Its first encrypted upload failed once with 401. A read-only check
|
| 193 |
+
confirmed **zero release assets** and no checkpoint namespace (Contents API 404).
|
| 194 |
+
No successful live recovery, incremental checkpoint or file-hash comparison is
|
| 195 |
+
claimed. The report is explicitly `status=blocked`, `recovery_verified=false`.
|
| 196 |
+
All prior recovery successes remain real age/SQLite/FFmpeg with **mocked GitHub**.
|
| 197 |
+
|
| 198 |
+
The live drill now persists sanitized diagnostics even on authentication failure.
|
| 199 |
+
After credentials work it is ready to record both committed checkpoint IDs, asset
|
| 200 |
+
counts, full restored file hashes before application startup, unchanged incremental
|
| 201 |
+
dedup, SQLite integrity/logical rows, fresh-process Library/Search/config/index,
|
| 202 |
+
authorized media Range and anonymous media denial, and actual FFmpeg media decode.
|
| 203 |
+
Search uses a deterministic synthetic query vector, not a live embedding provider;
|
| 204 |
+
the recovered index and application Search route perform real retrieval. The test
|
| 205 |
+
source and capture stage are deleted before restore, and recovery uses only GitHub
|
| 206 |
+
assets plus the isolated age key. Ephemeral test key destruction is intentional;
|
| 207 |
+
these disposable points are not production recovery assets.
|
| 208 |
+
|
| 209 |
+
### Phase 3: private HF staging preparation
|
| 210 |
+
|
| 211 |
+
Network reachability now works for the supplied production origin and HF metadata.
|
| 212 |
+
An anonymous HF metadata response reports **Leon4gr45/openweb private=false**, Docker,
|
| 213 |
+
CPU Basic, RUNNING, revision `5a12f98129417859ebddae6d7e74ca4a8d3dd5e4`. Its backup-status
|
| 214 |
+
route returns 404. This is not a private-boundary qualification and its privacy or
|
| 215 |
+
deployment was not changed. Anonymous metadata access to the proposed separate
|
| 216 |
+
staging target returns 401; existence/access cannot be concluded without credentials.
|
| 217 |
+
|
| 218 |
+
A concrete [private staging package](../deploy/hf-staging/README.md), restricted
|
| 219 |
+
probe `scripts/qualify_private_space.py`, and manual-only inactive
|
| 220 |
+
`qualify-private-staging.yml` are prepared. The probe rejects production targets,
|
| 221 |
+
requires private metadata, verifies anonymous denial plus correct HF/scoped-trigger
|
| 222 |
+
access and wrong-trigger rejection, and can run the bounded checkpoint coordinator.
|
| 223 |
+
It records an actual Actions run URL when executed on a runner and only claims a
|
| 224 |
+
sleep/wake test when initial runtime was sleeping and the authenticated route plus
|
| 225 |
+
new checkpoint succeeded. No restart/resume/hardware API is called.
|
| 226 |
+
|
| 227 |
+
The present task authorizes private staging qualification; it does not authorize
|
| 228 |
+
production changes. No HF credential is available, so staging deployment, authorized
|
| 229 |
+
HF access, actual runner dispatch and sleep/wake testing could not proceed. This is
|
| 230 |
+
a missing capability, not a request to reconfirm the task. The manual test workflow
|
| 231 |
+
and 12-hour production workflow both remain outside `.github/workflows`.
|
| 232 |
+
|
| 233 |
+
Evidence: [HF observation](evidence/hf-space-observation.json) and
|
| 234 |
+
[staging probe result](evidence/private-staging-live.json).
|
| 235 |
+
|
| 236 |
+
### Exact remaining credential and activation steps
|
| 237 |
+
|
| 238 |
+
1. In secure cloud environment settings, supply **VM_BACKUP_GITHUB_TOKEN**: a
|
| 239 |
+
fine-grained GitHub token authorized only for JsonLord/visual-mentor-backups,
|
| 240 |
+
**Contents read/write and Metadata read**. Configure proxy substitution for both
|
| 241 |
+
`api.github.com` and `uploads.github.com`; signed release downloads deliberately
|
| 242 |
+
receive no bearer token. Apply the saved environment configuration and confirm
|
| 243 |
+
both hosts with the same binding. Never paste values or expose the proxy-held
|
| 244 |
+
token. The existing integration's API access does not prove upload authentication.
|
| 245 |
+
2. Rerun `AGE_BIN=/path/to/age python tests/live_github_recovery.py`. Require a
|
| 246 |
+
passed evidence file with real asset readback, two committed checkpoints,
|
| 247 |
+
incremental manifest-only upload, all pre-startup file hashes equal, and fresh
|
| 248 |
+
application/media checks. Until then the sole-store live recovery gate fails.
|
| 249 |
+
3. Provision an HF identity with write access to the **separate private staging**
|
| 250 |
+
repository. Repository creation requires rights in the Leon4gr45 namespace; if
|
| 251 |
+
avoiding broad creation rights, create the private Docker staging repository
|
| 252 |
+
first and issue a token limited to its contents/settings. Use the staging package
|
| 253 |
+
and existing deployment allowlist, not a production upload. Set staging backup
|
| 254 |
+
namespace, protected key/trigger and independent role tokens in HF settings.
|
| 255 |
+
4. Give the coordinator a **separate read-only HF token** authorized to access the
|
| 256 |
+
private staging repository and its authenticated app route, plus its own staging
|
| 257 |
+
trigger token. Add the exact staging origin to allowed network/credential
|
| 258 |
+
destinations. No paid inference or production-repository write scope is needed.
|
| 259 |
+
Run the private probe locally, then install only the reviewed manual workflow
|
| 260 |
+
with its `private-backup-staging` environment, STAGING_SPACE_ID/URL variables and
|
| 261 |
+
STAGING_HF_TOKEN / STAGING_BACKUP_TRIGGER_TOKEN secrets. Keep workflow permissions
|
| 262 |
+
at Contents read; GitHub writes occur inside the Space with its scoped writer.
|
| 263 |
+
5. Dispatch the manual workflow, record its run URL and new committed point, then
|
| 264 |
+
test a naturally sleeping staging instance through the same authenticated route.
|
| 265 |
+
Destroy only staging working state and verify clean startup recovery. Do not
|
| 266 |
+
equate an awake run, a local mock, or enqueue acceptance with wake/schedule success.
|
| 267 |
+
6. Leave the production timer inactive and approval guards unset. Production Space
|
| 268 |
+
deployment or production scheduling still requires explicit approval.
|
| 269 |
+
|
| 270 |
+
Current follow-up regression: **116 backend tests passed** (20 backup tests),
|
| 271 |
+
including permanent-error retry suppression, transient reconciliation, sanitized
|
| 272 |
+
headers and private staging boundary mocks. These new tests do not convert the
|
| 273 |
+
blocked live gates into passes. No application UI or inference feature was redesigned.
|
| 274 |
+
|
| 275 |
+
|
| 276 |
+
## Dedicated credential recheck — 18 October 2026
|
| 277 |
+
|
| 278 |
+
The dedicated VM_BACKUP_GITHUB_TOKEN is now provisioned and ready. All requests
|
| 279 |
+
in this recheck used it, with no integration GH_TOKEN fallback. Private repository
|
| 280 |
+
read succeeded (200) and a new disposable release was created (201), but its first
|
| 281 |
+
age-encrypted asset upload returned **404** from uploads.github.com. Request ID:
|
| 282 |
+
`FE93:319F8B:36F88A:4FB435:6AD3F6B8`. The permanent rejection was not retried.
|
| 283 |
+
|
| 284 |
+
Namespace `test-recovery-20261017-745d0278` has zero assets and zero committed
|
| 285 |
+
checkpoints, verified after the failure. Recovery/dedup were not reached. No
|
| 286 |
+
existing remote release/checkpoint was changed. Scope versus host-specific
|
| 287 |
+
credential propagation remains undetermined until the same permitted token is
|
| 288 |
+
compared on an independent trusted machine; none is attached here. No security
|
| 289 |
+
bypass or credential extraction was attempted. HF credentials remain absent.
|
| 290 |
+
|
| 291 |
+
The first private staging deployment remains **NO-GO**. See
|
| 292 |
+
[credential qualification](credential-qualification.md) for sanitized evidence,
|
| 293 |
+
exact comparison steps and attribution boundaries. Earlier statements that the
|
| 294 |
+
dedicated binding was absent are historical and superseded by this recheck.
|
docs/original-research.md
ADDED
|
@@ -0,0 +1,129 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Video RAG with EmbeddingGemma 2 + Gemma 4 12B
|
| 2 |
+
|
| 3 |
+
▶️ **Video walkthrough:** https://youtu.be/Q8T9lvtGFNw
|
| 4 |
+
|
| 5 |
+
Multimodal RAG over the latest uploads on the channel. Every ~30s segment is embedded by
|
| 6 |
+
[EmbeddingGemma 2](https://huggingface.co/google/embeddinggemma-2) five different ways into one
|
| 7 |
+
shared 768d space, and Gemma 4 12B (QAT, on vLLM) answers questions by *watching* the retrieved
|
| 8 |
+
clips. A Next.js site shows the library, search, RAG answers, and the Matryoshka (MRL) experiments.
|
| 9 |
+
|
| 10 |
+
```
|
| 11 |
+
sams-videos/*.mp4 ─ ingest ─▶ 480p proxy + YouTube match ─ transcribe ─▶ word-timed transcript
|
| 12 |
+
│
|
| 13 |
+
segment (~30s, snapped to sentences) ◀──────────┘
|
| 14 |
+
│ clip.mp4 · audio.wav · keyframe.jpg
|
| 15 |
+
▼
|
| 16 |
+
EmbeddingGemma 2 ─▶ text | frame | video | audio | fused (768d, unit-norm)
|
| 17 |
+
│
|
| 18 |
+
query ─ SearchQuery ──┤ cosine @ 768/512/256/128 (truncate + renormalise)
|
| 19 |
+
▼
|
| 20 |
+
top-k segments ─▶ Gemma 4 12B (transcript + video frames [+ audio]) ─▶ cited answer
|
| 21 |
+
```
|
| 22 |
+
|
| 23 |
+
## Embedding views
|
| 24 |
+
|
| 25 |
+
| view | what is embedded | why |
|
| 26 |
+
|---|---|---|
|
| 27 |
+
| `text` | `title: {video title} \| text: {transcript}` | the classic text-RAG baseline |
|
| 28 |
+
| `frame` | one keyframe from the middle of the segment | cheapest visual signal |
|
| 29 |
+
| `video` | the clip's frames at 1fps (no audio, no text) | what's on screen over time |
|
| 30 |
+
| `audio` | raw 16kHz mono audio (no transcript) | can it replace ASR? |
|
| 31 |
+
| `fused` | transcript + `<\|video\|>` + `<\|audio\|>` interleaved in one input | all signals in a single vector |
|
| 32 |
+
| `hybrid` | reciprocal rank fusion of `text`, `video`, `audio` rankings | multi-vector alternative to `fused` |
|
| 33 |
+
|
| 34 |
+
Segments target 30s: about 30 frames × 140 tokens + 750 audio tokens + the transcript, which fits
|
| 35 |
+
inside the model's 8,192-token window.
|
| 36 |
+
|
| 37 |
+
## First results (15 videos, 427 segments, 228 synthetic queries)
|
| 38 |
+
|
| 39 |
+
recall@5 (exact source segment in top 5):
|
| 40 |
+
|
| 41 |
+
| view | spoken 768d | visual 768d | all 768d | all 256d | all 128d |
|
| 42 |
+
|---|---|---|---|---|---|
|
| 43 |
+
| text | **0.98** | 0.67 | **0.82** | **0.80** | 0.71 |
|
| 44 |
+
| frame | 0.39 | 0.60 | 0.49 | 0.46 | 0.41 |
|
| 45 |
+
| video | 0.53 | 0.75 | 0.64 | 0.59 | 0.52 |
|
| 46 |
+
| audio | 0.82 | 0.61 | 0.71 | 0.66 | 0.57 |
|
| 47 |
+
| fused | 0.84 | **0.78** | 0.81 | 0.77 | 0.69 |
|
| 48 |
+
| hybrid | 0.84 | 0.69 | 0.77 | 0.74 | **0.72** |
|
| 49 |
+
|
| 50 |
+
- **Spoken queries favour transcripts.** Text is near-perfect. Raw audio with no ASR reaches 0.82.
|
| 51 |
+
The spoken queries were written from transcripts, which biases the comparison toward the text view.
|
| 52 |
+
- **Visual queries favour pixels.** Fused (0.78) and video (0.75) beat text (0.67). This is where multimodal embedding pays off.
|
| 53 |
+
- **MRL:** 256d keeps about 95% of 768d quality across views. 128d costs 10 to 15 points, except hybrid,
|
| 54 |
+
where rank fusion absorbs most of the loss.
|
| 55 |
+
- Caveats: the queries are synthetic, Gemma 4 always produced a "visual" query even for talking-head shots,
|
| 56 |
+
and some spoken queries are vague. Add real queries to `data/eval/manual.jsonl` and re-run `vrag experiments`.
|
| 57 |
+
|
| 58 |
+
## Setup
|
| 59 |
+
|
| 60 |
+
Two separate uv environments, so vLLM's pinned torch/transformers don't clash with the embedding stack:
|
| 61 |
+
|
| 62 |
+
```bash
|
| 63 |
+
uv sync # pipeline + API (torch cu130, transformers 5.19, sentence-transformers 6.1)
|
| 64 |
+
(cd serve && uv sync) # vLLM 0.23.0 for Gemma 4 12B QAT
|
| 65 |
+
(cd web && npm install) # Next.js site
|
| 66 |
+
```
|
| 67 |
+
|
| 68 |
+
## Build the index
|
| 69 |
+
|
| 70 |
+
Put video files in `sams-videos/`, then:
|
| 71 |
+
|
| 72 |
+
```bash
|
| 73 |
+
serve/serve.sh --bg --mtp # Gemma 4 12B QAT on :8001 (needed for evalset + ask)
|
| 74 |
+
uv run vrag build # ingest -> transcribe -> segment -> embed
|
| 75 |
+
uv run vrag evalset # Gemma 4 watches sampled segments, writes spoken + visual queries
|
| 76 |
+
uv run vrag experiments # view x MRL-dim retrieval grid -> data/experiments/results.json
|
| 77 |
+
```
|
| 78 |
+
|
| 79 |
+
- **ingest** transcodes each file to a 480p proxy and matches it to its YouTube upload by duration
|
| 80 |
+
(YouTube lists durations rounded up), recovering title, ID, and view count with no per-video requests.
|
| 81 |
+
- **transcribe** uses Gemma 4 12B on the vLLM server by default (`VRAG_ASR=whisper` switches to Whisper
|
| 82 |
+
large-v3-turbo). Audio is cut at pauses into chunks of 28 seconds or less and padded to exactly 30 seconds.
|
| 83 |
+
Gemma only hears the first 30 seconds of any clip, and vLLM 0.23 crashes when it batches clips of different lengths.
|
| 84 |
+
Sentence times are interpolated within each chunk. Chunks that fall into a repetition loop are retried with a
|
| 85 |
+
repetition penalty. On this corpus it took about 100 seconds for 3.9 hours of audio (Whisper: about 35 minutes),
|
| 86 |
+
with 4.3% word disagreement between the two. `uv run vrag switch-transcripts gemma|whisper` swaps the transcripts
|
| 87 |
+
behind the text and fused views while keeping the segment boundaries fixed.
|
| 88 |
+
- `uv run vrag download` is an alternative that pulls straight from YouTube, but YouTube often bot-blocks it.
|
| 89 |
+
|
| 90 |
+
## Run the app
|
| 91 |
+
|
| 92 |
+
```bash
|
| 93 |
+
scripts/dev.sh # vLLM (if not running) + FastAPI :8000 + Next.js :3000
|
| 94 |
+
```
|
| 95 |
+
|
| 96 |
+
or each separately: `uv run vrag api`, `cd web && npm run dev`. To open the dev site from other
|
| 97 |
+
machines, list their hostnames/IPs in `web/.env.local` as `VRAG_DEV_ORIGINS=host1,host2,*.your-tailnet.ts.net`.
|
| 98 |
+
CLI question answering:
|
| 99 |
+
`uv run vrag ask "what did Sam say about 1.6-bit DeepSeek?"`.
|
| 100 |
+
|
| 101 |
+
## Gotchas found along the way
|
| 102 |
+
|
| 103 |
+
- EmbeddingGemma 2 must run in **bf16 or fp32**. fp16 silently produces NaN or degraded vectors.
|
| 104 |
+
- After MRL truncation, vectors **must be re-normalised** (`vrag/index.py:truncate`).
|
| 105 |
+
- Video input in sentence-transformers needs `torchcodec`, because torchvision removed `read_video`.
|
| 106 |
+
- vLLM needs `vllm[audio]`, otherwise audio inputs return "Invalid or unsupported audio file".
|
| 107 |
+
- Gemma 4 treats audio placed before the instruction as a conversation turn. Put the instruction first.
|
| 108 |
+
- Whisper's batched *chunk* timestamps can collapse into 5-minute cues. Word timestamps fix that, but Whisper
|
| 109 |
+
still invents or duplicates text where its 30-second windows meet, for example "I'll see you next time" mid-video.
|
| 110 |
+
- Gemma 4 occasionally gets stuck in a loop ("thought thought thought…"). These are detected and retried.
|
| 111 |
+
- `serve/serve.sh` carries over the Blackwell/VPN workarounds from `~/projects/gemma-4-qat`.
|
| 112 |
+
|
| 113 |
+
## Layout
|
| 114 |
+
|
| 115 |
+
```
|
| 116 |
+
vrag/ pipeline + API (python)
|
| 117 |
+
ingest.py local files -> data/videos/<id>/{video.mp4, thumb.jpg, meta.json}
|
| 118 |
+
transcribe.py transcript.json
|
| 119 |
+
segment.py segments.json + segments/NNN.{mp4,wav,jpg}
|
| 120 |
+
embedder.py EmbeddingGemma 2 wrapper, per-view inputs
|
| 121 |
+
embed.py data/index/<view>.npy + manifest.json
|
| 122 |
+
index.py cosine search, MRL truncation, hybrid RRF
|
| 123 |
+
rag.py retrieval + Gemma 4 generation (video_url / audio_url clips)
|
| 124 |
+
evalset.py synthetic spoken/visual queries (+ data/eval/manual.jsonl for your own)
|
| 125 |
+
experiments.py metrics grid
|
| 126 |
+
api.py FastAPI for the site
|
| 127 |
+
serve/ vLLM env + serve/stop scripts
|
| 128 |
+
web/ Next.js 16 site (Library, Search, Ask, Experiments)
|
| 129 |
+
```
|
docs/providers.md
ADDED
|
@@ -0,0 +1,53 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Remote provider contracts and v2 qualification
|
| 2 |
+
|
| 3 |
+
No large model runs inside this Space. Runtime readiness is not inferred from a
|
| 4 |
+
configured API key, health endpoint, placeholder model ID or mock result.
|
| 5 |
+
|
| 6 |
+
| Adapter | Request contract | Actual v2 evidence |
|
| 7 |
+
| --- | --- | --- |
|
| 8 |
+
| `laya-vision` | Router root + POST `/v1/system-one`; object `state`, typed `questions`, bare `imageBase64` | Live synthetic image passed 2026-10-17, 22.632s; no real camera |
|
| 9 |
+
| `detection-http` | Gateway `/detect`, normalized boxes plus matching frame/camera IDs and dimensions | Mock contract only; no NVIDIA endpoint/model configured |
|
| 10 |
+
| `openai-transcription` | `/audio/transcriptions`, multipart audio file + model | Mock HTTP and real FFmpeg audio extraction; no live Whisper credentials |
|
| 11 |
+
| `openai-embeddings` | `/embeddings`, model + input; finite nonzero vector | Mock pipeline, version/dimension isolation; no live model configured |
|
| 12 |
+
| `openai-chat-vision` | `/chat/completions`, bounded transcript/text and optional base64 image URLs | Mock answers/citations and image payload safety; no live VLM configured |
|
| 13 |
+
|
| 14 |
+
Laya source contract was recovered during donor review from
|
| 15 |
+
`JsonLord/laya-inference`, source revision
|
| 16 |
+
`77b324ac6c72efee883c41fd333e84007c55bed0`. The donor fixture/tests preserve that
|
| 17 |
+
source envelope and the observed live `results.answers` alternative. Upstream
|
| 18 |
+
README `image` is not the JSON gateway image field. The historical source had a
|
| 19 |
+
placeholder backend; actual live readiness says placeholder=false, and adapter
|
| 20 |
+
requires backend=laya-vision and model=thaitea/laya-vision. Only a fixed `noul`
|
| 21 |
+
question about image inspectability is implemented. It is not a captioner,
|
| 22 |
+
spatial detector or proof of semantic mentoring quality. No boxes are invented.
|
| 23 |
+
The current runtime had a Laya credential available despite the environment
|
| 24 |
+
configuration inventory not listing one. It was used through the normal adapter
|
| 25 |
+
without reading/logging its value. `authorization_header_sent` records that a
|
| 26 |
+
header was sent; authentication enforcement was not tested.
|
| 27 |
+
|
| 28 |
+
NVIDIA is an explicit gateway protocol, not an assumption that any NVIDIA/NIM
|
| 29 |
+
model supports spatial detection. A deployment must provide a verified gateway
|
| 30 |
+
matching `vrag/providers.py` or add an upstream-source-verified adapter after
|
| 31 |
+
review. Placeholder IDs cannot be enabled. OpenAI provider base URLs normally
|
| 32 |
+
include `/v1`; Laya uses its router root. Secrets stay in referenced server env vars.
|
| 33 |
+
|
| 34 |
+
Routing supports single, fallback, parallel (all/first_valid/quorum), sequential
|
| 35 |
+
(typed previous output), conditional predicates and spatial/semantic fusion.
|
| 36 |
+
Quorum failure reports no valid quorum; ordinary partial failures retain successful
|
| 37 |
+
outputs. Per-session overrides reuse configured provider identities/capabilities,
|
| 38 |
+
recheck destinations on dispatch, stop local previews when applied and discard
|
| 39 |
+
in-flight results if routing changes. They do not persist defaults.
|
| 40 |
+
|
| 41 |
+
Text indexes identify endpoint/model, full dimension, preprocessing, modality,
|
| 42 |
+
segmentation and corpus revision. Only declared MRL dimensions may be used. New
|
| 43 |
+
jobs atomically switch CURRENT after a complete revision, reusing compatible
|
| 44 |
+
unchanged segment vectors. Frame/video/audio/fused embedding adapters remain
|
| 45 |
+
**unimplemented and unavailable**: text similarity is not a substitute. Historical
|
| 46 |
+
compatible index files/charts are retained, not regenerated or requalified here.
|
| 47 |
+
|
| 48 |
+
Destination policy: exact approved HTTPS hosts, no URL credentials/query/fragment,
|
| 49 |
+
no private/metadata IPs, no redirects, TLS verification, bounded payloads/responses,
|
| 50 |
+
per-provider concurrency/circuit control. Direct requests pin validated public IPs
|
| 51 |
+
with original hostname/SNI. A configured trusted proxy owns DNS on that path and
|
| 52 |
+
must independently enforce public-only DNS/metadata denial. Real deployment egress
|
| 53 |
+
policy is an outstanding staging check.
|
docs/v2-evidence/backend-tests.txt
ADDED
|
@@ -0,0 +1,9 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
........................................................................ [ 75%]
|
| 2 |
+
........................ [100%]
|
| 3 |
+
=============================== warnings summary ===============================
|
| 4 |
+
../visual-mentor-venv/lib/python3.12/site-packages/fastapi/testclient.py:1
|
| 5 |
+
/workspace/visual-mentor-venv/lib/python3.12/site-packages/fastapi/testclient.py:1: StarletteDeprecationWarning: Using `httpx` with `starlette.testclient` is deprecated; install `httpx2` instead.
|
| 6 |
+
from starlette.testclient import TestClient as TestClient # noqa
|
| 7 |
+
|
| 8 |
+
-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html
|
| 9 |
+
96 passed, 1 warning in 2.63s
|
docs/v2-evidence/browser-tests.txt
ADDED
|
@@ -0,0 +1,33 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
|
| 2 |
+
Running 21 tests using 1 worker
|
| 3 |
+
|
| 4 |
+
(node:44947) Warning: The 'NO_COLOR' env is ignored due to the 'FORCE_COLOR' env being set.
|
| 5 |
+
(Use `node --trace-warnings ...` to show where the warning was created)
|
| 6 |
+
✓ 1 tests/fullstack.spec.ts:2:5 › real same-origin sample library, search, Ask, playback and Range (5.5s)
|
| 7 |
+
✓ 2 tests/live.spec.ts:2:5 › dual simulated previews require explicit transfer and stop on navigation (3.7s)
|
| 8 |
+
✓ 3 tests/live.spec.ts:63:5 › models studio validates and atomically switches a profile (1.3s)
|
| 9 |
+
✓ 4 tests/parity.spec.ts:10:7 › original UI desktop (6.2s)
|
| 10 |
+
✓ 5 tests/parity.spec.ts:10:7 › original UI laptop (7.2s)
|
| 11 |
+
✓ 6 tests/parity.spec.ts:10:7 › original UI tablet (8.8s)
|
| 12 |
+
✓ 7 tests/parity.spec.ts:10:7 › original UI mobile (6.7s)
|
| 13 |
+
✓ 8 tests/parity.spec.ts:48:5 › clip and citation interactions (1.7s)
|
| 14 |
+
✓ 9 tests/release.spec.ts:11:5 › permission denial shows an error without uploading (869ms)
|
| 15 |
+
✓ 10 tests/release.spec.ts:19:7 › rejects frame mismatch and permits a later matching observation (2.3s)
|
| 16 |
+
✓ 11 tests/release.spec.ts:19:7 › rejects config mismatch and permits a later matching observation (1.2s)
|
| 17 |
+
✓ 12 tests/release.spec.ts:19:7 › rejects dimensions mismatch and permits a later matching observation (1.2s)
|
| 18 |
+
✓ 13 tests/release.spec.ts:19:7 › rejects box mismatch and permits a later matching observation (1.1s)
|
| 19 |
+
✓ 14 tests/release.spec.ts:46:5 › stopping a camera discards its delayed result (1.1s)
|
| 20 |
+
✓ 15 tests/release.spec.ts:59:5 › a pending camera permission grant after navigation stops the returned tracks (1.1s)
|
| 21 |
+
✓ 16 tests/release.spec.ts:72:5 › device contention is reported and track end stops the preview (1.1s)
|
| 22 |
+
✓ 17 tests/release.spec.ts:84:5 › hiding the page stops microphone capture without uploading audio (1.0s)
|
| 23 |
+
✓ 18 tests/release.spec.ts:98:5 › recorder errors release microphone tracks and discard audio (926ms)
|
| 24 |
+
✓ 19 tests/studio.spec.ts:60:7 › native Live and Models desktop (2.6s)
|
| 25 |
+
✓ 20 tests/studio.spec.ts:60:7 › native Live and Models mobile (1.9s)
|
| 26 |
+
✓ 21 tests/studio.spec.ts:74:5 › import edits draft, validate gates apply, route form updates YAML (1.4s)
|
| 27 |
+
|
| 28 |
+
21 passed (1.1m)
|
| 29 |
+
npm notice
|
| 30 |
+
npm notice New major version of npm available! 11.9.0 -> 12.2.0
|
| 31 |
+
npm notice Changelog: https://github.com/npm/cli/releases/tag/v12.2.0
|
| 32 |
+
npm notice To update run: npm install -g npm@12.2.0
|
| 33 |
+
npm notice
|
docs/v2-evidence/build.txt
ADDED
|
@@ -0,0 +1,120 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#0 building with "default" instance using docker driver
|
| 2 |
+
|
| 3 |
+
#1 [internal] load build definition from Dockerfile
|
| 4 |
+
#1 transferring dockerfile: 1.89kB done
|
| 5 |
+
#1 DONE 0.0s
|
| 6 |
+
|
| 7 |
+
#2 [internal] load metadata for docker.io/library/python:3.12-slim-bookworm@sha256:34386ef0cb081344d7ec1c103ba398e6e9f64e9ab3a1509accc92a4e24a07258
|
| 8 |
+
#2 DONE 0.0s
|
| 9 |
+
|
| 10 |
+
#3 [internal] load metadata for docker.io/library/node:24-bookworm-slim@sha256:a5f4366e834695d5f363b920c104953a535d3a9864d1bdf74566a0ebe01daee5
|
| 11 |
+
#3 DONE 0.0s
|
| 12 |
+
|
| 13 |
+
#4 [internal] load .dockerignore
|
| 14 |
+
#4 transferring context: 396B done
|
| 15 |
+
#4 DONE 0.0s
|
| 16 |
+
|
| 17 |
+
#5 [web 1/6] FROM docker.io/library/node:24-bookworm-slim@sha256:a5f4366e834695d5f363b920c104953a535d3a9864d1bdf74566a0ebe01daee5
|
| 18 |
+
#5 DONE 0.0s
|
| 19 |
+
|
| 20 |
+
#6 [runtime 1/6] FROM docker.io/library/python:3.12-slim-bookworm@sha256:34386ef0cb081344d7ec1c103ba398e6e9f64e9ab3a1509accc92a4e24a07258
|
| 21 |
+
#6 DONE 0.0s
|
| 22 |
+
|
| 23 |
+
#7 [internal] load build context
|
| 24 |
+
#7 transferring context: 24.00kB done
|
| 25 |
+
#7 DONE 0.0s
|
| 26 |
+
|
| 27 |
+
#8 [web 3/6] COPY web/package*.json ./
|
| 28 |
+
#8 CACHED
|
| 29 |
+
|
| 30 |
+
#9 [web 2/6] WORKDIR /build/web
|
| 31 |
+
#9 CACHED
|
| 32 |
+
|
| 33 |
+
#10 [web 4/6] RUN --mount=type=secret,id=build_ca if [ -f /run/secrets/build_ca ]; then export NODE_EXTRA_CA_CERTS=/run/secrets/build_ca; fi; npm ci --no-audit --no-fund
|
| 34 |
+
#10 CACHED
|
| 35 |
+
|
| 36 |
+
#11 [web 5/6] COPY web/ ./
|
| 37 |
+
#11 DONE 11.3s
|
| 38 |
+
|
| 39 |
+
#12 [web 6/6] RUN npm run build
|
| 40 |
+
#12 2.338
|
| 41 |
+
#12 2.338 > web@0.1.0 build
|
| 42 |
+
#12 2.338 > next build
|
| 43 |
+
#12 2.338
|
| 44 |
+
#12 2.710 ▲ Next.js 16.4.0 (Turbopack)
|
| 45 |
+
#12 2.748 ✓ Running next.config.ts took 36ms
|
| 46 |
+
#12 2.765 - Cache Components enabled
|
| 47 |
+
#12 2.765 - Partial Prefetching enabled
|
| 48 |
+
#12 2.765 - Experiments (use with caution):
|
| 49 |
+
#12 2.765 · proxyClientMaxBodySize: "65mb"
|
| 50 |
+
#12 2.765 · proxyTimeout: 300000
|
| 51 |
+
#12 2.765
|
| 52 |
+
#12 2.785 Creating an optimized production build ...
|
| 53 |
+
#12 9.207 ✓ Compiled successfully in 6.4s
|
| 54 |
+
#12 9.211 Running TypeScript ...
|
| 55 |
+
#12 12.98 Finished TypeScript in 3.8s ...
|
| 56 |
+
#12 12.98 Collecting page data using 4 workers ...
|
| 57 |
+
#12 13.54 Generating static pages using 4 workers (0/11) ...
|
| 58 |
+
#12 13.91 Generating static pages using 4 workers (2/11)
|
| 59 |
+
#12 13.95 Generating static pages using 4 workers (5/11)
|
| 60 |
+
#12 13.95 Generating static pages using 4 workers (8/11)
|
| 61 |
+
#12 14.04 ✓ Generating static pages using 4 workers (11/11) in 501ms
|
| 62 |
+
#12 14.05 Finalizing page optimization ...
|
| 63 |
+
#12 14.22
|
| 64 |
+
#12 14.22 Route (app)
|
| 65 |
+
#12 14.22 ┌ ○ /
|
| 66 |
+
#12 14.22 ├ ○ /_not-found
|
| 67 |
+
#12 14.22 ├ ○ /architecture
|
| 68 |
+
#12 14.22 ├ ○ /ask
|
| 69 |
+
#12 14.22 ├ ○ /experiments
|
| 70 |
+
#12 14.22 ├ ○ /live
|
| 71 |
+
#12 14.22 ├ ○ /models
|
| 72 |
+
#12 14.22 ├ ○ /search
|
| 73 |
+
#12 14.22 └ /videos/[id]
|
| 74 |
+
#12 14.22 └ ◐ /videos/[id]
|
| 75 |
+
#12 14.22
|
| 76 |
+
#12 14.22
|
| 77 |
+
#12 14.22 ○ (Static) prerendered as static content
|
| 78 |
+
#12 14.22 ◐ (Partial Prerender) prerendered as static HTML with dynamic server-streamed content
|
| 79 |
+
#12 14.22
|
| 80 |
+
#12 DONE 14.3s
|
| 81 |
+
|
| 82 |
+
#13 [bundle 1/6] COPY vrag /app/vrag
|
| 83 |
+
#13 CACHED
|
| 84 |
+
|
| 85 |
+
#14 [bundle 2/6] COPY config /app/config
|
| 86 |
+
#14 CACHED
|
| 87 |
+
|
| 88 |
+
#15 [bundle 3/6] COPY scripts/cloud-start.py /app/scripts/cloud-start.py
|
| 89 |
+
#15 CACHED
|
| 90 |
+
|
| 91 |
+
#16 [bundle 4/6] COPY --from=web /build/web/.next/standalone /app/web/.next/standalone
|
| 92 |
+
#16 DONE 0.1s
|
| 93 |
+
|
| 94 |
+
#17 [bundle 5/6] COPY --from=web /build/web/.next/static /app/web/.next/standalone/.next/static
|
| 95 |
+
#17 DONE 0.1s
|
| 96 |
+
|
| 97 |
+
#18 [bundle 6/6] COPY --from=web /build/web/public /app/web/.next/standalone/public
|
| 98 |
+
#18 DONE 0.1s
|
| 99 |
+
|
| 100 |
+
#19 [runtime 4/6] RUN --mount=type=secret,id=build_ca export http_proxy="$HTTP_PROXY" https_proxy="$HTTPS_PROXY"; if [ -f /run/secrets/build_ca ]; then export PIP_CERT=/run/secrets/build_ca; fi; apt-get update && apt-get install -y --no-install-recommends ca-certificates libatomic1 ffmpeg age && pip install --no-cache-dir -r requirements-runtime.txt && rm -rf /var/lib/apt/lists/* && useradd -m -u 1000 mentor && mkdir -p /app/data/videos && chown -R mentor:mentor /app/data
|
| 101 |
+
#19 CACHED
|
| 102 |
+
|
| 103 |
+
#20 [runtime 2/6] WORKDIR /app
|
| 104 |
+
#20 CACHED
|
| 105 |
+
|
| 106 |
+
#21 [runtime 3/6] COPY requirements-runtime.txt ./
|
| 107 |
+
#21 CACHED
|
| 108 |
+
|
| 109 |
+
#22 [runtime 5/6] COPY --from=web /usr/local/bin/node /usr/local/bin/node
|
| 110 |
+
#22 CACHED
|
| 111 |
+
|
| 112 |
+
#23 [runtime 6/6] COPY --from=bundle --chown=1000:1000 /app /app
|
| 113 |
+
#23 DONE 3.8s
|
| 114 |
+
|
| 115 |
+
#24 exporting to image
|
| 116 |
+
#24 exporting layers
|
| 117 |
+
#24 exporting layers 0.6s done
|
| 118 |
+
#24 writing image sha256:51259fc8221de63f5b4dfaf9e247e64e1496806ca88ca72f85ec243fae1ef1a3 done
|
| 119 |
+
#24 naming to docker.io/library/video-rag-mentor:v2-review done
|
| 120 |
+
#24 DONE 0.6s
|
docs/v2-evidence/container.json
ADDED
|
@@ -0,0 +1,22 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"image_id": "sha256:51259fc8221de63f5b4dfaf9e247e64e1496806ca88ca72f85ec243fae1ef1a3",
|
| 3 |
+
"startup_seconds": 13.009,
|
| 4 |
+
"network": "none",
|
| 5 |
+
"published_ports": [],
|
| 6 |
+
"limits": "2 CPUs / 16 GiB",
|
| 7 |
+
"checks": {
|
| 8 |
+
"media_range_and_cookie": "passed",
|
| 9 |
+
"authentication": "passed",
|
| 10 |
+
"csrf_origin": "passed",
|
| 11 |
+
"config_apply_rollback": "passed",
|
| 12 |
+
"session_ownership": "passed",
|
| 13 |
+
"disabled_inference": "503 without local fallback",
|
| 14 |
+
"uid": 1000,
|
| 15 |
+
"forbidden_packages": [],
|
| 16 |
+
"model_weights": [],
|
| 17 |
+
"gpu_devices": []
|
| 18 |
+
},
|
| 19 |
+
"persistent_volume": {
|
| 20 |
+
"recreated_container_config_version": 4
|
| 21 |
+
}
|
| 22 |
+
}
|
docs/v2-evidence/laya-live.json
ADDED
|
@@ -0,0 +1,70 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"metadata": {
|
| 3 |
+
"/health": {
|
| 4 |
+
"status": "ok"
|
| 5 |
+
},
|
| 6 |
+
"/ready": {
|
| 7 |
+
"status": "ready",
|
| 8 |
+
"backends": {
|
| 9 |
+
"laya": {
|
| 10 |
+
"ready": true
|
| 11 |
+
},
|
| 12 |
+
"laya-vision": {
|
| 13 |
+
"ready": true,
|
| 14 |
+
"placeholder": false
|
| 15 |
+
}
|
| 16 |
+
}
|
| 17 |
+
},
|
| 18 |
+
"/v1/info": {
|
| 19 |
+
"service": "system-one",
|
| 20 |
+
"inference_endpoint": "/v1/system-one",
|
| 21 |
+
"routing": {
|
| 22 |
+
"text_or_structured": "laya",
|
| 23 |
+
"visual": "laya-vision"
|
| 24 |
+
},
|
| 25 |
+
"backends": {
|
| 26 |
+
"laya": {
|
| 27 |
+
"device": "cpu",
|
| 28 |
+
"generative": false,
|
| 29 |
+
"model": "receptron/laya-onnx",
|
| 30 |
+
"threads": 2,
|
| 31 |
+
"max_concurrency": 4
|
| 32 |
+
},
|
| 33 |
+
"laya-vision": {
|
| 34 |
+
"device": "cpu",
|
| 35 |
+
"generative": false,
|
| 36 |
+
"model": "thaitea/laya-vision",
|
| 37 |
+
"parameters": "201M",
|
| 38 |
+
"runtime": "python/laya",
|
| 39 |
+
"threads": 1,
|
| 40 |
+
"max_concurrency": 1,
|
| 41 |
+
"max_queue": 8
|
| 42 |
+
}
|
| 43 |
+
},
|
| 44 |
+
"capabilities": [
|
| 45 |
+
"choice",
|
| 46 |
+
"score",
|
| 47 |
+
"noul",
|
| 48 |
+
"vision"
|
| 49 |
+
]
|
| 50 |
+
}
|
| 51 |
+
},
|
| 52 |
+
"synthetic_inference_executed": true,
|
| 53 |
+
"elapsed_s": 22.632,
|
| 54 |
+
"output": {
|
| 55 |
+
"detections": [],
|
| 56 |
+
"decisions": [
|
| 57 |
+
{
|
| 58 |
+
"action": "visual_assessment",
|
| 59 |
+
"reason": "Image inspectability probability: 0.3431 (Laya noul score; advisory, not a spatial detection).",
|
| 60 |
+
"source": "laya"
|
| 61 |
+
}
|
| 62 |
+
],
|
| 63 |
+
"text": "",
|
| 64 |
+
"warnings": [],
|
| 65 |
+
"embedding": null
|
| 66 |
+
},
|
| 67 |
+
"authorization_header_sent": true,
|
| 68 |
+
"authentication_enforcement_tested": false,
|
| 69 |
+
"tested_at": "2026-10-17"
|
| 70 |
+
}
|