Upload 3 files
Browse files- README.md +104 -79
- requirements.txt +40 -0
- tools/sync_platform.mjs +186 -0
README.md
CHANGED
|
@@ -3,101 +3,126 @@ title: CLOAK API
|
|
| 3 |
emoji: 🛡️
|
| 4 |
colorFrom: blue
|
| 5 |
colorTo: indigo
|
| 6 |
-
sdk:
|
| 7 |
-
|
| 8 |
-
python_version: '3.11'
|
| 9 |
-
app_file: app.py
|
| 10 |
pinned: false
|
| 11 |
---
|
| 12 |
|
| 13 |
-
# CLOAK
|
| 14 |
|
| 15 |
-
|
| 16 |
-
|
| 17 |
|
| 18 |
-
|
| 19 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 20 |
|
| 21 |
## Deploy
|
| 22 |
-
|
| 23 |
-
|
| 24 |
-
|
| 25 |
-
|
| 26 |
-
|
| 27 |
-
|
| 28 |
-
|
| 29 |
-
|
| 30 |
-
-
|
| 31 |
-
|
| 32 |
-
|
| 33 |
-
|
| 34 |
-
|
| 35 |
-
|
| 36 |
-
|
| 37 |
-
|
| 38 |
-
|
| 39 |
-
|
| 40 |
-
|
| 41 |
-
|
| 42 |
-
|
| 43 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 44 |
→ `{entities: [{type, text, start, end, confidence}], nested, redacted_text, model, processing_time_ms}`
|
| 45 |
-
- Failures are
|
| 46 |
-
|
| 47 |
-
- `
|
| 48 |
-
- `GET /health` → `{status, model, engine, threads, classes, max_chars}` —
|
| 49 |
-
used by the Settings backend-status probe.
|
| 50 |
|
| 51 |
-
##
|
| 52 |
|
| 53 |
-
`/detect` finds personal details in text. The `/file` paths say what KIND of
|
| 54 |
-
|
| 55 |
-
CLOAK stores both; neither replaces the other.
|
| 56 |
|
| 57 |
| Path | Body | Runs |
|
| 58 |
|---|---|---|
|
| 59 |
| `POST /file/classify` | `{"text": "...", "column_headers": [...]}` | Layer 3 on text CLOAK already extracted (~5 ms) |
|
| 60 |
| `POST /file/analyze` | raw bytes, `Content-Type: application/octet-stream`, `X-Filename: <name>` | Layers 1+2+3: real type from bytes, archive contents, OCR for images and scanned PDFs |
|
| 61 |
|
| 62 |
-
|
| 63 |
-
|
| 64 |
-
|
| 65 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 66 |
|
| 67 |
```
|
| 68 |
-
app.py
|
| 69 |
-
docclass.py
|
| 70 |
-
|
| 71 |
-
|
| 72 |
-
|
| 73 |
-
|
| 74 |
-
Policy table class -> suggested action
|
| 75 |
-
API model loading, limits, /file routes
|
| 76 |
-
requirements.txt
|
| 77 |
-
tests/test_docclass.py contract + HTTP tests
|
| 78 |
```
|
| 79 |
-
|
| 80 |
-
Always load the model through `Layer3DocType`, never the raw pickle.
|
| 81 |
-
|
| 82 |
-
Deploy notes:
|
| 83 |
-
- **The Space must run Python 3.11** — set `python_version: "3.11"` in the Space
|
| 84 |
-
README front-matter. The model is a scikit-learn pickle built on 3.11.9 and
|
| 85 |
-
`requirements.txt` pins its exact libraries; other versions can load it and
|
| 86 |
-
silently behave differently.
|
| 87 |
-
- The model is fetched from `DOCCLASS_REPO` (default: `MODEL_REPO`) /
|
| 88 |
-
`DOCCLASS_FILE` (default `layer3_25class.joblib`) and **refused unless its
|
| 89 |
-
SHA-256 matches** `DOCCLASS_SHA256` (pinned in `docclass.py`). Upload a new
|
| 90 |
-
model → update the hash in the same change.
|
| 91 |
-
- If the model can't load, `/file/*` returns 503 and `/detect` is unaffected.
|
| 92 |
-
- Limits: 25 MB per file, archives 3 levels / 500 entries / 150 MB unpacked,
|
| 93 |
-
at most 2 OCR jobs at once (`DOCCLASS_OCR_CONCURRENCY`), first 3 pages of a
|
| 94 |
-
scanned PDF (`DOCCLASS_PDF_OCR_PAGES`).
|
| 95 |
-
- `/file/analyze` receives whole files (including photos of ID cards). For
|
| 96 |
-
client installs that must not send files to Hugging Face, run this same
|
| 97 |
-
Space container on their own host and point CLOAK at it.
|
| 98 |
-
|
| 99 |
-
Test locally:
|
| 100 |
-
|
| 101 |
-
```bash
|
| 102 |
-
DOCCLASS_MODEL_PATH=path/to/layer3_25class.joblib python -m pytest tests -q
|
| 103 |
-
```
|
|
|
|
| 3 |
emoji: 🛡️
|
| 4 |
colorFrom: blue
|
| 5 |
colorTo: indigo
|
| 6 |
+
sdk: docker
|
| 7 |
+
app_port: 7860
|
|
|
|
|
|
|
| 8 |
pinned: false
|
| 9 |
---
|
| 10 |
|
| 11 |
+
# CLOAK model API (Hugging Face Space)
|
| 12 |
|
| 13 |
+
The models and engines the CLOAK server calls, in one Space with one secret. API only: there is
|
| 14 |
+
no UI and no interactive API docs (`/docs` and `/openapi.json` are switched off on purpose).
|
| 15 |
|
| 16 |
+
| Path | What it does | Server caller |
|
| 17 |
+
|---|---|---|
|
| 18 |
+
| `POST /detect` | Personal data in text: nested NER, `Wardline/CLOAK_V3.12_onnx`, ONNX INT8 on CPU | `server/src/detection/pii.ts` |
|
| 19 |
+
| `POST /file/classify` · `POST /file/analyze` | What kind of document a text or file is (26 classes) | `server/src/detection/classify.ts` |
|
| 20 |
+
| `POST /copilot/parse` · `GET /copilot/examples` | Policy Copilot: a policy described in plain English → drafts the policy forms would save | `server/src/modules/org/routes/copilot.ts` |
|
| 21 |
+
| `GET /health` | Status of all three, with no secret needed and none revealed | Settings → backend status |
|
| 22 |
+
|
| 23 |
+
Every path except `/` and `/health` needs `Authorization: Bearer <API_SECRET>`.
|
| 24 |
|
| 25 |
## Deploy
|
| 26 |
+
|
| 27 |
+
1. Push this folder to the Space repo. It is a **Docker Space** (`sdk: docker` above): HF builds
|
| 28 |
+
`Dockerfile`, which pins Python 3.11.9 and runs as uid 1000. If the Space was created as a
|
| 29 |
+
Gradio Space, the front-matter above switches it; HF rebuilds on push.
|
| 30 |
+
2. Space secrets:
|
| 31 |
+
- `API_SECRET`: a long random value (32+ characters). CLOAK sends it as the Bearer token.
|
| 32 |
+
If unset it falls back to `HF_TOKEN`; that works, but puts the HF account token in CLOAK's
|
| 33 |
+
settings, so don't.
|
| 34 |
+
- `HF_TOKEN`: read access to the model repos.
|
| 35 |
+
- Optional: `MODEL_REPO`, `MODEL_FILE`, `DOCCLASS_*` (below), `LOG_LEVEL`.
|
| 36 |
+
3. In the CLOAK dashboard → Settings: **Backend URL** = the Space URL
|
| 37 |
+
(`https://<space>.hf.space`), **API key** = `API_SECRET`. The Policy Copilot uses the same
|
| 38 |
+
two settings; nothing else to configure.
|
| 39 |
+
|
| 40 |
+
The same image runs on a customer's own host, for installs that must not send text or files to
|
| 41 |
+
Hugging Face:
|
| 42 |
+
|
| 43 |
+
```bash
|
| 44 |
+
docker build -t cloak-model-api .
|
| 45 |
+
docker run -p 7860:7860 -e API_SECRET=... -e HF_TOKEN=... cloak-model-api
|
| 46 |
+
```
|
| 47 |
+
|
| 48 |
+
Point CLOAK's Backend URL at it. (`POLICY_COPILOT_URL` on the server can point the copilot at a
|
| 49 |
+
different host, if it ever has to run apart from the models.)
|
| 50 |
+
|
| 51 |
+
## Security
|
| 52 |
+
|
| 53 |
+
- One Bearer secret for every data path, compared in constant time.
|
| 54 |
+
- Bodies are bounded before they are read: `/detect` by `MAX_INPUT_CHARS`, `/copilot/*` at 2 MB
|
| 55 |
+
(text at most 2,000 characters); `/file/*` by docclass's own limits.
|
| 56 |
+
- Failures are real HTTP errors (401, 411, 413, 422, 503) with generic messages; the reason goes
|
| 57 |
+
to the Space log only.
|
| 58 |
+
- Responses carry `Cache-Control: no-store`, `nosniff`, `DENY` framing and no referrer.
|
| 59 |
+
- Nothing is stored. The admin's copilot text is never logged: only request id, outcome and timing.
|
| 60 |
+
- The HF token is passed to each download and never written to disk.
|
| 61 |
+
- The service user cannot modify the code in the image; only its model cache is writable.
|
| 62 |
+
|
| 63 |
+
## `/detect` contract (must stay in sync with `server/src/detection/pii.ts`)
|
| 64 |
+
|
| 65 |
+
- `POST /detect` · `{"text": "..."}`
|
| 66 |
→ `{entities: [{type, text, start, end, confidence}], nested, redacted_text, model, processing_time_ms}`
|
| 67 |
+
- Failures are never 200-with-empty, so DSPM records scan **errors**, not false "clean".
|
| 68 |
+
- `MAX_INPUT_CHARS = 80_000` here must be ≥ `DETECT_MAX_CHARS` in `pii.ts`.
|
| 69 |
+
- `GET /health` → `{status, model, engine, threads, classes, max_input_chars, auth_configured, docclass, copilot}`.
|
|
|
|
|
|
|
| 70 |
|
| 71 |
+
## Document classification (`docclass.py`)
|
| 72 |
|
| 73 |
+
`/detect` finds personal details in text. The `/file` paths say what KIND of document a file is
|
| 74 |
+
(26 classes, including `credentials`) and suggest an action. CLOAK stores both.
|
|
|
|
| 75 |
|
| 76 |
| Path | Body | Runs |
|
| 77 |
|---|---|---|
|
| 78 |
| `POST /file/classify` | `{"text": "...", "column_headers": [...]}` | Layer 3 on text CLOAK already extracted (~5 ms) |
|
| 79 |
| `POST /file/analyze` | raw bytes, `Content-Type: application/octet-stream`, `X-Filename: <name>` | Layers 1+2+3: real type from bytes, archive contents, OCR for images and scanned PDFs |
|
| 80 |
|
| 81 |
+
- **Python 3.11.9 only** (the Dockerfile pins it): the model is a scikit-learn pickle built on
|
| 82 |
+
3.11.9, and `requirements.txt` pins its exact libraries. Other versions can load it and
|
| 83 |
+
silently behave differently.
|
| 84 |
+
- The model is fetched from `DOCCLASS_REPO` (default: `MODEL_REPO`) / `DOCCLASS_FILE` (default
|
| 85 |
+
`layer3_25class.joblib`) and **refused unless its SHA-256 matches** `DOCCLASS_SHA256` (pinned in
|
| 86 |
+
`docclass.py`). Upload a new model → update the hash in the same change.
|
| 87 |
+
- If it can't load, `/file/*` returns 503 and the other paths are unaffected.
|
| 88 |
+
- Limits: 25 MB per file, archives 3 levels / 500 entries / 150 MB unpacked, at most 2 OCR jobs at
|
| 89 |
+
once (`DOCCLASS_OCR_CONCURRENCY`), first 3 pages of a scanned PDF (`DOCCLASS_PDF_OCR_PAGES`).
|
| 90 |
+
- Always load the model through `Layer3DocType`, never the raw pickle.
|
| 91 |
+
|
| 92 |
+
## Policy Copilot (`copilot/`, `copilot_api.py`)
|
| 93 |
+
|
| 94 |
+
Turns "block Aadhaar and PAN in ChatGPT for interns" into the exact body the policy form would
|
| 95 |
+
save, with the questions and warnings an admin needs before saving. Rules and vocabulary only:
|
| 96 |
+
no model, no network, a few milliseconds per request.
|
| 97 |
+
|
| 98 |
+
- **Stateless and write-free.** The CLOAK server sends the tenant's context with each request
|
| 99 |
+
(groups, rules, detectors, block lists, connectors, policies) and saves nothing until the admin
|
| 100 |
+
confirms in the dashboard, through the normal policy routes, with the admin's own permissions.
|
| 101 |
+
Drafts are always saved switched off.
|
| 102 |
+
- **Validated twice.** Every draft passes `copilot/check.py` against the form's schema before it
|
| 103 |
+
leaves the Space; the server then accepts only save steps to the three policy-creation routes.
|
| 104 |
+
- **Platform facts are synced, not hand-copied.** Built-in rules, the 42 built-in block lists, AI
|
| 105 |
+
sites and every form's defaults come from the cloakv2 source:
|
| 106 |
+
|
| 107 |
+
```bash
|
| 108 |
+
node tools/sync_platform.mjs .. # from hf-space/, with cloakv2 as the parent
|
| 109 |
+
```
|
| 110 |
+
|
| 111 |
+
Run it whenever a policy form, preset or block list changes, review `git diff copilot/platform`,
|
| 112 |
+
and redeploy the Space. Phrases live in `copilot/lexicon/vocab.yaml`; the loader refuses a phrase
|
| 113 |
+
that points at a rule, list or setting the snapshot doesn't have.
|
| 114 |
+
- If the copilot can't load, `/copilot/*` returns 503, the dashboard tells the admin to use the
|
| 115 |
+
form, and the other paths are unaffected.
|
| 116 |
+
|
| 117 |
+
Try it in a terminal: `python -m copilot.cli "block usb for interns"`.
|
| 118 |
+
|
| 119 |
+
## Files
|
| 120 |
|
| 121 |
```
|
| 122 |
+
app.py PII model, /detect, /health, shared auth and limits; mounts the other two
|
| 123 |
+
docclass.py document pipeline (type from bytes, archives, extraction, OCR, Layer 3)
|
| 124 |
+
copilot_api.py /copilot routes over copilot/
|
| 125 |
+
copilot/ the Policy Copilot engine; platform/ holds the synced snapshot
|
| 126 |
+
tools/ sync_platform.mjs (not shipped in the image)
|
| 127 |
+
Dockerfile the Space image; .dockerignore keeps tools and docs out
|
|
|
|
|
|
|
|
|
|
|
|
|
| 128 |
```
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
requirements.txt
ADDED
|
@@ -0,0 +1,40 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# CLOAK model API (Docker Space, Python 3.11 — see Dockerfile). No UI framework: API only.
|
| 2 |
+
|
| 3 |
+
# ── HTTP ─────────────────────────────────────────────────────────────────────
|
| 4 |
+
fastapi>=0.115,<1
|
| 5 |
+
uvicorn>=0.30,<1
|
| 6 |
+
pydantic>=2,<3
|
| 7 |
+
|
| 8 |
+
# ── PII model (app.py) ───────────────────────────────────────────────────────
|
| 9 |
+
onnxruntime>=1.18
|
| 10 |
+
# Held below 5: the PII model was validated on 4.x, and nothing pinned it before.
|
| 11 |
+
transformers>=4.44,<5
|
| 12 |
+
huggingface_hub
|
| 13 |
+
# Tokenizer deps — required when the model uses a SentencePiece tokenizer
|
| 14 |
+
# (DeBERTa/XLM family). Without torch installed there is no fallback path.
|
| 15 |
+
sentencepiece
|
| 16 |
+
protobuf
|
| 17 |
+
|
| 18 |
+
# ── Document classification (docclass.py) ────────────────────────────────────
|
| 19 |
+
# EXACT pins: the model is a pickled scikit-learn pipeline, built and pickled
|
| 20 |
+
# on Python 3.11.9. A different scikit-learn/numpy can load it cleanly and
|
| 21 |
+
# behave differently with no error.
|
| 22 |
+
numpy==2.3.0
|
| 23 |
+
scikit-learn==1.7.1
|
| 24 |
+
scipy==1.16.0
|
| 25 |
+
joblib==1.5.1
|
| 26 |
+
# Kept with the pickle's libraries: the Gradio UI no longer needs it, but the
|
| 27 |
+
# pickled pipeline may reference it. Remove only after loading the model without it.
|
| 28 |
+
pandas
|
| 29 |
+
# Text extraction for /file/analyze.
|
| 30 |
+
pypdf==6.9.1
|
| 31 |
+
python-docx==1.2.0
|
| 32 |
+
openpyxl==3.1.5
|
| 33 |
+
# OCR: images, and the rendered pages of scanned PDFs.
|
| 34 |
+
rapidocr-onnxruntime==1.4.4
|
| 35 |
+
pypdfium2==4.30.0
|
| 36 |
+
pillow
|
| 37 |
+
|
| 38 |
+
# ── Policy Copilot (copilot/, copilot_api.py) ─────────────────────────────────
|
| 39 |
+
# Rules and vocabulary only: no model, no network. pyyaml reads lexicon/vocab.yaml.
|
| 40 |
+
pyyaml>=6,<7
|
tools/sync_platform.mjs
ADDED
|
@@ -0,0 +1,186 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
// Copies the platform facts the copilot depends on out of a cloakv2 checkout.
|
| 2 |
+
//
|
| 3 |
+
// node tools/sync_platform.mjs <path-to-cloakv2> (from hf-space/: node tools/sync_platform.mjs ..)
|
| 4 |
+
//
|
| 5 |
+
// Reads the checkout, never writes to it. Writes copilot/platform/snapshot.json and
|
| 6 |
+
// copilot/platform/defaults/<type>.json in this project. Each source file's sha256 is
|
| 7 |
+
// recorded, so after a later sync `git diff` shows exactly which platform file moved.
|
| 8 |
+
//
|
| 9 |
+
// The TypeScript is not compiled: every constant we need is an object or array literal,
|
| 10 |
+
// so we cut the literal out by bracket matching, strip `as T` casts and comments, and
|
| 11 |
+
// evaluate it.
|
| 12 |
+
|
| 13 |
+
import { readFileSync, writeFileSync, mkdirSync, existsSync } from 'node:fs';
|
| 14 |
+
import { createHash } from 'node:crypto';
|
| 15 |
+
import { join, dirname } from 'node:path';
|
| 16 |
+
import { fileURLToPath } from 'node:url';
|
| 17 |
+
|
| 18 |
+
const HERE = dirname(fileURLToPath(import.meta.url));
|
| 19 |
+
const OUT = join(HERE, '..', 'copilot', 'platform');
|
| 20 |
+
|
| 21 |
+
const repo = process.argv[2];
|
| 22 |
+
if (!repo || !existsSync(join(repo, 'server', 'src'))) {
|
| 23 |
+
console.error('usage: node tools/sync_platform.mjs <path-to-cloakv2>');
|
| 24 |
+
process.exit(2);
|
| 25 |
+
}
|
| 26 |
+
|
| 27 |
+
const SOURCES = {
|
| 28 |
+
presets: 'server/src/modules/org/presets.ts',
|
| 29 |
+
blockLists:'server/src/modules/org/blockListPresets.ts',
|
| 30 |
+
pii: 'server/src/detection/pii.ts',
|
| 31 |
+
patterns: 'server/src/detection/patterns.json',
|
| 32 |
+
resolve: 'server/src/modules/pii/resolvePolicy.ts',
|
| 33 |
+
ide: 'server/src/modules/ide/policy.ts',
|
| 34 |
+
ideCat: 'server/src/modules/ide/catalog.ts',
|
| 35 |
+
aiSites: 'dashboard/lib/platformLogos.ts',
|
| 36 |
+
rulesUi: 'dashboard/features/policies/DetectionRulesPanel.tsx',
|
| 37 |
+
endpoint: 'dashboard/features/agent/PolicyForm.tsx',
|
| 38 |
+
ideForm: 'dashboard/features/agent/IdePolicySection.tsx',
|
| 39 |
+
email: 'dashboard/features/email/PolicyForm.tsx',
|
| 40 |
+
emailT: 'dashboard/features/email/types.ts',
|
| 41 |
+
slack: 'dashboard/features/channels/slack/PolicyForm.tsx',
|
| 42 |
+
teams: 'dashboard/features/channels/microsoft/teams/PolicyForm.tsx',
|
| 43 |
+
chat: 'dashboard/features/channels/google/chat/PolicyForm.tsx',
|
| 44 |
+
drive: 'dashboard/features/channels/google/drive/PolicyForm.tsx',
|
| 45 |
+
m365files: 'dashboard/features/channels/microsoft/files/PolicyForm.tsx',
|
| 46 |
+
s3: 'dashboard/features/channels/aws/s3/PolicyForm.tsx',
|
| 47 |
+
gcs: 'dashboard/features/channels/google/gcs/PolicyForm.tsx',
|
| 48 |
+
azure: 'dashboard/features/channels/azure/blob/PolicyForm.tsx',
|
| 49 |
+
github: 'dashboard/features/channels/github/PolicyForm.tsx',
|
| 50 |
+
};
|
| 51 |
+
|
| 52 |
+
const text = {};
|
| 53 |
+
const hashes = {};
|
| 54 |
+
for (const [k, rel] of Object.entries(SOURCES)) {
|
| 55 |
+
const buf = readFileSync(join(repo, rel));
|
| 56 |
+
text[k] = buf.toString('utf8');
|
| 57 |
+
hashes[rel] = createHash('sha256').update(buf).digest('hex');
|
| 58 |
+
}
|
| 59 |
+
|
| 60 |
+
// The literal that follows `<name> ... =`, from its opening bracket to the matching close.
|
| 61 |
+
function literal(src, name) {
|
| 62 |
+
const at = src.search(new RegExp(`\\b(const|let)\\s+${name}\\b[^=]*=`));
|
| 63 |
+
if (at < 0) throw new Error(`${name} not found`);
|
| 64 |
+
let i = src.indexOf('=', at) + 1;
|
| 65 |
+
while (!'[{'.includes(src[i])) i++;
|
| 66 |
+
const start = i;
|
| 67 |
+
let depth = 0, quote = null;
|
| 68 |
+
for (; i < src.length; i++) {
|
| 69 |
+
const c = src[i];
|
| 70 |
+
if (quote) {
|
| 71 |
+
if (c === '\\') { i++; continue; }
|
| 72 |
+
if (c === quote) quote = null;
|
| 73 |
+
continue;
|
| 74 |
+
}
|
| 75 |
+
if (c === '/' && src[i + 1] === '/') { i = src.indexOf('\n', i); continue; }
|
| 76 |
+
if (c === "'" || c === '"' || c === '`') quote = c;
|
| 77 |
+
else if (c === '[' || c === '{') depth++;
|
| 78 |
+
else if (c === ']' || c === '}') { depth--; if (depth === 0) return src.slice(start, i + 1); }
|
| 79 |
+
}
|
| 80 |
+
throw new Error(`${name}: unbalanced literal`);
|
| 81 |
+
}
|
| 82 |
+
|
| 83 |
+
function evaluate(lit, scope = {}) {
|
| 84 |
+
const js = lit
|
| 85 |
+
.replace(/(^|[\s,{[(])\/\/[^\n]*/g, '$1') // line comments, not the // inside URLs
|
| 86 |
+
.replace(/\s+as\s+(const|[A-Za-z_][\w]*(\[\])?|'[^']*'(\s*\|\s*'[^']*')*)/g, ''); // casts
|
| 87 |
+
return new Function(...Object.keys(scope), `return (${js});`)(...Object.values(scope));
|
| 88 |
+
}
|
| 89 |
+
|
| 90 |
+
const value = (k, name, scope) => evaluate(literal(text[k], name), scope);
|
| 91 |
+
|
| 92 |
+
// `new Set([...])` constants: read the array inside.
|
| 93 |
+
function setLiteral(k, name) {
|
| 94 |
+
const m = text[k].match(new RegExp(`const\\s+${name}\\s*=\\s*new Set\\(\\s*(\\[[\\s\\S]*?\\])\\s*\\)`));
|
| 95 |
+
if (!m) throw new Error(`${name} not found`);
|
| 96 |
+
return evaluate(m[1]);
|
| 97 |
+
}
|
| 98 |
+
|
| 99 |
+
const aiSites = value('aiSites', 'AI_PLATFORMS');
|
| 100 |
+
const allHosts = aiSites.map(s => s.host);
|
| 101 |
+
const emptyExt = value('endpoint', 'EMPTY_EXT');
|
| 102 |
+
const emptyAgent = value('endpoint', 'EMPTY_AGENT');
|
| 103 |
+
const emptyIdeForm = value('ideForm', 'EMPTY_IDE');
|
| 104 |
+
const ideDefaults = value('ide', 'IDE_DEFAULTS');
|
| 105 |
+
const excludes = value('github', 'DEFAULT_EXCLUDES');
|
| 106 |
+
|
| 107 |
+
const channelEmpty = {
|
| 108 |
+
email: value('email', 'EMPTY_EMAIL'),
|
| 109 |
+
slack: value('slack', 'EMPTY_SLACK'),
|
| 110 |
+
teams: value('teams', 'EMPTY_TEAMS'),
|
| 111 |
+
chat: value('chat', 'EMPTY_CHAT'),
|
| 112 |
+
drive: value('drive', 'EMPTY_DRIVE'),
|
| 113 |
+
m365files: value('m365files', 'EMPTY_M365'),
|
| 114 |
+
s3: value('s3', 'EMPTY_S3'),
|
| 115 |
+
gcs: value('gcs', 'EMPTY_GCS'),
|
| 116 |
+
azure: value('azure', 'EMPTY_AZURE'),
|
| 117 |
+
github: value('github', 'EMPTY_GH', { DEFAULT_EXCLUDES: excludes }),
|
| 118 |
+
};
|
| 119 |
+
|
| 120 |
+
// Bodies exactly as each form's handleSubmit builds them on create, before anything is
|
| 121 |
+
// edited. The endpoint form spreads the whole form state, so ideControl is sent both at
|
| 122 |
+
// the top level (ignored by the server) and inside agentControl (stored).
|
| 123 |
+
const defaults = {
|
| 124 |
+
endpoint: {
|
| 125 |
+
name: '', description: '', groupId: 'default', action: 'warn',
|
| 126 |
+
active: true, testMode: false,
|
| 127 |
+
enabledEntities: [], allowFileUpload: true, allowCodePaste: true, customSites: [],
|
| 128 |
+
extensionControl: emptyExt,
|
| 129 |
+
agentControl: { ...emptyAgent, allowedUsbDeviceIds: [], blockedApplications: [], blockedDomains: [], ideControl: emptyIdeForm },
|
| 130 |
+
ideControl: emptyIdeForm,
|
| 131 |
+
activeWebCategories: [], exceptions: [],
|
| 132 |
+
enabled: true,
|
| 133 |
+
enabledSites: allHosts,
|
| 134 |
+
channels: {},
|
| 135 |
+
},
|
| 136 |
+
};
|
| 137 |
+
for (const [ch, empty] of Object.entries(channelEmpty)) {
|
| 138 |
+
defaults[ch] = {
|
| 139 |
+
name: '', description: '', groupId: 'default', active: true, enabled: true,
|
| 140 |
+
enabledEntities: [], agentControl: {}, extensionControl: {},
|
| 141 |
+
channels: { [ch]: { ...empty, enabled: true } },
|
| 142 |
+
};
|
| 143 |
+
}
|
| 144 |
+
|
| 145 |
+
const ideCatalog = {
|
| 146 |
+
providers: value('ideCat', 'AI_PROVIDERS').map(p => ({ id: p.id, name: p.name, hosts: p.hosts })),
|
| 147 |
+
tools: value('ideCat', 'AI_TOOLS').map(t => ({ key: t.key, name: t.name, kind: t.kind, processes: t.processes, provider: t.provider || null })),
|
| 148 |
+
aiConfigFiles: value('ideCat', 'AI_CONFIG_FILES'),
|
| 149 |
+
};
|
| 150 |
+
|
| 151 |
+
const stamp = { syncedAt: new Date().toISOString(), sources: hashes };
|
| 152 |
+
|
| 153 |
+
mkdirSync(join(OUT, 'defaults'), { recursive: true });
|
| 154 |
+
for (const [type, body] of Object.entries(defaults)) {
|
| 155 |
+
writeFileSync(join(OUT, 'defaults', `${type}.json`), JSON.stringify({ _source: stamp, body }, null, 2) + '\n');
|
| 156 |
+
}
|
| 157 |
+
|
| 158 |
+
const snapshot = {
|
| 159 |
+
_source: stamp,
|
| 160 |
+
aiSites,
|
| 161 |
+
genaiPasteSites: value('endpoint', 'GENAI_PRESET_SITES'),
|
| 162 |
+
genaiPasteApps: value('endpoint', 'GENAI_PRESET_APPS'),
|
| 163 |
+
detectionRules: value('presets', 'DETECTION_RULE_PRESETS'),
|
| 164 |
+
blockLists: value('blockLists', 'BLOCK_LIST_PRESETS'),
|
| 165 |
+
entityGroupsUi: value('rulesUi', 'ENTITY_GROUPS'),
|
| 166 |
+
fileTypes: value('rulesUi', 'FILE_TYPES'),
|
| 167 |
+
patternLabels: [...new Set(JSON.parse(text.patterns).patterns.map(p => p.label))].sort(),
|
| 168 |
+
highRisk: setLiteral('pii', 'HIGH_RISK'),
|
| 169 |
+
mediumRisk: setLiteral('pii', 'MEDIUM_RISK'),
|
| 170 |
+
actionRank: value('resolve', 'ACTION_RANK'),
|
| 171 |
+
emailActions: value('emailT', 'EMAIL_ACTIONS').map(a => a.value),
|
| 172 |
+
ide: {
|
| 173 |
+
defaults: ideDefaults,
|
| 174 |
+
promptActions: value('ide', 'PROMPT_ACTIONS'),
|
| 175 |
+
accessActions: value('ide', 'ACCESS_ACTIONS'),
|
| 176 |
+
sensorActions: value('ide', 'SENSOR_ACTIONS'),
|
| 177 |
+
riskLevels: value('ide', 'RISK_LEVELS'),
|
| 178 |
+
...ideCatalog,
|
| 179 |
+
},
|
| 180 |
+
};
|
| 181 |
+
writeFileSync(join(OUT, 'snapshot.json'), JSON.stringify(snapshot, null, 2) + '\n');
|
| 182 |
+
|
| 183 |
+
console.log(`types ${Object.keys(defaults).length}, rules ${snapshot.detectionRules.length}, ` +
|
| 184 |
+
`block lists ${snapshot.blockLists.length}, AI sites ${aiSites.length}, ` +
|
| 185 |
+
`pattern labels ${snapshot.patternLabels.length}, IDE tools ${ideCatalog.tools.length}, ` +
|
| 186 |
+
`IDE providers ${ideCatalog.providers.length}`);
|