Spaces:
Sleeping
Sleeping
Deploy scoring-only Space: remove edit model and fix features
Browse files- Dockerfile +1 -22
- README.md +19 -44
- TELEMETRY.md +24 -65
- requirements.txt +0 -1
- server/app.py +7 -193
- server/backends.py +0 -332
- server/config.py +12 -92
- server/fix_loop.py +0 -351
- server/fix_queue.py +0 -89
- server/fixtures/mock_patches.json +0 -103
- server/github_pr.py +0 -381
- server/guards.py +0 -77
- server/hygiene.py +9 -4
- server/prompt.py +0 -24
- server/service.py +0 -152
- server/telemetry.py +52 -118
- server/vendored_guards.py +0 -163
- server/vendored_pipeline.py +0 -278
- server/windowing.py +0 -331
- web/build.mjs +0 -2
- web/src/board.html +2 -2
- web/src/board.js +7 -13
- web/src/bonsai-guards.js +0 -502
- web/src/bonsai-shaders.js +0 -595
- web/src/bonsai-tokenizer.js +0 -197
- web/src/bonsai-webgpu.js +0 -605
- web/src/bonsai-window.js +0 -337
- web/src/bonsai-worker.js +0 -485
- web/src/browser-refiner.js +0 -153
- web/src/diff.js +0 -90
- web/src/fix-run.js +0 -172
- web/src/github.js +14 -29
- web/src/index.html +6 -25
- web/src/labels.js +0 -33
- web/src/main.js +69 -496
- web/src/navigator.js +3 -13
- web/src/patch.js +0 -78
- web/src/privacy.html +68 -76
- web/src/pull-request.js +0 -99
- web/src/refiner.js +0 -70
- web/src/sample.js +10 -19
- web/src/sources.js +4 -28
- web/src/stage.js +2 -52
- web/src/styles.css +3 -83
- web/src/tabs.js +0 -53
- web/src/telemetry.js +14 -13
Dockerfile
CHANGED
|
@@ -13,14 +13,9 @@ RUN cd web && npm run build
|
|
| 13 |
|
| 14 |
FROM python:3.12-slim
|
| 15 |
ARG CODEHYGIENE_REF
|
| 16 |
-
ARG REFINER_BACKEND=zerogpu
|
| 17 |
-
ARG ZEROGPU_SPACE=brianbellx/codehygiene-zerogpu
|
| 18 |
-
ARG REFINER_MODEL_ID=brianbellx/codehygiene-bonsai-1.7b
|
| 19 |
-
ARG REFINER_REPO=Qwen/Qwen2.5-Coder-1.5B-Instruct-GGUF
|
| 20 |
-
ARG REFINER_FILE=qwen2.5-coder-1.5b-instruct-q4_k_m.gguf
|
| 21 |
|
| 22 |
RUN apt-get update \
|
| 23 |
-
&& apt-get install -y --no-install-recommends
|
| 24 |
&& rm -rf /var/lib/apt/lists/*
|
| 25 |
|
| 26 |
RUN useradd -m -u 1000 user
|
|
@@ -35,25 +30,9 @@ RUN pip install --no-cache-dir --user \
|
|
| 35 |
-r requirements.txt \
|
| 36 |
"codehygiene @ git+https://github.com/brianbell-x/codehygiene@${CODEHYGIENE_REF}"
|
| 37 |
|
| 38 |
-
RUN if [ "${REFINER_BACKEND}" = "llama" ]; then \
|
| 39 |
-
pip install --no-cache-dir --user \
|
| 40 |
-
--extra-index-url https://abetlen.github.io/llama-cpp-python/whl/cpu \
|
| 41 |
-
--only-binary=llama-cpp-python \
|
| 42 |
-
llama-cpp-python==0.3.36 && \
|
| 43 |
-
python -c "import sys; from huggingface_hub import hf_hub_download; hf_hub_download(sys.argv[1], sys.argv[2], local_dir='/home/user/models')" \
|
| 44 |
-
"${REFINER_REPO}" "${REFINER_FILE}"; \
|
| 45 |
-
fi
|
| 46 |
-
|
| 47 |
COPY --chown=user server/ server/
|
| 48 |
COPY --from=web --chown=user /build/web/dist web/dist
|
| 49 |
COPY --from=web --chown=user /build/web/vendor/presets.json web/vendor/presets.json
|
| 50 |
|
| 51 |
-
ENV REFINER_BACKEND=${REFINER_BACKEND} \
|
| 52 |
-
ZEROGPU_SPACE=${ZEROGPU_SPACE} \
|
| 53 |
-
REFINER_MODEL_ID=${REFINER_MODEL_ID} \
|
| 54 |
-
REFINER_REPO=${REFINER_REPO} \
|
| 55 |
-
REFINER_FILE=${REFINER_FILE} \
|
| 56 |
-
REFINER_MODEL_PATH=/home/user/models/${REFINER_FILE}
|
| 57 |
-
|
| 58 |
EXPOSE 7860
|
| 59 |
CMD ["python", "-m", "uvicorn", "server.app:app", "--host", "0.0.0.0", "--port", "7860", "--workers", "1", "--no-access-log"]
|
|
|
|
| 13 |
|
| 14 |
FROM python:3.12-slim
|
| 15 |
ARG CODEHYGIENE_REF
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 16 |
|
| 17 |
RUN apt-get update \
|
| 18 |
+
&& apt-get install -y --no-install-recommends git \
|
| 19 |
&& rm -rf /var/lib/apt/lists/*
|
| 20 |
|
| 21 |
RUN useradd -m -u 1000 user
|
|
|
|
| 30 |
-r requirements.txt \
|
| 31 |
"codehygiene @ git+https://github.com/brianbell-x/codehygiene@${CODEHYGIENE_REF}"
|
| 32 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 33 |
COPY --chown=user server/ server/
|
| 34 |
COPY --from=web --chown=user /build/web/dist web/dist
|
| 35 |
COPY --from=web --chown=user /build/web/vendor/presets.json web/vendor/presets.json
|
| 36 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 37 |
EXPOSE 7860
|
| 38 |
CMD ["python", "-m", "uvicorn", "server.app:app", "--host", "0.0.0.0", "--port", "7860", "--workers", "1", "--no-access-log"]
|
README.md
CHANGED
|
@@ -1,64 +1,39 @@
|
|
| 1 |
---
|
| 2 |
title: codehygiene
|
| 3 |
emoji: 🧹
|
| 4 |
-
colorFrom:
|
| 5 |
-
colorTo:
|
| 6 |
sdk: docker
|
| 7 |
app_port: 7860
|
| 8 |
-
short_description:
|
| 9 |
-
|
| 10 |
-
|
|
|
|
| 11 |
---
|
| 12 |
|
| 13 |
-
# codehygiene
|
| 14 |
|
| 15 |
-
Paste a GitHub link
|
| 16 |
|
| 17 |
-
The
|
| 18 |
|
| 19 |
## Settings
|
| 20 |
|
| 21 |
| Variable | Default | What it does |
|
| 22 |
|---|---|---|
|
| 23 |
-
| `
|
| 24 |
-
| `
|
| 25 |
-
| `
|
| 26 |
-
| `
|
| 27 |
-
| `
|
| 28 |
-
| `
|
| 29 |
-
| `
|
| 30 |
-
| `MAX_FILE_CHARS` | `131072` | Files longer than 128 KiB are skipped; files over 9 KB are cleaned via focused squiggle windows. |
|
| 31 |
-
| `WARMUP` | on for `llama`, off for `mock` | Fix the examples in the background after start. |
|
| 32 |
-
| `MOCK_PROMPT_DELAY` | `1.2` | Seconds before the mock starts streaming. |
|
| 33 |
-
| `MOCK_PIECES_PER_SECOND` | `60` | Mock streaming speed. |
|
| 34 |
|
| 35 |
-
|
| 36 |
|
| 37 |
## Run locally
|
| 38 |
|
| 39 |
-
Mock mode needs only FastAPI, uvicorn and numpy.
|
| 40 |
-
|
| 41 |
-
```bash
|
| 42 |
-
cd space
|
| 43 |
-
REFINER_BACKEND=mock python -m uvicorn server.app:app --port 7860
|
| 44 |
-
```
|
| 45 |
-
|
| 46 |
-
Build the UI first (`cd web && npm ci && npm run build`) to serve it at `/`. Without it, `/` returns a short note and the API still works under `/api`.
|
| 47 |
-
|
| 48 |
-
Run the tests:
|
| 49 |
-
|
| 50 |
-
```bash
|
| 51 |
-
cd space
|
| 52 |
-
python -m pytest tests -q
|
| 53 |
-
```
|
| 54 |
-
|
| 55 |
-
## Swap the model
|
| 56 |
-
|
| 57 |
-
Pass other build args. The model must follow the ChatML prompt format.
|
| 58 |
-
|
| 59 |
```bash
|
| 60 |
-
|
| 61 |
-
|
| 62 |
-
--build-arg REFINER_FILE=refiner-q4_k_m.gguf \
|
| 63 |
-
-t codehygiene-space .
|
| 64 |
```
|
|
|
|
| 1 |
---
|
| 2 |
title: codehygiene
|
| 3 |
emoji: 🧹
|
| 4 |
+
colorFrom: gray
|
| 5 |
+
colorTo: green
|
| 6 |
sdk: docker
|
| 7 |
app_port: 7860
|
| 8 |
+
short_description: Find hard-to-read code in any GitHub repo
|
| 9 |
+
tags:
|
| 10 |
+
- code-quality
|
| 11 |
+
- readability
|
| 12 |
---
|
| 13 |
|
| 14 |
+
# codehygiene
|
| 15 |
|
| 16 |
+
Paste a GitHub link, upload a local folder, or paste a snippet to inspect structural readability issues in milliseconds.
|
| 17 |
|
| 18 |
+
The `23.84 KiB` ternary neural scorer (`codehygiene` `v0.6.1`) runs locally inside Web Workers in your browser — source code is never uploaded to score it. Public repositories can also be ranked on the [Leaderboard](TELEMETRY.md#leaderboard).
|
| 19 |
|
| 20 |
## Settings
|
| 21 |
|
| 22 |
| Variable | Default | What it does |
|
| 23 |
|---|---|---|
|
| 24 |
+
| `CODEHYGIENE_REF` | pinned commit | Build arg. Version of `codehygiene` (`engine.js`, `weights.npz`, and Python package). |
|
| 25 |
+
| `HF_STATS_DATASET_REPO` | unset | Optional Hugging Face Dataset repo storing aggregate `stats.json` and `leaderboard.json`. |
|
| 26 |
+
| `HF_TOKEN` | unset | Optional dataset-scoped write token for persisting `stats.json` and `leaderboard.json`. |
|
| 27 |
+
| `STATS_SYNC_SECONDS` | `300` | Seconds between dataset flushes. |
|
| 28 |
+
| `GITHUB_TOKEN` | unset | Optional zero-permission token raising GitHub API read limits for leaderboard ranking. |
|
| 29 |
+
| `LEADERBOARD_SEED` | `1` | Rank canonical seed repos in the background on startup. |
|
| 30 |
+
| `LEADERBOARD_SEED_PAUSE` | `20` | Seconds between background seed rankings. |
|
|
|
|
|
|
|
|
|
|
|
|
|
| 31 |
|
| 32 |
+
Aggregate counters and privacy rules are documented in [TELEMETRY.md](TELEMETRY.md).
|
| 33 |
|
| 34 |
## Run locally
|
| 35 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 36 |
```bash
|
| 37 |
+
npm --prefix web run build
|
| 38 |
+
LEADERBOARD_SEED=0 python -m uvicorn server.app:app --port 7860
|
|
|
|
|
|
|
| 39 |
```
|
TELEMETRY.md
CHANGED
|
@@ -1,48 +1,42 @@
|
|
| 1 |
# Telemetry
|
| 2 |
|
| 3 |
-
The Space keeps a handful of aggregate counters. Source code, file names, paths, user names and IP addresses are never stored, logged or published. Repository names are stored only for public repos that someone asks to rank (see [Leaderboard](#leaderboard)).
|
| 4 |
|
| 5 |
-
Every number is public at
|
| 6 |
|
| 7 |
## What is counted
|
| 8 |
|
| 9 |
| Counter | Counted by | When it goes up |
|
| 10 |
|---|---|---|
|
| 11 |
| `files_scored` | Browser report, checked by the server | A file from a repo, folder or snippet is scored for the first time. |
|
| 12 |
-
| `
|
| 13 |
-
| `
|
| 14 |
-
| `avg_score_before`, `avg_score_after` | Server only | Mean score out of 100 of cleaned files, before and after. |
|
| 15 |
-
| `issues_fixed` | Derived | Sum of `squiggles_resolved_by_kind`. |
|
| 16 |
| `median_score` | Browser report | Median score out of 100 of scored files, from a 101-bin histogram. |
|
| 17 |
-
| `score_ms_median`, `score_ms_samples` | Browser report | Median time the browser took to score one file,
|
| 18 |
-
| `by_ext` |
|
| 19 |
| `by_loc_bucket` | Browser report | Files scored per size band: `0-50`, `50-100`, `100-300`, `300-1000`, `1000+` lines. |
|
| 20 |
-
| `
|
| 21 |
-
| `
|
| 22 |
-
| `prs_opened` | Server only | The GitHub App opens a pull request. |
|
| 23 |
-
| `prs_merged`, `prs_closed_unmerged` | Signed GitHub webhook | A pull request opened by the App is merged, or closed without merging. |
|
| 24 |
-
| `prs_decided`, `pr_merge_rate` | Derived | `prs_merged + prs_closed_unmerged`, and `prs_merged / prs_decided`. Pull requests still open are left out of the rate. |
|
| 25 |
-
| `leaderboard_entries` | Server only | Repos on the leaderboard. |
|
| 26 |
|
| 27 |
-
Nothing is counted for the bundled examples, on the page or on the server. The server also ignores the examples' exact contents under any other source.
|
| 28 |
|
| 29 |
-
`files_scored` and
|
| 30 |
|
| 31 |
| Banner item | Shown when |
|
| 32 |
|---|---|
|
| 33 |
-
|
|
| 34 |
-
|
|
| 35 |
| Time per file | `score_ms_samples` ≥ 1,000 |
|
| 36 |
| Leaderboard link | `leaderboard_entries` ≥ 5 |
|
| 37 |
| Data Retention link | Always |
|
| 38 |
|
| 39 |
The Stats tab of the leaderboard page shows every number from the first count, with its sample size.
|
| 40 |
|
| 41 |
-
The Data Retention link opens `/privacy/` ([`web/src/privacy.html`](web/src/privacy.html)), a plain-language summary of this file for visitors.
|
| 42 |
|
| 43 |
## What the browser sends
|
| 44 |
|
| 45 |
-
After a repo, folder or snippet is scored, one request goes to `POST /api/telemetry/scan`
|
| 46 |
|
| 47 |
```json
|
| 48 |
{
|
|
@@ -52,7 +46,7 @@ After a repo, folder or snippet is scored, one request goes to `POST /api/teleme
|
|
| 52 |
"digest16": "9f84c2a10e77b31d",
|
| 53 |
"ext": "py",
|
| 54 |
"loc_bucket": "100-300",
|
| 55 |
-
"
|
| 56 |
"squiggles": { "deep_nesting": 2, "swallowed_error": 1 },
|
| 57 |
"score_ms": 14
|
| 58 |
}
|
|
@@ -63,10 +57,10 @@ After a repo, folder or snippet is scored, one request goes to `POST /api/teleme
|
|
| 63 |
| Field | Meaning |
|
| 64 |
|---|---|
|
| 65 |
| `source_kind` | `repo`, `folder` or `snippet`. Example runs send nothing. |
|
| 66 |
-
| `digest16` | First 16 hex characters (64 bits) of `SHA-256(ext + "\0" + content)`, computed in the browser with Web Crypto.
|
| 67 |
| `ext` | Lower-case file extension. |
|
| 68 |
| `loc_bucket` | Size band, not the line count. |
|
| 69 |
-
| `
|
| 70 |
| `squiggles` | Issue counts per kind. |
|
| 71 |
| `score_ms` | Milliseconds the browser took to score the file, capped at 60,000. Optional. |
|
| 72 |
|
|
@@ -74,34 +68,18 @@ Any other field is rejected. A batch holds at most 200 files. Each address may s
|
|
| 74 |
|
| 75 |
## What is never kept
|
| 76 |
|
| 77 |
-
- **
|
| 78 |
-
- **
|
| 79 |
-
- **Logs** carry no content, diffs, file names, folder or repository names, cookies or addresses. The access log is turned off.
|
| 80 |
- **Rate limiting** keys each bucket by a salted hash of the caller's address. The salt is random per process and never stored, and the address itself is dropped after the check.
|
| 81 |
-
- **Digests** for
|
| 82 |
|
| 83 |
## Where the counters live
|
| 84 |
|
| 85 |
-
Counters are written to `stats.json` at `STATS_PATH` (default `/tmp/codehygiene_stats.json`). When `HF_STATS_DATASET_REPO` and `HF_TOKEN` are set, `stats.json` is
|
| 86 |
-
|
| 87 |
-
`stats.json` holds only the counters above, a version and a timestamp, plus `open_pr_keys`. Each key is `SHA-256(repo_id + ":" + pr_number)[:16]` for a pull request the App opened that has not closed yet. It lets merges be counted after a restart, and it is dropped once the pull request closes. There are no titles, branch names, repository names or user names in the file.
|
| 88 |
-
|
| 89 |
-
## Pull requests
|
| 90 |
-
|
| 91 |
-
`Create PR` appears only when the Space has a GitHub App configured, the run came from a GitHub link, and at least one fix was accepted. It is off in mock mode. `Download patch` needs no account and works for repos and folders.
|
| 92 |
-
|
| 93 |
-
1. A pop-up asks the visitor to authorize the App on GitHub.
|
| 94 |
-
2. The callback page hands a one-time ticket to the CodeHygiene page that opened it (`postMessage`, this origin only). A sign-in link sent by someone else cannot be redeemed by them.
|
| 95 |
-
3. The server checks that the visitor has push access (`permissions.push`) and that the App is installed on that one repository. Without push access the request fails with `403`; no fork is ever made. Without the App, the pop-up goes to its install page.
|
| 96 |
-
4. Each file is re-read at the exact commit that was scored and re-scored on the server. A file must have fewer issues than its base, or nothing is committed.
|
| 97 |
-
5. A branch `codehygiene/<id>` is created from that commit, and the pull request lists each file with its score before and after and the issues resolved.
|
| 98 |
-
6. The visitor's token is held in memory for at most 3 minutes, used for that one pull request, then revoked.
|
| 99 |
-
|
| 100 |
-
`POST /api/github/webhook` accepts only requests signed with `GITHUB_WEBHOOK_SECRET` (`X-Hub-Signature-256`, compared in constant time). Anything else gets `401`.
|
| 101 |
|
| 102 |
## Leaderboard
|
| 103 |
|
| 104 |
-
The leaderboard page (`/leaderboard/`, opened in a new tab from the banner) ranks public GitHub repos by score. A repo is listed only when someone presses the trophy button after loading it, or when it is one of the seed repos in `SEED_REPOS` in [`server/leaderboard.py`](server/leaderboard.py). Scanning a repo on the page does not list it.
|
| 105 |
|
| 106 |
The browser sends only the owner and repository name to `POST /api/leaderboard/rank`. Any other field, a score included, is rejected. The server then:
|
| 107 |
|
|
@@ -109,29 +87,10 @@ The browser sends only the owner and repository name to `POST /api/leaderboard/r
|
|
| 109 |
2. Downloads the default branch as a `tar.gz` archive from `codeload.github.com` (at most 80 MB) and reads it in memory. Nothing is written to disk. The commit SHA comes from the archive header.
|
| 110 |
3. Keeps hand-written Python, TypeScript, JavaScript, Go and Rust files. Tests, vendored and build folders, minified, generated and type-declaration files, files over 150 KB and binary files are skipped.
|
| 111 |
4. Requires at least 5 such files and 1,000 lines.
|
| 112 |
-
5. Scores a stable sample, chosen by path hash, of at most 300 files or 20,000 lines with the
|
| 113 |
6. Scores the same branch as it was a year earlier, when that commit can be found. **Most improved** ranks repos by the change between the two.
|
| 114 |
7. Refuses a commit that is already ranked under another name, which catches mirrors and forks.
|
| 115 |
|
| 116 |
A ranked repo is reused for 24 hours. Each address may ask for 2 ranks a minute, the whole Space 20, and at most 3 wait at once.
|
| 117 |
|
| 118 |
-
Each entry records the scorer that produced it: the `codehygiene` version plus the first 12 hex characters of the SHA-256 of `weights.npz`. Only entries from the running scorer are listed or counted
|
| 119 |
-
|
| 120 |
-
The public board shows the top 100 of each list. There is no search and no bottom of the list; a rank below 100 appears only in the reply to a rank request. `leaderboard.json` holds, per repo: the name, score, file and line counts, commit SHA, time ranked, and the past score and SHA. It is stored like `stats.json`, in the same dataset repo when one is set.
|
| 121 |
-
|
| 122 |
-
## Settings
|
| 123 |
-
|
| 124 |
-
| Variable | What it does |
|
| 125 |
-
|---|---|
|
| 126 |
-
| `STATS_PATH` | Local `stats.json` path. |
|
| 127 |
-
| `HF_STATS_DATASET_REPO`, `HF_TOKEN` | Dataset repo that keeps `stats.json` across restarts. The repo can be public. |
|
| 128 |
-
| `STATS_SYNC_SECONDS` | Sync interval, default `300`. |
|
| 129 |
-
| `ALLOW_MOCK_TELEMETRY` | Count fixes from the mock backend. Tests only. |
|
| 130 |
-
| `GITHUB_APP_ID`, `GITHUB_APP_CLIENT_ID`, `GITHUB_APP_CLIENT_SECRET`, `GITHUB_APP_PRIVATE_KEY`, `GITHUB_WEBHOOK_SECRET` | All five turn on `Create PR`. The private key may use `\n` for line breaks. |
|
| 131 |
-
| `GITHUB_APP_SLUG` | Optional. Read from GitHub when unset. |
|
| 132 |
-
| `GITHUB_TOKEN` | Optional. A fine-grained token with no permissions, used only for the leaderboard's GitHub API reads. It raises the API limit from 60 to 5,000 requests an hour; the Space's address is shared, so the anonymous limit may already be spent. |
|
| 133 |
-
| `LEADERBOARD_PATH` | Local `leaderboard.json` path, default `/tmp/codehygiene_leaderboard.json`. |
|
| 134 |
-
| `LEADERBOARD_SEED` | After start, rank the seed repos in the background, then re-rank entries from an older scorer. On by default with the real backend; seeds ranked by the running scorer in the last 7 days are skipped. |
|
| 135 |
-
| `LEADERBOARD_SEED_PAUSE` | Seconds between seed repos, default `20`. |
|
| 136 |
-
|
| 137 |
-
The App needs repository permissions **Contents: write** and **Pull requests: write**, the callback URL `https://<space>.hf.space/api/github/callback`, and the webhook URL `https://<space>.hf.space/api/github/webhook` with the **Pull request** event.
|
|
|
|
| 1 |
# Telemetry
|
| 2 |
|
| 3 |
+
The Space keeps a handful of aggregate scoring counters. Source code, file names, paths, user names and IP addresses are never stored, logged or published. Repository names are stored only for public repos that someone explicitly asks to rank (see [Leaderboard](#leaderboard)).
|
| 4 |
|
| 5 |
+
Every number is public at `/api/stats`, and the leaderboard at `/api/leaderboard`. Counting logic lives in [`server/telemetry.py`](server/telemetry.py) on the server and [`web/src/telemetry.js`](web/src/telemetry.js) in the browser.
|
| 6 |
|
| 7 |
## What is counted
|
| 8 |
|
| 9 |
| Counter | Counted by | When it goes up |
|
| 10 |
|---|---|---|
|
| 11 |
| `files_scored` | Browser report, checked by the server | A file from a repo, folder or snippet is scored for the first time. |
|
| 12 |
+
| `issues_found` | Derived | Sum of `squiggles_by_kind` across scored files. |
|
| 13 |
+
| `avg_score_scored` | Browser report | Mean score out of 100 of scored files. |
|
|
|
|
|
|
|
| 14 |
| `median_score` | Browser report | Median score out of 100 of scored files, from a 101-bin histogram. |
|
| 15 |
+
| `score_ms_median`, `score_ms_samples` | Browser report | Median time the browser took to score one file, and how many files were timed. |
|
| 16 |
+
| `by_ext` | Browser report | Files scored per extension and mean score out of 100. Unknown extensions count as `other`. |
|
| 17 |
| `by_loc_bucket` | Browser report | Files scored per size band: `0-50`, `50-100`, `100-300`, `300-1000`, `1000+` lines. |
|
| 18 |
+
| `squiggles_by_kind` | Browser report | Issues flagged in scored files, per kind. |
|
| 19 |
+
| `leaderboard_entries` | Server only | Public repos on the leaderboard. |
|
|
|
|
|
|
|
|
|
|
|
|
|
| 20 |
|
| 21 |
+
Nothing is counted for the bundled examples, on the page or on the server. The server also ignores the examples' exact contents under any other source.
|
| 22 |
|
| 23 |
+
`files_scored` and scoring times come from the browser, so they are best effort. The banner at the top of the page shows each number only once its sample is large enough:
|
| 24 |
|
| 25 |
| Banner item | Shown when |
|
| 26 |
|---|---|
|
| 27 |
+
| Files scored | `files_scored` ≥ 1,000 |
|
| 28 |
+
| Issues flagged | `files_scored` ≥ 1,000 and `issues_found` > 0 |
|
| 29 |
| Time per file | `score_ms_samples` ≥ 1,000 |
|
| 30 |
| Leaderboard link | `leaderboard_entries` ≥ 5 |
|
| 31 |
| Data Retention link | Always |
|
| 32 |
|
| 33 |
The Stats tab of the leaderboard page shows every number from the first count, with its sample size.
|
| 34 |
|
| 35 |
+
The Data Retention link opens `/privacy/` ([`web/src/privacy.html`](web/src/privacy.html)), a plain-language summary of this file for visitors. Any change to what is collected or kept must be reflected there.
|
| 36 |
|
| 37 |
## What the browser sends
|
| 38 |
|
| 39 |
+
After a repo, folder or snippet is scored locally in the browser, one aggregate request goes to `POST /api/telemetry/scan`:
|
| 40 |
|
| 41 |
```json
|
| 42 |
{
|
|
|
|
| 46 |
"digest16": "9f84c2a10e77b31d",
|
| 47 |
"ext": "py",
|
| 48 |
"loc_bucket": "100-300",
|
| 49 |
+
"score": 29,
|
| 50 |
"squiggles": { "deep_nesting": 2, "swallowed_error": 1 },
|
| 51 |
"score_ms": 14
|
| 52 |
}
|
|
|
|
| 57 |
| Field | Meaning |
|
| 58 |
|---|---|
|
| 59 |
| `source_kind` | `repo`, `folder` or `snippet`. Example runs send nothing. |
|
| 60 |
+
| `digest16` | First 16 hex characters (64 bits) of `SHA-256(ext + "\0" + content)`, computed in the browser with Web Crypto. Used only to count a file once in memory. |
|
| 61 |
| `ext` | Lower-case file extension. |
|
| 62 |
| `loc_bucket` | Size band, not the line count. |
|
| 63 |
+
| `score` | Score out of 100. |
|
| 64 |
| `squiggles` | Issue counts per kind. |
|
| 65 |
| `score_ms` | Milliseconds the browser took to score the file, capped at 60,000. Optional. |
|
| 66 |
|
|
|
|
| 68 |
|
| 69 |
## What is never kept
|
| 70 |
|
| 71 |
+
- **Source code** is scored inside Web Workers in the visitor's browser (`23.84 KiB` ternary weights) and is never uploaded by the UI.
|
| 72 |
+
- **Logs** carry no content, file names, folder or repository names, cookies or addresses. The access log is turned off.
|
|
|
|
| 73 |
- **Rate limiting** keys each bucket by a salted hash of the caller's address. The salt is random per process and never stored, and the address itself is dropped after the check.
|
| 74 |
+
- **Digests** for deduplicating repeated scans live in an in-memory rolling set of at most 100,000 entries and are never written to disk or the dataset.
|
| 75 |
|
| 76 |
## Where the counters live
|
| 77 |
|
| 78 |
+
Counters are written to `stats.json` at `STATS_PATH` (default `/tmp/codehygiene_stats.json`). When `HF_STATS_DATASET_REPO` and `HF_TOKEN` are set, `stats.json` is read from that Hugging Face Dataset repo on start and written back every 5 minutes and on shutdown. Nothing is uploaded until the existing copy has been read, so a failed read cannot reset the public numbers.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 79 |
|
| 80 |
## Leaderboard
|
| 81 |
|
| 82 |
+
The leaderboard page (`/leaderboard/`, opened in a new tab from the banner) ranks public GitHub repos by `codehygiene` score. A repo is listed only when someone presses the trophy button after loading it, or when it is one of the seed repos in `SEED_REPOS` in [`server/leaderboard.py`](server/leaderboard.py). Scanning a repo on the page does not list it.
|
| 83 |
|
| 84 |
The browser sends only the owner and repository name to `POST /api/leaderboard/rank`. Any other field, a score included, is rejected. The server then:
|
| 85 |
|
|
|
|
| 87 |
2. Downloads the default branch as a `tar.gz` archive from `codeload.github.com` (at most 80 MB) and reads it in memory. Nothing is written to disk. The commit SHA comes from the archive header.
|
| 88 |
3. Keeps hand-written Python, TypeScript, JavaScript, Go and Rust files. Tests, vendored and build folders, minified, generated and type-declaration files, files over 150 KB and binary files are skipped.
|
| 89 |
4. Requires at least 5 such files and 1,000 lines.
|
| 90 |
+
5. Scores a stable sample, chosen by path hash, of at most 300 files or 20,000 lines with the Python `codehygiene` scorer. The repo score is the line-weighted mean, out of 100.
|
| 91 |
6. Scores the same branch as it was a year earlier, when that commit can be found. **Most improved** ranks repos by the change between the two.
|
| 92 |
7. Refuses a commit that is already ranked under another name, which catches mirrors and forks.
|
| 93 |
|
| 94 |
A ranked repo is reused for 24 hours. Each address may ask for 2 ranks a minute, the whole Space 20, and at most 3 wait at once.
|
| 95 |
|
| 96 |
+
Each entry records the scorer that produced it: the `codehygiene` version plus the first 12 hex characters of the SHA-256 of `weights.npz`. Only entries from the running scorer are listed or counted. The Stats tab's "As a reward" figures come from [`web/src/reward-facts.json`](web/src/reward-facts.json), keyed by version.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
requirements.txt
CHANGED
|
@@ -2,5 +2,4 @@ fastapi==0.115.12
|
|
| 2 |
uvicorn==0.34.2
|
| 3 |
numpy>=1.26,<3
|
| 4 |
httpx==0.28.1
|
| 5 |
-
PyJWT[crypto]==2.10.1
|
| 6 |
huggingface_hub>=0.24
|
|
|
|
| 2 |
uvicorn==0.34.2
|
| 3 |
numpy>=1.26,<3
|
| 4 |
httpx==0.28.1
|
|
|
|
| 5 |
huggingface_hub>=0.24
|
server/app.py
CHANGED
|
@@ -1,68 +1,32 @@
|
|
| 1 |
from __future__ import annotations
|
| 2 |
|
| 3 |
import asyncio
|
| 4 |
-
import json
|
| 5 |
-
import logging
|
| 6 |
from collections.abc import AsyncIterator, Awaitable, Callable
|
| 7 |
from contextlib import asynccontextmanager
|
| 8 |
-
|
| 9 |
from typing import Any
|
| 10 |
|
| 11 |
import httpx
|
| 12 |
from fastapi import FastAPI, HTTPException, Request
|
| 13 |
from fastapi.middleware.cors import CORSMiddleware
|
| 14 |
-
from fastapi.responses import
|
| 15 |
from fastapi.staticfiles import StaticFiles
|
| 16 |
from pydantic import BaseModel, Field
|
| 17 |
|
| 18 |
-
from .backends import VISITOR_IP_TOKEN, Refiner, make_refiner
|
| 19 |
from .config import Settings, load_settings
|
| 20 |
-
from .
|
| 21 |
-
from .fix_queue import QueueFull
|
| 22 |
-
from .github_pr import CreatePrBody, PrError, PullRequests
|
| 23 |
-
from .hygiene import SCORER_VERSION, score_dict
|
| 24 |
from .leaderboard import BOARD_FILE, SEED_REPOS, Leaderboard, RankBody, RankError
|
| 25 |
-
from .
|
| 26 |
-
from .telemetry import RateLimiter, ScanBody, SourceKind, StatsStore, SyncedFile, Telemetry, preset_digests
|
| 27 |
|
| 28 |
log = logging.getLogger("space")
|
| 29 |
|
| 30 |
-
PING_SECONDS = 10.0
|
| 31 |
-
SSE_HEADERS = {"Cache-Control": "no-cache, no-transform", "X-Accel-Buffering": "no"}
|
| 32 |
MAX_BODY_CHARS = 200_000
|
| 33 |
-
QUEUE_FULL_MESSAGE = "The refiner queue is full. Try again shortly."
|
| 34 |
ROOT_NOTE = "codehygiene Space API is running. Build space/web to serve the UI here.\n"
|
| 35 |
-
STREAM_ONLY_KEYS = ("tokens", "tokens_per_second")
|
| 36 |
-
ERROR_STATUS = {"model_unavailable": 503, "queue_full": 429, "cancelled": 499}
|
| 37 |
SCAN_BATCHES_PER_MINUTE = 10
|
| 38 |
SCAN_BATCHES_PER_MINUTE_ALL = 600
|
| 39 |
-
|
| 40 |
RANKS_PER_MINUTE = 2
|
| 41 |
RANKS_PER_MINUTE_ALL = 20
|
| 42 |
-
|
| 43 |
STATS_HEADERS = {"Cache-Control": "public, max-age=60"}
|
| 44 |
-
CALLBACK_PAGE = """<!doctype html>
|
| 45 |
-
<html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width, initial-scale=1">
|
| 46 |
-
<meta name="referrer" content="no-referrer"><title>CodeHygiene</title><style>
|
| 47 |
-
:root{{color-scheme:light dark}}body{{margin:0;display:grid;place-items:center;min-height:100svh;
|
| 48 |
-
font:15px/22px -apple-system,BlinkMacSystemFont,"Segoe UI",sans-serif;
|
| 49 |
-
background:light-dark(#f7f7f5,#161615);color:light-dark(#4d4c48,#b5b3ac)}}
|
| 50 |
-
</style></head><body><p id="message">{message}</p><script>
|
| 51 |
-
const reply = {reply};
|
| 52 |
-
if (window.opener) {{
|
| 53 |
-
window.opener.postMessage({{ type: "codehygiene:github", ...reply }}, window.location.origin);
|
| 54 |
-
}} else if (reply.ticket) {{
|
| 55 |
-
document.getElementById("message").textContent = "Start from the CodeHygiene page.";
|
| 56 |
-
}}
|
| 57 |
-
</script></body></html>
|
| 58 |
-
"""
|
| 59 |
-
SIGNED_IN = "Signed in. Return to CodeHygiene."
|
| 60 |
-
CALLBACK_HEADERS = {"Cache-Control": "no-store", "Referrer-Policy": "no-referrer"}
|
| 61 |
-
|
| 62 |
-
|
| 63 |
-
def callback_page(message: str, reply: dict[str, str], status_code: int = 200) -> HTMLResponse:
|
| 64 |
-
page = CALLBACK_PAGE.format(message=escape(message), reply=json.dumps(reply))
|
| 65 |
-
return HTMLResponse(page, status_code=status_code, headers=CALLBACK_HEADERS)
|
| 66 |
|
| 67 |
|
| 68 |
class ScoreBody(BaseModel):
|
|
@@ -71,75 +35,23 @@ class ScoreBody(BaseModel):
|
|
| 71 |
threshold: float = Field(0.55, ge=0.0, le=1.0)
|
| 72 |
|
| 73 |
|
| 74 |
-
class FixBody(ScoreBody):
|
| 75 |
-
max_passes: int = Field(2, ge=1, le=2)
|
| 76 |
-
no_cache: bool = False
|
| 77 |
-
source_kind: SourceKind = "snippet"
|
| 78 |
-
|
| 79 |
-
def to_request(self) -> FixRequest:
|
| 80 |
-
return FixRequest(self.filename, self.content, self.threshold, self.max_passes)
|
| 81 |
-
|
| 82 |
-
|
| 83 |
-
Producer = Callable[[Emit], Awaitable[None]]
|
| 84 |
-
|
| 85 |
-
|
| 86 |
-
def sse_frame(name: str, data: dict[str, Any]) -> str:
|
| 87 |
-
return f"event: {name}\ndata: {json.dumps(data)}\n\n"
|
| 88 |
-
|
| 89 |
-
|
| 90 |
-
async def guarded(produce: Producer, emit: Emit) -> None:
|
| 91 |
-
try:
|
| 92 |
-
await produce(emit)
|
| 93 |
-
except Cancelled:
|
| 94 |
-
log.info("fix cancelled")
|
| 95 |
-
except QueueFull:
|
| 96 |
-
emit("error", {"code": "queue_full", "message": QUEUE_FULL_MESSAGE})
|
| 97 |
-
except RefinerUnavailable as exc:
|
| 98 |
-
log.warning("refiner unavailable: %s", exc)
|
| 99 |
-
emit("error", {"code": "model_unavailable", "message": str(exc)})
|
| 100 |
-
except Exception as exc:
|
| 101 |
-
log.error("fix failed: %s", type(exc).__name__)
|
| 102 |
-
emit("error", {"code": "internal", "message": f"{type(exc).__name__}: {exc}"})
|
| 103 |
-
|
| 104 |
-
|
| 105 |
-
async def sse_events(produce: Producer) -> AsyncIterator[str]:
|
| 106 |
-
events: asyncio.Queue[tuple[str, dict[str, Any]] | None] = asyncio.Queue()
|
| 107 |
-
task = asyncio.create_task(guarded(produce, lambda name, data: events.put_nowait((name, data))))
|
| 108 |
-
task.add_done_callback(lambda _: events.put_nowait(None))
|
| 109 |
-
try:
|
| 110 |
-
while True:
|
| 111 |
-
try:
|
| 112 |
-
item = await asyncio.wait_for(events.get(), PING_SECONDS)
|
| 113 |
-
except TimeoutError:
|
| 114 |
-
yield ": ping\n\n"
|
| 115 |
-
continue
|
| 116 |
-
if item is None:
|
| 117 |
-
return
|
| 118 |
-
yield sse_frame(*item)
|
| 119 |
-
finally:
|
| 120 |
-
task.cancel()
|
| 121 |
-
|
| 122 |
-
|
| 123 |
def caller_of(request: Request) -> str:
|
| 124 |
"""Address used only for the rate-limit bucket check, then discarded."""
|
| 125 |
forwarded = request.headers.get("x-forwarded-for", "").split(",")[0].strip()
|
| 126 |
return forwarded or (request.client.host if request.client else "")
|
| 127 |
|
| 128 |
|
| 129 |
-
def error_response(error:
|
| 130 |
return JSONResponse(error.payload(), status_code=error.status)
|
| 131 |
|
| 132 |
|
| 133 |
def create_app(
|
| 134 |
settings: Settings | None = None,
|
| 135 |
-
refiner: Refiner | None = None,
|
| 136 |
github_transport: httpx.AsyncBaseTransport | None = None,
|
| 137 |
board: Leaderboard | None = None,
|
| 138 |
) -> FastAPI:
|
| 139 |
settings = settings or load_settings()
|
| 140 |
telemetry = Telemetry(excluded=preset_digests(settings.presets_path))
|
| 141 |
-
service = FixService(settings, refiner or make_refiner(settings), telemetry)
|
| 142 |
-
pulls = PullRequests(settings, telemetry, github_transport)
|
| 143 |
board = board or Leaderboard(settings, github_transport)
|
| 144 |
dataset = (settings.stats_dataset_repo, settings.hf_token)
|
| 145 |
synced = (
|
|
@@ -162,32 +74,17 @@ def create_app(
|
|
| 162 |
for file in synced:
|
| 163 |
await file.load()
|
| 164 |
tasks = [asyncio.create_task(sync_files())]
|
| 165 |
-
if settings.backend in ("llama", "zerogpu") or settings.warmup:
|
| 166 |
-
tasks.append(asyncio.create_task(warm_up()))
|
| 167 |
if settings.leaderboard_seed:
|
| 168 |
tasks.append(asyncio.create_task(board.maintain(SEED_REPOS, settings.leaderboard_seed_pause)))
|
| 169 |
yield
|
| 170 |
for task in tasks:
|
| 171 |
task.cancel()
|
| 172 |
-
service.close()
|
| 173 |
-
await pulls.close()
|
| 174 |
await board.close()
|
| 175 |
for file in synced:
|
| 176 |
await file.flush()
|
| 177 |
|
| 178 |
-
async def warm_up() -> None:
|
| 179 |
-
try:
|
| 180 |
-
if settings.warmup:
|
| 181 |
-
await service.warm_presets(settings.presets_path)
|
| 182 |
-
else:
|
| 183 |
-
await asyncio.get_running_loop().run_in_executor(None, service.refiner.load)
|
| 184 |
-
except Exception:
|
| 185 |
-
log.exception("model load or warm-up failed")
|
| 186 |
-
|
| 187 |
app = FastAPI(title="codehygiene", lifespan=lifespan)
|
| 188 |
-
app.state.service = service
|
| 189 |
app.state.telemetry = telemetry
|
| 190 |
-
app.state.pulls = pulls
|
| 191 |
app.state.board = board
|
| 192 |
app.add_middleware(CORSMiddleware, allow_origins=["*"], allow_methods=["*"], allow_headers=["*"])
|
| 193 |
|
|
@@ -209,28 +106,12 @@ def create_app(
|
|
| 209 |
response.headers["Cache-Control"] = "no-cache"
|
| 210 |
return response
|
| 211 |
|
| 212 |
-
def producer_for(body: FixBody, paced: bool) -> Producer:
|
| 213 |
-
request = body.to_request()
|
| 214 |
-
cached = None if body.no_cache else service.cached(request, body.source_kind)
|
| 215 |
-
if cached is not None:
|
| 216 |
-
return lambda emit: service.replay(cached, emit, paced=paced)
|
| 217 |
-
if not service.queue.has_room:
|
| 218 |
-
raise HTTPException(status_code=429, detail=QUEUE_FULL_MESSAGE)
|
| 219 |
-
|
| 220 |
-
async def produce(emit: Emit) -> None:
|
| 221 |
-
await service.execute(request, service.queue.join(), emit, source_kind=body.source_kind)
|
| 222 |
-
|
| 223 |
-
return produce
|
| 224 |
-
|
| 225 |
@app.get("/api/health")
|
| 226 |
def health() -> dict[str, Any]:
|
| 227 |
return {
|
| 228 |
"status": "ok",
|
| 229 |
-
"
|
| 230 |
-
"
|
| 231 |
-
"backend": settings.backend,
|
| 232 |
-
"max_file_chars": settings.max_file_chars,
|
| 233 |
-
"github_pr_enabled": pulls.enabled,
|
| 234 |
}
|
| 235 |
|
| 236 |
@app.get("/api/stats")
|
|
@@ -239,7 +120,6 @@ def create_app(
|
|
| 239 |
**telemetry.public(),
|
| 240 |
"leaderboard_entries": len(board.current()),
|
| 241 |
"scorer_version": SCORER_VERSION,
|
| 242 |
-
"github_pr_enabled": pulls.enabled,
|
| 243 |
}
|
| 244 |
return JSONResponse(body, headers=STATS_HEADERS)
|
| 245 |
|
|
@@ -266,76 +146,10 @@ def create_app(
|
|
| 266 |
raise HTTPException(status_code=429, detail="Too many scan reports.")
|
| 267 |
return {"counted": telemetry.record_scan(body)}
|
| 268 |
|
| 269 |
-
def require_pulls() -> None:
|
| 270 |
-
if not pulls.enabled:
|
| 271 |
-
raise HTTPException(status_code=404, detail="Pull requests are not enabled.")
|
| 272 |
-
|
| 273 |
-
@app.get("/api/github/login", response_model=None)
|
| 274 |
-
def github_login() -> RedirectResponse | JSONResponse:
|
| 275 |
-
require_pulls()
|
| 276 |
-
try:
|
| 277 |
-
return RedirectResponse(pulls.login_url(), status_code=302, headers=CALLBACK_HEADERS)
|
| 278 |
-
except PrError as error:
|
| 279 |
-
return error_response(error)
|
| 280 |
-
|
| 281 |
-
@app.get("/api/github/callback")
|
| 282 |
-
async def github_callback(code: str = "", state: str = "") -> HTMLResponse:
|
| 283 |
-
require_pulls()
|
| 284 |
-
try:
|
| 285 |
-
ticket = await pulls.complete_login(code, state)
|
| 286 |
-
except PrError as error:
|
| 287 |
-
return callback_page(error.message, {"error": error.code}, error.status)
|
| 288 |
-
except httpx.HTTPError as error:
|
| 289 |
-
log.warning("GitHub sign-in failed: %s", type(error).__name__)
|
| 290 |
-
return callback_page("GitHub could not be reached.", {"error": "github_error"}, 502)
|
| 291 |
-
return callback_page(SIGNED_IN, {"ticket": ticket})
|
| 292 |
-
|
| 293 |
-
@app.post("/api/github/create-pr", response_model=None)
|
| 294 |
-
async def github_create_pr(body: CreatePrBody) -> dict[str, Any] | JSONResponse:
|
| 295 |
-
require_pulls()
|
| 296 |
-
try:
|
| 297 |
-
return await pulls.create(body)
|
| 298 |
-
except PrError as error:
|
| 299 |
-
return error_response(error)
|
| 300 |
-
except httpx.HTTPError as error:
|
| 301 |
-
log.warning("GitHub request failed: %s", type(error).__name__)
|
| 302 |
-
return error_response(PrError(502, "github_error", "GitHub could not be reached."))
|
| 303 |
-
|
| 304 |
-
@app.post("/api/github/webhook", response_model=None)
|
| 305 |
-
async def github_webhook(request: Request) -> dict[str, Any] | JSONResponse:
|
| 306 |
-
if not settings.github_webhook_secret:
|
| 307 |
-
raise HTTPException(status_code=404, detail="Webhooks are not enabled.")
|
| 308 |
-
body = await request.body()
|
| 309 |
-
try:
|
| 310 |
-
return pulls.webhook(
|
| 311 |
-
request.headers.get("x-github-event", ""),
|
| 312 |
-
body,
|
| 313 |
-
request.headers.get("x-hub-signature-256"),
|
| 314 |
-
)
|
| 315 |
-
except PrError as error:
|
| 316 |
-
return error_response(error)
|
| 317 |
-
|
| 318 |
@app.post("/api/score")
|
| 319 |
def score(body: ScoreBody) -> dict[str, Any]:
|
| 320 |
return score_dict(body.content, body.threshold)
|
| 321 |
|
| 322 |
-
@app.post("/api/fix")
|
| 323 |
-
async def fix(body: FixBody, request: Request) -> dict[str, Any]:
|
| 324 |
-
VISITOR_IP_TOKEN.set(request.headers.get("x-ip-token", "").strip())
|
| 325 |
-
collected: dict[str, dict[str, Any]] = {}
|
| 326 |
-
produce = producer_for(body, paced=False)
|
| 327 |
-
await guarded(produce, lambda name, data: collected.__setitem__(name, data))
|
| 328 |
-
if "done" in collected:
|
| 329 |
-
return {key: value for key, value in collected["done"].items() if key not in STREAM_ONLY_KEYS}
|
| 330 |
-
failure = collected.get("error", {"code": "cancelled", "message": "The fix was cancelled."})
|
| 331 |
-
raise HTTPException(status_code=ERROR_STATUS.get(failure["code"], 500), detail=failure)
|
| 332 |
-
|
| 333 |
-
@app.post("/api/fix/stream")
|
| 334 |
-
async def fix_stream(body: FixBody, request: Request) -> StreamingResponse:
|
| 335 |
-
VISITOR_IP_TOKEN.set(request.headers.get("x-ip-token", "").strip())
|
| 336 |
-
produce = producer_for(body, paced=True)
|
| 337 |
-
return StreamingResponse(sse_events(produce), media_type="text/event-stream", headers=SSE_HEADERS)
|
| 338 |
-
|
| 339 |
if (settings.web_dist / "index.html").is_file():
|
| 340 |
app.mount("/", StaticFiles(directory=settings.web_dist, html=True), name="web")
|
| 341 |
else:
|
|
|
|
| 1 |
from __future__ import annotations
|
| 2 |
|
| 3 |
import asyncio
|
|
|
|
|
|
|
| 4 |
from collections.abc import AsyncIterator, Awaitable, Callable
|
| 5 |
from contextlib import asynccontextmanager
|
| 6 |
+
import logging
|
| 7 |
from typing import Any
|
| 8 |
|
| 9 |
import httpx
|
| 10 |
from fastapi import FastAPI, HTTPException, Request
|
| 11 |
from fastapi.middleware.cors import CORSMiddleware
|
| 12 |
+
from fastapi.responses import JSONResponse, PlainTextResponse
|
| 13 |
from fastapi.staticfiles import StaticFiles
|
| 14 |
from pydantic import BaseModel, Field
|
| 15 |
|
|
|
|
| 16 |
from .config import Settings, load_settings
|
| 17 |
+
from .hygiene import SCORER_ID, SCORER_VERSION, score_dict
|
|
|
|
|
|
|
|
|
|
| 18 |
from .leaderboard import BOARD_FILE, SEED_REPOS, Leaderboard, RankBody, RankError
|
| 19 |
+
from .telemetry import RateLimiter, ScanBody, StatsStore, SyncedFile, Telemetry, preset_digests
|
|
|
|
| 20 |
|
| 21 |
log = logging.getLogger("space")
|
| 22 |
|
|
|
|
|
|
|
| 23 |
MAX_BODY_CHARS = 200_000
|
|
|
|
| 24 |
ROOT_NOTE = "codehygiene Space API is running. Build space/web to serve the UI here.\n"
|
|
|
|
|
|
|
| 25 |
SCAN_BATCHES_PER_MINUTE = 10
|
| 26 |
SCAN_BATCHES_PER_MINUTE_ALL = 600
|
|
|
|
| 27 |
RANKS_PER_MINUTE = 2
|
| 28 |
RANKS_PER_MINUTE_ALL = 20
|
|
|
|
| 29 |
STATS_HEADERS = {"Cache-Control": "public, max-age=60"}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 30 |
|
| 31 |
|
| 32 |
class ScoreBody(BaseModel):
|
|
|
|
| 35 |
threshold: float = Field(0.55, ge=0.0, le=1.0)
|
| 36 |
|
| 37 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 38 |
def caller_of(request: Request) -> str:
|
| 39 |
"""Address used only for the rate-limit bucket check, then discarded."""
|
| 40 |
forwarded = request.headers.get("x-forwarded-for", "").split(",")[0].strip()
|
| 41 |
return forwarded or (request.client.host if request.client else "")
|
| 42 |
|
| 43 |
|
| 44 |
+
def error_response(error: RankError) -> JSONResponse:
|
| 45 |
return JSONResponse(error.payload(), status_code=error.status)
|
| 46 |
|
| 47 |
|
| 48 |
def create_app(
|
| 49 |
settings: Settings | None = None,
|
|
|
|
| 50 |
github_transport: httpx.AsyncBaseTransport | None = None,
|
| 51 |
board: Leaderboard | None = None,
|
| 52 |
) -> FastAPI:
|
| 53 |
settings = settings or load_settings()
|
| 54 |
telemetry = Telemetry(excluded=preset_digests(settings.presets_path))
|
|
|
|
|
|
|
| 55 |
board = board or Leaderboard(settings, github_transport)
|
| 56 |
dataset = (settings.stats_dataset_repo, settings.hf_token)
|
| 57 |
synced = (
|
|
|
|
| 74 |
for file in synced:
|
| 75 |
await file.load()
|
| 76 |
tasks = [asyncio.create_task(sync_files())]
|
|
|
|
|
|
|
| 77 |
if settings.leaderboard_seed:
|
| 78 |
tasks.append(asyncio.create_task(board.maintain(SEED_REPOS, settings.leaderboard_seed_pause)))
|
| 79 |
yield
|
| 80 |
for task in tasks:
|
| 81 |
task.cancel()
|
|
|
|
|
|
|
| 82 |
await board.close()
|
| 83 |
for file in synced:
|
| 84 |
await file.flush()
|
| 85 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 86 |
app = FastAPI(title="codehygiene", lifespan=lifespan)
|
|
|
|
| 87 |
app.state.telemetry = telemetry
|
|
|
|
| 88 |
app.state.board = board
|
| 89 |
app.add_middleware(CORSMiddleware, allow_origins=["*"], allow_methods=["*"], allow_headers=["*"])
|
| 90 |
|
|
|
|
| 106 |
response.headers["Cache-Control"] = "no-cache"
|
| 107 |
return response
|
| 108 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 109 |
@app.get("/api/health")
|
| 110 |
def health() -> dict[str, Any]:
|
| 111 |
return {
|
| 112 |
"status": "ok",
|
| 113 |
+
"scorer_version": SCORER_VERSION,
|
| 114 |
+
"scorer_id": SCORER_ID,
|
|
|
|
|
|
|
|
|
|
| 115 |
}
|
| 116 |
|
| 117 |
@app.get("/api/stats")
|
|
|
|
| 120 |
**telemetry.public(),
|
| 121 |
"leaderboard_entries": len(board.current()),
|
| 122 |
"scorer_version": SCORER_VERSION,
|
|
|
|
| 123 |
}
|
| 124 |
return JSONResponse(body, headers=STATS_HEADERS)
|
| 125 |
|
|
|
|
| 146 |
raise HTTPException(status_code=429, detail="Too many scan reports.")
|
| 147 |
return {"counted": telemetry.record_scan(body)}
|
| 148 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 149 |
@app.post("/api/score")
|
| 150 |
def score(body: ScoreBody) -> dict[str, Any]:
|
| 151 |
return score_dict(body.content, body.threshold)
|
| 152 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 153 |
if (settings.web_dist / "index.html").is_file():
|
| 154 |
app.mount("/", StaticFiles(directory=settings.web_dist, html=True), name="web")
|
| 155 |
else:
|
server/backends.py
DELETED
|
@@ -1,332 +0,0 @@
|
|
| 1 |
-
from __future__ import annotations
|
| 2 |
-
|
| 3 |
-
import contextvars
|
| 4 |
-
import json
|
| 5 |
-
import logging
|
| 6 |
-
import re
|
| 7 |
-
import threading
|
| 8 |
-
import time
|
| 9 |
-
from collections.abc import Callable, Iterator, Sequence
|
| 10 |
-
from pathlib import Path
|
| 11 |
-
from typing import Any, Protocol
|
| 12 |
-
|
| 13 |
-
import httpx
|
| 14 |
-
|
| 15 |
-
from .config import Settings
|
| 16 |
-
from .prompt import content_key
|
| 17 |
-
|
| 18 |
-
log = logging.getLogger("space.refiner")
|
| 19 |
-
|
| 20 |
-
VISITOR_IP_TOKEN: contextvars.ContextVar[str] = contextvars.ContextVar("visitor_ip_token", default="")
|
| 21 |
-
|
| 22 |
-
STOP = "<|im_end|>"
|
| 23 |
-
PREFILL_CHUNK = 128
|
| 24 |
-
MOCK_PROGRESS_STEP = 0.2
|
| 25 |
-
_PIECE_RE = re.compile(r" ?\w{1,6}| ?[^\w\s]{1,3}|\s+")
|
| 26 |
-
|
| 27 |
-
Progress = Callable[[int, int], None]
|
| 28 |
-
|
| 29 |
-
|
| 30 |
-
class Refiner(Protocol):
|
| 31 |
-
model_id: str
|
| 32 |
-
|
| 33 |
-
@property
|
| 34 |
-
def loaded(self) -> bool: ...
|
| 35 |
-
|
| 36 |
-
def load(self) -> None: ...
|
| 37 |
-
|
| 38 |
-
def prompt_fits(self, prompt: str) -> bool: ...
|
| 39 |
-
|
| 40 |
-
def stream(
|
| 41 |
-
self,
|
| 42 |
-
prompt: str,
|
| 43 |
-
content: str,
|
| 44 |
-
temperature: float,
|
| 45 |
-
top_p: float,
|
| 46 |
-
cancel: threading.Event,
|
| 47 |
-
on_progress: Progress,
|
| 48 |
-
) -> Iterator[str]: ...
|
| 49 |
-
|
| 50 |
-
|
| 51 |
-
def shared_prefix(left: Sequence[int], right: Sequence[int]) -> int:
|
| 52 |
-
count = 0
|
| 53 |
-
for left_token, right_token in zip(left, right):
|
| 54 |
-
if left_token != right_token:
|
| 55 |
-
break
|
| 56 |
-
count += 1
|
| 57 |
-
return count
|
| 58 |
-
|
| 59 |
-
|
| 60 |
-
class LlamaRefiner:
|
| 61 |
-
def __init__(self, settings: Settings) -> None:
|
| 62 |
-
self.model_id = settings.model_id
|
| 63 |
-
self._model_path = settings.model_path
|
| 64 |
-
self._n_ctx = settings.n_ctx
|
| 65 |
-
self._n_threads = settings.n_threads
|
| 66 |
-
self._max_new_tokens = settings.max_new_tokens
|
| 67 |
-
self._llm: Any = None
|
| 68 |
-
self._load_lock = threading.Lock()
|
| 69 |
-
|
| 70 |
-
@property
|
| 71 |
-
def loaded(self) -> bool:
|
| 72 |
-
return self._llm is not None
|
| 73 |
-
|
| 74 |
-
def load(self) -> None:
|
| 75 |
-
with self._load_lock:
|
| 76 |
-
if self._llm is not None:
|
| 77 |
-
return
|
| 78 |
-
if not Path(self._model_path).is_file():
|
| 79 |
-
raise FileNotFoundError(f"GGUF not found at REFINER_MODEL_PATH={self._model_path!r}")
|
| 80 |
-
from llama_cpp import Llama
|
| 81 |
-
|
| 82 |
-
self._llm = Llama(
|
| 83 |
-
model_path=self._model_path,
|
| 84 |
-
n_ctx=self._n_ctx,
|
| 85 |
-
n_threads=self._n_threads,
|
| 86 |
-
verbose=False,
|
| 87 |
-
)
|
| 88 |
-
|
| 89 |
-
def _tokenize(self, prompt: str) -> list[int]:
|
| 90 |
-
return self._llm.tokenize(prompt.encode("utf-8"), add_bos=False, special=True)
|
| 91 |
-
|
| 92 |
-
def prompt_fits(self, prompt: str) -> bool:
|
| 93 |
-
return len(self._tokenize(prompt)) <= self._n_ctx - self._max_new_tokens
|
| 94 |
-
|
| 95 |
-
def _prefill(self, tokens: list[int], cancel: threading.Event, on_progress: Progress) -> None:
|
| 96 |
-
"""Evaluate all but the last prompt token so create_completion reuses them by prefix match."""
|
| 97 |
-
llm = self._llm
|
| 98 |
-
total = len(tokens)
|
| 99 |
-
end = total - 1
|
| 100 |
-
cached = list(llm.input_ids[: llm.n_tokens])
|
| 101 |
-
reuse = min(shared_prefix(cached, tokens), end)
|
| 102 |
-
if reuse == 0:
|
| 103 |
-
llm.reset()
|
| 104 |
-
else:
|
| 105 |
-
llm.n_tokens = reuse
|
| 106 |
-
on_progress(reuse, total)
|
| 107 |
-
for start in range(reuse, end, PREFILL_CHUNK):
|
| 108 |
-
if cancel.is_set():
|
| 109 |
-
return
|
| 110 |
-
stop = min(start + PREFILL_CHUNK, end)
|
| 111 |
-
llm.eval(tokens[start:stop])
|
| 112 |
-
on_progress(stop, total)
|
| 113 |
-
on_progress(total, total)
|
| 114 |
-
|
| 115 |
-
def stream(
|
| 116 |
-
self,
|
| 117 |
-
prompt: str,
|
| 118 |
-
content: str,
|
| 119 |
-
temperature: float,
|
| 120 |
-
top_p: float,
|
| 121 |
-
cancel: threading.Event,
|
| 122 |
-
on_progress: Progress,
|
| 123 |
-
) -> Iterator[str]:
|
| 124 |
-
tokens = self._tokenize(prompt)
|
| 125 |
-
completion_prompt: str | list[int] = tokens
|
| 126 |
-
try:
|
| 127 |
-
self._prefill(tokens, cancel, on_progress)
|
| 128 |
-
except Exception:
|
| 129 |
-
log.exception("chunked prompt evaluation failed; using plain create_completion")
|
| 130 |
-
self._llm.reset()
|
| 131 |
-
completion_prompt = prompt
|
| 132 |
-
if cancel.is_set():
|
| 133 |
-
return
|
| 134 |
-
chunks = self._llm.create_completion(
|
| 135 |
-
prompt=completion_prompt,
|
| 136 |
-
stream=True,
|
| 137 |
-
max_tokens=self._max_new_tokens,
|
| 138 |
-
temperature=temperature,
|
| 139 |
-
top_p=top_p,
|
| 140 |
-
stop=[STOP],
|
| 141 |
-
)
|
| 142 |
-
try:
|
| 143 |
-
for chunk in chunks:
|
| 144 |
-
if cancel.is_set():
|
| 145 |
-
return
|
| 146 |
-
text = chunk["choices"][0]["text"]
|
| 147 |
-
if text:
|
| 148 |
-
yield text
|
| 149 |
-
finally:
|
| 150 |
-
chunks.close()
|
| 151 |
-
|
| 152 |
-
|
| 153 |
-
def split_pieces(text: str) -> list[str]:
|
| 154 |
-
return _PIECE_RE.findall(text)
|
| 155 |
-
|
| 156 |
-
|
| 157 |
-
class MockRefiner:
|
| 158 |
-
def __init__(self, settings: Settings) -> None:
|
| 159 |
-
self.model_id = settings.model_id or "mock"
|
| 160 |
-
self._prompt_delay = settings.mock_prompt_delay
|
| 161 |
-
rate = settings.mock_pieces_per_second
|
| 162 |
-
self._interval = 1.0 / rate if rate > 0 else 0.0
|
| 163 |
-
self._prompt_budget = settings.n_ctx - settings.max_new_tokens
|
| 164 |
-
fixtures = json.loads(Path(settings.fixtures_path).read_text(encoding="utf-8"))["fixtures"]
|
| 165 |
-
self._replies = {fixture["key"]: fixture["reply"] for fixture in fixtures}
|
| 166 |
-
|
| 167 |
-
@property
|
| 168 |
-
def loaded(self) -> bool:
|
| 169 |
-
return True
|
| 170 |
-
|
| 171 |
-
def load(self) -> None:
|
| 172 |
-
return None
|
| 173 |
-
|
| 174 |
-
def prompt_tokens(self, prompt: str) -> int:
|
| 175 |
-
return len(prompt) // 3
|
| 176 |
-
|
| 177 |
-
def prompt_fits(self, prompt: str) -> bool:
|
| 178 |
-
return self.prompt_tokens(prompt) <= self._prompt_budget
|
| 179 |
-
|
| 180 |
-
def _read_prompt(self, total: int, cancel: threading.Event, on_progress: Progress) -> bool:
|
| 181 |
-
started = time.monotonic()
|
| 182 |
-
while (elapsed := time.monotonic() - started) < self._prompt_delay:
|
| 183 |
-
on_progress(int(total * elapsed / self._prompt_delay), total)
|
| 184 |
-
if cancel.wait(min(MOCK_PROGRESS_STEP, self._prompt_delay - elapsed)):
|
| 185 |
-
return False
|
| 186 |
-
on_progress(total, total)
|
| 187 |
-
return True
|
| 188 |
-
|
| 189 |
-
def stream(
|
| 190 |
-
self,
|
| 191 |
-
prompt: str,
|
| 192 |
-
content: str,
|
| 193 |
-
temperature: float,
|
| 194 |
-
top_p: float,
|
| 195 |
-
cancel: threading.Event,
|
| 196 |
-
on_progress: Progress,
|
| 197 |
-
) -> Iterator[str]:
|
| 198 |
-
if not self._read_prompt(self.prompt_tokens(prompt), cancel, on_progress):
|
| 199 |
-
return
|
| 200 |
-
started = time.monotonic()
|
| 201 |
-
for index, piece in enumerate(split_pieces(self._replies.get(content_key(content), ""))):
|
| 202 |
-
ahead = started + index * self._interval - time.monotonic()
|
| 203 |
-
if cancel.wait(max(0.0, ahead)):
|
| 204 |
-
return
|
| 205 |
-
yield piece
|
| 206 |
-
|
| 207 |
-
|
| 208 |
-
class ZeroGpuRefiner:
|
| 209 |
-
def __init__(
|
| 210 |
-
self,
|
| 211 |
-
settings: Settings,
|
| 212 |
-
transport: httpx.BaseTransport | None = None,
|
| 213 |
-
) -> None:
|
| 214 |
-
self.model_id = settings.model_id or settings.zerogpu_space
|
| 215 |
-
self._space_id = settings.zerogpu_space
|
| 216 |
-
self._hf_token = settings.zerogpu_hf_token
|
| 217 |
-
self._n_ctx = settings.n_ctx
|
| 218 |
-
self._max_new_tokens = settings.max_new_tokens
|
| 219 |
-
self._transport = transport
|
| 220 |
-
self._http: httpx.Client | None = None
|
| 221 |
-
self._load_lock = threading.Lock()
|
| 222 |
-
|
| 223 |
-
@property
|
| 224 |
-
def loaded(self) -> bool:
|
| 225 |
-
return self._http is not None
|
| 226 |
-
|
| 227 |
-
def _base_url(self) -> str:
|
| 228 |
-
if self._space_id.startswith(("http://", "https://")):
|
| 229 |
-
return self._space_id.rstrip("/")
|
| 230 |
-
if "/" in self._space_id:
|
| 231 |
-
owner, name = self._space_id.split("/", 1)
|
| 232 |
-
subdomain = f"{owner}-{name}".lower().replace("_", "-").replace(".", "-")
|
| 233 |
-
return f"https://{subdomain}.hf.space"
|
| 234 |
-
return f"https://{self._space_id}.hf.space"
|
| 235 |
-
|
| 236 |
-
def load(self) -> None:
|
| 237 |
-
with self._load_lock:
|
| 238 |
-
if self._http is not None:
|
| 239 |
-
return
|
| 240 |
-
transport = self._transport or httpx.HTTPTransport(local_address="0.0.0.0", retries=3)
|
| 241 |
-
self._http = httpx.Client(
|
| 242 |
-
base_url=self._base_url(),
|
| 243 |
-
transport=transport,
|
| 244 |
-
timeout=httpx.Timeout(90.0, connect=15.0),
|
| 245 |
-
follow_redirects=True,
|
| 246 |
-
)
|
| 247 |
-
|
| 248 |
-
def prompt_fits(self, prompt: str) -> bool:
|
| 249 |
-
return len(prompt) // 3 <= self._n_ctx - self._max_new_tokens
|
| 250 |
-
|
| 251 |
-
def _iter_sse_cumulative(
|
| 252 |
-
self,
|
| 253 |
-
event_id: str,
|
| 254 |
-
headers: dict[str, str],
|
| 255 |
-
cancel: threading.Event,
|
| 256 |
-
) -> Iterator[str]:
|
| 257 |
-
assert self._http is not None
|
| 258 |
-
sse_headers = {**headers, "Accept": "text/event-stream"}
|
| 259 |
-
url = f"/gradio_api/call/stream_patch/{event_id}"
|
| 260 |
-
with self._http.stream("GET", url, headers=sse_headers) as response:
|
| 261 |
-
response.raise_for_status()
|
| 262 |
-
event_name = ""
|
| 263 |
-
for raw_line in response.iter_lines():
|
| 264 |
-
if cancel.is_set():
|
| 265 |
-
return
|
| 266 |
-
if not raw_line:
|
| 267 |
-
continue
|
| 268 |
-
if raw_line.startswith("event:"):
|
| 269 |
-
event_name = raw_line[6:].strip()
|
| 270 |
-
continue
|
| 271 |
-
if not raw_line.startswith("data:"):
|
| 272 |
-
continue
|
| 273 |
-
payload_text = raw_line[5:].strip()
|
| 274 |
-
if event_name == "error":
|
| 275 |
-
raise RuntimeError(f"ZeroGPU error: {payload_text}")
|
| 276 |
-
if payload_text == "null":
|
| 277 |
-
continue
|
| 278 |
-
parsed = json.loads(payload_text)
|
| 279 |
-
if isinstance(parsed, list) and parsed:
|
| 280 |
-
yield str(parsed[0] or "")
|
| 281 |
-
if event_name == "complete":
|
| 282 |
-
return
|
| 283 |
-
|
| 284 |
-
def stream(
|
| 285 |
-
self,
|
| 286 |
-
prompt: str,
|
| 287 |
-
content: str,
|
| 288 |
-
temperature: float,
|
| 289 |
-
top_p: float,
|
| 290 |
-
cancel: threading.Event,
|
| 291 |
-
on_progress: Progress,
|
| 292 |
-
) -> Iterator[str]:
|
| 293 |
-
self.load()
|
| 294 |
-
assert self._http is not None
|
| 295 |
-
total = max(1, len(prompt) // 3)
|
| 296 |
-
on_progress(0, total)
|
| 297 |
-
if cancel.is_set():
|
| 298 |
-
return
|
| 299 |
-
ip_token = VISITOR_IP_TOKEN.get()
|
| 300 |
-
headers = {"X-IP-Token": ip_token} if ip_token else {}
|
| 301 |
-
if self._hf_token and not ip_token:
|
| 302 |
-
headers["Authorization"] = f"Bearer {self._hf_token}"
|
| 303 |
-
payload = {"data": [prompt, self._max_new_tokens, temperature, top_p]}
|
| 304 |
-
start_res = self._http.post("/gradio_api/call/stream_patch", json=payload, headers=headers)
|
| 305 |
-
if start_res.status_code in (401, 403) and "Authorization" in headers:
|
| 306 |
-
headers.pop("Authorization", None)
|
| 307 |
-
start_res = self._http.post("/gradio_api/call/stream_patch", json=payload, headers=headers)
|
| 308 |
-
start_res.raise_for_status()
|
| 309 |
-
event_id = start_res.json()["event_id"]
|
| 310 |
-
|
| 311 |
-
sent = 0
|
| 312 |
-
progress_done = False
|
| 313 |
-
for cumulative in self._iter_sse_cumulative(event_id, headers, cancel):
|
| 314 |
-
if not progress_done:
|
| 315 |
-
on_progress(total, total)
|
| 316 |
-
progress_done = True
|
| 317 |
-
if len(cumulative) <= sent:
|
| 318 |
-
continue
|
| 319 |
-
delta = cumulative[sent:].split(STOP, 1)[0]
|
| 320 |
-
sent = len(cumulative)
|
| 321 |
-
for piece in split_pieces(delta):
|
| 322 |
-
if cancel.is_set():
|
| 323 |
-
return
|
| 324 |
-
yield piece
|
| 325 |
-
|
| 326 |
-
|
| 327 |
-
def make_refiner(settings: Settings) -> Refiner:
|
| 328 |
-
if settings.backend == "mock":
|
| 329 |
-
return MockRefiner(settings)
|
| 330 |
-
if settings.backend == "zerogpu":
|
| 331 |
-
return ZeroGpuRefiner(settings)
|
| 332 |
-
return LlamaRefiner(settings)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
server/config.py
CHANGED
|
@@ -1,7 +1,7 @@
|
|
| 1 |
from __future__ import annotations
|
| 2 |
|
| 3 |
-
import os
|
| 4 |
from dataclasses import dataclass, field
|
|
|
|
| 5 |
from pathlib import Path
|
| 6 |
|
| 7 |
SPACE_DIR = Path(__file__).resolve().parents[1]
|
|
@@ -16,11 +16,6 @@ def _env_float(env: dict[str, str], name: str, default: float) -> float:
|
|
| 16 |
return float(raw) if raw else default
|
| 17 |
|
| 18 |
|
| 19 |
-
def _env_int(env: dict[str, str], name: str, default: int) -> int:
|
| 20 |
-
raw = env.get(name, "").strip()
|
| 21 |
-
return int(raw) if raw else default
|
| 22 |
-
|
| 23 |
-
|
| 24 |
def _env_flag(env: dict[str, str], name: str, default: bool) -> bool:
|
| 25 |
raw = env.get(name, "").strip().lower()
|
| 26 |
if not raw:
|
|
@@ -28,89 +23,33 @@ def _env_flag(env: dict[str, str], name: str, default: bool) -> bool:
|
|
| 28 |
return raw in ("1", "true", "yes", "on")
|
| 29 |
|
| 30 |
|
|
|
|
|
|
|
|
|
|
|
|
|
| 31 |
def _first_existing(*candidates: Path) -> Path | None:
|
| 32 |
return next((path for path in candidates if path.is_file()), None)
|
| 33 |
|
| 34 |
|
| 35 |
@dataclass(frozen=True)
|
| 36 |
class Settings:
|
| 37 |
-
backend: str = "llama"
|
| 38 |
-
model_path: str = ""
|
| 39 |
-
model_id: str = ""
|
| 40 |
-
zerogpu_space: str = "brianbellx/codehygiene-zerogpu"
|
| 41 |
-
n_ctx: int = 8192
|
| 42 |
-
n_threads: int = 2
|
| 43 |
-
max_new_tokens: int = 1280
|
| 44 |
-
max_file_chars: int = 131_072
|
| 45 |
-
|
| 46 |
-
fixtures_path: Path = SPACE_DIR / "server" / "fixtures" / "mock_patches.json"
|
| 47 |
-
mock_prompt_delay: float = 1.2
|
| 48 |
-
mock_pieces_per_second: float = 60.0
|
| 49 |
-
replay_tokens_per_second: float = 120.0
|
| 50 |
-
|
| 51 |
-
warmup: bool = True
|
| 52 |
presets_path: Path | None = None
|
| 53 |
web_dist: Path = SPACE_DIR / "web" / "dist"
|
| 54 |
-
max_waiters: int = 5
|
| 55 |
-
cache_size: int = 128
|
| 56 |
|
| 57 |
stats_path: Path = Path("/tmp/codehygiene_stats.json")
|
| 58 |
stats_dataset_repo: str = ""
|
| 59 |
hf_token: str = field(default="", repr=False)
|
| 60 |
-
zerogpu_hf_token: str = field(default="", repr=False)
|
| 61 |
stats_sync_seconds: float = 300.0
|
| 62 |
-
allow_mock_telemetry: bool = False
|
| 63 |
-
|
| 64 |
-
github_app_id: str = ""
|
| 65 |
-
github_app_client_id: str = ""
|
| 66 |
-
github_app_slug: str = ""
|
| 67 |
-
github_app_client_secret: str = field(default="", repr=False)
|
| 68 |
-
github_app_private_key: str = field(default="", repr=False)
|
| 69 |
-
github_webhook_secret: str = field(default="", repr=False)
|
| 70 |
|
| 71 |
github_token: str = field(default="", repr=False)
|
| 72 |
leaderboard_path: Path = Path("/tmp/codehygiene_leaderboard.json")
|
| 73 |
-
leaderboard_seed: bool =
|
| 74 |
leaderboard_seed_pause: float = 20.0
|
| 75 |
|
| 76 |
-
@property
|
| 77 |
-
def counts_fixes(self) -> bool:
|
| 78 |
-
return self.backend != "mock" or self.allow_mock_telemetry
|
| 79 |
-
|
| 80 |
-
@property
|
| 81 |
-
def github_pr_enabled(self) -> bool:
|
| 82 |
-
credentials = (
|
| 83 |
-
self.github_app_id,
|
| 84 |
-
self.github_app_client_id,
|
| 85 |
-
self.github_app_client_secret,
|
| 86 |
-
self.github_app_private_key,
|
| 87 |
-
self.github_webhook_secret,
|
| 88 |
-
)
|
| 89 |
-
return all(credentials) and self.backend != "mock"
|
| 90 |
-
|
| 91 |
-
|
| 92 |
-
def _env_text(env: dict[str, str], name: str) -> str:
|
| 93 |
-
return env.get(name, "").strip()
|
| 94 |
-
|
| 95 |
|
| 96 |
def load_settings(env: dict[str, str] | None = None) -> Settings:
|
| 97 |
env = dict(os.environ if env is None else env)
|
| 98 |
-
|
| 99 |
-
if backend not in ("llama", "zerogpu", "mock"):
|
| 100 |
-
raise ValueError(f"REFINER_BACKEND must be 'llama', 'zerogpu' or 'mock', got {backend!r}")
|
| 101 |
-
|
| 102 |
-
repo_id = env.get("REFINER_REPO", "").strip()
|
| 103 |
-
file_name = env.get("REFINER_FILE", "").strip()
|
| 104 |
-
model_path = env.get("REFINER_MODEL_PATH", "").strip()
|
| 105 |
-
zerogpu_space = env.get("ZEROGPU_SPACE", "brianbellx/codehygiene-zerogpu").strip()
|
| 106 |
-
default_id = (
|
| 107 |
-
zerogpu_space if backend == "zerogpu"
|
| 108 |
-
else f"{repo_id}/{file_name}" if repo_id and file_name
|
| 109 |
-
else Path(model_path).name
|
| 110 |
-
)
|
| 111 |
-
model_id = env.get("REFINER_MODEL_ID", "").strip() or ("mock" if backend == "mock" else default_id)
|
| 112 |
-
|
| 113 |
-
presets_env = env.get("PRESETS_PATH", "").strip()
|
| 114 |
presets_path = _first_existing(
|
| 115 |
*([Path(presets_env)] if presets_env else []),
|
| 116 |
SPACE_DIR / "web" / "vendor" / "presets.json",
|
|
@@ -119,35 +58,16 @@ def load_settings(env: dict[str, str] | None = None) -> Settings:
|
|
| 119 |
)
|
| 120 |
defaults = Settings()
|
| 121 |
return Settings(
|
| 122 |
-
backend=backend,
|
| 123 |
-
model_path=model_path,
|
| 124 |
-
model_id=model_id,
|
| 125 |
-
zerogpu_space=zerogpu_space,
|
| 126 |
-
n_threads=_env_int(env, "REFINER_THREADS", defaults.n_threads),
|
| 127 |
-
max_file_chars=_env_int(env, "MAX_FILE_CHARS", defaults.max_file_chars),
|
| 128 |
-
fixtures_path=Path(env.get("MOCK_FIXTURES", "").strip() or defaults.fixtures_path),
|
| 129 |
-
mock_prompt_delay=_env_float(env, "MOCK_PROMPT_DELAY", defaults.mock_prompt_delay),
|
| 130 |
-
mock_pieces_per_second=_env_float(env, "MOCK_PIECES_PER_SECOND", defaults.mock_pieces_per_second),
|
| 131 |
-
replay_tokens_per_second=_env_float(
|
| 132 |
-
env, "REPLAY_TOKENS_PER_SECOND", defaults.replay_tokens_per_second
|
| 133 |
-
),
|
| 134 |
-
warmup=_env_flag(env, "WARMUP", backend == "llama"),
|
| 135 |
presets_path=presets_path,
|
| 136 |
-
web_dist=Path(
|
| 137 |
stats_path=Path(_env_text(env, "STATS_PATH") or defaults.stats_path),
|
| 138 |
stats_dataset_repo=_env_text(env, "HF_STATS_DATASET_REPO"),
|
| 139 |
hf_token=_env_text(env, "HF_TOKEN"),
|
| 140 |
-
zerogpu_hf_token=_env_text(env, "ZEROGPU_HF_TOKEN"),
|
| 141 |
stats_sync_seconds=_env_float(env, "STATS_SYNC_SECONDS", defaults.stats_sync_seconds),
|
| 142 |
-
allow_mock_telemetry=_env_flag(env, "ALLOW_MOCK_TELEMETRY", False),
|
| 143 |
-
github_app_id=_env_text(env, "GITHUB_APP_ID"),
|
| 144 |
-
github_app_client_id=_env_text(env, "GITHUB_APP_CLIENT_ID"),
|
| 145 |
-
github_app_slug=_env_text(env, "GITHUB_APP_SLUG"),
|
| 146 |
-
github_app_client_secret=_env_text(env, "GITHUB_APP_CLIENT_SECRET"),
|
| 147 |
-
github_app_private_key=_env_text(env, "GITHUB_APP_PRIVATE_KEY").replace("\\n", "\n"),
|
| 148 |
-
github_webhook_secret=_env_text(env, "GITHUB_WEBHOOK_SECRET"),
|
| 149 |
github_token=_env_text(env, "GITHUB_TOKEN"),
|
| 150 |
leaderboard_path=Path(_env_text(env, "LEADERBOARD_PATH") or defaults.leaderboard_path),
|
| 151 |
-
leaderboard_seed=_env_flag(env, "LEADERBOARD_SEED",
|
| 152 |
-
leaderboard_seed_pause=_env_float(
|
|
|
|
|
|
|
| 153 |
)
|
|
|
|
| 1 |
from __future__ import annotations
|
| 2 |
|
|
|
|
| 3 |
from dataclasses import dataclass, field
|
| 4 |
+
import os
|
| 5 |
from pathlib import Path
|
| 6 |
|
| 7 |
SPACE_DIR = Path(__file__).resolve().parents[1]
|
|
|
|
| 16 |
return float(raw) if raw else default
|
| 17 |
|
| 18 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 19 |
def _env_flag(env: dict[str, str], name: str, default: bool) -> bool:
|
| 20 |
raw = env.get(name, "").strip().lower()
|
| 21 |
if not raw:
|
|
|
|
| 23 |
return raw in ("1", "true", "yes", "on")
|
| 24 |
|
| 25 |
|
| 26 |
+
def _env_text(env: dict[str, str], name: str) -> str:
|
| 27 |
+
return env.get(name, "").strip()
|
| 28 |
+
|
| 29 |
+
|
| 30 |
def _first_existing(*candidates: Path) -> Path | None:
|
| 31 |
return next((path for path in candidates if path.is_file()), None)
|
| 32 |
|
| 33 |
|
| 34 |
@dataclass(frozen=True)
|
| 35 |
class Settings:
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 36 |
presets_path: Path | None = None
|
| 37 |
web_dist: Path = SPACE_DIR / "web" / "dist"
|
|
|
|
|
|
|
| 38 |
|
| 39 |
stats_path: Path = Path("/tmp/codehygiene_stats.json")
|
| 40 |
stats_dataset_repo: str = ""
|
| 41 |
hf_token: str = field(default="", repr=False)
|
|
|
|
| 42 |
stats_sync_seconds: float = 300.0
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 43 |
|
| 44 |
github_token: str = field(default="", repr=False)
|
| 45 |
leaderboard_path: Path = Path("/tmp/codehygiene_leaderboard.json")
|
| 46 |
+
leaderboard_seed: bool = True
|
| 47 |
leaderboard_seed_pause: float = 20.0
|
| 48 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 49 |
|
| 50 |
def load_settings(env: dict[str, str] | None = None) -> Settings:
|
| 51 |
env = dict(os.environ if env is None else env)
|
| 52 |
+
presets_env = _env_text(env, "PRESETS_PATH")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 53 |
presets_path = _first_existing(
|
| 54 |
*([Path(presets_env)] if presets_env else []),
|
| 55 |
SPACE_DIR / "web" / "vendor" / "presets.json",
|
|
|
|
| 58 |
)
|
| 59 |
defaults = Settings()
|
| 60 |
return Settings(
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 61 |
presets_path=presets_path,
|
| 62 |
+
web_dist=Path(_env_text(env, "WEB_DIST") or defaults.web_dist),
|
| 63 |
stats_path=Path(_env_text(env, "STATS_PATH") or defaults.stats_path),
|
| 64 |
stats_dataset_repo=_env_text(env, "HF_STATS_DATASET_REPO"),
|
| 65 |
hf_token=_env_text(env, "HF_TOKEN"),
|
|
|
|
| 66 |
stats_sync_seconds=_env_float(env, "STATS_SYNC_SECONDS", defaults.stats_sync_seconds),
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 67 |
github_token=_env_text(env, "GITHUB_TOKEN"),
|
| 68 |
leaderboard_path=Path(_env_text(env, "LEADERBOARD_PATH") or defaults.leaderboard_path),
|
| 69 |
+
leaderboard_seed=_env_flag(env, "LEADERBOARD_SEED", defaults.leaderboard_seed),
|
| 70 |
+
leaderboard_seed_pause=_env_float(
|
| 71 |
+
env, "LEADERBOARD_SEED_PAUSE", defaults.leaderboard_seed_pause
|
| 72 |
+
),
|
| 73 |
)
|
server/fix_loop.py
DELETED
|
@@ -1,351 +0,0 @@
|
|
| 1 |
-
from __future__ import annotations
|
| 2 |
-
|
| 3 |
-
import threading
|
| 4 |
-
import time
|
| 5 |
-
from collections.abc import Callable
|
| 6 |
-
from dataclasses import dataclass
|
| 7 |
-
from typing import Any
|
| 8 |
-
|
| 9 |
-
from .backends import Refiner
|
| 10 |
-
from .guards import Verdict
|
| 11 |
-
from .hygiene import score_dict
|
| 12 |
-
from .prompt import build_prompt
|
| 13 |
-
from .vendored_pipeline import apply_changes
|
| 14 |
-
from .windowing import (
|
| 15 |
-
CodeWindow,
|
| 16 |
-
build_chunk_window,
|
| 17 |
-
extract_squiggle_windows,
|
| 18 |
-
select_code_window,
|
| 19 |
-
should_chunk_file,
|
| 20 |
-
verify_window_patch,
|
| 21 |
-
)
|
| 22 |
-
|
| 23 |
-
Emit = Callable[[str, dict[str, Any]], None]
|
| 24 |
-
|
| 25 |
-
GREEDY = (0.0, 1.0)
|
| 26 |
-
SAMPLED = (0.45, 0.92)
|
| 27 |
-
PREVIEW_INTERVAL = 1.0 / 12
|
| 28 |
-
PROGRESS_INTERVAL = 1.0 / 5
|
| 29 |
-
CLOSE_BLOCK = "\n>>>>>>> REPLACE"
|
| 30 |
-
_OPEN_MARKER = "<<<<<<< SEARCH"
|
| 31 |
-
_PARTIAL_MARKERS = ("=======", ">>>>>>> REPLACE")
|
| 32 |
-
|
| 33 |
-
|
| 34 |
-
class Cancelled(Exception):
|
| 35 |
-
pass
|
| 36 |
-
|
| 37 |
-
|
| 38 |
-
class RefinerUnavailable(Exception):
|
| 39 |
-
pass
|
| 40 |
-
|
| 41 |
-
|
| 42 |
-
@dataclass(frozen=True)
|
| 43 |
-
class FixRequest:
|
| 44 |
-
filename: str
|
| 45 |
-
content: str
|
| 46 |
-
threshold: float = 0.55
|
| 47 |
-
max_passes: int = 2
|
| 48 |
-
|
| 49 |
-
|
| 50 |
-
def preview_patch_text(raw: str) -> str:
|
| 51 |
-
open_at = raw.rfind(_OPEN_MARKER)
|
| 52 |
-
if open_at == -1 or ">>>>>>> REPLACE" in raw[open_at:]:
|
| 53 |
-
return raw
|
| 54 |
-
text = raw[:-1] if raw.endswith("\n") else raw
|
| 55 |
-
head, newline, tail = text.rpartition("\n")
|
| 56 |
-
if newline and tail != "=======" and any(marker.startswith(tail) for marker in _PARTIAL_MARKERS):
|
| 57 |
-
text = head
|
| 58 |
-
return text + CLOSE_BLOCK
|
| 59 |
-
|
| 60 |
-
|
| 61 |
-
def preview_content(base: str, raw: str) -> tuple[str, int] | None:
|
| 62 |
-
try:
|
| 63 |
-
content, applied = apply_changes(base, preview_patch_text(raw))
|
| 64 |
-
except ValueError:
|
| 65 |
-
return None
|
| 66 |
-
if applied == 0:
|
| 67 |
-
return None
|
| 68 |
-
return content, applied
|
| 69 |
-
|
| 70 |
-
|
| 71 |
-
class _Attempt:
|
| 72 |
-
def __init__(self, emit: Emit, cancel: threading.Event, window: CodeWindow) -> None:
|
| 73 |
-
self.emit = emit
|
| 74 |
-
self.cancel = cancel
|
| 75 |
-
self.window = window
|
| 76 |
-
self.raw = ""
|
| 77 |
-
self.tokens = 0
|
| 78 |
-
self.seconds = 0.0
|
| 79 |
-
self._last_preview_at = 0.0
|
| 80 |
-
self._last_preview = window.full_content
|
| 81 |
-
self._last_progress_at = -PROGRESS_INTERVAL
|
| 82 |
-
self._prompt_total: int | None = None
|
| 83 |
-
self._read_reported = False
|
| 84 |
-
|
| 85 |
-
def progress(self, done: int, total: int) -> None:
|
| 86 |
-
if self.tokens or self._read_reported:
|
| 87 |
-
return
|
| 88 |
-
self._prompt_total = total
|
| 89 |
-
finished = done >= total
|
| 90 |
-
if not finished and time.monotonic() - self._last_progress_at < PROGRESS_INTERVAL:
|
| 91 |
-
return
|
| 92 |
-
self._last_progress_at = time.monotonic()
|
| 93 |
-
self._read_reported = finished
|
| 94 |
-
self.emit("progress", {"phase": "read", "done": min(done, total), "total": total})
|
| 95 |
-
|
| 96 |
-
def add(self, piece: str) -> None:
|
| 97 |
-
if not self.tokens and self._prompt_total is not None:
|
| 98 |
-
self.progress(self._prompt_total, self._prompt_total)
|
| 99 |
-
self.raw += piece
|
| 100 |
-
self.tokens += 1
|
| 101 |
-
self.emit("token", {"text": piece})
|
| 102 |
-
if time.monotonic() - self._last_preview_at >= PREVIEW_INTERVAL:
|
| 103 |
-
self._preview()
|
| 104 |
-
|
| 105 |
-
def flush(self) -> None:
|
| 106 |
-
wait = PREVIEW_INTERVAL - (time.monotonic() - self._last_preview_at)
|
| 107 |
-
if wait > 0 and self.cancel.wait(wait):
|
| 108 |
-
return
|
| 109 |
-
self._preview()
|
| 110 |
-
|
| 111 |
-
def _preview(self) -> None:
|
| 112 |
-
self._last_preview_at = time.monotonic()
|
| 113 |
-
found = preview_content(self.window.excerpt, self.raw)
|
| 114 |
-
if found is None:
|
| 115 |
-
return
|
| 116 |
-
patched_excerpt, applied = found
|
| 117 |
-
full_preview = self.window.splice(patched_excerpt)
|
| 118 |
-
if full_preview == self._last_preview:
|
| 119 |
-
return
|
| 120 |
-
self._last_preview = full_preview
|
| 121 |
-
self.emit("preview", {"content": self._last_preview, "applied_blocks": applied})
|
| 122 |
-
|
| 123 |
-
|
| 124 |
-
class FixRun:
|
| 125 |
-
def __init__(
|
| 126 |
-
self,
|
| 127 |
-
refiner: Refiner,
|
| 128 |
-
request: FixRequest,
|
| 129 |
-
cancel: threading.Event,
|
| 130 |
-
emit: Emit,
|
| 131 |
-
max_file_chars: int,
|
| 132 |
-
) -> None:
|
| 133 |
-
self.refiner = refiner
|
| 134 |
-
self.request = request
|
| 135 |
-
self.cancel = cancel
|
| 136 |
-
self.emit = emit
|
| 137 |
-
self.max_file_chars = max_file_chars
|
| 138 |
-
self.attempts = 0
|
| 139 |
-
self.tokens = 0
|
| 140 |
-
self.generation_seconds = 0.0
|
| 141 |
-
|
| 142 |
-
def run(self) -> dict[str, Any]:
|
| 143 |
-
started = time.monotonic()
|
| 144 |
-
request = self.request
|
| 145 |
-
before = score_dict(request.content, request.threshold)
|
| 146 |
-
self.emit("start", {"before_hygiene": before, "model_id": self.refiner.model_id, "cached": False})
|
| 147 |
-
outcome = self._outcome(before)
|
| 148 |
-
done = self._done_payload(before, outcome, started)
|
| 149 |
-
self.emit("done", done)
|
| 150 |
-
return done
|
| 151 |
-
|
| 152 |
-
def _outcome(self, before: dict[str, Any]) -> dict[str, Any]:
|
| 153 |
-
request = self.request
|
| 154 |
-
outcome: dict[str, Any] = {"accepted": [], "reason": None, "passes": 0}
|
| 155 |
-
if not before["squiggles"]:
|
| 156 |
-
return outcome
|
| 157 |
-
if len(request.content) > self.max_file_chars:
|
| 158 |
-
outcome["reason"] = "too_long"
|
| 159 |
-
return outcome
|
| 160 |
-
self._load_refiner()
|
| 161 |
-
if should_chunk_file(request.content):
|
| 162 |
-
return self._chunked_outcome(before)
|
| 163 |
-
|
| 164 |
-
first = self._attempt(1, request.content, before, sampled=False)
|
| 165 |
-
outcome["passes"] = 1
|
| 166 |
-
if first.accepted:
|
| 167 |
-
outcome["accepted"].append(first)
|
| 168 |
-
if request.max_passes < 2:
|
| 169 |
-
outcome["reason"] = first.reason
|
| 170 |
-
return outcome
|
| 171 |
-
|
| 172 |
-
if first.accepted and first.squiggles_after:
|
| 173 |
-
outcome["passes"] = 2
|
| 174 |
-
second = self._attempt(2, first.content, first.hygiene, sampled=False)
|
| 175 |
-
if second.accepted:
|
| 176 |
-
outcome["accepted"].append(second)
|
| 177 |
-
elif not first.accepted:
|
| 178 |
-
outcome["passes"] = 2
|
| 179 |
-
outcome["reason"], best = self._sampled_retry(before)
|
| 180 |
-
if best is not None:
|
| 181 |
-
outcome["accepted"].append(best)
|
| 182 |
-
if not outcome["accepted"] and outcome["reason"] is None:
|
| 183 |
-
outcome["reason"] = first.reason
|
| 184 |
-
return outcome
|
| 185 |
-
|
| 186 |
-
def _run_single_chunk(
|
| 187 |
-
self,
|
| 188 |
-
cur_code: str,
|
| 189 |
-
cur_hyg: dict[str, Any],
|
| 190 |
-
w_lo: int,
|
| 191 |
-
w_hi: int,
|
| 192 |
-
allow_second_pass: bool,
|
| 193 |
-
) -> tuple[list[Verdict], int, str | None]:
|
| 194 |
-
request = self.request
|
| 195 |
-
window = build_chunk_window(cur_code, cur_hyg, w_lo, w_hi, request.threshold)
|
| 196 |
-
first = self._attempt_window(1, window, cur_hyg, sampled=False)
|
| 197 |
-
if not first.accepted:
|
| 198 |
-
if not allow_second_pass or first.reason == "too_long":
|
| 199 |
-
return [], 1, first.reason
|
| 200 |
-
retry = self._attempt_window(2, window, cur_hyg, sampled=True)
|
| 201 |
-
return ([retry] if retry.accepted else []), 2, retry.reason
|
| 202 |
-
|
| 203 |
-
kept = [first]
|
| 204 |
-
passes = 1
|
| 205 |
-
new_hi = w_hi + (first.content.count("\n") - cur_code.count("\n"))
|
| 206 |
-
still_flagged = any(w_lo < int(sq["line"]) <= new_hi for sq in first.hygiene["squiggles"])
|
| 207 |
-
if allow_second_pass and still_flagged:
|
| 208 |
-
passes = 2
|
| 209 |
-
follow = build_chunk_window(
|
| 210 |
-
first.content, first.hygiene, w_lo, new_hi, request.threshold
|
| 211 |
-
)
|
| 212 |
-
second = self._attempt_window(2, follow, first.hygiene, sampled=False)
|
| 213 |
-
if second.accepted:
|
| 214 |
-
kept.append(second)
|
| 215 |
-
return kept, passes, None
|
| 216 |
-
|
| 217 |
-
def _chunked_outcome(self, before: dict[str, Any]) -> dict[str, Any]:
|
| 218 |
-
request = self.request
|
| 219 |
-
windows = extract_squiggle_windows(request.content, before["squiggles"])
|
| 220 |
-
cur_code = request.content
|
| 221 |
-
cur_hyg = before
|
| 222 |
-
line_shift = 0
|
| 223 |
-
accepted: list[Verdict] = []
|
| 224 |
-
passes_used = 0
|
| 225 |
-
last_reason: str | None = None
|
| 226 |
-
allow_second_pass = request.max_passes >= 2 and len(windows) <= 4
|
| 227 |
-
|
| 228 |
-
for orig_lo, orig_hi in windows:
|
| 229 |
-
total_lines = cur_code.count("\n") + 1
|
| 230 |
-
w_lo = max(0, min(total_lines, orig_lo + line_shift))
|
| 231 |
-
w_hi = max(w_lo, min(total_lines, orig_hi + line_shift))
|
| 232 |
-
has_sq = any(w_lo < int(sq["line"]) <= w_hi for sq in cur_hyg["squiggles"])
|
| 233 |
-
if not has_sq or w_lo >= w_hi:
|
| 234 |
-
continue
|
| 235 |
-
|
| 236 |
-
chunk_kept, chunk_passes, reason = self._run_single_chunk(
|
| 237 |
-
cur_code, cur_hyg, w_lo, w_hi, allow_second_pass
|
| 238 |
-
)
|
| 239 |
-
passes_used = max(passes_used, chunk_passes)
|
| 240 |
-
if chunk_kept:
|
| 241 |
-
latest = chunk_kept[-1]
|
| 242 |
-
line_shift += latest.content.count("\n") - cur_code.count("\n")
|
| 243 |
-
cur_code = latest.content
|
| 244 |
-
cur_hyg = latest.hygiene
|
| 245 |
-
accepted.extend(chunk_kept)
|
| 246 |
-
else:
|
| 247 |
-
last_reason = reason
|
| 248 |
-
|
| 249 |
-
return {
|
| 250 |
-
"accepted": accepted,
|
| 251 |
-
"reason": None if accepted else last_reason,
|
| 252 |
-
"passes": passes_used,
|
| 253 |
-
}
|
| 254 |
-
|
| 255 |
-
def _sampled_retry(self, before: dict[str, Any]) -> tuple[str | None, Verdict | None]:
|
| 256 |
-
tries = [self._attempt(2, self.request.content, before, sampled=True) for _ in range(2)]
|
| 257 |
-
valid = [verdict for verdict in tries if verdict.accepted]
|
| 258 |
-
if not valid:
|
| 259 |
-
return tries[-1].reason, None
|
| 260 |
-
best = min(valid, key=lambda verdict: (verdict.squiggles_after, -(verdict.score_after or 0.0)))
|
| 261 |
-
return None, best
|
| 262 |
-
|
| 263 |
-
def _load_refiner(self) -> None:
|
| 264 |
-
try:
|
| 265 |
-
self.refiner.load()
|
| 266 |
-
except Exception as exc:
|
| 267 |
-
raise RefinerUnavailable(f"{type(exc).__name__}: {exc}") from exc
|
| 268 |
-
|
| 269 |
-
def _attempt(self, pass_no: int, base: str, hygiene: dict[str, Any], sampled: bool) -> Verdict:
|
| 270 |
-
window = select_code_window(base, hygiene, self.request.threshold)
|
| 271 |
-
return self._attempt_window(pass_no, window, hygiene, sampled)
|
| 272 |
-
|
| 273 |
-
def _attempt_window(
|
| 274 |
-
self, pass_no: int, window: CodeWindow, hygiene: dict[str, Any], sampled: bool
|
| 275 |
-
) -> Verdict:
|
| 276 |
-
if self.cancel.is_set():
|
| 277 |
-
raise Cancelled()
|
| 278 |
-
self.attempts += 1
|
| 279 |
-
number = self.attempts
|
| 280 |
-
self.emit("attempt", {"n": number, "pass": pass_no, "sampled": sampled})
|
| 281 |
-
prompt = build_prompt(window.excerpt, window.prompt_hygiene, self.request.filename)
|
| 282 |
-
if self.refiner.prompt_fits(prompt):
|
| 283 |
-
reply = self._generate(prompt, window, sampled)
|
| 284 |
-
verdict = verify_window_patch(
|
| 285 |
-
window,
|
| 286 |
-
reply,
|
| 287 |
-
self.request.filename,
|
| 288 |
-
self.request.threshold,
|
| 289 |
-
hygiene,
|
| 290 |
-
self.request.content,
|
| 291 |
-
)
|
| 292 |
-
verdict.reply = reply
|
| 293 |
-
else:
|
| 294 |
-
verdict = Verdict(
|
| 295 |
-
False, "too_long", window.full_content, squiggles_before=len(hygiene["squiggles"])
|
| 296 |
-
)
|
| 297 |
-
self.emit("verdict", {
|
| 298 |
-
"n": number,
|
| 299 |
-
"accepted": verdict.accepted,
|
| 300 |
-
"reason": verdict.reason,
|
| 301 |
-
"squiggles_before": verdict.squiggles_before,
|
| 302 |
-
"squiggles_after": verdict.squiggles_after,
|
| 303 |
-
"score_after": verdict.score_after,
|
| 304 |
-
"applied_blocks": verdict.applied_blocks,
|
| 305 |
-
})
|
| 306 |
-
return verdict
|
| 307 |
-
|
| 308 |
-
def _generate(self, prompt: str, window: CodeWindow, sampled: bool) -> str:
|
| 309 |
-
temperature, top_p = SAMPLED if sampled else GREEDY
|
| 310 |
-
attempt = _Attempt(self.emit, self.cancel, window)
|
| 311 |
-
first_token_at: float | None = None
|
| 312 |
-
pieces = self.refiner.stream(
|
| 313 |
-
prompt, window.excerpt, temperature, top_p, self.cancel, attempt.progress
|
| 314 |
-
)
|
| 315 |
-
try:
|
| 316 |
-
for piece in pieces:
|
| 317 |
-
first_token_at = first_token_at or time.monotonic()
|
| 318 |
-
attempt.add(piece)
|
| 319 |
-
except Exception as exc:
|
| 320 |
-
if self.cancel.is_set():
|
| 321 |
-
raise Cancelled() from exc
|
| 322 |
-
raise RefinerUnavailable(f"{type(exc).__name__}: {exc}") from exc
|
| 323 |
-
if self.cancel.is_set():
|
| 324 |
-
raise Cancelled()
|
| 325 |
-
if first_token_at is not None:
|
| 326 |
-
self.generation_seconds += time.monotonic() - first_token_at
|
| 327 |
-
attempt.flush()
|
| 328 |
-
self.tokens += attempt.tokens
|
| 329 |
-
return attempt.raw
|
| 330 |
-
|
| 331 |
-
def _done_payload(
|
| 332 |
-
self, before: dict[str, Any], outcome: dict[str, Any], started: float
|
| 333 |
-
) -> dict[str, Any]:
|
| 334 |
-
accepted: list[Verdict] = outcome["accepted"]
|
| 335 |
-
final = accepted[-1] if accepted else None
|
| 336 |
-
rate = self.tokens / self.generation_seconds if self.generation_seconds > 0 else 0.0
|
| 337 |
-
return {
|
| 338 |
-
"filename": self.request.filename,
|
| 339 |
-
"changes": "\n\n".join(verdict.reply for verdict in accepted),
|
| 340 |
-
"applied_blocks": sum(verdict.applied_blocks for verdict in accepted),
|
| 341 |
-
"updated_content": final.content if final else self.request.content,
|
| 342 |
-
"before_hygiene": before,
|
| 343 |
-
"after_hygiene": final.hygiene if final else before,
|
| 344 |
-
"passes_used": outcome["passes"],
|
| 345 |
-
"latency_ms": round((time.monotonic() - started) * 1000),
|
| 346 |
-
"accepted": final is not None,
|
| 347 |
-
"reason": None if final else outcome["reason"],
|
| 348 |
-
"cached": False,
|
| 349 |
-
"tokens": self.tokens,
|
| 350 |
-
"tokens_per_second": round(rate, 1),
|
| 351 |
-
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
server/fix_queue.py
DELETED
|
@@ -1,89 +0,0 @@
|
|
| 1 |
-
from __future__ import annotations
|
| 2 |
-
|
| 3 |
-
import asyncio
|
| 4 |
-
import threading
|
| 5 |
-
from collections.abc import Callable
|
| 6 |
-
from dataclasses import dataclass, field
|
| 7 |
-
|
| 8 |
-
|
| 9 |
-
class QueueFull(Exception):
|
| 10 |
-
pass
|
| 11 |
-
|
| 12 |
-
|
| 13 |
-
@dataclass(eq=False)
|
| 14 |
-
class Ticket:
|
| 15 |
-
background: bool = False
|
| 16 |
-
cancel: threading.Event = field(default_factory=threading.Event)
|
| 17 |
-
|
| 18 |
-
|
| 19 |
-
class FixQueue:
|
| 20 |
-
"""One running fix at a time; visitors wait in order ahead of background warm-up jobs."""
|
| 21 |
-
|
| 22 |
-
def __init__(self, max_waiters: int) -> None:
|
| 23 |
-
self.max_waiters = max_waiters
|
| 24 |
-
self._waiting: list[Ticket] = []
|
| 25 |
-
self._busy: Ticket | None = None
|
| 26 |
-
self._changed = asyncio.Event()
|
| 27 |
-
|
| 28 |
-
@property
|
| 29 |
-
def idle(self) -> bool:
|
| 30 |
-
return self._busy is None and not self._waiting
|
| 31 |
-
|
| 32 |
-
@property
|
| 33 |
-
def visitors_waiting(self) -> int:
|
| 34 |
-
return sum(1 for ticket in self._waiting if not ticket.background)
|
| 35 |
-
|
| 36 |
-
@property
|
| 37 |
-
def has_room(self) -> bool:
|
| 38 |
-
return self._busy is None or self.visitors_waiting < self.max_waiters
|
| 39 |
-
|
| 40 |
-
def join(self, background: bool = False) -> Ticket:
|
| 41 |
-
if not background and not self.has_room:
|
| 42 |
-
raise QueueFull()
|
| 43 |
-
ticket = Ticket(background=background)
|
| 44 |
-
if background:
|
| 45 |
-
self._waiting.append(ticket)
|
| 46 |
-
else:
|
| 47 |
-
first_background = next(
|
| 48 |
-
(idx for idx, queued in enumerate(self._waiting) if queued.background),
|
| 49 |
-
len(self._waiting),
|
| 50 |
-
)
|
| 51 |
-
self._waiting.insert(first_background, ticket)
|
| 52 |
-
if self._busy is not None and self._busy.background:
|
| 53 |
-
self._busy.cancel.set()
|
| 54 |
-
self._notify()
|
| 55 |
-
return ticket
|
| 56 |
-
|
| 57 |
-
async def wait_turn(self, ticket: Ticket, on_position: Callable[[int], None]) -> None:
|
| 58 |
-
last_position = -1
|
| 59 |
-
while True:
|
| 60 |
-
changed = self._changed
|
| 61 |
-
if self._busy is None and self._waiting and self._waiting[0] is ticket:
|
| 62 |
-
self._waiting.pop(0)
|
| 63 |
-
self._busy = ticket
|
| 64 |
-
self._notify()
|
| 65 |
-
return
|
| 66 |
-
position = self._waiting.index(ticket) + (self._busy is not None)
|
| 67 |
-
if position != last_position:
|
| 68 |
-
on_position(position)
|
| 69 |
-
last_position = position
|
| 70 |
-
await changed.wait()
|
| 71 |
-
|
| 72 |
-
async def wait_idle(self) -> None:
|
| 73 |
-
while not self.idle:
|
| 74 |
-
await self._changed.wait()
|
| 75 |
-
|
| 76 |
-
def release(self, ticket: Ticket) -> None:
|
| 77 |
-
if self._busy is ticket:
|
| 78 |
-
self._busy = None
|
| 79 |
-
elif ticket in self._waiting:
|
| 80 |
-
self._waiting.remove(ticket)
|
| 81 |
-
self._notify()
|
| 82 |
-
|
| 83 |
-
def cancel_all(self) -> None:
|
| 84 |
-
for ticket in [*self._waiting, *([self._busy] if self._busy else [])]:
|
| 85 |
-
ticket.cancel.set()
|
| 86 |
-
|
| 87 |
-
def _notify(self) -> None:
|
| 88 |
-
self._changed.set()
|
| 89 |
-
self._changed = asyncio.Event()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
server/fixtures/mock_patches.json
DELETED
|
@@ -1,103 +0,0 @@
|
|
| 1 |
-
{
|
| 2 |
-
"fixtures": [
|
| 3 |
-
{
|
| 4 |
-
"key": "603901a66c358a271f8bea8305951afa99fa1fec4581a303fbd3d3cd392e8ab0",
|
| 5 |
-
"filename": "send_receipt.py",
|
| 6 |
-
"pass": 1,
|
| 7 |
-
"source": "handwritten",
|
| 8 |
-
"origin": "docs/presets.json before",
|
| 9 |
-
"squiggles_before": 3,
|
| 10 |
-
"squiggles_after": 0,
|
| 11 |
-
"content": "def send_receipt(order: Order) -> None:\n try:\n if order is not None:\n if order.paid and not order.refunded:\n # Extract email address from order\n email = order.email\n if email is not None:\n if len(email.strip()) > 0:\n mailer.send(email, order.id)\n except Exception:\n pass",
|
| 12 |
-
"reply": "<<<<<<< SEARCH\ndef send_receipt(order: Order) -> None:\n try:\n if order is not None:\n if order.paid and not order.refunded:\n # Extract email address from order\n email = order.email\n if email is not None:\n if len(email.strip()) > 0:\n mailer.send(email, order.id)\n except Exception:\n pass\n=======\nimport logging\n\nlogger = logging.getLogger(__name__)\n\n\ndef send_receipt(order: Order) -> None:\n if order is None or not order.paid or order.refunded:\n return\n email = order.email\n if not email or not email.strip():\n return\n try:\n mailer.send(email, order.id)\n except Exception:\n logger.exception(\"Could not send receipt for order %s\", order.id)\n>>>>>>> REPLACE\n"
|
| 13 |
-
},
|
| 14 |
-
{
|
| 15 |
-
"key": "fb26a694b102dfe43772450032b23dd08ecc96f56672ea4be0f7bad66322a32d",
|
| 16 |
-
"filename": "pricing.ts",
|
| 17 |
-
"pass": 1,
|
| 18 |
-
"source": "handwritten",
|
| 19 |
-
"origin": "docs/presets.json before",
|
| 20 |
-
"squiggles_before": 4,
|
| 21 |
-
"squiggles_after": 2,
|
| 22 |
-
"content": "export class DiscountStrategyFactoryManager {\n public computeTotalAmount(cartItems: CartItem[], promoRate: number): number {\n try {\n // Step 1: Compute discounted subtotal from cart items\n const total = cartItems.filter((item) => item.active && item.qty > 0).map((item) => item.price * item.qty * (promoRate > 0 ? (promoRate > 0.5 ? 0.5 : 1 - promoRate) : 1)).reduce((acc, val) => acc + val, 0);\n return total;\n } catch (err) {\n console.log(err);\n return 0.0;\n }\n }\n}",
|
| 23 |
-
"reply": "<<<<<<< SEARCH\n // Step 1: Compute discounted subtotal from cart items\n const total = cartItems.filter((item) => item.active && item.qty > 0).map((item) => item.price * item.qty * (promoRate > 0 ? (promoRate > 0.5 ? 0.5 : 1 - promoRate) : 1)).reduce((acc, val) => acc + val, 0);\n return total;\n=======\n const multiplier = promoRate > 0 ? 1 - Math.min(promoRate, 0.5) : 1;\n const activeItems = cartItems.filter((item) => item.active && item.qty > 0);\n return activeItems.reduce((acc, item) => acc + item.price * item.qty * multiplier, 0);\n>>>>>>> REPLACE\n"
|
| 24 |
-
},
|
| 25 |
-
{
|
| 26 |
-
"key": "1b7c59b3968bb5426ae5916be0eea950aa72d7a36f610efe3d3d555c064c02ff",
|
| 27 |
-
"filename": "frame.rs",
|
| 28 |
-
"pass": 1,
|
| 29 |
-
"source": "handwritten",
|
| 30 |
-
"origin": "docs/presets.json before",
|
| 31 |
-
"squiggles_before": 10,
|
| 32 |
-
"squiggles_after": 8,
|
| 33 |
-
"content": "#[allow(clippy::many_single_char_names)]\npub fn decode_frame(buf: &[u8]) -> Option<(u32, u16, u8)> {\n let mut hd = [0u8; 16];\n hd[0] = buf[12] ^ buf[28];\n hd[1] = buf[14] ^ buf[30];\n hd[2] = buf[18] ^ buf[34];\n hd[3] = buf[22] ^ buf[38];\n let mg = u16::from_be_bytes([buf[10], buf[11]]);\n let sq = u32::from_be_bytes([buf[16], buf[17], buf[18], buf[19]]);\n let ln = u16::from_be_bytes([buf[24], buf[25]]);\n let fl = buf[36] ^ buf[42];\n let ck = buf[44] ^ buf[47];\n let vr = hd[0].wrapping_add(hd[1]).wrapping_add(hd[2]);\n let ok = mg == 23112 && vr > 0 && (fl & 4) != 0 && ck != 0;\n if !ok { return None; }\n Some((sq, ln, ck))\n}",
|
| 34 |
-
"reply": "<<<<<<< SEARCH\n let mg = u16::from_be_bytes([buf[10], buf[11]]);\n let sq = u32::from_be_bytes([buf[16], buf[17], buf[18], buf[19]]);\n let ln = u16::from_be_bytes([buf[24], buf[25]]);\n let fl = buf[36] ^ buf[42];\n let ck = buf[44] ^ buf[47];\n let vr = hd[0].wrapping_add(hd[1]).wrapping_add(hd[2]);\n let ok = mg == 23112 && vr > 0 && (fl & 4) != 0 && ck != 0;\n if !ok { return None; }\n Some((sq, ln, ck))\n=======\n const MAGIC_POS: usize = 10;\n const SEQ_POS: usize = 16;\n const LEN_POS: usize = 24;\n const FLAGS_POS: (usize, usize) = (36, 42);\n const CHECK_POS: (usize, usize) = (44, 47);\n let magic = u16::from_be_bytes([buf[MAGIC_POS], buf[MAGIC_POS + 1]]);\n let sequence = u32::from_be_bytes(buf[SEQ_POS..SEQ_POS + 4].try_into().unwrap());\n let length = u16::from_be_bytes([buf[LEN_POS], buf[LEN_POS + 1]]);\n let flags = buf[FLAGS_POS.0] ^ buf[FLAGS_POS.1];\n let checksum = buf[CHECK_POS.0] ^ buf[CHECK_POS.1];\n let header_sum = hd[0].wrapping_add(hd[1]).wrapping_add(hd[2]);\n if magic != 23112 || header_sum == 0 || (flags & 4) == 0 || checksum == 0 {\n return None;\n }\n Some((sequence, length, checksum))\n>>>>>>> REPLACE\n"
|
| 35 |
-
},
|
| 36 |
-
{
|
| 37 |
-
"key": "c18b0b04d282fac62340e8c4c5eb3556972dc80047bacd887ccdfa76a4e9a7ba",
|
| 38 |
-
"filename": "frame.rs",
|
| 39 |
-
"pass": 2,
|
| 40 |
-
"source": "handwritten",
|
| 41 |
-
"origin": "pass-1 output",
|
| 42 |
-
"squiggles_before": 8,
|
| 43 |
-
"squiggles_after": 1,
|
| 44 |
-
"content": "#[allow(clippy::many_single_char_names)]\npub fn decode_frame(buf: &[u8]) -> Option<(u32, u16, u8)> {\n let mut hd = [0u8; 16];\n hd[0] = buf[12] ^ buf[28];\n hd[1] = buf[14] ^ buf[30];\n hd[2] = buf[18] ^ buf[34];\n hd[3] = buf[22] ^ buf[38];\n const MAGIC_POS: usize = 10;\n const SEQ_POS: usize = 16;\n const LEN_POS: usize = 24;\n const FLAGS_POS: (usize, usize) = (36, 42);\n const CHECK_POS: (usize, usize) = (44, 47);\n let magic = u16::from_be_bytes([buf[MAGIC_POS], buf[MAGIC_POS + 1]]);\n let sequence = u32::from_be_bytes(buf[SEQ_POS..SEQ_POS + 4].try_into().unwrap());\n let length = u16::from_be_bytes([buf[LEN_POS], buf[LEN_POS + 1]]);\n let flags = buf[FLAGS_POS.0] ^ buf[FLAGS_POS.1];\n let checksum = buf[CHECK_POS.0] ^ buf[CHECK_POS.1];\n let header_sum = hd[0].wrapping_add(hd[1]).wrapping_add(hd[2]);\n if magic != 23112 || header_sum == 0 || (flags & 4) == 0 || checksum == 0 {\n return None;\n }\n Some((sequence, length, checksum))\n}",
|
| 45 |
-
"reply": "<<<<<<< SEARCH\n#[allow(clippy::many_single_char_names)]\npub fn decode_frame(buf: &[u8]) -> Option<(u32, u16, u8)> {\n let mut hd = [0u8; 16];\n hd[0] = buf[12] ^ buf[28];\n hd[1] = buf[14] ^ buf[30];\n hd[2] = buf[18] ^ buf[34];\n hd[3] = buf[22] ^ buf[38];\n=======\npub fn decode_frame(buf: &[u8]) -> Option<(u32, u16, u8)> {\n const HEADER_PAIRS: [(usize, usize); 4] = [(12, 28), (14, 30), (18, 34), (22, 38)];\n let mut header = [0u8; 16];\n for (idx, &(left, right)) in HEADER_PAIRS.iter().enumerate() {\n header[idx] = buf[left] ^ buf[right];\n }\n>>>>>>> REPLACE\n<<<<<<< SEARCH\n let header_sum = hd[0].wrapping_add(hd[1]).wrapping_add(hd[2]);\n=======\n let header_sum = header.iter().take(3).copied().fold(0u8, u8::wrapping_add);\n>>>>>>> REPLACE\n"
|
| 46 |
-
},
|
| 47 |
-
{
|
| 48 |
-
"key": "0e8168b89bab1d90bc459c5daaeaffcd14c49fd368aff7099136e17a4d460ad0",
|
| 49 |
-
"filename": "webhook.go",
|
| 50 |
-
"pass": 1,
|
| 51 |
-
"source": "handwritten",
|
| 52 |
-
"origin": "docs/presets.json before",
|
| 53 |
-
"squiggles_before": 7,
|
| 54 |
-
"squiggles_after": 2,
|
| 55 |
-
"content": "func resolveEndpoint(region string) string {\n if region == \"us-east-1\" { return \"https://east.hooks.internal\" }\n if region == \"us-west-2\" { return \"https://west.hooks.internal\" }\n if region == \"eu-central-1\" { return \"https://eu.hooks.internal\" }\n if region == \"ap-northeast-1\" { return \"https://ap.hooks.internal\" }\n if region == \"sa-east-1\" { return \"https://sa.hooks.internal\" }\n return \"https://default.hooks.internal\"\n}\n\nfunc publishAuditEvent(httpClient *Client, region string, auditItem Event) {\n targetUrl := resolveEndpoint(region)\n bodyBytes := serializeJsonBody(auditItem)\n authToken := computeHmacSignature(bodyBytes, httpClient.signingSecret)\n statusCode := transmitHttpPost(httpClient, targetUrl, bodyBytes, authToken)\n recordTelemetryCounter(statusCode, region)\n}\n\nfunc publishBillingEvent(httpClient *Client, region string, billingItem Event) {\n targetUrl := resolveEndpoint(region)\n bodyBytes := serializeJsonBody(billingItem)\n authToken := computeHmacSignature(bodyBytes, httpClient.signingSecret)\n statusCode := transmitHttpPost(httpClient, targetUrl, bodyBytes, authToken)\n recordTelemetryCounter(statusCode, region)\n}",
|
| 56 |
-
"reply": "<<<<<<< SEARCH\nfunc resolveEndpoint(region string) string {\n if region == \"us-east-1\" { return \"https://east.hooks.internal\" }\n if region == \"us-west-2\" { return \"https://west.hooks.internal\" }\n if region == \"eu-central-1\" { return \"https://eu.hooks.internal\" }\n if region == \"ap-northeast-1\" { return \"https://ap.hooks.internal\" }\n if region == \"sa-east-1\" { return \"https://sa.hooks.internal\" }\n return \"https://default.hooks.internal\"\n}\n=======\nvar regionEndpoints = map[string]string{\n \"us-east-1\": \"https://east.hooks.internal\",\n \"us-west-2\": \"https://west.hooks.internal\",\n \"eu-central-1\": \"https://eu.hooks.internal\",\n \"ap-northeast-1\": \"https://ap.hooks.internal\",\n \"sa-east-1\": \"https://sa.hooks.internal\",\n}\n\nfunc resolveEndpoint(region string) string {\n if endpoint, ok := regionEndpoints[region]; ok {\n return endpoint\n }\n return \"https://default.hooks.internal\"\n}\n>>>>>>> REPLACE\n"
|
| 57 |
-
},
|
| 58 |
-
{
|
| 59 |
-
"key": "def4e6ca446326c5e5e83b5127daff28b28d01d51ecd3b2576bd4e7b3293e5a4",
|
| 60 |
-
"filename": "webhook.go",
|
| 61 |
-
"pass": 2,
|
| 62 |
-
"source": "handwritten",
|
| 63 |
-
"origin": "pass-1 output",
|
| 64 |
-
"squiggles_before": 2,
|
| 65 |
-
"squiggles_after": 0,
|
| 66 |
-
"content": "var regionEndpoints = map[string]string{\n \"us-east-1\": \"https://east.hooks.internal\",\n \"us-west-2\": \"https://west.hooks.internal\",\n \"eu-central-1\": \"https://eu.hooks.internal\",\n \"ap-northeast-1\": \"https://ap.hooks.internal\",\n \"sa-east-1\": \"https://sa.hooks.internal\",\n}\n\nfunc resolveEndpoint(region string) string {\n if endpoint, ok := regionEndpoints[region]; ok {\n return endpoint\n }\n return \"https://default.hooks.internal\"\n}\n\nfunc publishAuditEvent(httpClient *Client, region string, auditItem Event) {\n targetUrl := resolveEndpoint(region)\n bodyBytes := serializeJsonBody(auditItem)\n authToken := computeHmacSignature(bodyBytes, httpClient.signingSecret)\n statusCode := transmitHttpPost(httpClient, targetUrl, bodyBytes, authToken)\n recordTelemetryCounter(statusCode, region)\n}\n\nfunc publishBillingEvent(httpClient *Client, region string, billingItem Event) {\n targetUrl := resolveEndpoint(region)\n bodyBytes := serializeJsonBody(billingItem)\n authToken := computeHmacSignature(bodyBytes, httpClient.signingSecret)\n statusCode := transmitHttpPost(httpClient, targetUrl, bodyBytes, authToken)\n recordTelemetryCounter(statusCode, region)\n}",
|
| 67 |
-
"reply": "<<<<<<< SEARCH\nfunc publishAuditEvent(httpClient *Client, region string, auditItem Event) {\n targetUrl := resolveEndpoint(region)\n bodyBytes := serializeJsonBody(auditItem)\n authToken := computeHmacSignature(bodyBytes, httpClient.signingSecret)\n statusCode := transmitHttpPost(httpClient, targetUrl, bodyBytes, authToken)\n recordTelemetryCounter(statusCode, region)\n}\n\nfunc publishBillingEvent(httpClient *Client, region string, billingItem Event) {\n targetUrl := resolveEndpoint(region)\n bodyBytes := serializeJsonBody(billingItem)\n authToken := computeHmacSignature(bodyBytes, httpClient.signingSecret)\n statusCode := transmitHttpPost(httpClient, targetUrl, bodyBytes, authToken)\n recordTelemetryCounter(statusCode, region)\n}\n=======\nfunc publishEvent(httpClient *Client, region string, item Event) {\n endpoint := resolveEndpoint(region)\n body := serializeJsonBody(item)\n signature := computeHmacSignature(body, httpClient.signingSecret)\n status := transmitHttpPost(httpClient, endpoint, body, signature)\n recordTelemetryCounter(status, region)\n}\n\nfunc publishAuditEvent(httpClient *Client, region string, auditItem Event) {\n publishEvent(httpClient, region, auditItem)\n}\n\nfunc publishBillingEvent(httpClient *Client, region string, billingItem Event) {\n publishEvent(httpClient, region, billingItem)\n}\n>>>>>>> REPLACE\n"
|
| 68 |
-
},
|
| 69 |
-
{
|
| 70 |
-
"key": "e06eb41fc26eac74527f6145b19c1796c95d167092e4be3202d71ae449601e02",
|
| 71 |
-
"filename": "engine.js",
|
| 72 |
-
"pass": 1,
|
| 73 |
-
"source": "handwritten",
|
| 74 |
-
"origin": "docs/presets.json before",
|
| 75 |
-
"squiggles_before": 7,
|
| 76 |
-
"squiggles_after": 5,
|
| 77 |
-
"content": "export function unpack(tb, vb, ms) {\n let sp, ft, bo, vo, bv, ds;\n // TODO: clean up golfed trit unpacking\n sp = ms.split(\"|\");\n ft = new Int8Array(tb.length * 5);\n bo = 0;\n vo = 0;\n for (let i = 0; i < tb.length; i++) {\n bv = tb[i];\n ds = i * 5;\n ft[ds] = (((bv % 3) + 1) % 3) - 1;\n ft[ds + 1] = (((Math.floor(bv / 3) % 3) + 1) % 3) - 1;\n ft[ds + 2] = (((Math.floor(bv / 9) % 3) + 1) % 3) - 1;\n ft[ds + 3] = (((Math.floor(bv / 27) % 3) + 1) % 3) - 1;\n ft[ds + 4] = (((Math.floor(bv / 81) % 3) + 1) % 3) - 1;\n }\n try {\n return sp[0].length > 0 && bo >= vo ? ft : (vb || []);\n } catch (e) {}\n}",
|
| 78 |
-
"reply": "<<<<<<< SEARCH\n try {\n return sp[0].length > 0 && bo >= vo ? ft : (vb || []);\n } catch (e) {}\n=======\n if (sp[0].length > 0 && bo >= vo) {\n return ft;\n }\n return vb ? vb : [];\n>>>>>>> REPLACE\n"
|
| 79 |
-
},
|
| 80 |
-
{
|
| 81 |
-
"key": "5026dc2109a6c6eeceb284475e808c2dc1ff30327232235e4296a4fe404b5ce8",
|
| 82 |
-
"filename": "engine.js",
|
| 83 |
-
"pass": 2,
|
| 84 |
-
"source": "handwritten",
|
| 85 |
-
"origin": "pass-1 output",
|
| 86 |
-
"squiggles_before": 5,
|
| 87 |
-
"squiggles_after": 0,
|
| 88 |
-
"content": "export function unpack(tb, vb, ms) {\n let sp, ft, bo, vo, bv, ds;\n // TODO: clean up golfed trit unpacking\n sp = ms.split(\"|\");\n ft = new Int8Array(tb.length * 5);\n bo = 0;\n vo = 0;\n for (let i = 0; i < tb.length; i++) {\n bv = tb[i];\n ds = i * 5;\n ft[ds] = (((bv % 3) + 1) % 3) - 1;\n ft[ds + 1] = (((Math.floor(bv / 3) % 3) + 1) % 3) - 1;\n ft[ds + 2] = (((Math.floor(bv / 9) % 3) + 1) % 3) - 1;\n ft[ds + 3] = (((Math.floor(bv / 27) % 3) + 1) % 3) - 1;\n ft[ds + 4] = (((Math.floor(bv / 81) % 3) + 1) % 3) - 1;\n }\n if (sp[0].length > 0 && bo >= vo) {\n return ft;\n }\n return vb ? vb : [];\n}",
|
| 89 |
-
"reply": "<<<<<<< SEARCH\nexport function unpack(tb, vb, ms) {\n let sp, ft, bo, vo, bv, ds;\n // TODO: clean up golfed trit unpacking\n sp = ms.split(\"|\");\n ft = new Int8Array(tb.length * 5);\n bo = 0;\n vo = 0;\n for (let i = 0; i < tb.length; i++) {\n bv = tb[i];\n ds = i * 5;\n ft[ds] = (((bv % 3) + 1) % 3) - 1;\n ft[ds + 1] = (((Math.floor(bv / 3) % 3) + 1) % 3) - 1;\n ft[ds + 2] = (((Math.floor(bv / 9) % 3) + 1) % 3) - 1;\n ft[ds + 3] = (((Math.floor(bv / 27) % 3) + 1) % 3) - 1;\n ft[ds + 4] = (((Math.floor(bv / 81) % 3) + 1) % 3) - 1;\n }\n if (sp[0].length > 0 && bo >= vo) {\n return ft;\n }\n return vb ? vb : [];\n=======\nexport function unpack(tritBytes, fallback, manifest) {\n const sections = manifest.split(\"|\");\n const trits = new Int8Array(tritBytes.length * 5);\n for (let i = 0; i < tritBytes.length; i++) {\n const packed = tritBytes[i];\n const base = i * 5;\n trits[base] = (((packed % 3) + 1) % 3) - 1;\n trits[base + 1] = (((Math.floor(packed / 3) % 3) + 1) % 3) - 1;\n trits[base + 2] = (((Math.floor(packed / 9) % 3) + 1) % 3) - 1;\n trits[base + 3] = (((Math.floor(packed / 27) % 3) + 1) % 3) - 1;\n trits[base + 4] = (((Math.floor(packed / 81) % 3) + 1) % 3) - 1;\n }\n if (sections[0].length > 0) {\n return trits;\n }\n return fallback ? fallback : [];\n>>>>>>> REPLACE\n"
|
| 90 |
-
},
|
| 91 |
-
{
|
| 92 |
-
"key": "924c910f843823ebb2889d773d89212f28bb465f7197f360417cac41e8cd2e8e",
|
| 93 |
-
"filename": "index.js",
|
| 94 |
-
"pass": 1,
|
| 95 |
-
"source": "handwritten",
|
| 96 |
-
"origin": "github:lukeed/clsx@925494cf31bcd97d3337aacd34e659e80cae7fe2:src/index.js",
|
| 97 |
-
"squiggles_before": 2,
|
| 98 |
-
"squiggles_after": 0,
|
| 99 |
-
"content": "function toVal(mix) {\n\tvar k, y, str='';\n\n\tif (typeof mix === 'string' || typeof mix === 'number') {\n\t\tstr += mix;\n\t} else if (typeof mix === 'object') {\n\t\tif (Array.isArray(mix)) {\n\t\t\tvar len=mix.length;\n\t\t\tfor (k=0; k < len; k++) {\n\t\t\t\tif (mix[k]) {\n\t\t\t\t\tif (y = toVal(mix[k])) {\n\t\t\t\t\t\tstr && (str += ' ');\n\t\t\t\t\t\tstr += y;\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t} else {\n\t\t\tfor (y in mix) {\n\t\t\t\tif (mix[y]) {\n\t\t\t\t\tstr && (str += ' ');\n\t\t\t\t\tstr += y;\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n\n\treturn str;\n}\n\nexport function clsx() {\n\tvar i=0, tmp, x, str='', len=arguments.length;\n\tfor (; i < len; i++) {\n\t\tif (tmp = arguments[i]) {\n\t\t\tif (x = toVal(tmp)) {\n\t\t\t\tstr && (str += ' ');\n\t\t\t\tstr += x\n\t\t\t}\n\t\t}\n\t}\n\treturn str;\n}\n\nexport default clsx;\n",
|
| 100 |
-
"reply": "<<<<<<< SEARCH\n\t\t\t\tif (mix[k]) {\n\t\t\t\t\tif (y = toVal(mix[k])) {\n\t\t\t\t\t\tstr && (str += ' ');\n\t\t\t\t\t\tstr += y;\n\t\t\t\t\t}\n\t\t\t\t}\n=======\n\t\t\t\tif (mix[k] && (y = toVal(mix[k]))) {\n\t\t\t\t\tstr && (str += ' ');\n\t\t\t\t\tstr += y;\n\t\t\t\t}\n>>>>>>> REPLACE"
|
| 101 |
-
}
|
| 102 |
-
]
|
| 103 |
-
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
server/github_pr.py
DELETED
|
@@ -1,381 +0,0 @@
|
|
| 1 |
-
"""Single-repo pull requests through a GitHub App, plus signed webhook merge tracking.
|
| 2 |
-
|
| 3 |
-
The visitor authorizes the App in a pop-up. The callback page hands a one-time ticket to the
|
| 4 |
-
CodeHygiene page that opened it (`postMessage` restricted to this origin), so a sign-in link
|
| 5 |
-
sent by someone else can never be redeemed by them. The user token lives only in this
|
| 6 |
-
process's memory, is used for one pull request on a repository the visitor can push to, and
|
| 7 |
-
is then revoked.
|
| 8 |
-
"""
|
| 9 |
-
|
| 10 |
-
from __future__ import annotations
|
| 11 |
-
|
| 12 |
-
import asyncio
|
| 13 |
-
import hashlib
|
| 14 |
-
import hmac
|
| 15 |
-
import json
|
| 16 |
-
import logging
|
| 17 |
-
import secrets
|
| 18 |
-
import time
|
| 19 |
-
from dataclasses import dataclass
|
| 20 |
-
from typing import Any
|
| 21 |
-
from urllib.parse import quote, urlencode
|
| 22 |
-
|
| 23 |
-
import httpx
|
| 24 |
-
import jwt
|
| 25 |
-
from pydantic import BaseModel, ConfigDict, Field
|
| 26 |
-
|
| 27 |
-
from .config import Settings
|
| 28 |
-
from .hygiene import score_dict
|
| 29 |
-
from .telemetry import Telemetry, pr_key
|
| 30 |
-
|
| 31 |
-
log = logging.getLogger("space.github")
|
| 32 |
-
|
| 33 |
-
API_URL = "https://api.github.com"
|
| 34 |
-
AUTHORIZE_URL = "https://github.com/login/oauth/authorize"
|
| 35 |
-
TOKEN_URL = "https://github.com/login/oauth/access_token"
|
| 36 |
-
PROJECT_URL = "https://github.com/brianbell-x/codehygiene"
|
| 37 |
-
STATE_TTL = 600.0
|
| 38 |
-
TICKET_TTL = 180.0
|
| 39 |
-
MAX_PENDING = 256
|
| 40 |
-
MAX_PR_FILES = 20
|
| 41 |
-
THRESHOLD = 0.55
|
| 42 |
-
NAME_PATTERN = r"^[A-Za-z0-9_.-]{1,100}$"
|
| 43 |
-
TICKET_PATTERN = r"^[0-9a-f]{32}$"
|
| 44 |
-
DEFAULT_MODE = "100644"
|
| 45 |
-
BLOB_MODES = frozenset({DEFAULT_MODE, "100755"})
|
| 46 |
-
|
| 47 |
-
|
| 48 |
-
class PrError(Exception):
|
| 49 |
-
def __init__(self, status: int, code: str, message: str, **extra: Any) -> None:
|
| 50 |
-
super().__init__(message)
|
| 51 |
-
self.status = status
|
| 52 |
-
self.code = code
|
| 53 |
-
self.message = message
|
| 54 |
-
self.extra = extra
|
| 55 |
-
|
| 56 |
-
def payload(self) -> dict[str, Any]:
|
| 57 |
-
return {"code": self.code, "message": self.message, **self.extra}
|
| 58 |
-
|
| 59 |
-
|
| 60 |
-
class PrFile(BaseModel):
|
| 61 |
-
model_config = ConfigDict(extra="forbid")
|
| 62 |
-
|
| 63 |
-
path: str = Field(min_length=1, max_length=400)
|
| 64 |
-
content_after: str = Field(max_length=200_000)
|
| 65 |
-
score_before: int = Field(ge=0, le=100)
|
| 66 |
-
score_after: int = Field(ge=0, le=100)
|
| 67 |
-
squiggles_fixed: int = Field(ge=0, le=10_000)
|
| 68 |
-
|
| 69 |
-
|
| 70 |
-
class CreatePrBody(BaseModel):
|
| 71 |
-
model_config = ConfigDict(extra="forbid")
|
| 72 |
-
|
| 73 |
-
ticket: str = Field(pattern=TICKET_PATTERN)
|
| 74 |
-
owner: str = Field(pattern=NAME_PATTERN)
|
| 75 |
-
repo: str = Field(pattern=NAME_PATTERN)
|
| 76 |
-
base_branch: str = Field("", max_length=255)
|
| 77 |
-
base_commit_sha: str = Field(pattern=r"^[0-9a-f]{40}$")
|
| 78 |
-
files: list[PrFile] = Field(min_length=1, max_length=MAX_PR_FILES)
|
| 79 |
-
|
| 80 |
-
|
| 81 |
-
@dataclass(frozen=True)
|
| 82 |
-
class CheckedFile:
|
| 83 |
-
path: str
|
| 84 |
-
mode: str
|
| 85 |
-
content: str
|
| 86 |
-
score_before: int
|
| 87 |
-
score_after: int
|
| 88 |
-
resolved: int
|
| 89 |
-
|
| 90 |
-
|
| 91 |
-
def signature_valid(secret: str, body: bytes, header: str | None) -> bool:
|
| 92 |
-
if not secret or not header or not header.startswith("sha256="):
|
| 93 |
-
return False
|
| 94 |
-
expected = "sha256=" + hmac.new(secret.encode(), body, hashlib.sha256).hexdigest()
|
| 95 |
-
return hmac.compare_digest(expected, header)
|
| 96 |
-
|
| 97 |
-
|
| 98 |
-
def safe_path(path: str) -> bool:
|
| 99 |
-
parts = path.split("/")
|
| 100 |
-
if path.startswith("/") or "\\" in path or "\0" in path:
|
| 101 |
-
return False
|
| 102 |
-
return all(part not in ("", ".", "..", ".git") for part in parts)
|
| 103 |
-
|
| 104 |
-
|
| 105 |
-
def file_mode(modes: dict[str, str] | None, path: str) -> str | None:
|
| 106 |
-
"""Git mode of a base file. A truncated tree has no listing; reading the file later proves it exists."""
|
| 107 |
-
if modes is None:
|
| 108 |
-
return DEFAULT_MODE
|
| 109 |
-
return modes.get(path)
|
| 110 |
-
|
| 111 |
-
|
| 112 |
-
def pr_body(files: list[CheckedFile]) -> str:
|
| 113 |
-
rows = "\n".join(
|
| 114 |
-
f"| `{item.path}` | {item.score_before} → {item.score_after} | {item.resolved} |" for item in files
|
| 115 |
-
)
|
| 116 |
-
return (
|
| 117 |
-
f"Readability cleanup suggested by [CodeHygiene]({PROJECT_URL}).\n\n"
|
| 118 |
-
"> [!WARNING]\n"
|
| 119 |
-
"> **REVIEW AND PASSING TESTS ARE REQUIRED BEFORE MERGING.**\n"
|
| 120 |
-
"> Each file was cleaned separately. Cross-file interactions and application behavior "
|
| 121 |
-
"were not verified. These edits must not be treated as drop-in fixes.\n"
|
| 122 |
-
"> Every change must be reviewed, and all relevant tests must be run and confirmed passing. "
|
| 123 |
-
"**Serious bugs, data loss, or outages may result if these checks are skipped.**\n\n"
|
| 124 |
-
"| File | Score (of 100) | Issues resolved |\n"
|
| 125 |
-
"| --- | --- | --- |\n"
|
| 126 |
-
f"{rows}\n\n"
|
| 127 |
-
"Readability scores were checked automatically. Functional correctness and passing "
|
| 128 |
-
"tests were not verified.\n"
|
| 129 |
-
)
|
| 130 |
-
|
| 131 |
-
|
| 132 |
-
def pr_title(count: int) -> str:
|
| 133 |
-
return "Clean up readability in 1 file" if count == 1 else f"Clean up readability in {count} files"
|
| 134 |
-
|
| 135 |
-
|
| 136 |
-
class PullRequests:
|
| 137 |
-
def __init__(
|
| 138 |
-
self,
|
| 139 |
-
settings: Settings,
|
| 140 |
-
telemetry: Telemetry,
|
| 141 |
-
transport: httpx.AsyncBaseTransport | None = None,
|
| 142 |
-
) -> None:
|
| 143 |
-
self.settings = settings
|
| 144 |
-
self.telemetry = telemetry
|
| 145 |
-
self.http = httpx.AsyncClient(
|
| 146 |
-
timeout=20.0,
|
| 147 |
-
transport=transport,
|
| 148 |
-
headers={
|
| 149 |
-
"Accept": "application/vnd.github+json",
|
| 150 |
-
"X-GitHub-Api-Version": "2022-11-28",
|
| 151 |
-
"User-Agent": "codehygiene-space",
|
| 152 |
-
},
|
| 153 |
-
)
|
| 154 |
-
self._states: dict[str, float] = {}
|
| 155 |
-
self._tickets: dict[str, tuple[str, float]] = {}
|
| 156 |
-
self._slug = settings.github_app_slug
|
| 157 |
-
|
| 158 |
-
@property
|
| 159 |
-
def enabled(self) -> bool:
|
| 160 |
-
return self.settings.github_pr_enabled
|
| 161 |
-
|
| 162 |
-
async def close(self) -> None:
|
| 163 |
-
await self.http.aclose()
|
| 164 |
-
|
| 165 |
-
def _sweep(self) -> None:
|
| 166 |
-
now = time.monotonic()
|
| 167 |
-
self._states = {state: expires for state, expires in self._states.items() if expires > now}
|
| 168 |
-
self._tickets = {key: entry for key, entry in self._tickets.items() if entry[1] > now}
|
| 169 |
-
|
| 170 |
-
def login_url(self) -> str:
|
| 171 |
-
self._sweep()
|
| 172 |
-
if len(self._states) >= MAX_PENDING:
|
| 173 |
-
raise PrError(429, "busy", "Too many sign-ins in progress. Try again shortly.")
|
| 174 |
-
state = secrets.token_urlsafe(24)
|
| 175 |
-
self._states[state] = time.monotonic() + STATE_TTL
|
| 176 |
-
query = urlencode({"client_id": self.settings.github_app_client_id, "state": state})
|
| 177 |
-
return f"{AUTHORIZE_URL}?{query}"
|
| 178 |
-
|
| 179 |
-
async def complete_login(self, code: str, state: str) -> str:
|
| 180 |
-
"""Exchange the code and return a one-time ticket for the page that opened the pop-up."""
|
| 181 |
-
self._sweep()
|
| 182 |
-
if self._states.pop(state, None) is None:
|
| 183 |
-
raise PrError(400, "expired", "This sign-in expired. Start again from CodeHygiene.")
|
| 184 |
-
if len(self._tickets) >= MAX_PENDING:
|
| 185 |
-
raise PrError(429, "busy", "Too many sign-ins in progress. Try again shortly.")
|
| 186 |
-
return self.issue_ticket(await self._exchange_code(code))
|
| 187 |
-
|
| 188 |
-
def issue_ticket(self, token: str) -> str:
|
| 189 |
-
"""Hold a user token in memory under a fresh one-time ticket."""
|
| 190 |
-
ticket = secrets.token_hex(16)
|
| 191 |
-
self._tickets[ticket] = (token, time.monotonic() + TICKET_TTL)
|
| 192 |
-
return ticket
|
| 193 |
-
|
| 194 |
-
async def create(self, body: CreatePrBody) -> dict[str, Any]:
|
| 195 |
-
self._sweep()
|
| 196 |
-
entry = self._tickets.pop(body.ticket, None)
|
| 197 |
-
if entry is None:
|
| 198 |
-
raise PrError(401, "login_required", "Sign in with GitHub again.")
|
| 199 |
-
token = entry[0]
|
| 200 |
-
try:
|
| 201 |
-
return await self._open(token, body)
|
| 202 |
-
finally:
|
| 203 |
-
await self._revoke(token)
|
| 204 |
-
|
| 205 |
-
async def _exchange_code(self, code: str) -> str:
|
| 206 |
-
response = await self.http.post(
|
| 207 |
-
TOKEN_URL,
|
| 208 |
-
data={
|
| 209 |
-
"client_id": self.settings.github_app_client_id,
|
| 210 |
-
"client_secret": self.settings.github_app_client_secret,
|
| 211 |
-
"code": code,
|
| 212 |
-
},
|
| 213 |
-
headers={"Accept": "application/json"},
|
| 214 |
-
)
|
| 215 |
-
token = response.json().get("access_token") if response.status_code == 200 else None
|
| 216 |
-
if not token:
|
| 217 |
-
raise PrError(401, "login_failed", "GitHub sign-in failed.")
|
| 218 |
-
return token
|
| 219 |
-
|
| 220 |
-
async def _revoke(self, token: str) -> None:
|
| 221 |
-
"""Best effort: the token is dropped from memory either way and expires on GitHub's side."""
|
| 222 |
-
try:
|
| 223 |
-
response = await self.http.request(
|
| 224 |
-
"DELETE",
|
| 225 |
-
f"{API_URL}/applications/{self.settings.github_app_client_id}/token",
|
| 226 |
-
auth=(self.settings.github_app_client_id, self.settings.github_app_client_secret),
|
| 227 |
-
json={"access_token": token},
|
| 228 |
-
)
|
| 229 |
-
except httpx.HTTPError as error:
|
| 230 |
-
log.warning("token revoke failed: %s", type(error).__name__)
|
| 231 |
-
return
|
| 232 |
-
if response.status_code not in (204, 404):
|
| 233 |
-
log.warning("token revoke returned HTTP %d", response.status_code)
|
| 234 |
-
|
| 235 |
-
def _app_jwt(self) -> str:
|
| 236 |
-
now = int(time.time())
|
| 237 |
-
claims = {"iat": now - 60, "exp": now + 540, "iss": self.settings.github_app_id}
|
| 238 |
-
return jwt.encode(claims, self.settings.github_app_private_key, algorithm="RS256")
|
| 239 |
-
|
| 240 |
-
async def _call(
|
| 241 |
-
self, method: str, path: str, *, token: str | None = None, app: bool = False, **kwargs: Any
|
| 242 |
-
) -> httpx.Response:
|
| 243 |
-
bearer = self._app_jwt() if app else token
|
| 244 |
-
headers = {**kwargs.pop("headers", {}), "Authorization": f"Bearer {bearer}"}
|
| 245 |
-
return await self.http.request(method, f"{API_URL}{path}", headers=headers, **kwargs)
|
| 246 |
-
|
| 247 |
-
async def _json(self, method: str, path: str, *, token: str, expect: int = 200, **kwargs: Any) -> Any:
|
| 248 |
-
response = await self._call(method, path, token=token, **kwargs)
|
| 249 |
-
if response.status_code != expect:
|
| 250 |
-
raise PrError(502, "github_error", f"GitHub returned HTTP {response.status_code}.")
|
| 251 |
-
return response.json()
|
| 252 |
-
|
| 253 |
-
async def _app_slug(self) -> str:
|
| 254 |
-
if not self._slug:
|
| 255 |
-
response = await self._call("GET", "/app", app=True)
|
| 256 |
-
self._slug = response.json().get("slug", "") if response.status_code == 200 else ""
|
| 257 |
-
return self._slug
|
| 258 |
-
|
| 259 |
-
async def _install_url(self, repo: dict[str, Any] | None) -> str:
|
| 260 |
-
url = f"https://github.com/apps/{await self._app_slug()}/installations/new"
|
| 261 |
-
if repo is None:
|
| 262 |
-
return url
|
| 263 |
-
query = urlencode({"suggested_target_id": repo["owner"]["id"], "repository_ids[]": repo["id"]})
|
| 264 |
-
return f"{url}/permissions?{query}"
|
| 265 |
-
|
| 266 |
-
async def _installed(self, slug: str) -> bool:
|
| 267 |
-
response = await self._call("GET", f"/repos/{slug}/installation", app=True)
|
| 268 |
-
return response.status_code == 200
|
| 269 |
-
|
| 270 |
-
async def _open(self, token: str, body: CreatePrBody) -> dict[str, Any]:
|
| 271 |
-
slug = f"{body.owner}/{body.repo}"
|
| 272 |
-
response = await self._call("GET", f"/repos/{slug}", token=token)
|
| 273 |
-
if response.status_code == 404:
|
| 274 |
-
if not await self._installed(slug):
|
| 275 |
-
raise PrError(409, "not_installed", "Install the app on this repository first.",
|
| 276 |
-
install_url=await self._install_url(None))
|
| 277 |
-
raise PrError(404, "not_found", "Repository not found.")
|
| 278 |
-
if response.status_code != 200:
|
| 279 |
-
raise PrError(502, "github_error", f"GitHub returned HTTP {response.status_code}.")
|
| 280 |
-
repo = response.json()
|
| 281 |
-
if not repo.get("permissions", {}).get("push"):
|
| 282 |
-
raise PrError(403, "no_push", "You need write access to open a pull request here.")
|
| 283 |
-
if not await self._installed(slug):
|
| 284 |
-
raise PrError(409, "not_installed", "Install the app on this repository first.",
|
| 285 |
-
install_url=await self._install_url(repo))
|
| 286 |
-
|
| 287 |
-
base_branch = body.base_branch or repo["default_branch"]
|
| 288 |
-
branch_path = f"/repos/{slug}/branches/{quote(base_branch, safe='')}"
|
| 289 |
-
branch = await self._call("GET", branch_path, token=token)
|
| 290 |
-
if branch.status_code != 200:
|
| 291 |
-
raise PrError(422, "bad_base", "The base must be a branch.")
|
| 292 |
-
commit = await self._call("GET", f"/repos/{slug}/git/commits/{body.base_commit_sha}", token=token)
|
| 293 |
-
if commit.status_code != 200:
|
| 294 |
-
raise PrError(422, "bad_base", "The base commit was not found.")
|
| 295 |
-
base_tree = commit.json()["tree"]["sha"]
|
| 296 |
-
|
| 297 |
-
files = await self._checked_files(token, slug, base_tree, body)
|
| 298 |
-
head = await self._commit(token, slug, base_tree, body.base_commit_sha, files)
|
| 299 |
-
created = await self._json("POST", f"/repos/{slug}/pulls", token=token, expect=201, json={
|
| 300 |
-
"title": pr_title(len(files)),
|
| 301 |
-
"head": head,
|
| 302 |
-
"base": base_branch,
|
| 303 |
-
"body": pr_body(files),
|
| 304 |
-
"maintainer_can_modify": True,
|
| 305 |
-
})
|
| 306 |
-
self.telemetry.record_pr_opened(pr_key(repo["id"], created["number"]))
|
| 307 |
-
return {"url": created["html_url"], "number": created["number"]}
|
| 308 |
-
|
| 309 |
-
async def _modes(self, token: str, slug: str, base_tree: str) -> dict[str, str] | None:
|
| 310 |
-
tree = await self._json("GET", f"/repos/{slug}/git/trees/{base_tree}?recursive=1", token=token)
|
| 311 |
-
if tree.get("truncated"):
|
| 312 |
-
return None
|
| 313 |
-
return {node["path"]: node["mode"] for node in tree.get("tree", []) if node.get("type") == "blob"}
|
| 314 |
-
|
| 315 |
-
async def _checked_files(
|
| 316 |
-
self, token: str, slug: str, base_tree: str, body: CreatePrBody
|
| 317 |
-
) -> list[CheckedFile]:
|
| 318 |
-
"""Re-read each base file at the commit and re-score both versions; the client's numbers are not used."""
|
| 319 |
-
if len({item.path for item in body.files}) != len(body.files):
|
| 320 |
-
raise PrError(422, "bad_path", "Each file may appear once.")
|
| 321 |
-
modes = await self._modes(token, slug, base_tree)
|
| 322 |
-
checked: list[CheckedFile] = []
|
| 323 |
-
for item in body.files:
|
| 324 |
-
mode = file_mode(modes, item.path)
|
| 325 |
-
if not safe_path(item.path) or mode not in BLOB_MODES:
|
| 326 |
-
raise PrError(422, "bad_path", "A file is not in the base commit.")
|
| 327 |
-
raw = await self._call(
|
| 328 |
-
"GET", f"/repos/{slug}/contents/{quote(item.path)}?ref={body.base_commit_sha}",
|
| 329 |
-
token=token, headers={"Accept": "application/vnd.github.raw"},
|
| 330 |
-
)
|
| 331 |
-
if raw.status_code != 200:
|
| 332 |
-
raise PrError(422, "bad_path", "A file is not in the base commit.")
|
| 333 |
-
before, after = await asyncio.gather(
|
| 334 |
-
asyncio.to_thread(score_dict, raw.text, THRESHOLD),
|
| 335 |
-
asyncio.to_thread(score_dict, item.content_after, THRESHOLD),
|
| 336 |
-
)
|
| 337 |
-
resolved = len(before["squiggles"]) - len(after["squiggles"])
|
| 338 |
-
if item.content_after == raw.text or resolved <= 0:
|
| 339 |
-
raise PrError(422, "not_cleaner", "Every file must have fewer issues than its base.")
|
| 340 |
-
checked.append(CheckedFile(
|
| 341 |
-
item.path, mode, item.content_after,
|
| 342 |
-
round(before["score"] * 100), round(after["score"] * 100), resolved,
|
| 343 |
-
))
|
| 344 |
-
return checked
|
| 345 |
-
|
| 346 |
-
async def _commit(
|
| 347 |
-
self, token: str, slug: str, base_tree: str, base_sha: str, files: list[CheckedFile]
|
| 348 |
-
) -> str:
|
| 349 |
-
entries = []
|
| 350 |
-
for item in files:
|
| 351 |
-
blob = await self._json("POST", f"/repos/{slug}/git/blobs", token=token, expect=201,
|
| 352 |
-
json={"content": item.content, "encoding": "utf-8"})
|
| 353 |
-
entries.append({"path": item.path, "mode": item.mode, "type": "blob", "sha": blob["sha"]})
|
| 354 |
-
tree = await self._json("POST", f"/repos/{slug}/git/trees", token=token, expect=201,
|
| 355 |
-
json={"base_tree": base_tree, "tree": entries})
|
| 356 |
-
commit = await self._json("POST", f"/repos/{slug}/git/commits", token=token, expect=201, json={
|
| 357 |
-
"message": pr_title(len(files)),
|
| 358 |
-
"tree": tree["sha"],
|
| 359 |
-
"parents": [base_sha],
|
| 360 |
-
})
|
| 361 |
-
for _ in range(3):
|
| 362 |
-
branch = f"codehygiene/{secrets.token_hex(4)}"
|
| 363 |
-
ref = await self._call("POST", f"/repos/{slug}/git/refs", token=token,
|
| 364 |
-
json={"ref": f"refs/heads/{branch}", "sha": commit["sha"]})
|
| 365 |
-
if ref.status_code == 201:
|
| 366 |
-
return branch
|
| 367 |
-
if ref.status_code != 422:
|
| 368 |
-
break
|
| 369 |
-
raise PrError(502, "github_error", "Could not create the branch.")
|
| 370 |
-
|
| 371 |
-
def webhook(self, event: str, body: bytes, signature: str | None) -> dict[str, Any]:
|
| 372 |
-
if not signature_valid(self.settings.github_webhook_secret, body, signature):
|
| 373 |
-
raise PrError(401, "bad_signature", "Invalid signature.")
|
| 374 |
-
if event != "pull_request":
|
| 375 |
-
return {"counted": False}
|
| 376 |
-
payload = json.loads(body)
|
| 377 |
-
if payload.get("action") != "closed" or "pull_request" not in payload:
|
| 378 |
-
return {"counted": False}
|
| 379 |
-
pull = payload["pull_request"]
|
| 380 |
-
key = pr_key(payload["repository"]["id"], pull["number"])
|
| 381 |
-
return {"counted": self.telemetry.record_pr_closed(key, merged=bool(pull.get("merged")))}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
server/guards.py
DELETED
|
@@ -1,77 +0,0 @@
|
|
| 1 |
-
from __future__ import annotations
|
| 2 |
-
|
| 3 |
-
from dataclasses import dataclass, field
|
| 4 |
-
from typing import Any
|
| 5 |
-
|
| 6 |
-
from .hygiene import score_dict
|
| 7 |
-
from .vendored_guards import (
|
| 8 |
-
check_basic_syntax,
|
| 9 |
-
check_reference_integrity,
|
| 10 |
-
is_bloated,
|
| 11 |
-
is_gutted,
|
| 12 |
-
)
|
| 13 |
-
from .vendored_pipeline import apply_changes
|
| 14 |
-
|
| 15 |
-
|
| 16 |
-
@dataclass
|
| 17 |
-
class Verdict:
|
| 18 |
-
accepted: bool
|
| 19 |
-
reason: str | None
|
| 20 |
-
content: str
|
| 21 |
-
applied_blocks: int = 0
|
| 22 |
-
squiggles_before: int = 0
|
| 23 |
-
squiggles_after: int | None = None
|
| 24 |
-
hygiene: dict[str, Any] = field(default_factory=dict)
|
| 25 |
-
reply: str = ""
|
| 26 |
-
|
| 27 |
-
@property
|
| 28 |
-
def score_after(self) -> float | None:
|
| 29 |
-
return self.hygiene.get("score")
|
| 30 |
-
|
| 31 |
-
|
| 32 |
-
def verify_patch(
|
| 33 |
-
content: str,
|
| 34 |
-
reply: str,
|
| 35 |
-
filename: str,
|
| 36 |
-
threshold: float,
|
| 37 |
-
before_hygiene: dict[str, Any],
|
| 38 |
-
) -> Verdict:
|
| 39 |
-
squiggles_before = len(before_hygiene["squiggles"])
|
| 40 |
-
rejected = Verdict(False, None, content, squiggles_before=squiggles_before)
|
| 41 |
-
if not reply.strip():
|
| 42 |
-
rejected.reason = "empty_output"
|
| 43 |
-
return rejected
|
| 44 |
-
try:
|
| 45 |
-
updated, applied = apply_changes(content, reply)
|
| 46 |
-
except ValueError:
|
| 47 |
-
rejected.reason = "search_miss"
|
| 48 |
-
return rejected
|
| 49 |
-
rejected.applied_blocks = applied
|
| 50 |
-
if applied == 0 or updated == content:
|
| 51 |
-
rejected.reason = "no_blocks_applied"
|
| 52 |
-
return rejected
|
| 53 |
-
if filename and not check_basic_syntax(updated, filename):
|
| 54 |
-
rejected.reason = "invalid_syntax"
|
| 55 |
-
return rejected
|
| 56 |
-
if filename and not check_reference_integrity(content, updated, filename):
|
| 57 |
-
rejected.reason = "invalid_syntax"
|
| 58 |
-
return rejected
|
| 59 |
-
if is_gutted(content, updated):
|
| 60 |
-
rejected.reason = "gutted_file"
|
| 61 |
-
return rejected
|
| 62 |
-
if is_bloated(content, updated):
|
| 63 |
-
rejected.reason = "line_bloat"
|
| 64 |
-
return rejected
|
| 65 |
-
|
| 66 |
-
after = score_dict(updated, threshold)
|
| 67 |
-
squiggles_after = len(after["squiggles"])
|
| 68 |
-
accepted = squiggles_after < squiggles_before
|
| 69 |
-
return Verdict(
|
| 70 |
-
accepted=accepted,
|
| 71 |
-
reason=None if accepted else "no_improvement",
|
| 72 |
-
content=updated if accepted else content,
|
| 73 |
-
applied_blocks=applied,
|
| 74 |
-
squiggles_before=squiggles_before,
|
| 75 |
-
squiggles_after=squiggles_after,
|
| 76 |
-
hygiene=after,
|
| 77 |
-
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
server/hygiene.py
CHANGED
|
@@ -6,10 +6,15 @@ import sys
|
|
| 6 |
from pathlib import Path
|
| 7 |
from typing import Any
|
| 8 |
|
| 9 |
-
|
| 10 |
-
|
| 11 |
-
|
| 12 |
-
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 13 |
LOCAL_PACKAGE = REPO_ROOT / "codehygiene" / "__init__.py"
|
| 14 |
|
| 15 |
if importlib.util.find_spec("codehygiene") is None and LOCAL_PACKAGE.is_file():
|
|
|
|
| 6 |
from pathlib import Path
|
| 7 |
from typing import Any
|
| 8 |
|
| 9 |
+
def _find_repo_root() -> Path:
|
| 10 |
+
current = Path(__file__).resolve().parent
|
| 11 |
+
for parent in current.parents:
|
| 12 |
+
if (parent / "codehygiene" / "__init__.py").is_file():
|
| 13 |
+
return parent
|
| 14 |
+
return current.parent.parent
|
| 15 |
+
|
| 16 |
+
|
| 17 |
+
REPO_ROOT = _find_repo_root()
|
| 18 |
LOCAL_PACKAGE = REPO_ROOT / "codehygiene" / "__init__.py"
|
| 19 |
|
| 20 |
if importlib.util.find_spec("codehygiene") is None and LOCAL_PACKAGE.is_file():
|
server/prompt.py
DELETED
|
@@ -1,24 +0,0 @@
|
|
| 1 |
-
from __future__ import annotations
|
| 2 |
-
|
| 3 |
-
import hashlib
|
| 4 |
-
from typing import Any
|
| 5 |
-
|
| 6 |
-
from .vendored_pipeline import build_inference_messages
|
| 7 |
-
|
| 8 |
-
|
| 9 |
-
def render_chatml(messages: list[dict[str, str]]) -> str:
|
| 10 |
-
system, user = messages
|
| 11 |
-
return (
|
| 12 |
-
f"<|im_start|>system\n{system['content']}<|im_end|>\n"
|
| 13 |
-
f"<|im_start|>user\n{user['content']}<|im_end|>\n"
|
| 14 |
-
"<|im_start|>assistant\n<think>\n\n</think>\n\n"
|
| 15 |
-
)
|
| 16 |
-
|
| 17 |
-
|
| 18 |
-
def build_prompt(content: str, hygiene: dict[str, Any], filename: str) -> str:
|
| 19 |
-
return render_chatml(build_inference_messages(content, hygiene, filename))
|
| 20 |
-
|
| 21 |
-
|
| 22 |
-
def content_key(content: str) -> str:
|
| 23 |
-
normalized = content.replace("\r\n", "\n").rstrip("\n")
|
| 24 |
-
return hashlib.sha256(normalized.encode("utf-8")).hexdigest()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
server/service.py
DELETED
|
@@ -1,152 +0,0 @@
|
|
| 1 |
-
from __future__ import annotations
|
| 2 |
-
|
| 3 |
-
import asyncio
|
| 4 |
-
import contextvars
|
| 5 |
-
import json
|
| 6 |
-
import logging
|
| 7 |
-
import time
|
| 8 |
-
from collections import OrderedDict
|
| 9 |
-
from concurrent.futures import ThreadPoolExecutor
|
| 10 |
-
from pathlib import Path
|
| 11 |
-
from typing import Any
|
| 12 |
-
|
| 13 |
-
from .backends import Refiner
|
| 14 |
-
from .config import Settings
|
| 15 |
-
from .fix_loop import PREVIEW_INTERVAL, Cancelled, Emit, FixRequest, FixRun
|
| 16 |
-
from .fix_queue import FixQueue, Ticket
|
| 17 |
-
from .telemetry import PRIVATE_SOURCES, Telemetry
|
| 18 |
-
|
| 19 |
-
log = logging.getLogger("space.fix")
|
| 20 |
-
|
| 21 |
-
Event = tuple[str, dict[str, Any]]
|
| 22 |
-
CacheKey = tuple[str, str, str, float, int]
|
| 23 |
-
|
| 24 |
-
|
| 25 |
-
def final_previews(events: list[Event]) -> set[int]:
|
| 26 |
-
"""Indices of the last preview before each verdict, which a paced replay must not drop."""
|
| 27 |
-
keep: set[int] = set()
|
| 28 |
-
latest = None
|
| 29 |
-
for index, (name, _) in enumerate(events):
|
| 30 |
-
if name == "preview":
|
| 31 |
-
latest = index
|
| 32 |
-
elif name == "verdict" and latest is not None:
|
| 33 |
-
keep.add(latest)
|
| 34 |
-
latest = None
|
| 35 |
-
return keep
|
| 36 |
-
|
| 37 |
-
|
| 38 |
-
class ResultCache:
|
| 39 |
-
def __init__(self, size: int) -> None:
|
| 40 |
-
self.size = size
|
| 41 |
-
self._items: OrderedDict[CacheKey, list[Event]] = OrderedDict()
|
| 42 |
-
|
| 43 |
-
def get(self, key: CacheKey) -> list[Event] | None:
|
| 44 |
-
events = self._items.get(key)
|
| 45 |
-
if events is not None:
|
| 46 |
-
self._items.move_to_end(key)
|
| 47 |
-
return events
|
| 48 |
-
|
| 49 |
-
def put(self, key: CacheKey, events: list[Event]) -> None:
|
| 50 |
-
self._items[key] = events
|
| 51 |
-
self._items.move_to_end(key)
|
| 52 |
-
while len(self._items) > self.size:
|
| 53 |
-
self._items.popitem(last=False)
|
| 54 |
-
|
| 55 |
-
|
| 56 |
-
def retains(source_kind: str) -> bool:
|
| 57 |
-
"""Uploaded folders and pasted snippets live only in request memory, never in the cache."""
|
| 58 |
-
return source_kind not in PRIVATE_SOURCES
|
| 59 |
-
|
| 60 |
-
|
| 61 |
-
class FixService:
|
| 62 |
-
def __init__(self, settings: Settings, refiner: Refiner, telemetry: Telemetry | None = None) -> None:
|
| 63 |
-
self.settings = settings
|
| 64 |
-
self.refiner = refiner
|
| 65 |
-
self.telemetry = telemetry or Telemetry()
|
| 66 |
-
self.queue = FixQueue(settings.max_waiters)
|
| 67 |
-
self.cache = ResultCache(settings.cache_size)
|
| 68 |
-
self._executor = ThreadPoolExecutor(max_workers=2, thread_name_prefix="refiner")
|
| 69 |
-
|
| 70 |
-
def cache_key(self, request: FixRequest) -> CacheKey:
|
| 71 |
-
model_id = self.refiner.model_id
|
| 72 |
-
return (model_id, request.filename, request.content, request.threshold, request.max_passes)
|
| 73 |
-
|
| 74 |
-
def cached(self, request: FixRequest, source_kind: str) -> list[Event] | None:
|
| 75 |
-
if not retains(source_kind):
|
| 76 |
-
return None
|
| 77 |
-
return self.cache.get(self.cache_key(request))
|
| 78 |
-
|
| 79 |
-
async def execute(
|
| 80 |
-
self, request: FixRequest, ticket: Ticket, emit: Emit, source_kind: str = "snippet"
|
| 81 |
-
) -> list[Event]:
|
| 82 |
-
loop = asyncio.get_running_loop()
|
| 83 |
-
events: list[Event] = []
|
| 84 |
-
|
| 85 |
-
def thread_emit(name: str, data: dict[str, Any]) -> None:
|
| 86 |
-
events.append((name, data))
|
| 87 |
-
loop.call_soon_threadsafe(emit, name, data)
|
| 88 |
-
|
| 89 |
-
future = None
|
| 90 |
-
try:
|
| 91 |
-
await self.queue.wait_turn(ticket, lambda position: emit("queue", {"position": position}))
|
| 92 |
-
run = FixRun(self.refiner, request, ticket.cancel, thread_emit, self.settings.max_file_chars)
|
| 93 |
-
ctx = contextvars.copy_context()
|
| 94 |
-
future = self._executor.submit(ctx.run, run.run)
|
| 95 |
-
future.add_done_callback(lambda _: loop.call_soon_threadsafe(self.queue.release, ticket))
|
| 96 |
-
await asyncio.wrap_future(future)
|
| 97 |
-
finally:
|
| 98 |
-
if future is None:
|
| 99 |
-
self.queue.release(ticket)
|
| 100 |
-
elif not future.done():
|
| 101 |
-
ticket.cancel.set()
|
| 102 |
-
if retains(source_kind):
|
| 103 |
-
self.cache.put(self.cache_key(request), events)
|
| 104 |
-
if source_kind != "examples" and self.settings.counts_fixes:
|
| 105 |
-
self.telemetry.record_clean(request.filename, request.content, future.result())
|
| 106 |
-
return events
|
| 107 |
-
|
| 108 |
-
async def replay(self, events: list[Event], emit: Emit, paced: bool = True) -> None:
|
| 109 |
-
rate = self.settings.replay_tokens_per_second if paced else 0.0
|
| 110 |
-
keep = final_previews(events)
|
| 111 |
-
started = time.monotonic()
|
| 112 |
-
sent = 0
|
| 113 |
-
last_preview_at = -PREVIEW_INTERVAL
|
| 114 |
-
for index, (name, data) in enumerate(events):
|
| 115 |
-
if name in ("queue", "progress"):
|
| 116 |
-
continue
|
| 117 |
-
if name == "preview" and rate > 0 and index not in keep:
|
| 118 |
-
if time.monotonic() - last_preview_at < PREVIEW_INTERVAL:
|
| 119 |
-
continue
|
| 120 |
-
last_preview_at = time.monotonic()
|
| 121 |
-
emit(name, {**data, "cached": True} if name in ("start", "done") else data)
|
| 122 |
-
if name != "token" or rate <= 0:
|
| 123 |
-
continue
|
| 124 |
-
sent += 1
|
| 125 |
-
ahead = started + sent / rate - time.monotonic()
|
| 126 |
-
if ahead > 0:
|
| 127 |
-
await asyncio.sleep(ahead)
|
| 128 |
-
|
| 129 |
-
async def warm_presets(self, presets_path: Path | None) -> None:
|
| 130 |
-
await asyncio.get_running_loop().run_in_executor(self._executor, self.refiner.load)
|
| 131 |
-
if presets_path is None:
|
| 132 |
-
return
|
| 133 |
-
presets = json.loads(presets_path.read_text(encoding="utf-8"))
|
| 134 |
-
pending = [FixRequest(preset["filename"], preset["before"]) for preset in presets]
|
| 135 |
-
while pending:
|
| 136 |
-
request = pending[0]
|
| 137 |
-
if self.cache.get(self.cache_key(request)) is not None:
|
| 138 |
-
pending.pop(0)
|
| 139 |
-
continue
|
| 140 |
-
await self.queue.wait_idle()
|
| 141 |
-
ticket = self.queue.join(background=True)
|
| 142 |
-
try:
|
| 143 |
-
await self.execute(request, ticket, lambda name, data: None, source_kind="examples")
|
| 144 |
-
except Cancelled:
|
| 145 |
-
log.info("warm-up yielded to a visitor")
|
| 146 |
-
continue
|
| 147 |
-
pending.pop(0)
|
| 148 |
-
log.info("warmed %d of %d examples", len(presets) - len(pending), len(presets))
|
| 149 |
-
|
| 150 |
-
def close(self) -> None:
|
| 151 |
-
self.queue.cancel_all()
|
| 152 |
-
self._executor.shutdown(wait=False, cancel_futures=True)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
server/telemetry.py
CHANGED
|
@@ -1,18 +1,18 @@
|
|
| 1 |
-
"""Aggregate, code-free
|
| 2 |
|
| 3 |
from __future__ import annotations
|
| 4 |
|
| 5 |
import asyncio
|
|
|
|
|
|
|
|
|
|
| 6 |
import hashlib
|
| 7 |
import hmac
|
| 8 |
import json
|
| 9 |
import logging
|
|
|
|
| 10 |
import secrets
|
| 11 |
import time
|
| 12 |
-
from collections import Counter, OrderedDict
|
| 13 |
-
from collections.abc import Iterable, Mapping
|
| 14 |
-
from datetime import datetime, timezone
|
| 15 |
-
from pathlib import Path
|
| 16 |
from typing import Annotated, Any, Literal, Protocol
|
| 17 |
|
| 18 |
from pydantic import BaseModel, ConfigDict, Field
|
|
@@ -21,14 +21,18 @@ log = logging.getLogger("space.telemetry")
|
|
| 21 |
|
| 22 |
DIGEST_HEX = 16
|
| 23 |
SEEN_LIMIT = 100_000
|
| 24 |
-
|
| 25 |
-
STATS_VERSION = 1
|
| 26 |
STATS_FILE = "stats.json"
|
| 27 |
-
PRIVATE_SOURCES = frozenset({"folder", "snippet"})
|
| 28 |
|
| 29 |
SQUIGGLE_KINDS = (
|
| 30 |
-
"deep_nesting",
|
| 31 |
-
"
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 32 |
)
|
| 33 |
LOC_BUCKETS = ("0-50", "50-100", "100-300", "300-1000", "1000+")
|
| 34 |
CODE_EXTENSIONS = frozenset({
|
|
@@ -42,13 +46,14 @@ MS_DECADES = 4
|
|
| 42 |
|
| 43 |
|
| 44 |
def _two_figures(value: float) -> int:
|
| 45 |
-
"""
|
| 46 |
digits = len(str(int(value)))
|
| 47 |
return int(round(value, 2 - digits))
|
| 48 |
|
| 49 |
|
| 50 |
MS_EDGES = tuple(sorted({
|
| 51 |
-
_two_figures(10 ** (step / MS_STEPS_PER_DECADE))
|
|
|
|
| 52 |
}))
|
| 53 |
MS_OVERFLOW = f"{MS_EDGES[-1]}+"
|
| 54 |
MS_KEYS = frozenset(str(edge) for edge in MS_EDGES) | {MS_OVERFLOW}
|
|
@@ -60,22 +65,20 @@ SquiggleKind = Literal[
|
|
| 60 |
]
|
| 61 |
Count = Annotated[int, Field(ge=0, le=10_000)]
|
| 62 |
|
| 63 |
-
COUNTERS = (
|
| 64 |
-
"files_scored", "files_cleaned", "fix_runs", "scan_batches",
|
| 65 |
-
"score_scored_sum", "score_before_sum", "score_after_sum",
|
| 66 |
-
"prs_opened", "prs_merged", "prs_closed_unmerged",
|
| 67 |
-
)
|
| 68 |
TALLIES = (
|
| 69 |
-
"scored_by_ext",
|
| 70 |
-
"
|
|
|
|
|
|
|
|
|
|
|
|
|
| 71 |
)
|
| 72 |
TALLY_KEYS: dict[str, frozenset[str]] = {
|
| 73 |
"scored_by_ext": CODE_EXTENSIONS | {OTHER_EXT},
|
| 74 |
-
"cleaned_by_ext": CODE_EXTENSIONS | {OTHER_EXT},
|
| 75 |
"score_sum_by_ext": CODE_EXTENSIONS | {OTHER_EXT},
|
| 76 |
"scored_by_loc_bucket": frozenset(LOC_BUCKETS),
|
| 77 |
-
"
|
| 78 |
-
"squiggles_resolved_by_kind": frozenset(SQUIGGLE_KINDS),
|
| 79 |
"score_hist": SCORE_KEYS,
|
| 80 |
"score_ms_hist": MS_KEYS,
|
| 81 |
}
|
|
@@ -87,7 +90,7 @@ class ScanFile(BaseModel):
|
|
| 87 |
digest16: str = Field(pattern=r"^[0-9a-f]{16}$")
|
| 88 |
ext: str = Field("", max_length=8)
|
| 89 |
loc_bucket: Literal["0-50", "50-100", "100-300", "300-1000", "1000+"]
|
| 90 |
-
|
| 91 |
squiggles: dict[SquiggleKind, Count] = Field(default_factory=dict)
|
| 92 |
score_ms: int | None = Field(None, ge=0, le=60_000)
|
| 93 |
|
|
@@ -100,35 +103,23 @@ class ScanBody(BaseModel):
|
|
| 100 |
|
| 101 |
|
| 102 |
def extension_of(filename: str) -> str:
|
| 103 |
-
"""Lower-case extension of the base name,
|
| 104 |
base = filename.rsplit("/", 1)[-1]
|
| 105 |
stem, dot, ext = base.rpartition(".")
|
| 106 |
return ext.lower() if dot and stem else ""
|
| 107 |
|
| 108 |
|
| 109 |
def known_ext(ext: str) -> str:
|
| 110 |
-
"""
|
| 111 |
return ext if ext in CODE_EXTENSIONS else OTHER_EXT
|
| 112 |
|
| 113 |
|
| 114 |
def content_digest(ext: str, content: str) -> str:
|
| 115 |
-
"""First 64 bits of SHA-256(ext + NUL + content),
|
| 116 |
payload = f"{ext}\0{content}".encode("utf-8", errors="replace")
|
| 117 |
return hashlib.sha256(payload).hexdigest()[:DIGEST_HEX]
|
| 118 |
|
| 119 |
|
| 120 |
-
def pr_key(repo_id: int, pr_number: int) -> str:
|
| 121 |
-
return hashlib.sha256(f"{repo_id}:{pr_number}".encode()).hexdigest()[:DIGEST_HEX]
|
| 122 |
-
|
| 123 |
-
|
| 124 |
-
def kind_counts(squiggles: Iterable[Mapping[str, Any]]) -> Counter[str]:
|
| 125 |
-
return Counter(item["kind"] for item in squiggles if item.get("kind") in SQUIGGLE_KINDS)
|
| 126 |
-
|
| 127 |
-
|
| 128 |
-
def to_points(score: float) -> int:
|
| 129 |
-
return round(score * 100)
|
| 130 |
-
|
| 131 |
-
|
| 132 |
class DigestSet:
|
| 133 |
"""Rolling set of the most recent 64-bit digests, oldest dropped first."""
|
| 134 |
|
|
@@ -154,7 +145,6 @@ def empty_stats() -> dict[str, Any]:
|
|
| 154 |
stats: dict[str, Any] = {"version": STATS_VERSION}
|
| 155 |
stats.update({name: 0 for name in COUNTERS})
|
| 156 |
stats.update({name: {} for name in TALLIES})
|
| 157 |
-
stats["open_pr_keys"] = []
|
| 158 |
return stats
|
| 159 |
|
| 160 |
|
|
@@ -163,7 +153,7 @@ def _clean_count(value: Any) -> int:
|
|
| 163 |
|
| 164 |
|
| 165 |
def sanitize_stats(raw: Any) -> dict[str, Any]:
|
| 166 |
-
"""Keep only known counters
|
| 167 |
stats = empty_stats()
|
| 168 |
if not isinstance(raw, dict):
|
| 169 |
return stats
|
|
@@ -171,13 +161,11 @@ def sanitize_stats(raw: Any) -> dict[str, Any]:
|
|
| 171 |
stats[name] = _clean_count(raw.get(name))
|
| 172 |
for name in TALLIES:
|
| 173 |
tally = raw.get(name)
|
|
|
|
|
|
|
| 174 |
if isinstance(tally, dict):
|
| 175 |
allowed = TALLY_KEYS[name]
|
| 176 |
-
stats[name] = {key: _clean_count(
|
| 177 |
-
keys = raw.get("open_pr_keys")
|
| 178 |
-
if isinstance(keys, list):
|
| 179 |
-
digests = [key for key in keys if isinstance(key, str) and len(key) == DIGEST_HEX]
|
| 180 |
-
stats["open_pr_keys"] = [key for key in digests if all(ch in "0123456789abcdef" for ch in key)]
|
| 181 |
return stats
|
| 182 |
|
| 183 |
|
|
@@ -218,15 +206,13 @@ def median_ms(tally: Mapping[str, int]) -> int | None:
|
|
| 218 |
|
| 219 |
|
| 220 |
class Telemetry:
|
| 221 |
-
"""Aggregate counters. Mutated only on the event loop;
|
| 222 |
|
| 223 |
def __init__(self, excluded: Iterable[str] = ()) -> None:
|
| 224 |
self.stats = empty_stats()
|
| 225 |
self.dirty = False
|
| 226 |
self.excluded = frozenset(excluded)
|
| 227 |
self.seen_scored = DigestSet()
|
| 228 |
-
self.seen_cleaned = DigestSet()
|
| 229 |
-
self._open_prs: set[str] = set()
|
| 230 |
|
| 231 |
def merge(self, raw: Any) -> None:
|
| 232 |
loaded = sanitize_stats(raw)
|
|
@@ -235,8 +221,6 @@ class Telemetry:
|
|
| 235 |
for name in TALLIES:
|
| 236 |
for key, value in loaded[name].items():
|
| 237 |
_bump(self.stats[name], key, value)
|
| 238 |
-
self._open_prs.update(loaded["open_pr_keys"])
|
| 239 |
-
self._trim_open_prs()
|
| 240 |
|
| 241 |
def record_scan(self, body: ScanBody) -> int:
|
| 242 |
if body.source_kind == "examples":
|
|
@@ -247,105 +231,49 @@ class Telemetry:
|
|
| 247 |
continue
|
| 248 |
counted += 1
|
| 249 |
ext = known_ext(item.ext)
|
| 250 |
-
self.stats["score_scored_sum"] += item.
|
| 251 |
_bump(self.stats["scored_by_ext"], ext)
|
| 252 |
-
_bump(self.stats["score_sum_by_ext"], ext, item.
|
| 253 |
_bump(self.stats["scored_by_loc_bucket"], item.loc_bucket)
|
| 254 |
-
_bump(self.stats["score_hist"], str(item.
|
| 255 |
if item.score_ms is not None:
|
| 256 |
_bump(self.stats["score_ms_hist"], ms_bucket(item.score_ms))
|
| 257 |
for kind, amount in item.squiggles.items():
|
| 258 |
-
_bump(self.stats["
|
| 259 |
self.stats["files_scored"] += counted
|
| 260 |
self.stats["scan_batches"] += 1
|
| 261 |
self.dirty = True
|
| 262 |
return counted
|
| 263 |
|
| 264 |
-
def record_clean(self, filename: str, content: str, done: Mapping[str, Any]) -> bool:
|
| 265 |
-
"""Count one accepted refiner run; the file counts once however often it is cleaned."""
|
| 266 |
-
ext = extension_of(filename)
|
| 267 |
-
digest = content_digest(ext, content)
|
| 268 |
-
if not done.get("accepted") or digest in self.excluded:
|
| 269 |
-
return False
|
| 270 |
-
self.stats["fix_runs"] += 1
|
| 271 |
-
self.dirty = True
|
| 272 |
-
if not self.seen_cleaned.add(digest):
|
| 273 |
-
return False
|
| 274 |
-
before, after = done["before_hygiene"], done["after_hygiene"]
|
| 275 |
-
self.stats["files_cleaned"] += 1
|
| 276 |
-
self.stats["score_before_sum"] += to_points(before["score"])
|
| 277 |
-
self.stats["score_after_sum"] += to_points(after["score"])
|
| 278 |
-
_bump(self.stats["cleaned_by_ext"], known_ext(ext))
|
| 279 |
-
remaining = kind_counts(after["squiggles"])
|
| 280 |
-
for kind, amount in kind_counts(before["squiggles"]).items():
|
| 281 |
-
_bump(self.stats["squiggles_resolved_by_kind"], kind, amount - remaining[kind])
|
| 282 |
-
return True
|
| 283 |
-
|
| 284 |
-
def record_pr_opened(self, key: str) -> bool:
|
| 285 |
-
if key in self._open_prs:
|
| 286 |
-
return False
|
| 287 |
-
self._open_prs.add(key)
|
| 288 |
-
self._trim_open_prs()
|
| 289 |
-
self.stats["prs_opened"] += 1
|
| 290 |
-
self.dirty = True
|
| 291 |
-
return True
|
| 292 |
-
|
| 293 |
-
def record_pr_closed(self, key: str, merged: bool) -> bool:
|
| 294 |
-
if key not in self._open_prs:
|
| 295 |
-
return False
|
| 296 |
-
self._open_prs.discard(key)
|
| 297 |
-
self.stats["prs_merged" if merged else "prs_closed_unmerged"] += 1
|
| 298 |
-
self.dirty = True
|
| 299 |
-
return True
|
| 300 |
-
|
| 301 |
-
def _trim_open_prs(self) -> None:
|
| 302 |
-
while len(self._open_prs) > OPEN_PR_LIMIT:
|
| 303 |
-
self._open_prs.pop()
|
| 304 |
-
|
| 305 |
def snapshot(self) -> str:
|
| 306 |
-
stats = {**self.stats
|
| 307 |
stats["updated_at"] = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
|
| 308 |
return json.dumps(stats, indent=2, sort_keys=True) + "\n"
|
| 309 |
|
| 310 |
def _by_ext(self) -> dict[str, dict[str, Any]]:
|
| 311 |
stats = self.stats
|
| 312 |
-
extensions = sorted(set(stats["scored_by_ext"]) | set(stats["cleaned_by_ext"]))
|
| 313 |
rows = {}
|
| 314 |
-
for ext in
|
| 315 |
scored = stats["scored_by_ext"].get(ext, 0)
|
| 316 |
rows[ext] = {
|
| 317 |
"scored": scored,
|
| 318 |
-
"cleaned": stats["cleaned_by_ext"].get(ext, 0),
|
| 319 |
"avg_score": _mean(stats["score_sum_by_ext"].get(ext, 0), scored),
|
| 320 |
}
|
| 321 |
return rows
|
| 322 |
|
| 323 |
def public(self) -> dict[str, Any]:
|
| 324 |
stats = self.stats
|
| 325 |
-
|
| 326 |
-
merged = stats["prs_merged"]
|
| 327 |
-
decided = merged + stats["prs_closed_unmerged"]
|
| 328 |
-
resolved = stats["squiggles_resolved_by_kind"]
|
| 329 |
return {
|
| 330 |
"files_scored": stats["files_scored"],
|
| 331 |
-
"
|
| 332 |
-
"fix_runs": stats["fix_runs"],
|
| 333 |
-
"issues_fixed": sum(resolved.values()),
|
| 334 |
"avg_score_scored": _mean(stats["score_scored_sum"], stats["files_scored"]),
|
| 335 |
-
"avg_score_before": _mean(stats["score_before_sum"], cleaned),
|
| 336 |
-
"avg_score_after": _mean(stats["score_after_sum"], cleaned),
|
| 337 |
"median_score": median_score(stats["score_hist"]),
|
| 338 |
"score_ms_median": median_ms(stats["score_ms_hist"]),
|
| 339 |
"score_ms_samples": sum(stats["score_ms_hist"].values()),
|
| 340 |
"by_ext": self._by_ext(),
|
| 341 |
"by_loc_bucket": dict(stats["scored_by_loc_bucket"]),
|
| 342 |
-
"
|
| 343 |
-
"squiggles_resolved_by_kind": dict(resolved),
|
| 344 |
-
"prs_opened": stats["prs_opened"],
|
| 345 |
-
"prs_merged": merged,
|
| 346 |
-
"prs_closed_unmerged": stats["prs_closed_unmerged"],
|
| 347 |
-
"prs_decided": decided,
|
| 348 |
-
"pr_merge_rate": round(merged / decided, 3) if decided else None,
|
| 349 |
}
|
| 350 |
|
| 351 |
|
|
@@ -415,7 +343,10 @@ class StatsStore:
|
|
| 415 |
|
| 416 |
try:
|
| 417 |
path = hf_hub_download(
|
| 418 |
-
self.dataset_repo,
|
|
|
|
|
|
|
|
|
|
| 419 |
force_download=True,
|
| 420 |
)
|
| 421 |
except EntryNotFoundError:
|
|
@@ -454,11 +385,14 @@ class SyncedFile:
|
|
| 454 |
self.store = store
|
| 455 |
self.model = model
|
| 456 |
|
|
|
|
|
|
|
|
|
|
| 457 |
async def load(self) -> None:
|
| 458 |
try:
|
| 459 |
loaded = await asyncio.to_thread(self.store.load)
|
| 460 |
except Exception as error:
|
| 461 |
-
|
| 462 |
return
|
| 463 |
self.model.merge(loaded)
|
| 464 |
|
|
@@ -473,4 +407,4 @@ class SyncedFile:
|
|
| 473 |
await asyncio.to_thread(self.store.save, text)
|
| 474 |
except Exception as error:
|
| 475 |
self.model.dirty = True
|
| 476 |
-
|
|
|
|
| 1 |
+
"""Aggregate, code-free scoring counters for the Space. See space/TELEMETRY.md."""
|
| 2 |
|
| 3 |
from __future__ import annotations
|
| 4 |
|
| 5 |
import asyncio
|
| 6 |
+
from collections import OrderedDict
|
| 7 |
+
from collections.abc import Iterable, Mapping
|
| 8 |
+
from datetime import datetime, timezone
|
| 9 |
import hashlib
|
| 10 |
import hmac
|
| 11 |
import json
|
| 12 |
import logging
|
| 13 |
+
from pathlib import Path
|
| 14 |
import secrets
|
| 15 |
import time
|
|
|
|
|
|
|
|
|
|
|
|
|
| 16 |
from typing import Annotated, Any, Literal, Protocol
|
| 17 |
|
| 18 |
from pydantic import BaseModel, ConfigDict, Field
|
|
|
|
| 21 |
|
| 22 |
DIGEST_HEX = 16
|
| 23 |
SEEN_LIMIT = 100_000
|
| 24 |
+
STATS_VERSION = 2
|
|
|
|
| 25 |
STATS_FILE = "stats.json"
|
|
|
|
| 26 |
|
| 27 |
SQUIGGLE_KINDS = (
|
| 28 |
+
"deep_nesting",
|
| 29 |
+
"poor_naming",
|
| 30 |
+
"noisy_comment",
|
| 31 |
+
"swallowed_error",
|
| 32 |
+
"dense_line",
|
| 33 |
+
"special_casing",
|
| 34 |
+
"duplicate_block",
|
| 35 |
+
"cramped_layout",
|
| 36 |
)
|
| 37 |
LOC_BUCKETS = ("0-50", "50-100", "100-300", "300-1000", "1000+")
|
| 38 |
CODE_EXTENSIONS = frozenset({
|
|
|
|
| 46 |
|
| 47 |
|
| 48 |
def _two_figures(value: float) -> int:
|
| 49 |
+
"""Round to two significant figures so bucket edges read as plain numbers."""
|
| 50 |
digits = len(str(int(value)))
|
| 51 |
return int(round(value, 2 - digits))
|
| 52 |
|
| 53 |
|
| 54 |
MS_EDGES = tuple(sorted({
|
| 55 |
+
_two_figures(10 ** (step / MS_STEPS_PER_DECADE))
|
| 56 |
+
for step in range(MS_STEPS_PER_DECADE * MS_DECADES + 1)
|
| 57 |
}))
|
| 58 |
MS_OVERFLOW = f"{MS_EDGES[-1]}+"
|
| 59 |
MS_KEYS = frozenset(str(edge) for edge in MS_EDGES) | {MS_OVERFLOW}
|
|
|
|
| 65 |
]
|
| 66 |
Count = Annotated[int, Field(ge=0, le=10_000)]
|
| 67 |
|
| 68 |
+
COUNTERS = ("files_scored", "scan_batches", "score_scored_sum")
|
|
|
|
|
|
|
|
|
|
|
|
|
| 69 |
TALLIES = (
|
| 70 |
+
"scored_by_ext",
|
| 71 |
+
"score_sum_by_ext",
|
| 72 |
+
"scored_by_loc_bucket",
|
| 73 |
+
"squiggles_by_kind",
|
| 74 |
+
"score_hist",
|
| 75 |
+
"score_ms_hist",
|
| 76 |
)
|
| 77 |
TALLY_KEYS: dict[str, frozenset[str]] = {
|
| 78 |
"scored_by_ext": CODE_EXTENSIONS | {OTHER_EXT},
|
|
|
|
| 79 |
"score_sum_by_ext": CODE_EXTENSIONS | {OTHER_EXT},
|
| 80 |
"scored_by_loc_bucket": frozenset(LOC_BUCKETS),
|
| 81 |
+
"squiggles_by_kind": frozenset(SQUIGGLE_KINDS),
|
|
|
|
| 82 |
"score_hist": SCORE_KEYS,
|
| 83 |
"score_ms_hist": MS_KEYS,
|
| 84 |
}
|
|
|
|
| 90 |
digest16: str = Field(pattern=r"^[0-9a-f]{16}$")
|
| 91 |
ext: str = Field("", max_length=8)
|
| 92 |
loc_bucket: Literal["0-50", "50-100", "100-300", "300-1000", "1000+"]
|
| 93 |
+
score: int = Field(ge=0, le=100)
|
| 94 |
squiggles: dict[SquiggleKind, Count] = Field(default_factory=dict)
|
| 95 |
score_ms: int | None = Field(None, ge=0, le=60_000)
|
| 96 |
|
|
|
|
| 103 |
|
| 104 |
|
| 105 |
def extension_of(filename: str) -> str:
|
| 106 |
+
"""Lower-case extension of the base name, matching the browser computation."""
|
| 107 |
base = filename.rsplit("/", 1)[-1]
|
| 108 |
stem, dot, ext = base.rpartition(".")
|
| 109 |
return ext.lower() if dot and stem else ""
|
| 110 |
|
| 111 |
|
| 112 |
def known_ext(ext: str) -> str:
|
| 113 |
+
"""Bounded tally key: an extension from the fixed code set, or `other`."""
|
| 114 |
return ext if ext in CODE_EXTENSIONS else OTHER_EXT
|
| 115 |
|
| 116 |
|
| 117 |
def content_digest(ext: str, content: str) -> str:
|
| 118 |
+
"""First 64 bits of SHA-256(ext + NUL + content), matching the browser digest."""
|
| 119 |
payload = f"{ext}\0{content}".encode("utf-8", errors="replace")
|
| 120 |
return hashlib.sha256(payload).hexdigest()[:DIGEST_HEX]
|
| 121 |
|
| 122 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 123 |
class DigestSet:
|
| 124 |
"""Rolling set of the most recent 64-bit digests, oldest dropped first."""
|
| 125 |
|
|
|
|
| 145 |
stats: dict[str, Any] = {"version": STATS_VERSION}
|
| 146 |
stats.update({name: 0 for name in COUNTERS})
|
| 147 |
stats.update({name: {} for name in TALLIES})
|
|
|
|
| 148 |
return stats
|
| 149 |
|
| 150 |
|
|
|
|
| 153 |
|
| 154 |
|
| 155 |
def sanitize_stats(raw: Any) -> dict[str, Any]:
|
| 156 |
+
"""Keep only known scoring counters and bounded tally keys from a stats file."""
|
| 157 |
stats = empty_stats()
|
| 158 |
if not isinstance(raw, dict):
|
| 159 |
return stats
|
|
|
|
| 161 |
stats[name] = _clean_count(raw.get(name))
|
| 162 |
for name in TALLIES:
|
| 163 |
tally = raw.get(name)
|
| 164 |
+
if name == "squiggles_by_kind" and not isinstance(tally, dict):
|
| 165 |
+
tally = raw.get("squiggles_before_by_kind")
|
| 166 |
if isinstance(tally, dict):
|
| 167 |
allowed = TALLY_KEYS[name]
|
| 168 |
+
stats[name] = {key: _clean_count(val) for key, val in tally.items() if key in allowed}
|
|
|
|
|
|
|
|
|
|
|
|
|
| 169 |
return stats
|
| 170 |
|
| 171 |
|
|
|
|
| 206 |
|
| 207 |
|
| 208 |
class Telemetry:
|
| 209 |
+
"""Aggregate scoring counters. Mutated only on the event loop; zero code, names or IPs."""
|
| 210 |
|
| 211 |
def __init__(self, excluded: Iterable[str] = ()) -> None:
|
| 212 |
self.stats = empty_stats()
|
| 213 |
self.dirty = False
|
| 214 |
self.excluded = frozenset(excluded)
|
| 215 |
self.seen_scored = DigestSet()
|
|
|
|
|
|
|
| 216 |
|
| 217 |
def merge(self, raw: Any) -> None:
|
| 218 |
loaded = sanitize_stats(raw)
|
|
|
|
| 221 |
for name in TALLIES:
|
| 222 |
for key, value in loaded[name].items():
|
| 223 |
_bump(self.stats[name], key, value)
|
|
|
|
|
|
|
| 224 |
|
| 225 |
def record_scan(self, body: ScanBody) -> int:
|
| 226 |
if body.source_kind == "examples":
|
|
|
|
| 231 |
continue
|
| 232 |
counted += 1
|
| 233 |
ext = known_ext(item.ext)
|
| 234 |
+
self.stats["score_scored_sum"] += item.score
|
| 235 |
_bump(self.stats["scored_by_ext"], ext)
|
| 236 |
+
_bump(self.stats["score_sum_by_ext"], ext, item.score)
|
| 237 |
_bump(self.stats["scored_by_loc_bucket"], item.loc_bucket)
|
| 238 |
+
_bump(self.stats["score_hist"], str(item.score))
|
| 239 |
if item.score_ms is not None:
|
| 240 |
_bump(self.stats["score_ms_hist"], ms_bucket(item.score_ms))
|
| 241 |
for kind, amount in item.squiggles.items():
|
| 242 |
+
_bump(self.stats["squiggles_by_kind"], kind, amount)
|
| 243 |
self.stats["files_scored"] += counted
|
| 244 |
self.stats["scan_batches"] += 1
|
| 245 |
self.dirty = True
|
| 246 |
return counted
|
| 247 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 248 |
def snapshot(self) -> str:
|
| 249 |
+
stats = {**self.stats}
|
| 250 |
stats["updated_at"] = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
|
| 251 |
return json.dumps(stats, indent=2, sort_keys=True) + "\n"
|
| 252 |
|
| 253 |
def _by_ext(self) -> dict[str, dict[str, Any]]:
|
| 254 |
stats = self.stats
|
|
|
|
| 255 |
rows = {}
|
| 256 |
+
for ext in sorted(stats["scored_by_ext"]):
|
| 257 |
scored = stats["scored_by_ext"].get(ext, 0)
|
| 258 |
rows[ext] = {
|
| 259 |
"scored": scored,
|
|
|
|
| 260 |
"avg_score": _mean(stats["score_sum_by_ext"].get(ext, 0), scored),
|
| 261 |
}
|
| 262 |
return rows
|
| 263 |
|
| 264 |
def public(self) -> dict[str, Any]:
|
| 265 |
stats = self.stats
|
| 266 |
+
squiggles = stats["squiggles_by_kind"]
|
|
|
|
|
|
|
|
|
|
| 267 |
return {
|
| 268 |
"files_scored": stats["files_scored"],
|
| 269 |
+
"issues_found": sum(squiggles.values()),
|
|
|
|
|
|
|
| 270 |
"avg_score_scored": _mean(stats["score_scored_sum"], stats["files_scored"]),
|
|
|
|
|
|
|
| 271 |
"median_score": median_score(stats["score_hist"]),
|
| 272 |
"score_ms_median": median_ms(stats["score_ms_hist"]),
|
| 273 |
"score_ms_samples": sum(stats["score_ms_hist"].values()),
|
| 274 |
"by_ext": self._by_ext(),
|
| 275 |
"by_loc_bucket": dict(stats["scored_by_loc_bucket"]),
|
| 276 |
+
"squiggles_by_kind": dict(squiggles),
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 277 |
}
|
| 278 |
|
| 279 |
|
|
|
|
| 343 |
|
| 344 |
try:
|
| 345 |
path = hf_hub_download(
|
| 346 |
+
self.dataset_repo,
|
| 347 |
+
self.filename,
|
| 348 |
+
repo_type="dataset",
|
| 349 |
+
token=self.token,
|
| 350 |
force_download=True,
|
| 351 |
)
|
| 352 |
except EntryNotFoundError:
|
|
|
|
| 385 |
self.store = store
|
| 386 |
self.model = model
|
| 387 |
|
| 388 |
+
def _warn(self, action: str, error: Exception) -> None:
|
| 389 |
+
log.warning("%s %s failed: %s", self.store.filename, action, type(error).__name__)
|
| 390 |
+
|
| 391 |
async def load(self) -> None:
|
| 392 |
try:
|
| 393 |
loaded = await asyncio.to_thread(self.store.load)
|
| 394 |
except Exception as error:
|
| 395 |
+
self._warn("load", error)
|
| 396 |
return
|
| 397 |
self.model.merge(loaded)
|
| 398 |
|
|
|
|
| 407 |
await asyncio.to_thread(self.store.save, text)
|
| 408 |
except Exception as error:
|
| 409 |
self.model.dirty = True
|
| 410 |
+
self._warn("sync", error)
|
server/vendored_guards.py
DELETED
|
@@ -1,163 +0,0 @@
|
|
| 1 |
-
# Vendored verbatim from prototype/train_browser_model.py (helpers and grade_patch_candidate thresholds).
|
| 2 |
-
from __future__ import annotations
|
| 3 |
-
|
| 4 |
-
import ast
|
| 5 |
-
import builtins
|
| 6 |
-
from pathlib import Path
|
| 7 |
-
import re
|
| 8 |
-
import symtable
|
| 9 |
-
|
| 10 |
-
_JS_METHOD_SKIP = {
|
| 11 |
-
"if",
|
| 12 |
-
"for",
|
| 13 |
-
"while",
|
| 14 |
-
"switch",
|
| 15 |
-
"catch",
|
| 16 |
-
"with",
|
| 17 |
-
"return",
|
| 18 |
-
"function",
|
| 19 |
-
"constructor",
|
| 20 |
-
"super",
|
| 21 |
-
}
|
| 22 |
-
_PY_BUILTINS = set(dir(builtins)) | {
|
| 23 |
-
"__file__",
|
| 24 |
-
"__name__",
|
| 25 |
-
"__doc__",
|
| 26 |
-
"__package__",
|
| 27 |
-
"__spec__",
|
| 28 |
-
"__annotations__",
|
| 29 |
-
}
|
| 30 |
-
_LOCAL_DECL_RE = re.compile(
|
| 31 |
-
r"(?:^\s*(?:export\s+)?(?:async\s+)?(?:function|class|fn|struct)\s+([A-Za-z_$][A-Za-z0-9_$]*)\b"
|
| 32 |
-
r"|^\s*func\s+(?:\([^)\n]+\)\s*)?([A-Za-z_$][A-Za-z0-9_$]*)\b"
|
| 33 |
-
r"|^\s{2,8}(?:private\s+|public\s+|protected\s+|readonly\s+|static\s+|async\s+|pub\s+)*"
|
| 34 |
-
r"(?:fn\s+)?([A-Za-z_$][A-Za-z0-9_$]*)(?:<[^>(\n]+>)?\s*\()",
|
| 35 |
-
re.MULTILINE,
|
| 36 |
-
)
|
| 37 |
-
_RECEIVER_CALL_RE = re.compile(r"\b(?:this|self)\.([A-Za-z_$][A-Za-z0-9_$]*)\s*\(")
|
| 38 |
-
_PY_DEF_RE = re.compile(r"^\s*(?:async\s+)?def\s+([A-Za-z_][A-Za-z0-9_]*)\s*\(", re.MULTILINE)
|
| 39 |
-
|
| 40 |
-
|
| 41 |
-
def infer_lang_from_name(filename: str) -> str:
|
| 42 |
-
ext = Path(filename.lower()).suffix
|
| 43 |
-
mapping = {
|
| 44 |
-
".py": "python",
|
| 45 |
-
".ts": "typescript",
|
| 46 |
-
".mts": "typescript",
|
| 47 |
-
".js": "javascript",
|
| 48 |
-
".mjs": "javascript",
|
| 49 |
-
".rs": "rust",
|
| 50 |
-
".go": "go",
|
| 51 |
-
}
|
| 52 |
-
return mapping.get(ext, "other")
|
| 53 |
-
|
| 54 |
-
|
| 55 |
-
def count_non_comment_lines(code: str) -> int:
|
| 56 |
-
total = 0
|
| 57 |
-
for raw in code.splitlines():
|
| 58 |
-
stripped = raw.strip()
|
| 59 |
-
if stripped and not stripped.startswith(("#", "//", "/*", "*")):
|
| 60 |
-
total += 1
|
| 61 |
-
return total
|
| 62 |
-
|
| 63 |
-
|
| 64 |
-
def _unresolved_py_globals(code: str) -> set[str]:
|
| 65 |
-
try:
|
| 66 |
-
top = symtable.symtable(code, "<mem>", "exec")
|
| 67 |
-
except SyntaxError:
|
| 68 |
-
return set()
|
| 69 |
-
top_defs = {
|
| 70 |
-
sym.get_name()
|
| 71 |
-
for sym in top.get_symbols()
|
| 72 |
-
if sym.is_assigned() or sym.is_imported() or sym.is_namespace()
|
| 73 |
-
}
|
| 74 |
-
missing = {
|
| 75 |
-
sym.get_name()
|
| 76 |
-
for sym in top.get_symbols()
|
| 77 |
-
if sym.is_referenced()
|
| 78 |
-
and sym.get_name() not in top_defs
|
| 79 |
-
and sym.get_name() not in _PY_BUILTINS
|
| 80 |
-
}
|
| 81 |
-
stack = list(top.get_children())
|
| 82 |
-
while stack:
|
| 83 |
-
tbl = stack.pop()
|
| 84 |
-
stack.extend(tbl.get_children())
|
| 85 |
-
if tbl.get_type() != "function":
|
| 86 |
-
continue
|
| 87 |
-
for sym in tbl.get_symbols():
|
| 88 |
-
if not (sym.is_global() and sym.is_referenced()):
|
| 89 |
-
continue
|
| 90 |
-
name = sym.get_name()
|
| 91 |
-
if name not in top_defs and name not in _PY_BUILTINS:
|
| 92 |
-
missing.add(name)
|
| 93 |
-
return missing
|
| 94 |
-
|
| 95 |
-
|
| 96 |
-
def _extract_local_defs(code: str) -> set[str]:
|
| 97 |
-
found: set[str] = set()
|
| 98 |
-
for match in _LOCAL_DECL_RE.finditer(code):
|
| 99 |
-
sym = match.group(1) or match.group(2) or match.group(3) or ""
|
| 100 |
-
if sym and sym not in _JS_METHOD_SKIP:
|
| 101 |
-
found.add(sym)
|
| 102 |
-
return found
|
| 103 |
-
|
| 104 |
-
|
| 105 |
-
def check_reference_integrity(before_code: str, after_code: str, filename: str) -> bool:
|
| 106 |
-
lang = infer_lang_from_name(filename)
|
| 107 |
-
if lang == "python":
|
| 108 |
-
if _unresolved_py_globals(after_code) - _unresolved_py_globals(before_code):
|
| 109 |
-
return False
|
| 110 |
-
py_defs_b = set(_PY_DEF_RE.findall(before_code))
|
| 111 |
-
py_defs_a = set(_PY_DEF_RE.findall(after_code))
|
| 112 |
-
self_b = set(_RECEIVER_CALL_RE.findall(before_code))
|
| 113 |
-
self_a = set(_RECEIVER_CALL_RE.findall(after_code))
|
| 114 |
-
if (py_defs_b - py_defs_a) & self_a:
|
| 115 |
-
return False
|
| 116 |
-
if (self_a - self_b) - py_defs_a:
|
| 117 |
-
return False
|
| 118 |
-
return True
|
| 119 |
-
|
| 120 |
-
if lang in ("javascript", "typescript", "rust", "go"):
|
| 121 |
-
defs_b = _extract_local_defs(before_code)
|
| 122 |
-
defs_a = _extract_local_defs(after_code)
|
| 123 |
-
for dropped in defs_b - defs_a:
|
| 124 |
-
if re.search(rf"(?:^|[^.\w$]){re.escape(dropped)}\s*\(", after_code):
|
| 125 |
-
return False
|
| 126 |
-
recv_b = set(_RECEIVER_CALL_RE.findall(before_code))
|
| 127 |
-
recv_a = set(_RECEIVER_CALL_RE.findall(after_code))
|
| 128 |
-
if (recv_a - recv_b) - defs_a:
|
| 129 |
-
return False
|
| 130 |
-
return True
|
| 131 |
-
|
| 132 |
-
|
| 133 |
-
def check_basic_syntax(code: str, filename: str) -> bool:
|
| 134 |
-
if not code.strip():
|
| 135 |
-
return False
|
| 136 |
-
lang = infer_lang_from_name(filename)
|
| 137 |
-
if lang == "python":
|
| 138 |
-
try:
|
| 139 |
-
ast.parse(code)
|
| 140 |
-
return True
|
| 141 |
-
except SyntaxError:
|
| 142 |
-
return False
|
| 143 |
-
if lang in ("typescript", "javascript", "rust", "go"):
|
| 144 |
-
return code.count("{") == code.count("}") and code.count("(") == code.count(")")
|
| 145 |
-
return True
|
| 146 |
-
|
| 147 |
-
|
| 148 |
-
def is_gutted(before_code: str, updated: str) -> bool:
|
| 149 |
-
orig_code_n = count_non_comment_lines(before_code)
|
| 150 |
-
new_code_n = count_non_comment_lines(updated)
|
| 151 |
-
if orig_code_n >= 1 and new_code_n == 0:
|
| 152 |
-
return True
|
| 153 |
-
if orig_code_n >= 10 and new_code_n < max(5, int(orig_code_n * 0.55)):
|
| 154 |
-
return True
|
| 155 |
-
return False
|
| 156 |
-
|
| 157 |
-
|
| 158 |
-
def is_bloated(before_code: str, updated: str) -> bool:
|
| 159 |
-
before_nonempty = sum(1 for ln in before_code.splitlines() if ln.strip())
|
| 160 |
-
after_nonempty = sum(1 for ln in updated.splitlines() if ln.strip())
|
| 161 |
-
if after_nonempty > max(40, int(before_nonempty * 1.60)):
|
| 162 |
-
return True
|
| 163 |
-
return False
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
server/vendored_pipeline.py
DELETED
|
@@ -1,278 +0,0 @@
|
|
| 1 |
-
# Vendored verbatim from prototype/pipeline.py; keep byte-identical (tests/test_vendored.py).
|
| 2 |
-
from __future__ import annotations
|
| 3 |
-
|
| 4 |
-
import difflib
|
| 5 |
-
import json
|
| 6 |
-
import re
|
| 7 |
-
from typing import Any
|
| 8 |
-
|
| 9 |
-
TASK_INSTRUCTIONS = """\
|
| 10 |
-
Refactor the file to resolve every squiggle in the codehygiene response while preserving exact runtime behavior and non-squiggled public/exported function, class, and method names.
|
| 11 |
-
Keep edits minimal and idiomatic; do not add redundant comments, wrapper ceremony, or line bloat.
|
| 12 |
-
Wrap dense lines under 88 columns and rename cryptic 1-2 character identifiers, ceremonial *Helper/*Manager/*Factory names, or magic-index expressions flagged by poor_naming into clear domain names across their scope.
|
| 13 |
-
Return ONLY one or more SEARCH/REPLACE blocks in this exact format (no prose outside blocks):
|
| 14 |
-
|
| 15 |
-
<<<<<<< SEARCH
|
| 16 |
-
exact existing lines from the file
|
| 17 |
-
=======
|
| 18 |
-
replacement lines
|
| 19 |
-
>>>>>>> REPLACE"""
|
| 20 |
-
|
| 21 |
-
_PATCH_BLOCK_RE = re.compile(
|
| 22 |
-
r"<<<<<<< SEARCH\r?\n(.*?)\r?\n=======\r?\n(.*?)>>>>>>> REPLACE",
|
| 23 |
-
re.DOTALL,
|
| 24 |
-
)
|
| 25 |
-
|
| 26 |
-
|
| 27 |
-
def enrich_hygiene_with_source_lines(
|
| 28 |
-
content: str,
|
| 29 |
-
hygiene_response: dict[str, Any],
|
| 30 |
-
) -> dict[str, Any]:
|
| 31 |
-
src_lines = content.replace("\r\n", "\n").split("\n")
|
| 32 |
-
enriched_sq: list[dict[str, Any]] = []
|
| 33 |
-
for sq in hygiene_response.get("squiggles", []):
|
| 34 |
-
line_no = int(sq.get("line", 1))
|
| 35 |
-
src_line = src_lines[line_no - 1] if 1 <= line_no <= len(src_lines) else ""
|
| 36 |
-
enriched_sq.append(
|
| 37 |
-
{
|
| 38 |
-
"line": line_no,
|
| 39 |
-
"source_line": src_line,
|
| 40 |
-
"col_start": sq.get("col_start", 0),
|
| 41 |
-
"col_end": sq.get("col_end", len(src_line)),
|
| 42 |
-
"probability": sq.get("probability", 1.0),
|
| 43 |
-
"kind": sq.get("kind", ""),
|
| 44 |
-
"message": sq.get("message", ""),
|
| 45 |
-
}
|
| 46 |
-
)
|
| 47 |
-
return {
|
| 48 |
-
"score": hygiene_response.get("score", 0.0),
|
| 49 |
-
"rl_hygiene": hygiene_response.get("rl_hygiene", 0.0),
|
| 50 |
-
"scores": hygiene_response.get("scores", {}),
|
| 51 |
-
"archetype": hygiene_response.get("archetype", ""),
|
| 52 |
-
"squiggles": enriched_sq,
|
| 53 |
-
}
|
| 54 |
-
|
| 55 |
-
|
| 56 |
-
def build_user_context(
|
| 57 |
-
content: str,
|
| 58 |
-
hygiene_response: dict[str, Any],
|
| 59 |
-
filename: str = "",
|
| 60 |
-
) -> str:
|
| 61 |
-
file_line = f"File: {filename}\n\n" if filename else ""
|
| 62 |
-
enriched = enrich_hygiene_with_source_lines(content, hygiene_response)
|
| 63 |
-
hygiene_text = json.dumps(enriched, indent=2)
|
| 64 |
-
return (
|
| 65 |
-
f"{file_line}"
|
| 66 |
-
f"<file_content>\n{content}\n</file_content>\n\n"
|
| 67 |
-
f"<codehygiene_response>\n{hygiene_text}\n</codehygiene_response>"
|
| 68 |
-
)
|
| 69 |
-
|
| 70 |
-
|
| 71 |
-
def build_inference_messages(
|
| 72 |
-
content: str,
|
| 73 |
-
hygiene_response: dict[str, Any],
|
| 74 |
-
filename: str = "",
|
| 75 |
-
) -> list[dict[str, str]]:
|
| 76 |
-
"""Build the exact [system, user] message list used for both teacher API and browser LLM."""
|
| 77 |
-
return [
|
| 78 |
-
{"role": "system", "content": TASK_INSTRUCTIONS},
|
| 79 |
-
{"role": "user", "content": build_user_context(content, hygiene_response, filename)},
|
| 80 |
-
]
|
| 81 |
-
|
| 82 |
-
|
| 83 |
-
def _strip_single_trailing_newline(text: str) -> str:
|
| 84 |
-
if text.endswith("\r\n"):
|
| 85 |
-
return text[:-2]
|
| 86 |
-
if text.endswith("\n"):
|
| 87 |
-
return text[:-1]
|
| 88 |
-
return text
|
| 89 |
-
|
| 90 |
-
|
| 91 |
-
def parse_changes(response_text: str) -> list[tuple[str, str]]:
|
| 92 |
-
blocks: list[tuple[str, str]] = []
|
| 93 |
-
for match in _PATCH_BLOCK_RE.finditer(response_text):
|
| 94 |
-
search_part = match.group(1).replace("\r\n", "\n")
|
| 95 |
-
replace_part = _strip_single_trailing_newline(match.group(2)).replace("\r\n", "\n")
|
| 96 |
-
blocks.append((search_part, replace_part))
|
| 97 |
-
return blocks
|
| 98 |
-
|
| 99 |
-
|
| 100 |
-
def _replace_rstrip_lines(source: str, search: str, replace: str) -> str | None:
|
| 101 |
-
src_lines = source.split("\n")
|
| 102 |
-
needle = [ln.rstrip() for ln in search.split("\n")]
|
| 103 |
-
span = len(needle)
|
| 104 |
-
if span == 0 or span > len(src_lines):
|
| 105 |
-
return None
|
| 106 |
-
|
| 107 |
-
matches = [
|
| 108 |
-
idx
|
| 109 |
-
for idx in range(len(src_lines) - span + 1)
|
| 110 |
-
if [ln.rstrip() for ln in src_lines[idx : idx + span]] == needle
|
| 111 |
-
]
|
| 112 |
-
if len(matches) != 1:
|
| 113 |
-
return None
|
| 114 |
-
|
| 115 |
-
pos = matches[0]
|
| 116 |
-
repl_lines = replace.split("\n") if replace else []
|
| 117 |
-
merged = src_lines[:pos] + repl_lines + src_lines[pos + span :]
|
| 118 |
-
return "\n".join(merged)
|
| 119 |
-
|
| 120 |
-
|
| 121 |
-
def _leading_spaces(line: str) -> int:
|
| 122 |
-
return len(line) - len(line.lstrip(" "))
|
| 123 |
-
|
| 124 |
-
|
| 125 |
-
def _shift_indent(text: str, delta: int) -> str:
|
| 126 |
-
if not text or delta == 0:
|
| 127 |
-
return text
|
| 128 |
-
shifted: list[str] = []
|
| 129 |
-
for raw in text.split("\n"):
|
| 130 |
-
if not raw.strip():
|
| 131 |
-
shifted.append("")
|
| 132 |
-
elif delta > 0:
|
| 133 |
-
shifted.append((" " * delta) + raw)
|
| 134 |
-
else:
|
| 135 |
-
trim = min(-delta, _leading_spaces(raw))
|
| 136 |
-
shifted.append(raw[trim:])
|
| 137 |
-
return "\n".join(shifted)
|
| 138 |
-
|
| 139 |
-
|
| 140 |
-
def _replace_stripped_unique(source: str, search: str, replace: str) -> str | None:
|
| 141 |
-
src_lines = source.split("\n")
|
| 142 |
-
s_lines = search.split("\n")
|
| 143 |
-
needle = [ln.strip() for ln in s_lines]
|
| 144 |
-
span = len(needle)
|
| 145 |
-
if span == 0 or span > len(src_lines) or not any(needle):
|
| 146 |
-
return None
|
| 147 |
-
if span == 1 and len(needle[0]) < 14:
|
| 148 |
-
return None
|
| 149 |
-
|
| 150 |
-
matches = [
|
| 151 |
-
idx
|
| 152 |
-
for idx in range(len(src_lines) - span + 1)
|
| 153 |
-
if [ln.strip() for ln in src_lines[idx : idx + span]] == needle
|
| 154 |
-
]
|
| 155 |
-
if len(matches) != 1:
|
| 156 |
-
return None
|
| 157 |
-
|
| 158 |
-
pos = matches[0]
|
| 159 |
-
delta = 0
|
| 160 |
-
for src_ln, srch_ln in zip(src_lines[pos : pos + span], s_lines):
|
| 161 |
-
if src_ln.strip() and srch_ln.strip() and "\t" not in src_ln and "\t" not in srch_ln:
|
| 162 |
-
delta = _leading_spaces(src_ln) - _leading_spaces(srch_ln)
|
| 163 |
-
break
|
| 164 |
-
|
| 165 |
-
adj_replace = _shift_indent(replace, delta)
|
| 166 |
-
repl_lines = adj_replace.split("\n") if adj_replace else []
|
| 167 |
-
merged = src_lines[:pos] + repl_lines + src_lines[pos + span :]
|
| 168 |
-
return "\n".join(merged)
|
| 169 |
-
|
| 170 |
-
|
| 171 |
-
def _score_fuzzy_candidates(
|
| 172 |
-
src_norm: list[str],
|
| 173 |
-
s_nonempty: list[str],
|
| 174 |
-
needle_text: str,
|
| 175 |
-
span: int,
|
| 176 |
-
) -> tuple[int, float, float]:
|
| 177 |
-
best_pos = -1
|
| 178 |
-
best_ratio = 0.0
|
| 179 |
-
runner_up = 0.0
|
| 180 |
-
if span < 1 or span > len(src_norm):
|
| 181 |
-
return best_pos, best_ratio, runner_up
|
| 182 |
-
|
| 183 |
-
for pos in range(len(src_norm) - span + 1):
|
| 184 |
-
window = src_norm[pos : pos + span]
|
| 185 |
-
c_nonempty = [ln for ln in window if ln]
|
| 186 |
-
if not c_nonempty:
|
| 187 |
-
continue
|
| 188 |
-
first_ok = difflib.SequenceMatcher(None, s_nonempty[0], c_nonempty[0]).ratio() >= 0.78
|
| 189 |
-
last_ok = difflib.SequenceMatcher(None, s_nonempty[-1], c_nonempty[-1]).ratio() >= 0.78
|
| 190 |
-
if not (first_ok and last_ok):
|
| 191 |
-
continue
|
| 192 |
-
|
| 193 |
-
cand_text = "\n".join(window)
|
| 194 |
-
matcher = difflib.SequenceMatcher(None, needle_text, cand_text)
|
| 195 |
-
if matcher.quick_ratio() < 0.88:
|
| 196 |
-
continue
|
| 197 |
-
ratio = matcher.ratio()
|
| 198 |
-
if ratio > best_ratio:
|
| 199 |
-
if best_pos >= 0 and abs(pos - best_pos) >= max(2, span // 2):
|
| 200 |
-
runner_up = max(runner_up, best_ratio)
|
| 201 |
-
best_ratio = ratio
|
| 202 |
-
best_pos = pos
|
| 203 |
-
elif abs(pos - best_pos) >= max(2, span // 2):
|
| 204 |
-
runner_up = max(runner_up, ratio)
|
| 205 |
-
return best_pos, best_ratio, runner_up
|
| 206 |
-
|
| 207 |
-
|
| 208 |
-
def _replace_fuzzy_unique(source: str, search: str, replace: str) -> str | None:
|
| 209 |
-
src_lines = source.split("\n")
|
| 210 |
-
s_lines = search.split("\n")
|
| 211 |
-
s_norm = [ln.strip() for ln in s_lines]
|
| 212 |
-
s_nonempty = [ln for ln in s_norm if ln]
|
| 213 |
-
span = len(s_norm)
|
| 214 |
-
if span == 0 or span > len(src_lines) or not s_nonempty:
|
| 215 |
-
return None
|
| 216 |
-
|
| 217 |
-
needle_text = "\n".join(s_norm)
|
| 218 |
-
min_chars = 12 if span >= 2 else 24
|
| 219 |
-
if len(needle_text) < min_chars:
|
| 220 |
-
return None
|
| 221 |
-
|
| 222 |
-
src_norm = [ln.strip() for ln in src_lines]
|
| 223 |
-
pos, best_ratio, runner_up = _score_fuzzy_candidates(src_norm, s_nonempty, needle_text, span)
|
| 224 |
-
min_ratio = 0.90 if span >= 3 else 0.93
|
| 225 |
-
if pos < 0 or best_ratio < min_ratio or (best_ratio - runner_up) < 0.05:
|
| 226 |
-
return None
|
| 227 |
-
|
| 228 |
-
delta = 0
|
| 229 |
-
for src_ln, srch_ln in zip(src_lines[pos : pos + span], s_lines):
|
| 230 |
-
if src_ln.strip() and srch_ln.strip() and "\t" not in src_ln and "\t" not in srch_ln:
|
| 231 |
-
delta = _leading_spaces(src_ln) - _leading_spaces(srch_ln)
|
| 232 |
-
break
|
| 233 |
-
|
| 234 |
-
adj_replace = _shift_indent(replace, delta)
|
| 235 |
-
repl_lines = adj_replace.split("\n") if adj_replace else []
|
| 236 |
-
merged = src_lines[:pos] + repl_lines + src_lines[pos + span :]
|
| 237 |
-
return "\n".join(merged)
|
| 238 |
-
|
| 239 |
-
|
| 240 |
-
def apply_changes(content: str, response_text: str) -> tuple[str, int]:
|
| 241 |
-
blocks = parse_changes(response_text)
|
| 242 |
-
if not blocks:
|
| 243 |
-
return content, 0
|
| 244 |
-
|
| 245 |
-
newline = "\r\n" if "\r\n" in content else "\n"
|
| 246 |
-
working = content.replace("\r\n", "\n")
|
| 247 |
-
applied = 0
|
| 248 |
-
first_miss: str | None = None
|
| 249 |
-
|
| 250 |
-
for search, replace in blocks:
|
| 251 |
-
if not search or search == replace:
|
| 252 |
-
continue
|
| 253 |
-
if replace == "" and f"{search}\n" in working:
|
| 254 |
-
working = working.replace(f"{search}\n", "", 1)
|
| 255 |
-
applied += 1
|
| 256 |
-
continue
|
| 257 |
-
if search in working:
|
| 258 |
-
working = working.replace(search, replace, 1)
|
| 259 |
-
applied += 1
|
| 260 |
-
continue
|
| 261 |
-
|
| 262 |
-
fallback = _replace_rstrip_lines(working, search, replace)
|
| 263 |
-
if fallback is None:
|
| 264 |
-
fallback = _replace_stripped_unique(working, search, replace)
|
| 265 |
-
if fallback is None:
|
| 266 |
-
fallback = _replace_fuzzy_unique(working, search, replace)
|
| 267 |
-
if fallback is not None:
|
| 268 |
-
working = fallback
|
| 269 |
-
applied += 1
|
| 270 |
-
elif first_miss is None:
|
| 271 |
-
first_miss = search.splitlines()[0][:60]
|
| 272 |
-
|
| 273 |
-
if applied == 0 and first_miss is not None:
|
| 274 |
-
raise ValueError(f"SEARCH block did not match file content: {first_miss!r}")
|
| 275 |
-
|
| 276 |
-
if newline == "\r\n":
|
| 277 |
-
working = working.replace("\n", "\r\n")
|
| 278 |
-
return working, applied
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
server/windowing.py
DELETED
|
@@ -1,331 +0,0 @@
|
|
| 1 |
-
from __future__ import annotations
|
| 2 |
-
|
| 3 |
-
from dataclasses import dataclass
|
| 4 |
-
import re
|
| 5 |
-
from typing import Any
|
| 6 |
-
|
| 7 |
-
from .guards import Verdict, verify_patch
|
| 8 |
-
from .hygiene import score_dict
|
| 9 |
-
from .vendored_guards import (
|
| 10 |
-
check_basic_syntax,
|
| 11 |
-
check_reference_integrity,
|
| 12 |
-
is_bloated,
|
| 13 |
-
is_gutted,
|
| 14 |
-
)
|
| 15 |
-
from .vendored_pipeline import apply_changes
|
| 16 |
-
|
| 17 |
-
CHUNK_LINE_THRESHOLD = 160
|
| 18 |
-
CHUNK_CHAR_THRESHOLD = 8_500
|
| 19 |
-
MAX_WINDOW_LINES = 84
|
| 20 |
-
MAX_CHUNK_CHARS = 7_500
|
| 21 |
-
MAX_PROMPT_SQUIGGLES = 16
|
| 22 |
-
MAX_CHUNKS_PER_FILE = 12
|
| 23 |
-
|
| 24 |
-
_COL0_DECL_PREFIXES = (
|
| 25 |
-
"def ",
|
| 26 |
-
"async def ",
|
| 27 |
-
"class ",
|
| 28 |
-
"export ",
|
| 29 |
-
"function ",
|
| 30 |
-
"async function ",
|
| 31 |
-
"pub fn ",
|
| 32 |
-
"pub async fn ",
|
| 33 |
-
"pub struct ",
|
| 34 |
-
"impl ",
|
| 35 |
-
"fn ",
|
| 36 |
-
"func ",
|
| 37 |
-
"static ",
|
| 38 |
-
"void ",
|
| 39 |
-
"int ",
|
| 40 |
-
"bool ",
|
| 41 |
-
)
|
| 42 |
-
_METHOD_DECL_RE = re.compile(
|
| 43 |
-
r"^[ \t]{2,8}(?:async\s+def|def|pub\s+(?:async\s+)?fn|fn|"
|
| 44 |
-
r"(?:public|private|protected|static|async)\s+[A-Za-z_$])\b"
|
| 45 |
-
)
|
| 46 |
-
_LOCAL_ASSIGN_RE = re.compile(
|
| 47 |
-
r"^\s*(?:(?:let|const|var)\s+([A-Za-z_$][A-Za-z0-9_$]*)\b"
|
| 48 |
-
r"|([A-Za-z_][A-Za-z0-9_]*)\s*:=)",
|
| 49 |
-
re.MULTILINE,
|
| 50 |
-
)
|
| 51 |
-
|
| 52 |
-
|
| 53 |
-
@dataclass(frozen=True)
|
| 54 |
-
class CodeWindow:
|
| 55 |
-
full_content: str
|
| 56 |
-
excerpt: str
|
| 57 |
-
prompt_hygiene: dict[str, Any]
|
| 58 |
-
start_line: int
|
| 59 |
-
end_line: int
|
| 60 |
-
is_sliced: bool
|
| 61 |
-
|
| 62 |
-
def splice(self, updated_excerpt: str) -> str:
|
| 63 |
-
if not self.is_sliced:
|
| 64 |
-
return updated_excerpt
|
| 65 |
-
lines = self.full_content.split("\n")
|
| 66 |
-
patched_lines = updated_excerpt.split("\n")
|
| 67 |
-
return "\n".join([*lines[: self.start_line], *patched_lines, *lines[self.end_line :]])
|
| 68 |
-
|
| 69 |
-
|
| 70 |
-
def should_chunk_file(content: str) -> bool:
|
| 71 |
-
if len(content) > CHUNK_CHAR_THRESHOLD:
|
| 72 |
-
return True
|
| 73 |
-
return content.count("\n") + 1 > CHUNK_LINE_THRESHOLD
|
| 74 |
-
|
| 75 |
-
|
| 76 |
-
def _is_col0_decl(line: str) -> bool:
|
| 77 |
-
return line.startswith(_COL0_DECL_PREFIXES)
|
| 78 |
-
|
| 79 |
-
|
| 80 |
-
def _is_decorator_or_attr(line: str) -> bool:
|
| 81 |
-
stripped = line.strip()
|
| 82 |
-
return stripped.startswith("@") or stripped.startswith("#[")
|
| 83 |
-
|
| 84 |
-
|
| 85 |
-
def _snap_chunk_start(src_lines: list[str], cluster_lo: int, pad: int) -> int:
|
| 86 |
-
floor = max(0, cluster_lo - 55)
|
| 87 |
-
for scan in range(cluster_lo, floor - 1, -1):
|
| 88 |
-
if _is_col0_decl(src_lines[scan]):
|
| 89 |
-
while scan > 0 and _is_decorator_or_attr(src_lines[scan - 1]):
|
| 90 |
-
scan -= 1
|
| 91 |
-
return scan
|
| 92 |
-
for scan in range(cluster_lo, floor - 1, -1):
|
| 93 |
-
if _METHOD_DECL_RE.match(src_lines[scan]):
|
| 94 |
-
while scan > 0 and _is_decorator_or_attr(src_lines[scan - 1]):
|
| 95 |
-
scan -= 1
|
| 96 |
-
return scan
|
| 97 |
-
return max(0, cluster_lo - pad)
|
| 98 |
-
|
| 99 |
-
|
| 100 |
-
def _snap_chunk_end(src_lines: list[str], cluster_hi: int, pad: int, use_methods: bool) -> int:
|
| 101 |
-
total = len(src_lines)
|
| 102 |
-
limit = min(total, cluster_hi + 55)
|
| 103 |
-
end = min(total, cluster_hi + pad + 1)
|
| 104 |
-
for scan in range(cluster_hi + 1, limit):
|
| 105 |
-
line = src_lines[scan]
|
| 106 |
-
is_method = use_methods and bool(_METHOD_DECL_RE.match(line))
|
| 107 |
-
if _is_col0_decl(line) or is_method:
|
| 108 |
-
end = scan
|
| 109 |
-
while end > cluster_hi + 1 and _is_decorator_or_attr(src_lines[end - 1]):
|
| 110 |
-
end -= 1
|
| 111 |
-
break
|
| 112 |
-
while end > cluster_hi + 1 and not src_lines[end - 1].strip():
|
| 113 |
-
end -= 1
|
| 114 |
-
return end
|
| 115 |
-
|
| 116 |
-
|
| 117 |
-
def _clamp_chunk_chars(
|
| 118 |
-
src_lines: list[str],
|
| 119 |
-
w_lo: int,
|
| 120 |
-
w_hi: int,
|
| 121 |
-
c_lo: int,
|
| 122 |
-
c_hi: int,
|
| 123 |
-
max_chars: int = MAX_CHUNK_CHARS,
|
| 124 |
-
) -> tuple[int, int]:
|
| 125 |
-
char_total = sum(len(src_lines[idx]) + 1 for idx in range(w_lo, w_hi))
|
| 126 |
-
while w_hi - w_lo > 12 and char_total > max_chars:
|
| 127 |
-
if w_hi - 1 > c_hi and (w_hi - c_hi) >= (c_lo - w_lo):
|
| 128 |
-
w_hi -= 1
|
| 129 |
-
char_total -= len(src_lines[w_hi]) + 1
|
| 130 |
-
elif w_lo < c_lo:
|
| 131 |
-
char_total -= len(src_lines[w_lo]) + 1
|
| 132 |
-
w_lo += 1
|
| 133 |
-
else:
|
| 134 |
-
w_hi -= 1
|
| 135 |
-
char_total -= len(src_lines[w_hi]) + 1
|
| 136 |
-
return w_lo, w_hi
|
| 137 |
-
|
| 138 |
-
|
| 139 |
-
def extract_squiggle_windows(
|
| 140 |
-
content: str,
|
| 141 |
-
squiggles: list[dict[str, Any]],
|
| 142 |
-
max_gap: int = 16,
|
| 143 |
-
pad: int = 14,
|
| 144 |
-
max_window_lines: int = MAX_WINDOW_LINES,
|
| 145 |
-
) -> list[tuple[int, int]]:
|
| 146 |
-
"""Partition squiggles into ordered, disjoint 0-based [w_lo, w_hi) declaration chunks."""
|
| 147 |
-
src_lines = content.replace("\r\n", "\n").split("\n")
|
| 148 |
-
n_lines = len(src_lines)
|
| 149 |
-
sq_indices = sorted(
|
| 150 |
-
{max(0, min(n_lines - 1, int(sq["line"]) - 1)) for sq in squiggles if "line" in sq}
|
| 151 |
-
)
|
| 152 |
-
if not sq_indices:
|
| 153 |
-
return []
|
| 154 |
-
|
| 155 |
-
clusters: list[tuple[int, int]] = []
|
| 156 |
-
for idx in sq_indices:
|
| 157 |
-
prev_ok = bool(clusters) and idx - clusters[-1][1] <= max_gap
|
| 158 |
-
span_ok = bool(clusters) and idx - clusters[-1][0] <= max_window_lines - 16
|
| 159 |
-
if prev_ok and span_ok:
|
| 160 |
-
clusters[-1] = (clusters[-1][0], idx)
|
| 161 |
-
else:
|
| 162 |
-
clusters.append((idx, idx))
|
| 163 |
-
|
| 164 |
-
windows: list[tuple[int, int]] = []
|
| 165 |
-
for c_lo, c_hi in clusters:
|
| 166 |
-
w_lo = _snap_chunk_start(src_lines, c_lo, pad)
|
| 167 |
-
started_on_method = bool(_METHOD_DECL_RE.match(src_lines[w_lo]))
|
| 168 |
-
w_hi = _snap_chunk_end(src_lines, c_hi, pad, use_methods=started_on_method)
|
| 169 |
-
w_lo, w_hi = _clamp_chunk_chars(src_lines, w_lo, w_hi, c_lo, c_hi)
|
| 170 |
-
|
| 171 |
-
if windows and w_lo <= windows[-1][1]:
|
| 172 |
-
merged_hi = max(windows[-1][1], w_hi)
|
| 173 |
-
merged_chars = sum(len(src_lines[i]) + 1 for i in range(windows[-1][0], merged_hi))
|
| 174 |
-
if merged_hi - windows[-1][0] <= max_window_lines and merged_chars <= MAX_CHUNK_CHARS:
|
| 175 |
-
windows[-1] = (windows[-1][0], merged_hi)
|
| 176 |
-
else:
|
| 177 |
-
split_lo = max(windows[-1][1], max(0, c_lo - 4))
|
| 178 |
-
if split_lo < w_hi:
|
| 179 |
-
windows.append((split_lo, w_hi))
|
| 180 |
-
else:
|
| 181 |
-
windows.append((w_lo, w_hi))
|
| 182 |
-
|
| 183 |
-
if len(windows) <= MAX_CHUNKS_PER_FILE:
|
| 184 |
-
return windows
|
| 185 |
-
|
| 186 |
-
scored = sorted(
|
| 187 |
-
windows,
|
| 188 |
-
key=lambda span: sum(1 for idx in sq_indices if span[0] <= idx < span[1]),
|
| 189 |
-
reverse=True,
|
| 190 |
-
)
|
| 191 |
-
return sorted(scored[:MAX_CHUNKS_PER_FILE], key=lambda span: span[0])
|
| 192 |
-
|
| 193 |
-
|
| 194 |
-
def build_chunk_window(
|
| 195 |
-
full_content: str,
|
| 196 |
-
full_hygiene: dict[str, Any],
|
| 197 |
-
w_lo: int,
|
| 198 |
-
w_hi: int,
|
| 199 |
-
threshold: float = 0.55,
|
| 200 |
-
) -> CodeWindow:
|
| 201 |
-
src_lines = full_content.split("\n")
|
| 202 |
-
n_lines = len(src_lines)
|
| 203 |
-
is_sliced = w_lo > 0 or w_hi < n_lines
|
| 204 |
-
if not is_sliced:
|
| 205 |
-
squiggles = list(full_hygiene.get("squiggles", []))[:MAX_PROMPT_SQUIGGLES]
|
| 206 |
-
return CodeWindow(full_content, full_content, {**full_hygiene, "squiggles": squiggles}, 0, n_lines, False)
|
| 207 |
-
|
| 208 |
-
excerpt = "\n".join(src_lines[w_lo:w_hi])
|
| 209 |
-
win_res = score_dict(excerpt, threshold)
|
| 210 |
-
shifted = [
|
| 211 |
-
{**sq, "line": int(sq["line"]) - w_lo}
|
| 212 |
-
for sq in full_hygiene.get("squiggles", [])
|
| 213 |
-
if w_lo < int(sq.get("line", 0)) <= w_hi
|
| 214 |
-
][:MAX_PROMPT_SQUIGGLES]
|
| 215 |
-
win_res["squiggles"] = shifted or win_res.get("squiggles", [])[:MAX_PROMPT_SQUIGGLES]
|
| 216 |
-
return CodeWindow(full_content, excerpt, win_res, w_lo, w_hi, True)
|
| 217 |
-
|
| 218 |
-
|
| 219 |
-
def select_code_window(
|
| 220 |
-
content: str,
|
| 221 |
-
hygiene: dict[str, Any],
|
| 222 |
-
threshold: float = 0.55,
|
| 223 |
-
) -> CodeWindow:
|
| 224 |
-
"""Select the first squiggle chunk window (or whole file when under the chunk threshold)."""
|
| 225 |
-
squiggles = list(hygiene.get("squiggles", []))
|
| 226 |
-
if not should_chunk_file(content) or not squiggles:
|
| 227 |
-
total = content.count("\n") + 1
|
| 228 |
-
capped = {**hygiene, "squiggles": squiggles[:MAX_PROMPT_SQUIGGLES]}
|
| 229 |
-
return CodeWindow(content, content, capped, 0, total, False)
|
| 230 |
-
windows = extract_squiggle_windows(content, squiggles)
|
| 231 |
-
if not windows:
|
| 232 |
-
total = content.count("\n") + 1
|
| 233 |
-
return CodeWindow(content, content, hygiene, 0, total, False)
|
| 234 |
-
w_lo, w_hi = windows[0]
|
| 235 |
-
return build_chunk_window(content, hygiene, w_lo, w_hi, threshold)
|
| 236 |
-
|
| 237 |
-
|
| 238 |
-
def _local_vars_intact_across_boundary(
|
| 239 |
-
full_lines: list[str],
|
| 240 |
-
w_lo: int,
|
| 241 |
-
w_hi: int,
|
| 242 |
-
chunk_before: str,
|
| 243 |
-
chunk_after: str,
|
| 244 |
-
) -> bool:
|
| 245 |
-
before_vars = {m.group(1) or m.group(2) for m in _LOCAL_ASSIGN_RE.finditer(chunk_before)}
|
| 246 |
-
after_vars = {m.group(1) or m.group(2) for m in _LOCAL_ASSIGN_RE.finditer(chunk_after)}
|
| 247 |
-
dropped = {name for name in (before_vars - after_vars) if name and len(name) > 1}
|
| 248 |
-
if not dropped:
|
| 249 |
-
return True
|
| 250 |
-
neighbor_lines = [
|
| 251 |
-
*full_lines[max(0, w_lo - 40) : w_lo],
|
| 252 |
-
*full_lines[w_hi : min(len(full_lines), w_hi + 50)],
|
| 253 |
-
]
|
| 254 |
-
neighbor_text = "\n".join(neighbor_lines)
|
| 255 |
-
for name in dropped:
|
| 256 |
-
if re.search(rf"\b{re.escape(name)}\b", neighbor_text):
|
| 257 |
-
return False
|
| 258 |
-
return True
|
| 259 |
-
|
| 260 |
-
|
| 261 |
-
def verify_window_patch(
|
| 262 |
-
window: CodeWindow,
|
| 263 |
-
reply: str,
|
| 264 |
-
filename: str,
|
| 265 |
-
threshold: float,
|
| 266 |
-
before_hygiene: dict[str, Any],
|
| 267 |
-
original_full_content: str | None = None,
|
| 268 |
-
) -> Verdict:
|
| 269 |
-
if not window.is_sliced:
|
| 270 |
-
return verify_patch(window.full_content, reply, filename, threshold, before_hygiene)
|
| 271 |
-
|
| 272 |
-
cur_full = window.full_content
|
| 273 |
-
root_full = original_full_content if original_full_content is not None else cur_full
|
| 274 |
-
squiggles_before = len(before_hygiene["squiggles"])
|
| 275 |
-
rejected = Verdict(False, None, cur_full, squiggles_before=squiggles_before)
|
| 276 |
-
if not reply.strip():
|
| 277 |
-
rejected.reason = "empty_output"
|
| 278 |
-
return rejected
|
| 279 |
-
try:
|
| 280 |
-
updated_excerpt, applied = apply_changes(window.excerpt, reply)
|
| 281 |
-
except ValueError:
|
| 282 |
-
rejected.reason = "search_miss"
|
| 283 |
-
return rejected
|
| 284 |
-
rejected.applied_blocks = applied
|
| 285 |
-
if applied == 0 or updated_excerpt == window.excerpt:
|
| 286 |
-
rejected.reason = "no_blocks_applied"
|
| 287 |
-
return rejected
|
| 288 |
-
if is_gutted(window.excerpt, updated_excerpt):
|
| 289 |
-
rejected.reason = "gutted_file"
|
| 290 |
-
return rejected
|
| 291 |
-
if is_bloated(window.excerpt, updated_excerpt):
|
| 292 |
-
rejected.reason = "line_bloat"
|
| 293 |
-
return rejected
|
| 294 |
-
|
| 295 |
-
cur_lines = cur_full.split("\n")
|
| 296 |
-
if not _local_vars_intact_across_boundary(
|
| 297 |
-
cur_lines, window.start_line, window.end_line, window.excerpt, updated_excerpt
|
| 298 |
-
):
|
| 299 |
-
rejected.reason = "invalid_syntax"
|
| 300 |
-
return rejected
|
| 301 |
-
|
| 302 |
-
updated_full = window.splice(updated_excerpt)
|
| 303 |
-
if filename and not check_basic_syntax(updated_full, filename):
|
| 304 |
-
rejected.reason = "invalid_syntax"
|
| 305 |
-
return rejected
|
| 306 |
-
if filename and not check_reference_integrity(cur_full, updated_full, filename):
|
| 307 |
-
rejected.reason = "invalid_syntax"
|
| 308 |
-
return rejected
|
| 309 |
-
if root_full != cur_full and filename:
|
| 310 |
-
if not check_reference_integrity(root_full, updated_full, filename):
|
| 311 |
-
rejected.reason = "invalid_syntax"
|
| 312 |
-
return rejected
|
| 313 |
-
if is_gutted(root_full, updated_full):
|
| 314 |
-
rejected.reason = "gutted_file"
|
| 315 |
-
return rejected
|
| 316 |
-
if is_bloated(root_full, updated_full):
|
| 317 |
-
rejected.reason = "line_bloat"
|
| 318 |
-
return rejected
|
| 319 |
-
|
| 320 |
-
after = score_dict(updated_full, threshold)
|
| 321 |
-
squiggles_after = len(after["squiggles"])
|
| 322 |
-
accepted = squiggles_after < squiggles_before
|
| 323 |
-
return Verdict(
|
| 324 |
-
accepted=accepted,
|
| 325 |
-
reason=None if accepted else "no_improvement",
|
| 326 |
-
content=updated_full if accepted else cur_full,
|
| 327 |
-
applied_blocks=applied,
|
| 328 |
-
squiggles_before=squiggles_before,
|
| 329 |
-
squiggles_after=squiggles_after,
|
| 330 |
-
hygiene=after,
|
| 331 |
-
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
web/build.mjs
CHANGED
|
@@ -50,7 +50,6 @@ const HASHED_ASSETS = [
|
|
| 50 |
"app.js",
|
| 51 |
"board.js",
|
| 52 |
"score-worker.js",
|
| 53 |
-
"bonsai-worker.js",
|
| 54 |
"styles.css",
|
| 55 |
"board.css",
|
| 56 |
"privacy.css",
|
|
@@ -107,7 +106,6 @@ async function main() {
|
|
| 107 |
app: path.join(srcDir, "main.js"),
|
| 108 |
board: path.join(srcDir, "board.js"),
|
| 109 |
"score-worker": path.join(srcDir, "score-worker.js"),
|
| 110 |
-
"bonsai-worker": path.join(srcDir, "bonsai-worker.js"),
|
| 111 |
styles: path.join(srcDir, "styles.css"),
|
| 112 |
privacy: path.join(srcDir, "privacy.css"),
|
| 113 |
},
|
|
|
|
| 50 |
"app.js",
|
| 51 |
"board.js",
|
| 52 |
"score-worker.js",
|
|
|
|
| 53 |
"styles.css",
|
| 54 |
"board.css",
|
| 55 |
"privacy.css",
|
|
|
|
| 106 |
app: path.join(srcDir, "main.js"),
|
| 107 |
board: path.join(srcDir, "board.js"),
|
| 108 |
"score-worker": path.join(srcDir, "score-worker.js"),
|
|
|
|
| 109 |
styles: path.join(srcDir, "styles.css"),
|
| 110 |
privacy: path.join(srcDir, "privacy.css"),
|
| 111 |
},
|
web/src/board.html
CHANGED
|
@@ -43,8 +43,8 @@
|
|
| 43 |
<div class="tiles" id="tiles"></div>
|
| 44 |
<div class="charts">
|
| 45 |
<section class="panel">
|
| 46 |
-
<h2 class="panel-title">Most
|
| 47 |
-
<p class="panel-note">Issues
|
| 48 |
<div class="bars" id="chart-kinds"></div>
|
| 49 |
</section>
|
| 50 |
<section class="panel">
|
|
|
|
| 43 |
<div class="tiles" id="tiles"></div>
|
| 44 |
<div class="charts">
|
| 45 |
<section class="panel">
|
| 46 |
+
<h2 class="panel-title">Most flagged</h2>
|
| 47 |
+
<p class="panel-note">Issues found by kind</p>
|
| 48 |
<div class="bars" id="chart-kinds"></div>
|
| 49 |
</section>
|
| 50 |
<section class="panel">
|
web/src/board.js
CHANGED
|
@@ -103,23 +103,16 @@ function tile(label, value, note) {
|
|
| 103 |
]);
|
| 104 |
}
|
| 105 |
|
| 106 |
-
function mergeTile(stats) {
|
| 107 |
-
const decided = stats.prs_decided;
|
| 108 |
-
if (!decided) {
|
| 109 |
-
return tile("PRs merged", "—", "No PRs yet");
|
| 110 |
-
}
|
| 111 |
-
const rate = `${Math.round(stats.pr_merge_rate * 100)}%`;
|
| 112 |
-
return tile("PRs merged", rate, `${formatCount(stats.prs_merged)} of ${formatCount(decided)}`);
|
| 113 |
-
}
|
| 114 |
-
|
| 115 |
function statTiles(stats) {
|
| 116 |
const timed = stats.score_ms_median === null ? "—" : formatMs(stats.score_ms_median);
|
| 117 |
const typical = stats.median_score === null ? "—" : String(stats.median_score);
|
|
|
|
|
|
|
| 118 |
return [
|
| 119 |
-
|
| 120 |
-
tile("Issues
|
| 121 |
tile("Scoring time", timed, "per file, median"),
|
| 122 |
-
tile("Typical file", typical, `of 100, ${formatCount(stats.
|
| 123 |
];
|
| 124 |
}
|
| 125 |
|
|
@@ -136,7 +129,8 @@ function emptyChart() {
|
|
| 136 |
}
|
| 137 |
|
| 138 |
function kindBars(stats) {
|
| 139 |
-
const
|
|
|
|
| 140 |
.filter(([kind, count]) => count > 0 && kind in NOTES)
|
| 141 |
.sort((left, right) => right[1] - left[1])
|
| 142 |
.slice(0, MAX_KINDS);
|
|
|
|
| 103 |
]);
|
| 104 |
}
|
| 105 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 106 |
function statTiles(stats) {
|
| 107 |
const timed = stats.score_ms_median === null ? "—" : formatMs(stats.score_ms_median);
|
| 108 |
const typical = stats.median_score === null ? "—" : String(stats.median_score);
|
| 109 |
+
const avgNote =
|
| 110 |
+
stats.avg_score_scored === null ? "No files yet" : `${Math.round(stats.avg_score_scored)} avg score`;
|
| 111 |
return [
|
| 112 |
+
tile("Files scored", formatCount(stats.files_scored), avgNote),
|
| 113 |
+
tile("Issues flagged", formatCount(stats.issues_found ?? 0), "across scored files"),
|
| 114 |
tile("Scoring time", timed, "per file, median"),
|
| 115 |
+
tile("Typical file", typical, `of 100, ${formatCount(stats.leaderboard_entries)} repos ranked`),
|
| 116 |
];
|
| 117 |
}
|
| 118 |
|
|
|
|
| 129 |
}
|
| 130 |
|
| 131 |
function kindBars(stats) {
|
| 132 |
+
const source = stats.squiggles_by_kind ?? {};
|
| 133 |
+
const kinds = Object.entries(source)
|
| 134 |
.filter(([kind, count]) => count > 0 && kind in NOTES)
|
| 135 |
.sort((left, right) => right[1] - left[1])
|
| 136 |
.slice(0, MAX_KINDS);
|
web/src/bonsai-guards.js
DELETED
|
@@ -1,502 +0,0 @@
|
|
| 1 |
-
// Browser-side prompt builder, SEARCH/REPLACE patcher, and hygiene guards matching server/guards.py.
|
| 2 |
-
import { formatJsonResult, scoreCode } from "shared:engine";
|
| 3 |
-
|
| 4 |
-
export const TASK_INSTRUCTIONS = [
|
| 5 |
-
"Refactor the file to resolve every squiggle in the codehygiene response while preserving exact runtime behavior and non-squiggled public/exported function, class, and method names.",
|
| 6 |
-
"Keep edits minimal and idiomatic; do not add redundant comments, wrapper ceremony, or line bloat.",
|
| 7 |
-
"Wrap dense lines under 88 columns and rename cryptic 1-2 character identifiers, ceremonial *Helper/*Manager/*Factory names, or magic-index expressions flagged by poor_naming into clear domain names across their scope.",
|
| 8 |
-
"Return ONLY one or more SEARCH/REPLACE blocks in this exact format (no prose outside blocks):",
|
| 9 |
-
"",
|
| 10 |
-
"<<<<<<< SEARCH",
|
| 11 |
-
"exact existing lines from the file",
|
| 12 |
-
"=======",
|
| 13 |
-
"replacement lines",
|
| 14 |
-
">>>>>>> REPLACE",
|
| 15 |
-
].join("\n");
|
| 16 |
-
|
| 17 |
-
const PATCH_BLOCK_RE = /<<<<<<< SEARCH\r?\n([\s\S]*?)\r?\n=======\r?\n([\s\S]*?)>>>>>>> REPLACE/g;
|
| 18 |
-
const OPEN_MARKER = "<<<<<<< SEARCH";
|
| 19 |
-
const PARTIAL_MARKERS = ["=======", ">>>>>>> REPLACE"];
|
| 20 |
-
const CLOSE_BLOCK = "\n>>>>>>> REPLACE";
|
| 21 |
-
|
| 22 |
-
const JS_METHOD_SKIP = new Set([
|
| 23 |
-
"if", "for", "while", "switch", "catch", "with", "return", "function", "constructor", "super",
|
| 24 |
-
]);
|
| 25 |
-
const LOCAL_DECL_RE = new RegExp(
|
| 26 |
-
[
|
| 27 |
-
"^\\s*(?:export\\s+)?(?:async\\s+)?(?:function|class|fn|struct)\\s+([A-Za-z_$][A-Za-z0-9_$]*)\\b",
|
| 28 |
-
"^\\s*func\\s+(?:\\([^)\\n]+\\)\\s*)?([A-Za-z_$][A-Za-z0-9_$]*)\\b",
|
| 29 |
-
"^\\s{2,8}(?:private\\s+|public\\s+|protected\\s+|readonly\\s+|static\\s+|async\\s+|pub\\s+)*(?:fn\\s+)?([A-Za-z_$][A-Za-z0-9_$]*)(?:<[^>(]+>)?\\s*\\(",
|
| 30 |
-
].join("|"),
|
| 31 |
-
"gm",
|
| 32 |
-
);
|
| 33 |
-
const RECEIVER_CALL_RE = /\b(?:this|self)\.([A-Za-z_$][A-Za-z0-9_$]*)\s*\(/g;
|
| 34 |
-
const PY_DEF_RE = /^\s*(?:async\s+)?def\s+([A-Za-z_][A-Za-z0-9_]*)\s*\(/gm;
|
| 35 |
-
|
| 36 |
-
export function scoreHygieneDict(content, weights, threshold = 0.55) {
|
| 37 |
-
return formatJsonResult(scoreCode(content, weights, threshold));
|
| 38 |
-
}
|
| 39 |
-
|
| 40 |
-
export function enrichHygieneWithSourceLines(content, hygiene) {
|
| 41 |
-
const srcLines = content.replace(/\r\n/g, "\n").split("\n");
|
| 42 |
-
const enrichedSq = (hygiene.squiggles ?? []).map((sq) => {
|
| 43 |
-
const lineNo = Number(sq.line ?? 1);
|
| 44 |
-
const srcLine = lineNo >= 1 && lineNo <= srcLines.length ? srcLines[lineNo - 1] : "";
|
| 45 |
-
return {
|
| 46 |
-
line: lineNo,
|
| 47 |
-
source_line: srcLine,
|
| 48 |
-
col_start: sq.col_start ?? 0,
|
| 49 |
-
col_end: sq.col_end ?? srcLine.length,
|
| 50 |
-
probability: sq.probability ?? 1.0,
|
| 51 |
-
kind: sq.kind ?? "",
|
| 52 |
-
message: sq.message ?? "",
|
| 53 |
-
};
|
| 54 |
-
});
|
| 55 |
-
return {
|
| 56 |
-
score: hygiene.score ?? 0.0,
|
| 57 |
-
rl_hygiene: hygiene.rl_hygiene ?? 0.0,
|
| 58 |
-
scores: hygiene.scores ?? {},
|
| 59 |
-
archetype: hygiene.archetype ?? "",
|
| 60 |
-
squiggles: enrichedSq,
|
| 61 |
-
};
|
| 62 |
-
}
|
| 63 |
-
|
| 64 |
-
export function buildPrompt(content, hygiene, filename = "") {
|
| 65 |
-
const filePrefix = filename ? `File: ${filename}\n\n` : "";
|
| 66 |
-
const enriched = enrichHygieneWithSourceLines(content, hygiene);
|
| 67 |
-
const userBody = [
|
| 68 |
-
`${filePrefix}<file_content>`,
|
| 69 |
-
content,
|
| 70 |
-
"</file_content>",
|
| 71 |
-
"",
|
| 72 |
-
"<codehygiene_response>",
|
| 73 |
-
JSON.stringify(enriched, null, 2),
|
| 74 |
-
"</codehygiene_response>",
|
| 75 |
-
].join("\n");
|
| 76 |
-
return (
|
| 77 |
-
`<|im_start|>system\n${TASK_INSTRUCTIONS}<|im_end|>\n` +
|
| 78 |
-
`<|im_start|>user\n${userBody}<|im_end|>\n` +
|
| 79 |
-
"<|im_start|>assistant\n<think>\n\n</think>\n\n"
|
| 80 |
-
);
|
| 81 |
-
}
|
| 82 |
-
|
| 83 |
-
function stripSingleTrailingNewline(text) {
|
| 84 |
-
if (text.endsWith("\r\n")) return text.slice(0, -2);
|
| 85 |
-
if (text.endsWith("\n")) return text.slice(0, -1);
|
| 86 |
-
return text;
|
| 87 |
-
}
|
| 88 |
-
|
| 89 |
-
export function parseChanges(responseText) {
|
| 90 |
-
const blocks = [];
|
| 91 |
-
PATCH_BLOCK_RE.lastIndex = 0;
|
| 92 |
-
let match = PATCH_BLOCK_RE.exec(responseText);
|
| 93 |
-
while (match !== null) {
|
| 94 |
-
const searchPart = match[1].replace(/\r\n/g, "\n");
|
| 95 |
-
const replacePart = stripSingleTrailingNewline(match[2]).replace(/\r\n/g, "\n");
|
| 96 |
-
blocks.push([searchPart, replacePart]);
|
| 97 |
-
match = PATCH_BLOCK_RE.exec(responseText);
|
| 98 |
-
}
|
| 99 |
-
return blocks;
|
| 100 |
-
}
|
| 101 |
-
|
| 102 |
-
function leadingSpaces(line) {
|
| 103 |
-
let count = 0;
|
| 104 |
-
while (count < line.length && line[count] === " ") count += 1;
|
| 105 |
-
return count;
|
| 106 |
-
}
|
| 107 |
-
|
| 108 |
-
function shiftIndent(text, delta) {
|
| 109 |
-
if (!text || delta === 0) return text;
|
| 110 |
-
const prefix = delta > 0 ? " ".repeat(delta) : "";
|
| 111 |
-
return text.split("\n").map((raw) => {
|
| 112 |
-
if (!raw.trim()) return "";
|
| 113 |
-
if (delta > 0) return prefix + raw;
|
| 114 |
-
return raw.slice(Math.min(-delta, leadingSpaces(raw)));
|
| 115 |
-
}).join("\n");
|
| 116 |
-
}
|
| 117 |
-
|
| 118 |
-
function findUniqueSpan(srcLines, needle, normFn) {
|
| 119 |
-
const span = needle.length;
|
| 120 |
-
if (span === 0 || span > srcLines.length) return -1;
|
| 121 |
-
let foundPos = -1;
|
| 122 |
-
for (let idx = 0; idx <= srcLines.length - span; idx += 1) {
|
| 123 |
-
let matches = true;
|
| 124 |
-
for (let k = 0; k < span; k += 1) {
|
| 125 |
-
if (normFn(srcLines[idx + k]) !== needle[k]) {
|
| 126 |
-
matches = false;
|
| 127 |
-
break;
|
| 128 |
-
}
|
| 129 |
-
}
|
| 130 |
-
if (matches) {
|
| 131 |
-
if (foundPos !== -1) return -1;
|
| 132 |
-
foundPos = idx;
|
| 133 |
-
}
|
| 134 |
-
}
|
| 135 |
-
return foundPos;
|
| 136 |
-
}
|
| 137 |
-
|
| 138 |
-
function spliceMatchedSpan(srcLines, searchLines, pos, replace, alignIndent) {
|
| 139 |
-
const span = searchLines.length;
|
| 140 |
-
let delta = 0;
|
| 141 |
-
if (alignIndent) {
|
| 142 |
-
for (let k = 0; k < span; k += 1) {
|
| 143 |
-
const sLine = srcLines[pos + k];
|
| 144 |
-
const qLine = searchLines[k];
|
| 145 |
-
if (sLine.trim() && qLine.trim() && !sLine.includes("\t") && !qLine.includes("\t")) {
|
| 146 |
-
delta = leadingSpaces(sLine) - leadingSpaces(qLine);
|
| 147 |
-
break;
|
| 148 |
-
}
|
| 149 |
-
}
|
| 150 |
-
}
|
| 151 |
-
const adjusted = alignIndent ? shiftIndent(replace, delta) : replace;
|
| 152 |
-
const replLines = adjusted ? adjusted.split("\n") : [];
|
| 153 |
-
return [...srcLines.slice(0, pos), ...replLines, ...srcLines.slice(pos + span)].join("\n");
|
| 154 |
-
}
|
| 155 |
-
|
| 156 |
-
function replaceRstripLines(source, search, replace) {
|
| 157 |
-
const srcLines = source.split("\n");
|
| 158 |
-
const searchLines = search.split("\n");
|
| 159 |
-
const pos = findUniqueSpan(srcLines, searchLines.map((ln) => ln.trimEnd()), (ln) => ln.trimEnd());
|
| 160 |
-
return pos === -1 ? null : spliceMatchedSpan(srcLines, searchLines, pos, replace, false);
|
| 161 |
-
}
|
| 162 |
-
|
| 163 |
-
function replaceStrippedUnique(source, search, replace) {
|
| 164 |
-
const srcLines = source.split("\n");
|
| 165 |
-
const searchLines = search.split("\n");
|
| 166 |
-
const needle = searchLines.map((ln) => ln.trim());
|
| 167 |
-
if (!needle.some(Boolean) || (needle.length === 1 && needle[0].length < 14)) return null;
|
| 168 |
-
const pos = findUniqueSpan(srcLines, needle, (ln) => ln.trim());
|
| 169 |
-
return pos === -1 ? null : spliceMatchedSpan(srcLines, searchLines, pos, replace, true);
|
| 170 |
-
}
|
| 171 |
-
|
| 172 |
-
function similarityRatio(left, right) {
|
| 173 |
-
if (left === right) return 1.0;
|
| 174 |
-
const total = left.length + right.length;
|
| 175 |
-
if (total === 0) return 1.0;
|
| 176 |
-
const n = left.length;
|
| 177 |
-
const m = right.length;
|
| 178 |
-
let prev = new Uint16Array(m + 1);
|
| 179 |
-
let curr = new Uint16Array(m + 1);
|
| 180 |
-
for (let i = 1; i <= n; i += 1) {
|
| 181 |
-
const ch = left.charCodeAt(i - 1);
|
| 182 |
-
for (let j = 1; j <= m; j += 1) {
|
| 183 |
-
curr[j] = ch === right.charCodeAt(j - 1) ? prev[j - 1] + 1 : Math.max(prev[j], curr[j - 1]);
|
| 184 |
-
}
|
| 185 |
-
const swap = prev;
|
| 186 |
-
prev = curr;
|
| 187 |
-
curr = swap;
|
| 188 |
-
curr.fill(0);
|
| 189 |
-
}
|
| 190 |
-
return (2.0 * prev[m]) / total;
|
| 191 |
-
}
|
| 192 |
-
|
| 193 |
-
function endpointsMatch(firstSearch, lastSearch, nonEmpty) {
|
| 194 |
-
if (nonEmpty.length === 0) return false;
|
| 195 |
-
const firstRatio = similarityRatio(firstSearch, nonEmpty[0]);
|
| 196 |
-
if (firstRatio < 0.78) return false;
|
| 197 |
-
return similarityRatio(lastSearch, nonEmpty[nonEmpty.length - 1]) >= 0.78;
|
| 198 |
-
}
|
| 199 |
-
|
| 200 |
-
function scoreFuzzyCandidates(srcNorm, searchNonempty, needleText, span) {
|
| 201 |
-
let bestPos = -1;
|
| 202 |
-
let bestRatio = 0.0;
|
| 203 |
-
let runnerUp = 0.0;
|
| 204 |
-
const minGap = Math.max(2, Math.floor(span / 2));
|
| 205 |
-
const firstSearch = searchNonempty[0];
|
| 206 |
-
const lastSearch = searchNonempty[searchNonempty.length - 1];
|
| 207 |
-
|
| 208 |
-
for (let pos = 0; pos <= srcNorm.length - span; pos += 1) {
|
| 209 |
-
const win = srcNorm.slice(pos, pos + span);
|
| 210 |
-
if (!endpointsMatch(firstSearch, lastSearch, win.filter(Boolean))) continue;
|
| 211 |
-
const ratio = similarityRatio(needleText, win.join("\n"));
|
| 212 |
-
if (ratio > bestRatio) {
|
| 213 |
-
if (bestPos >= 0 && Math.abs(pos - bestPos) >= minGap) {
|
| 214 |
-
runnerUp = Math.max(runnerUp, bestRatio);
|
| 215 |
-
}
|
| 216 |
-
bestRatio = ratio;
|
| 217 |
-
bestPos = pos;
|
| 218 |
-
} else if (Math.abs(pos - bestPos) >= minGap) {
|
| 219 |
-
runnerUp = Math.max(runnerUp, ratio);
|
| 220 |
-
}
|
| 221 |
-
}
|
| 222 |
-
return { bestPos, bestRatio, runnerUp };
|
| 223 |
-
}
|
| 224 |
-
|
| 225 |
-
function replaceFuzzyUnique(source, search, replace) {
|
| 226 |
-
const srcLines = source.split("\n");
|
| 227 |
-
const searchLines = search.split("\n");
|
| 228 |
-
const searchNorm = searchLines.map((ln) => ln.trim());
|
| 229 |
-
const searchNonempty = searchNorm.filter(Boolean);
|
| 230 |
-
const span = searchNorm.length;
|
| 231 |
-
if (span === 0 || span > srcLines.length || searchNonempty.length === 0) return null;
|
| 232 |
-
|
| 233 |
-
const needleText = searchNorm.join("\n");
|
| 234 |
-
if (needleText.length < (span >= 2 ? 12 : 24) || needleText.length > 4000) return null;
|
| 235 |
-
|
| 236 |
-
const srcNorm = srcLines.map((ln) => ln.trim());
|
| 237 |
-
const { bestPos, bestRatio, runnerUp } = scoreFuzzyCandidates(srcNorm, searchNonempty, needleText, span);
|
| 238 |
-
const minRatio = span >= 3 ? 0.9 : 0.93;
|
| 239 |
-
if (bestPos < 0 || bestRatio < minRatio || bestRatio - runnerUp < 0.05) return null;
|
| 240 |
-
return spliceMatchedSpan(srcLines, searchLines, bestPos, replace, true);
|
| 241 |
-
}
|
| 242 |
-
|
| 243 |
-
export function applyChanges(content, responseText) {
|
| 244 |
-
const blocks = parseChanges(responseText);
|
| 245 |
-
if (blocks.length === 0) return { content, applied: 0 };
|
| 246 |
-
|
| 247 |
-
const hasCrLf = content.includes("\r\n");
|
| 248 |
-
let working = content.replace(/\r\n/g, "\n");
|
| 249 |
-
let applied = 0;
|
| 250 |
-
let firstMiss = null;
|
| 251 |
-
|
| 252 |
-
for (const [search, replace] of blocks) {
|
| 253 |
-
if (!search || search === replace) continue;
|
| 254 |
-
if (replace === "" && working.includes(`${search}\n`)) {
|
| 255 |
-
working = working.replace(`${search}\n`, "");
|
| 256 |
-
applied += 1;
|
| 257 |
-
continue;
|
| 258 |
-
}
|
| 259 |
-
if (working.includes(search)) {
|
| 260 |
-
working = working.replace(search, () => replace);
|
| 261 |
-
applied += 1;
|
| 262 |
-
continue;
|
| 263 |
-
}
|
| 264 |
-
let fallback = replaceRstripLines(working, search, replace);
|
| 265 |
-
if (fallback === null) {
|
| 266 |
-
fallback = replaceStrippedUnique(working, search, replace);
|
| 267 |
-
}
|
| 268 |
-
if (fallback === null) {
|
| 269 |
-
fallback = replaceFuzzyUnique(working, search, replace);
|
| 270 |
-
}
|
| 271 |
-
if (fallback !== null) {
|
| 272 |
-
working = fallback;
|
| 273 |
-
applied += 1;
|
| 274 |
-
} else if (firstMiss === null) {
|
| 275 |
-
firstMiss = search.split("\n")[0].slice(0, 60);
|
| 276 |
-
}
|
| 277 |
-
}
|
| 278 |
-
if (applied === 0 && firstMiss !== null) {
|
| 279 |
-
throw new Error(`SEARCH block did not match file content: ${firstMiss}`);
|
| 280 |
-
}
|
| 281 |
-
return { content: hasCrLf ? working.replace(/\n/g, "\r\n") : working, applied };
|
| 282 |
-
}
|
| 283 |
-
|
| 284 |
-
export function previewContent(base, raw) {
|
| 285 |
-
const openAt = raw.lastIndexOf(OPEN_MARKER);
|
| 286 |
-
let patchedRaw = raw;
|
| 287 |
-
if (openAt !== -1 && !raw.slice(openAt).includes(">>>>>>> REPLACE")) {
|
| 288 |
-
let text = raw.endsWith("\n") ? raw.slice(0, -1) : raw;
|
| 289 |
-
const nl = text.lastIndexOf("\n");
|
| 290 |
-
if (nl >= 0) {
|
| 291 |
-
const tail = text.slice(nl + 1);
|
| 292 |
-
if (tail !== "=======" && PARTIAL_MARKERS.some((m) => m.startsWith(tail))) {
|
| 293 |
-
text = text.slice(0, nl);
|
| 294 |
-
}
|
| 295 |
-
}
|
| 296 |
-
patchedRaw = text + CLOSE_BLOCK;
|
| 297 |
-
}
|
| 298 |
-
try {
|
| 299 |
-
const { content, applied } = applyChanges(base, patchedRaw);
|
| 300 |
-
return applied > 0 ? { content, applied } : null;
|
| 301 |
-
} catch {
|
| 302 |
-
return null;
|
| 303 |
-
}
|
| 304 |
-
}
|
| 305 |
-
|
| 306 |
-
function inferLang(filename) {
|
| 307 |
-
const dot = filename.lastIndexOf(".");
|
| 308 |
-
const ext = dot >= 0 ? filename.slice(dot).toLowerCase() : "";
|
| 309 |
-
const map = {
|
| 310 |
-
".py": "python",
|
| 311 |
-
".ts": "typescript",
|
| 312 |
-
".mts": "typescript",
|
| 313 |
-
".js": "javascript",
|
| 314 |
-
".mjs": "javascript",
|
| 315 |
-
".rs": "rust",
|
| 316 |
-
".go": "go",
|
| 317 |
-
};
|
| 318 |
-
return map[ext] ?? "other";
|
| 319 |
-
}
|
| 320 |
-
|
| 321 |
-
function hasBalancedDelimiters(code, commentPrefix) {
|
| 322 |
-
const stack = [];
|
| 323 |
-
const pairs = { "(": ")", "[": "]", "{": "}" };
|
| 324 |
-
const closers = new Set([")", "]", "}"]);
|
| 325 |
-
let quote = null;
|
| 326 |
-
for (let i = 0; i < code.length; i += 1) {
|
| 327 |
-
const ch = code[i];
|
| 328 |
-
if (quote) {
|
| 329 |
-
if (ch === "\\") {
|
| 330 |
-
i += 1;
|
| 331 |
-
} else if (ch === quote) {
|
| 332 |
-
quote = null;
|
| 333 |
-
}
|
| 334 |
-
continue;
|
| 335 |
-
}
|
| 336 |
-
if (commentPrefix && code.startsWith(commentPrefix, i)) {
|
| 337 |
-
const nl = code.indexOf("\n", i);
|
| 338 |
-
if (nl === -1) break;
|
| 339 |
-
i = nl;
|
| 340 |
-
continue;
|
| 341 |
-
}
|
| 342 |
-
if (ch === '"' || ch === "'" || ch === "`") {
|
| 343 |
-
quote = ch;
|
| 344 |
-
} else if (pairs[ch]) {
|
| 345 |
-
stack.push(pairs[ch]);
|
| 346 |
-
} else if (closers.has(ch) && stack.pop() !== ch) {
|
| 347 |
-
return false;
|
| 348 |
-
}
|
| 349 |
-
}
|
| 350 |
-
return quote === null && stack.length === 0;
|
| 351 |
-
}
|
| 352 |
-
|
| 353 |
-
function checkPythonStructure(code) {
|
| 354 |
-
if (!hasBalancedDelimiters(code, "#")) return false;
|
| 355 |
-
const openTries = [];
|
| 356 |
-
for (const raw of code.split("\n")) {
|
| 357 |
-
const trimmed = raw.trim();
|
| 358 |
-
if (!trimmed || trimmed.startsWith("#")) continue;
|
| 359 |
-
const indent = leadingSpaces(raw);
|
| 360 |
-
while (openTries.length > 0 && indent <= openTries[openTries.length - 1]) {
|
| 361 |
-
const tryIndent = openTries[openTries.length - 1];
|
| 362 |
-
if (indent === tryIndent && /^(?:except\b|finally\s*:)/.test(trimmed)) {
|
| 363 |
-
openTries.pop();
|
| 364 |
-
break;
|
| 365 |
-
}
|
| 366 |
-
if (indent <= tryIndent) {
|
| 367 |
-
return false;
|
| 368 |
-
}
|
| 369 |
-
}
|
| 370 |
-
if (/^try\s*:/.test(trimmed)) {
|
| 371 |
-
openTries.push(indent);
|
| 372 |
-
}
|
| 373 |
-
}
|
| 374 |
-
return openTries.length === 0;
|
| 375 |
-
}
|
| 376 |
-
|
| 377 |
-
export function checkBasicSyntax(code, filename) {
|
| 378 |
-
if (!code.trim()) return false;
|
| 379 |
-
const lang = inferLang(filename);
|
| 380 |
-
if (lang === "python") return checkPythonStructure(code);
|
| 381 |
-
if (lang === "javascript" || lang === "typescript" || lang === "rust" || lang === "go") {
|
| 382 |
-
return hasBalancedDelimiters(code, "//");
|
| 383 |
-
}
|
| 384 |
-
return true;
|
| 385 |
-
}
|
| 386 |
-
|
| 387 |
-
function collectRegexMatches(re, text) {
|
| 388 |
-
const found = new Set();
|
| 389 |
-
re.lastIndex = 0;
|
| 390 |
-
let m = re.exec(text);
|
| 391 |
-
while (m !== null) {
|
| 392 |
-
const sym = m[1] || m[2] || m[3] || "";
|
| 393 |
-
if (sym && !JS_METHOD_SKIP.has(sym)) found.add(sym);
|
| 394 |
-
m = re.exec(text);
|
| 395 |
-
}
|
| 396 |
-
return found;
|
| 397 |
-
}
|
| 398 |
-
|
| 399 |
-
export function checkReferenceIntegrity(beforeCode, afterCode, filename) {
|
| 400 |
-
const lang = inferLang(filename);
|
| 401 |
-
if (lang === "python") {
|
| 402 |
-
const defsBefore = collectRegexMatches(PY_DEF_RE, beforeCode);
|
| 403 |
-
const defsAfter = collectRegexMatches(PY_DEF_RE, afterCode);
|
| 404 |
-
const selfBefore = collectRegexMatches(RECEIVER_CALL_RE, beforeCode);
|
| 405 |
-
const selfAfter = collectRegexMatches(RECEIVER_CALL_RE, afterCode);
|
| 406 |
-
for (const sym of selfAfter) {
|
| 407 |
-
if (defsBefore.has(sym) && !defsAfter.has(sym)) return false;
|
| 408 |
-
if (!selfBefore.has(sym) && !defsAfter.has(sym)) return false;
|
| 409 |
-
}
|
| 410 |
-
return true;
|
| 411 |
-
}
|
| 412 |
-
if (lang === "javascript" || lang === "typescript" || lang === "rust" || lang === "go") {
|
| 413 |
-
const defsBefore = collectRegexMatches(LOCAL_DECL_RE, beforeCode);
|
| 414 |
-
const defsAfter = collectRegexMatches(LOCAL_DECL_RE, afterCode);
|
| 415 |
-
for (const dropped of defsBefore) {
|
| 416 |
-
if (!defsAfter.has(dropped)) {
|
| 417 |
-
const callRe = new RegExp(`(?:^|[^.\\w$])${dropped.replace(/\$/g, "\\$")}\\s*\\(`, "m");
|
| 418 |
-
if (callRe.test(afterCode)) return false;
|
| 419 |
-
}
|
| 420 |
-
}
|
| 421 |
-
const recvBefore = collectRegexMatches(RECEIVER_CALL_RE, beforeCode);
|
| 422 |
-
const recvAfter = collectRegexMatches(RECEIVER_CALL_RE, afterCode);
|
| 423 |
-
for (const sym of recvAfter) {
|
| 424 |
-
if (!recvBefore.has(sym) && !defsAfter.has(sym)) return false;
|
| 425 |
-
}
|
| 426 |
-
}
|
| 427 |
-
return true;
|
| 428 |
-
}
|
| 429 |
-
|
| 430 |
-
function countNonCommentLines(code) {
|
| 431 |
-
let total = 0;
|
| 432 |
-
for (const raw of code.split("\n")) {
|
| 433 |
-
const s = raw.trim();
|
| 434 |
-
if (s && !s.startsWith("#") && !s.startsWith("//") && !s.startsWith("/*") && !s.startsWith("*")) {
|
| 435 |
-
total += 1;
|
| 436 |
-
}
|
| 437 |
-
}
|
| 438 |
-
return total;
|
| 439 |
-
}
|
| 440 |
-
|
| 441 |
-
export function isGutted(beforeCode, updated) {
|
| 442 |
-
const origN = countNonCommentLines(beforeCode);
|
| 443 |
-
const newN = countNonCommentLines(updated);
|
| 444 |
-
if (origN >= 1 && newN === 0) return true;
|
| 445 |
-
return origN >= 10 && newN < Math.max(5, Math.trunc(origN * 0.55));
|
| 446 |
-
}
|
| 447 |
-
|
| 448 |
-
export function isBloated(beforeCode, updated) {
|
| 449 |
-
const beforeN = beforeCode.split("\n").filter((ln) => ln.trim()).length;
|
| 450 |
-
const afterN = updated.split("\n").filter((ln) => ln.trim()).length;
|
| 451 |
-
return afterN > Math.max(40, Math.trunc(beforeN * 1.6));
|
| 452 |
-
}
|
| 453 |
-
|
| 454 |
-
export function verifyPatch(content, reply, filename, threshold, beforeHygiene, weights) {
|
| 455 |
-
const squigglesBefore = beforeHygiene.squiggles.length;
|
| 456 |
-
const reject = (reason, applied = 0) => ({
|
| 457 |
-
accepted: false,
|
| 458 |
-
reason,
|
| 459 |
-
content,
|
| 460 |
-
appliedBlocks: applied,
|
| 461 |
-
squigglesBefore,
|
| 462 |
-
squigglesAfter: null,
|
| 463 |
-
hygiene: {},
|
| 464 |
-
scoreAfter: null,
|
| 465 |
-
reply: "",
|
| 466 |
-
});
|
| 467 |
-
|
| 468 |
-
if (!reply.trim()) return reject("empty_output");
|
| 469 |
-
let patched;
|
| 470 |
-
try {
|
| 471 |
-
patched = applyChanges(content, reply);
|
| 472 |
-
} catch {
|
| 473 |
-
return reject("search_miss");
|
| 474 |
-
}
|
| 475 |
-
if (patched.applied === 0 || patched.content === content) {
|
| 476 |
-
return reject("no_blocks_applied", patched.applied);
|
| 477 |
-
}
|
| 478 |
-
if (filename) {
|
| 479 |
-
const syntaxOk = checkBasicSyntax(patched.content, filename);
|
| 480 |
-
const refsOk = syntaxOk && checkReferenceIntegrity(content, patched.content, filename);
|
| 481 |
-
if (!refsOk) {
|
| 482 |
-
return reject("invalid_syntax", patched.applied);
|
| 483 |
-
}
|
| 484 |
-
}
|
| 485 |
-
if (isGutted(content, patched.content)) return reject("gutted_file", patched.applied);
|
| 486 |
-
if (isBloated(content, patched.content)) return reject("line_bloat", patched.applied);
|
| 487 |
-
|
| 488 |
-
const after = scoreHygieneDict(patched.content, weights, threshold);
|
| 489 |
-
const squigglesAfter = after.squiggles.length;
|
| 490 |
-
const accepted = squigglesAfter < squigglesBefore;
|
| 491 |
-
return {
|
| 492 |
-
accepted,
|
| 493 |
-
reason: accepted ? null : "no_improvement",
|
| 494 |
-
content: accepted ? patched.content : content,
|
| 495 |
-
appliedBlocks: patched.applied,
|
| 496 |
-
squigglesBefore,
|
| 497 |
-
squigglesAfter,
|
| 498 |
-
hygiene: after,
|
| 499 |
-
scoreAfter: after.score ?? null,
|
| 500 |
-
reply,
|
| 501 |
-
};
|
| 502 |
-
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
web/src/bonsai-shaders.js
DELETED
|
@@ -1,595 +0,0 @@
|
|
| 1 |
-
// Baseline WebGPU 1.0 WGSL compute kernels for 2-bit packed ternary Qwen3-1.7B (group_size = 32).
|
| 2 |
-
|
| 3 |
-
const DECODE_HELPER_WGSL = `
|
| 4 |
-
fn decode_trit4(word: u32, shift: u32) -> vec4<f32> {
|
| 5 |
-
let c0 = (word >> shift) & 3u;
|
| 6 |
-
let c1 = (word >> (shift + 2u)) & 3u;
|
| 7 |
-
let c2 = (word >> (shift + 4u)) & 3u;
|
| 8 |
-
let c3 = (word >> (shift + 6u)) & 3u;
|
| 9 |
-
return vec4<f32>(
|
| 10 |
-
f32(i32(c0 & 1u) - i32(c0 >> 1u)),
|
| 11 |
-
f32(i32(c1 & 1u) - i32(c1 >> 1u)),
|
| 12 |
-
f32(i32(c2 & 1u) - i32(c2 >> 1u)),
|
| 13 |
-
f32(i32(c3 & 1u) - i32(c3 >> 1u)),
|
| 14 |
-
);
|
| 15 |
-
}
|
| 16 |
-
|
| 17 |
-
fn read_scale(scales: ptr<storage, array<u32>, read>, base_u32: u32, group_idx: u32) -> f32 {
|
| 18 |
-
let pair = unpack2x16float((*scales)[base_u32 + (group_idx >> 1u)]);
|
| 19 |
-
return select(pair.x, pair.y, (group_idx & 1u) == 1u);
|
| 20 |
-
}
|
| 21 |
-
`;
|
| 22 |
-
|
| 23 |
-
export const EMBED_WGSL = `
|
| 24 |
-
struct StepParams {
|
| 25 |
-
batch_len: u32,
|
| 26 |
-
start_pos: u32,
|
| 27 |
-
max_seq: u32,
|
| 28 |
-
pad0: u32,
|
| 29 |
-
};
|
| 30 |
-
|
| 31 |
-
@group(0) @binding(0) var<uniform> step: StepParams;
|
| 32 |
-
@group(0) @binding(1) var<storage, read> token_ids: array<u32>;
|
| 33 |
-
@group(0) @binding(2) var<storage, read> embed_trits: array<u32>;
|
| 34 |
-
@group(0) @binding(3) var<storage, read> embed_scales: array<u32>;
|
| 35 |
-
@group(0) @binding(4) var<storage, read_write> out_x: array<f32>;
|
| 36 |
-
|
| 37 |
-
${DECODE_HELPER_WGSL}
|
| 38 |
-
|
| 39 |
-
@compute @workgroup_size(64, 1, 1)
|
| 40 |
-
fn main(
|
| 41 |
-
@builtin(workgroup_id) wg: vec3<u32>,
|
| 42 |
-
@builtin(local_invocation_id) lid: vec3<u32>,
|
| 43 |
-
) {
|
| 44 |
-
let b = wg.x;
|
| 45 |
-
if (b >= step.batch_len) {
|
| 46 |
-
return;
|
| 47 |
-
}
|
| 48 |
-
let col4 = wg.y * 64u + lid.x;
|
| 49 |
-
let row = token_ids[b];
|
| 50 |
-
let group_idx = col4 >> 3u;
|
| 51 |
-
let word_idx = (col4 >> 2u) & 1u;
|
| 52 |
-
let shift = (col4 & 3u) * 8u;
|
| 53 |
-
let word = embed_trits[row * 128u + group_idx * 2u + word_idx];
|
| 54 |
-
let scale = read_scale(&embed_scales, row * 32u, group_idx);
|
| 55 |
-
let vals = decode_trit4(word, shift) * scale;
|
| 56 |
-
let dst = b * 2048u + col4 * 4u;
|
| 57 |
-
out_x[dst] = vals.x;
|
| 58 |
-
out_x[dst + 1u] = vals.y;
|
| 59 |
-
out_x[dst + 2u] = vals.z;
|
| 60 |
-
out_x[dst + 3u] = vals.w;
|
| 61 |
-
}
|
| 62 |
-
`;
|
| 63 |
-
|
| 64 |
-
export const RMSNORM_WGSL = `
|
| 65 |
-
struct StepParams {
|
| 66 |
-
batch_len: u32,
|
| 67 |
-
start_pos: u32,
|
| 68 |
-
max_seq: u32,
|
| 69 |
-
pad0: u32,
|
| 70 |
-
};
|
| 71 |
-
|
| 72 |
-
struct NormOp {
|
| 73 |
-
norm_offset: u32,
|
| 74 |
-
dim: u32,
|
| 75 |
-
pad0: u32,
|
| 76 |
-
pad1: u32,
|
| 77 |
-
};
|
| 78 |
-
|
| 79 |
-
@group(0) @binding(0) var<uniform> step: StepParams;
|
| 80 |
-
@group(0) @binding(1) var<uniform> op: NormOp;
|
| 81 |
-
@group(0) @binding(2) var<storage, read> in_x: array<f32>;
|
| 82 |
-
@group(0) @binding(3) var<storage, read> norms: array<f32>;
|
| 83 |
-
@group(0) @binding(4) var<storage, read_write> out_y: array<f32>;
|
| 84 |
-
|
| 85 |
-
var<workgroup> shared_sq: array<f32, 256>;
|
| 86 |
-
|
| 87 |
-
@compute @workgroup_size(256, 1, 1)
|
| 88 |
-
fn main(
|
| 89 |
-
@builtin(workgroup_id) wg: vec3<u32>,
|
| 90 |
-
@builtin(local_invocation_id) lid: vec3<u32>,
|
| 91 |
-
) {
|
| 92 |
-
let b = wg.x;
|
| 93 |
-
let tid = lid.x;
|
| 94 |
-
var sq_sum = 0.0;
|
| 95 |
-
if (b < step.batch_len) {
|
| 96 |
-
let base = b * op.dim;
|
| 97 |
-
for (var idx = tid; idx < op.dim; idx += 256u) {
|
| 98 |
-
let v = in_x[base + idx];
|
| 99 |
-
sq_sum += v * v;
|
| 100 |
-
}
|
| 101 |
-
}
|
| 102 |
-
shared_sq[tid] = sq_sum;
|
| 103 |
-
workgroupBarrier();
|
| 104 |
-
|
| 105 |
-
for (var stride = 128u; stride > 0u; stride >>= 1u) {
|
| 106 |
-
if (tid < stride) {
|
| 107 |
-
shared_sq[tid] += shared_sq[tid + stride];
|
| 108 |
-
}
|
| 109 |
-
workgroupBarrier();
|
| 110 |
-
}
|
| 111 |
-
|
| 112 |
-
if (b < step.batch_len) {
|
| 113 |
-
let inv_rms = inverseSqrt(shared_sq[0] / f32(op.dim) + 1e-6);
|
| 114 |
-
let base = b * op.dim;
|
| 115 |
-
for (var idx = tid; idx < op.dim; idx += 256u) {
|
| 116 |
-
out_y[base + idx] = in_x[base + idx] * inv_rms * norms[op.norm_offset + idx];
|
| 117 |
-
}
|
| 118 |
-
}
|
| 119 |
-
}
|
| 120 |
-
`;
|
| 121 |
-
|
| 122 |
-
export const TERNARY_GEMV_WGSL = `
|
| 123 |
-
struct MatOp {
|
| 124 |
-
rows: u32,
|
| 125 |
-
in_groups: u32,
|
| 126 |
-
trit_offset: u32,
|
| 127 |
-
scale_offset: u32,
|
| 128 |
-
add_residual: u32,
|
| 129 |
-
pad0: u32,
|
| 130 |
-
pad1: u32,
|
| 131 |
-
pad2: u32,
|
| 132 |
-
};
|
| 133 |
-
|
| 134 |
-
@group(0) @binding(0) var<uniform> op: MatOp;
|
| 135 |
-
@group(0) @binding(1) var<storage, read> in_x: array<vec4<f32>>;
|
| 136 |
-
@group(0) @binding(2) var<storage, read> trits: array<u32>;
|
| 137 |
-
@group(0) @binding(3) var<storage, read> scales: array<u32>;
|
| 138 |
-
@group(0) @binding(4) var<storage, read_write> out_y: array<f32>;
|
| 139 |
-
|
| 140 |
-
${DECODE_HELPER_WGSL}
|
| 141 |
-
|
| 142 |
-
var<workgroup> shared_x: array<vec4<f32>, 512>;
|
| 143 |
-
var<workgroup> partial_sum: array<f32, 256>;
|
| 144 |
-
|
| 145 |
-
@compute @workgroup_size(64, 4, 1)
|
| 146 |
-
fn main(
|
| 147 |
-
@builtin(workgroup_id) wg: vec3<u32>,
|
| 148 |
-
@builtin(local_invocation_id) lid: vec3<u32>,
|
| 149 |
-
) {
|
| 150 |
-
let gx = lid.x;
|
| 151 |
-
let ry = lid.y;
|
| 152 |
-
let flat_tid = ry * 64u + gx;
|
| 153 |
-
let row = wg.x * 4u + ry;
|
| 154 |
-
let num_tiles = op.in_groups / 64u;
|
| 155 |
-
|
| 156 |
-
var acc = 0.0;
|
| 157 |
-
for (var tile = 0u; tile < num_tiles; tile += 1u) {
|
| 158 |
-
let x_vec_base = tile * 512u;
|
| 159 |
-
shared_x[flat_tid] = in_x[x_vec_base + flat_tid];
|
| 160 |
-
shared_x[flat_tid + 256u] = in_x[x_vec_base + flat_tid + 256u];
|
| 161 |
-
workgroupBarrier();
|
| 162 |
-
|
| 163 |
-
if (row < op.rows) {
|
| 164 |
-
let g = tile * 64u + gx;
|
| 165 |
-
let row_trit_base = op.trit_offset + row * op.in_groups * 2u;
|
| 166 |
-
let row_scale_base = op.scale_offset + row * (op.in_groups >> 1u);
|
| 167 |
-
let w0 = trits[row_trit_base + g * 2u];
|
| 168 |
-
let w1 = trits[row_trit_base + g * 2u + 1u];
|
| 169 |
-
let scale = read_scale(&scales, row_scale_base, g);
|
| 170 |
-
let x_off = gx * 8u;
|
| 171 |
-
let s0 = dot(decode_trit4(w0, 0u), shared_x[x_off])
|
| 172 |
-
+ dot(decode_trit4(w0, 8u), shared_x[x_off + 1u])
|
| 173 |
-
+ dot(decode_trit4(w0, 16u), shared_x[x_off + 2u])
|
| 174 |
-
+ dot(decode_trit4(w0, 24u), shared_x[x_off + 3u]);
|
| 175 |
-
let s1 = dot(decode_trit4(w1, 0u), shared_x[x_off + 4u])
|
| 176 |
-
+ dot(decode_trit4(w1, 8u), shared_x[x_off + 5u])
|
| 177 |
-
+ dot(decode_trit4(w1, 16u), shared_x[x_off + 6u])
|
| 178 |
-
+ dot(decode_trit4(w1, 24u), shared_x[x_off + 7u]);
|
| 179 |
-
acc += (s0 + s1) * scale;
|
| 180 |
-
}
|
| 181 |
-
workgroupBarrier();
|
| 182 |
-
}
|
| 183 |
-
|
| 184 |
-
partial_sum[flat_tid] = acc;
|
| 185 |
-
workgroupBarrier();
|
| 186 |
-
|
| 187 |
-
for (var stride = 32u; stride > 0u; stride >>= 1u) {
|
| 188 |
-
if (gx < stride) {
|
| 189 |
-
partial_sum[flat_tid] += partial_sum[flat_tid + stride];
|
| 190 |
-
}
|
| 191 |
-
workgroupBarrier();
|
| 192 |
-
}
|
| 193 |
-
|
| 194 |
-
if (gx == 0u && row < op.rows) {
|
| 195 |
-
let total = partial_sum[ry * 64u];
|
| 196 |
-
if (op.add_residual == 1u) {
|
| 197 |
-
out_y[row] += total;
|
| 198 |
-
} else {
|
| 199 |
-
out_y[row] = total;
|
| 200 |
-
}
|
| 201 |
-
}
|
| 202 |
-
}
|
| 203 |
-
`;
|
| 204 |
-
|
| 205 |
-
export const TERNARY_GEMM_WGSL = `
|
| 206 |
-
struct StepParams {
|
| 207 |
-
batch_len: u32,
|
| 208 |
-
start_pos: u32,
|
| 209 |
-
max_seq: u32,
|
| 210 |
-
pad0: u32,
|
| 211 |
-
};
|
| 212 |
-
|
| 213 |
-
struct MatOp {
|
| 214 |
-
rows: u32,
|
| 215 |
-
in_groups: u32,
|
| 216 |
-
trit_offset: u32,
|
| 217 |
-
scale_offset: u32,
|
| 218 |
-
add_residual: u32,
|
| 219 |
-
pad0: u32,
|
| 220 |
-
pad1: u32,
|
| 221 |
-
pad2: u32,
|
| 222 |
-
};
|
| 223 |
-
|
| 224 |
-
@group(0) @binding(0) var<uniform> step: StepParams;
|
| 225 |
-
@group(0) @binding(1) var<uniform> op: MatOp;
|
| 226 |
-
@group(0) @binding(2) var<storage, read> in_x: array<vec4<f32>>;
|
| 227 |
-
@group(0) @binding(3) var<storage, read> trits: array<u32>;
|
| 228 |
-
@group(0) @binding(4) var<storage, read> scales: array<u32>;
|
| 229 |
-
@group(0) @binding(5) var<storage, read_write> out_y: array<f32>;
|
| 230 |
-
|
| 231 |
-
${DECODE_HELPER_WGSL}
|
| 232 |
-
|
| 233 |
-
var<workgroup> tile_x: array<vec4<f32>, 128>;
|
| 234 |
-
var<workgroup> tile_w: array<vec4<f32>, 128>;
|
| 235 |
-
|
| 236 |
-
@compute @workgroup_size(16, 16, 1)
|
| 237 |
-
fn main(
|
| 238 |
-
@builtin(workgroup_id) wg: vec3<u32>,
|
| 239 |
-
@builtin(local_invocation_id) lid: vec3<u32>,
|
| 240 |
-
) {
|
| 241 |
-
let rx = lid.x;
|
| 242 |
-
let by = lid.y;
|
| 243 |
-
let row = wg.x * 16u + rx;
|
| 244 |
-
let b = wg.y * 16u + by;
|
| 245 |
-
let x_row_vecs = op.in_groups * 8u;
|
| 246 |
-
|
| 247 |
-
var acc = 0.0;
|
| 248 |
-
for (var g = 0u; g < op.in_groups; g += 1u) {
|
| 249 |
-
if (rx < 8u) {
|
| 250 |
-
if (b < step.batch_len) {
|
| 251 |
-
tile_x[by * 8u + rx] = in_x[b * x_row_vecs + g * 8u + rx];
|
| 252 |
-
} else {
|
| 253 |
-
tile_x[by * 8u + rx] = vec4<f32>(0.0);
|
| 254 |
-
}
|
| 255 |
-
}
|
| 256 |
-
if (by < 8u) {
|
| 257 |
-
if (row < op.rows) {
|
| 258 |
-
let row_trit_base = op.trit_offset + row * op.in_groups * 2u;
|
| 259 |
-
let row_scale_base = op.scale_offset + row * (op.in_groups >> 1u);
|
| 260 |
-
let word = trits[row_trit_base + g * 2u + (by >> 2u)];
|
| 261 |
-
let shift = (by & 3u) * 8u;
|
| 262 |
-
let scale = read_scale(&scales, row_scale_base, g);
|
| 263 |
-
tile_w[rx * 8u + by] = decode_trit4(word, shift) * scale;
|
| 264 |
-
} else {
|
| 265 |
-
tile_w[rx * 8u + by] = vec4<f32>(0.0);
|
| 266 |
-
}
|
| 267 |
-
}
|
| 268 |
-
workgroupBarrier();
|
| 269 |
-
|
| 270 |
-
let w_off = rx * 8u;
|
| 271 |
-
let x_off = by * 8u;
|
| 272 |
-
acc += dot(tile_w[w_off], tile_x[x_off])
|
| 273 |
-
+ dot(tile_w[w_off + 1u], tile_x[x_off + 1u])
|
| 274 |
-
+ dot(tile_w[w_off + 2u], tile_x[x_off + 2u])
|
| 275 |
-
+ dot(tile_w[w_off + 3u], tile_x[x_off + 3u])
|
| 276 |
-
+ dot(tile_w[w_off + 4u], tile_x[x_off + 4u])
|
| 277 |
-
+ dot(tile_w[w_off + 5u], tile_x[x_off + 5u])
|
| 278 |
-
+ dot(tile_w[w_off + 6u], tile_x[x_off + 6u])
|
| 279 |
-
+ dot(tile_w[w_off + 7u], tile_x[x_off + 7u]);
|
| 280 |
-
workgroupBarrier();
|
| 281 |
-
}
|
| 282 |
-
|
| 283 |
-
if (b < step.batch_len && row < op.rows) {
|
| 284 |
-
let dst = b * op.rows + row;
|
| 285 |
-
if (op.add_residual == 1u) {
|
| 286 |
-
out_y[dst] += acc;
|
| 287 |
-
} else {
|
| 288 |
-
out_y[dst] = acc;
|
| 289 |
-
}
|
| 290 |
-
}
|
| 291 |
-
}
|
| 292 |
-
`;
|
| 293 |
-
|
| 294 |
-
export const QK_ROPE_KV_WGSL = `
|
| 295 |
-
struct StepParams {
|
| 296 |
-
batch_len: u32,
|
| 297 |
-
start_pos: u32,
|
| 298 |
-
max_seq: u32,
|
| 299 |
-
pad0: u32,
|
| 300 |
-
};
|
| 301 |
-
|
| 302 |
-
struct AttnOp {
|
| 303 |
-
local_layer: u32,
|
| 304 |
-
q_norm_offset: u32,
|
| 305 |
-
k_norm_offset: u32,
|
| 306 |
-
cos_offset: u32,
|
| 307 |
-
sin_offset: u32,
|
| 308 |
-
pad0: u32,
|
| 309 |
-
pad1: u32,
|
| 310 |
-
pad2: u32,
|
| 311 |
-
};
|
| 312 |
-
|
| 313 |
-
@group(0) @binding(0) var<uniform> step: StepParams;
|
| 314 |
-
@group(0) @binding(1) var<uniform> op: AttnOp;
|
| 315 |
-
@group(0) @binding(2) var<storage, read> norms: array<f32>;
|
| 316 |
-
@group(0) @binding(3) var<storage, read_write> q_buf: array<f32>;
|
| 317 |
-
@group(0) @binding(4) var<storage, read> k_in: array<f32>;
|
| 318 |
-
@group(0) @binding(5) var<storage, read> v_in: array<f32>;
|
| 319 |
-
@group(0) @binding(6) var<storage, read_write> k_cache: array<u32>;
|
| 320 |
-
@group(0) @binding(7) var<storage, read_write> v_cache: array<u32>;
|
| 321 |
-
|
| 322 |
-
var<workgroup> head_sq: array<f32, 64>;
|
| 323 |
-
|
| 324 |
-
@compute @workgroup_size(64, 1, 1)
|
| 325 |
-
fn main(
|
| 326 |
-
@builtin(workgroup_id) wg: vec3<u32>,
|
| 327 |
-
@builtin(local_invocation_id) lid: vec3<u32>,
|
| 328 |
-
) {
|
| 329 |
-
let b = wg.x;
|
| 330 |
-
let head_id = wg.y;
|
| 331 |
-
let tid = lid.x;
|
| 332 |
-
if (b >= step.batch_len) {
|
| 333 |
-
return;
|
| 334 |
-
}
|
| 335 |
-
|
| 336 |
-
let is_q = head_id < 16u;
|
| 337 |
-
var x0 = 0.0;
|
| 338 |
-
var x1 = 0.0;
|
| 339 |
-
if (is_q) {
|
| 340 |
-
let base = b * 2048u + head_id * 128u;
|
| 341 |
-
x0 = q_buf[base + tid];
|
| 342 |
-
x1 = q_buf[base + tid + 64u];
|
| 343 |
-
} else {
|
| 344 |
-
let kv_h = head_id - 16u;
|
| 345 |
-
let base = b * 1024u + kv_h * 128u;
|
| 346 |
-
x0 = k_in[base + tid];
|
| 347 |
-
x1 = k_in[base + tid + 64u];
|
| 348 |
-
}
|
| 349 |
-
|
| 350 |
-
head_sq[tid] = x0 * x0 + x1 * x1;
|
| 351 |
-
workgroupBarrier();
|
| 352 |
-
for (var stride = 32u; stride > 0u; stride >>= 1u) {
|
| 353 |
-
if (tid < stride) {
|
| 354 |
-
head_sq[tid] += head_sq[tid + stride];
|
| 355 |
-
}
|
| 356 |
-
workgroupBarrier();
|
| 357 |
-
}
|
| 358 |
-
|
| 359 |
-
let inv_rms = inverseSqrt(head_sq[0] / 128.0 + 1e-6);
|
| 360 |
-
let norm_off = select(op.k_norm_offset, op.q_norm_offset, is_q);
|
| 361 |
-
let n0 = x0 * inv_rms * norms[norm_off + tid];
|
| 362 |
-
let n1 = x1 * inv_rms * norms[norm_off + tid + 64u];
|
| 363 |
-
|
| 364 |
-
let pos = step.start_pos + b;
|
| 365 |
-
let c = norms[op.cos_offset + pos * 64u + tid];
|
| 366 |
-
let s = norms[op.sin_offset + pos * 64u + tid];
|
| 367 |
-
let r0 = n0 * c - n1 * s;
|
| 368 |
-
let r1 = n1 * c + n0 * s;
|
| 369 |
-
|
| 370 |
-
if (is_q) {
|
| 371 |
-
let base = b * 2048u + head_id * 128u;
|
| 372 |
-
q_buf[base + tid] = r0;
|
| 373 |
-
q_buf[base + tid + 64u] = r1;
|
| 374 |
-
} else {
|
| 375 |
-
let kv_h = head_id - 16u;
|
| 376 |
-
let cache_slot = ((op.local_layer * 8u + kv_h) * step.max_seq + pos) * 64u;
|
| 377 |
-
if (tid < 32u) {
|
| 378 |
-
head_sq[tid] = r0;
|
| 379 |
-
head_sq[tid + 32u] = r1;
|
| 380 |
-
}
|
| 381 |
-
workgroupBarrier();
|
| 382 |
-
if (tid >= 32u) {
|
| 383 |
-
// Wait cleanly while first half writes; pack using contiguous pairs below.
|
| 384 |
-
}
|
| 385 |
-
// Pack contiguous (2*tid, 2*tid+1) for both K and V so GQA reads vec2 linearly.
|
| 386 |
-
workgroupBarrier();
|
| 387 |
-
head_sq[tid] = r0;
|
| 388 |
-
workgroupBarrier();
|
| 389 |
-
if (tid < 32u) {
|
| 390 |
-
let k_even = head_sq[tid * 2u];
|
| 391 |
-
let k_odd = head_sq[tid * 2u + 1u];
|
| 392 |
-
k_cache[cache_slot + tid] = pack2x16float(vec2<f32>(k_even, k_odd));
|
| 393 |
-
}
|
| 394 |
-
workgroupBarrier();
|
| 395 |
-
head_sq[tid] = r1;
|
| 396 |
-
workgroupBarrier();
|
| 397 |
-
if (tid < 32u) {
|
| 398 |
-
let k_even2 = head_sq[tid * 2u];
|
| 399 |
-
let k_odd2 = head_sq[tid * 2u + 1u];
|
| 400 |
-
k_cache[cache_slot + 32u + tid] = pack2x16float(vec2<f32>(k_even2, k_odd2));
|
| 401 |
-
}
|
| 402 |
-
|
| 403 |
-
let v_base = b * 1024u + kv_h * 128u + tid * 2u;
|
| 404 |
-
v_cache[cache_slot + tid] = pack2x16float(vec2<f32>(v_in[v_base], v_in[v_base + 1u]));
|
| 405 |
-
}
|
| 406 |
-
}
|
| 407 |
-
`;
|
| 408 |
-
|
| 409 |
-
export const GQA_ATTN_WGSL = `
|
| 410 |
-
struct StepParams {
|
| 411 |
-
batch_len: u32,
|
| 412 |
-
start_pos: u32,
|
| 413 |
-
max_seq: u32,
|
| 414 |
-
pad0: u32,
|
| 415 |
-
};
|
| 416 |
-
|
| 417 |
-
struct AttnOp {
|
| 418 |
-
local_layer: u32,
|
| 419 |
-
q_norm_offset: u32,
|
| 420 |
-
k_norm_offset: u32,
|
| 421 |
-
cos_offset: u32,
|
| 422 |
-
sin_offset: u32,
|
| 423 |
-
pad0: u32,
|
| 424 |
-
pad1: u32,
|
| 425 |
-
pad2: u32,
|
| 426 |
-
};
|
| 427 |
-
|
| 428 |
-
@group(0) @binding(0) var<uniform> step: StepParams;
|
| 429 |
-
@group(0) @binding(1) var<uniform> op: AttnOp;
|
| 430 |
-
@group(0) @binding(2) var<storage, read> q_buf: array<f32>;
|
| 431 |
-
@group(0) @binding(3) var<storage, read> k_cache: array<u32>;
|
| 432 |
-
@group(0) @binding(4) var<storage, read> v_cache: array<u32>;
|
| 433 |
-
@group(0) @binding(5) var<storage, read_write> attn_out: array<f32>;
|
| 434 |
-
|
| 435 |
-
var<workgroup> q_sh: array<vec2<f32>, 64>;
|
| 436 |
-
var<workgroup> tile_scores: array<f32, 64>;
|
| 437 |
-
|
| 438 |
-
@compute @workgroup_size(64, 1, 1)
|
| 439 |
-
fn main(
|
| 440 |
-
@builtin(workgroup_id) wg: vec3<u32>,
|
| 441 |
-
@builtin(local_invocation_id) lid: vec3<u32>,
|
| 442 |
-
) {
|
| 443 |
-
let b = wg.x;
|
| 444 |
-
let q_head = wg.y;
|
| 445 |
-
let tid = lid.x;
|
| 446 |
-
if (b >= step.batch_len) {
|
| 447 |
-
return;
|
| 448 |
-
}
|
| 449 |
-
|
| 450 |
-
let kv_head = q_head >> 1u;
|
| 451 |
-
let q_base = b * 2048u + q_head * 128u + tid * 2u;
|
| 452 |
-
q_sh[tid] = vec2<f32>(q_buf[q_base], q_buf[q_base + 1u]);
|
| 453 |
-
workgroupBarrier();
|
| 454 |
-
|
| 455 |
-
let max_t = step.start_pos + b;
|
| 456 |
-
let kv_head_base = (op.local_layer * 8u + kv_head) * step.max_seq;
|
| 457 |
-
let inv_sqrt_d = 0.08838834764831845;
|
| 458 |
-
|
| 459 |
-
var m = -1e30;
|
| 460 |
-
var l = 0.0;
|
| 461 |
-
var acc = vec2<f32>(0.0, 0.0);
|
| 462 |
-
|
| 463 |
-
for (var t_base = 0u; t_base <= max_t; t_base += 64u) {
|
| 464 |
-
let t = t_base + tid;
|
| 465 |
-
var score = -1e30;
|
| 466 |
-
if (t <= max_t) {
|
| 467 |
-
let k_slot = (kv_head_base + t) * 64u;
|
| 468 |
-
var dot_sum = 0.0;
|
| 469 |
-
for (var d = 0u; d < 64u; d += 1u) {
|
| 470 |
-
dot_sum += dot(q_sh[d], unpack2x16float(k_cache[k_slot + d]));
|
| 471 |
-
}
|
| 472 |
-
score = dot_sum * inv_sqrt_d;
|
| 473 |
-
}
|
| 474 |
-
tile_scores[tid] = score;
|
| 475 |
-
workgroupBarrier();
|
| 476 |
-
|
| 477 |
-
let count = min(64u, max_t - t_base + 1u);
|
| 478 |
-
for (var j = 0u; j < count; j += 1u) {
|
| 479 |
-
let s = tile_scores[j];
|
| 480 |
-
let m_next = max(m, s);
|
| 481 |
-
let alpha = exp(m - m_next);
|
| 482 |
-
let beta = exp(s - m_next);
|
| 483 |
-
let v_pair = unpack2x16float(v_cache[(kv_head_base + t_base + j) * 64u + tid]);
|
| 484 |
-
acc = acc * alpha + v_pair * beta;
|
| 485 |
-
l = l * alpha + beta;
|
| 486 |
-
m = m_next;
|
| 487 |
-
}
|
| 488 |
-
workgroupBarrier();
|
| 489 |
-
}
|
| 490 |
-
|
| 491 |
-
let out_pair = acc / max(l, 1e-20);
|
| 492 |
-
attn_out[q_base] = out_pair.x;
|
| 493 |
-
attn_out[q_base + 1u] = out_pair.y;
|
| 494 |
-
}
|
| 495 |
-
`;
|
| 496 |
-
|
| 497 |
-
export const SILU_MUL_WGSL = `
|
| 498 |
-
struct StepParams {
|
| 499 |
-
batch_len: u32,
|
| 500 |
-
start_pos: u32,
|
| 501 |
-
max_seq: u32,
|
| 502 |
-
pad0: u32,
|
| 503 |
-
};
|
| 504 |
-
|
| 505 |
-
@group(0) @binding(0) var<uniform> step: StepParams;
|
| 506 |
-
@group(0) @binding(1) var<storage, read> gate_buf: array<f32>;
|
| 507 |
-
@group(0) @binding(2) var<storage, read> up_buf: array<f32>;
|
| 508 |
-
@group(0) @binding(3) var<storage, read_write> out_buf: array<f32>;
|
| 509 |
-
|
| 510 |
-
@compute @workgroup_size(256, 1, 1)
|
| 511 |
-
fn main(@builtin(global_invocation_id) gid: vec3<u32>) {
|
| 512 |
-
let idx = gid.x;
|
| 513 |
-
let total = step.batch_len * 6144u;
|
| 514 |
-
if (idx >= total) {
|
| 515 |
-
return;
|
| 516 |
-
}
|
| 517 |
-
let g = gate_buf[idx];
|
| 518 |
-
out_buf[idx] = (g / (1.0 + exp(-g))) * up_buf[idx];
|
| 519 |
-
}
|
| 520 |
-
`;
|
| 521 |
-
|
| 522 |
-
export const COPY_LAST_ROW_WGSL = `
|
| 523 |
-
struct StepParams {
|
| 524 |
-
batch_len: u32,
|
| 525 |
-
start_pos: u32,
|
| 526 |
-
max_seq: u32,
|
| 527 |
-
pad0: u32,
|
| 528 |
-
};
|
| 529 |
-
|
| 530 |
-
@group(0) @binding(0) var<uniform> step: StepParams;
|
| 531 |
-
@group(0) @binding(1) var<storage, read> in_x: array<f32>;
|
| 532 |
-
@group(0) @binding(2) var<storage, read_write> last_x: array<f32>;
|
| 533 |
-
|
| 534 |
-
@compute @workgroup_size(256, 1, 1)
|
| 535 |
-
fn main(@builtin(global_invocation_id) gid: vec3<u32>) {
|
| 536 |
-
let idx = gid.x;
|
| 537 |
-
if (idx < 2048u) {
|
| 538 |
-
let base = (step.batch_len - 1u) * 2048u;
|
| 539 |
-
last_x[idx] = in_x[base + idx];
|
| 540 |
-
}
|
| 541 |
-
}
|
| 542 |
-
`;
|
| 543 |
-
|
| 544 |
-
export const TOPK_BLOCKS_WGSL = `
|
| 545 |
-
const VOCAB_SIZE: u32 = 151669u;
|
| 546 |
-
|
| 547 |
-
struct Candidate {
|
| 548 |
-
logit: f32,
|
| 549 |
-
token_id: u32,
|
| 550 |
-
};
|
| 551 |
-
|
| 552 |
-
@group(0) @binding(0) var<storage, read> logits: array<f32>;
|
| 553 |
-
@group(0) @binding(1) var<storage, read_write> out_top: array<Candidate>;
|
| 554 |
-
|
| 555 |
-
var<workgroup> sh_val: array<f32, 256>;
|
| 556 |
-
var<workgroup> sh_idx: array<u32, 256>;
|
| 557 |
-
|
| 558 |
-
@compute @workgroup_size(256, 1, 1)
|
| 559 |
-
fn main(
|
| 560 |
-
@builtin(workgroup_id) wg: vec3<u32>,
|
| 561 |
-
@builtin(local_invocation_id) lid: vec3<u32>,
|
| 562 |
-
) {
|
| 563 |
-
let block = wg.x;
|
| 564 |
-
let tid = lid.x;
|
| 565 |
-
|
| 566 |
-
var best_v = -1e30;
|
| 567 |
-
var best_i = 0u;
|
| 568 |
-
for (var idx = block + tid * 64u; idx < VOCAB_SIZE; idx += 16384u) {
|
| 569 |
-
let v = logits[idx];
|
| 570 |
-
if (v > best_v) {
|
| 571 |
-
best_v = v;
|
| 572 |
-
best_i = idx;
|
| 573 |
-
}
|
| 574 |
-
}
|
| 575 |
-
sh_val[tid] = best_v;
|
| 576 |
-
sh_idx[tid] = best_i;
|
| 577 |
-
workgroupBarrier();
|
| 578 |
-
|
| 579 |
-
for (var stride = 128u; stride > 0u; stride >>= 1u) {
|
| 580 |
-
if (tid < stride) {
|
| 581 |
-
let other_v = sh_val[tid + stride];
|
| 582 |
-
let other_i = sh_idx[tid + stride];
|
| 583 |
-
if (other_v > sh_val[tid] || (other_v == sh_val[tid] && other_i < sh_idx[tid])) {
|
| 584 |
-
sh_val[tid] = other_v;
|
| 585 |
-
sh_idx[tid] = other_i;
|
| 586 |
-
}
|
| 587 |
-
}
|
| 588 |
-
workgroupBarrier();
|
| 589 |
-
}
|
| 590 |
-
|
| 591 |
-
if (tid == 0u) {
|
| 592 |
-
out_top[block] = Candidate(sh_val[0], sh_idx[0]);
|
| 593 |
-
}
|
| 594 |
-
}
|
| 595 |
-
`;
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
web/src/bonsai-tokenizer.js
DELETED
|
@@ -1,197 +0,0 @@
|
|
| 1 |
-
// Pure-JS Qwen3 ByteLevel BPE tokenizer matching Hugging Face tokenizers / llama.cpp.
|
| 2 |
-
|
| 3 |
-
const PRE_TOKENIZE_RE = new RegExp(
|
| 4 |
-
[
|
| 5 |
-
"(?:'[sStTrRdDmM]|'[rR][eE]|'[vV][eE]|'[lL][lL])",
|
| 6 |
-
"[^\\r\\n\\p{L}\\p{N}]?\\p{L}+",
|
| 7 |
-
"\\p{N}",
|
| 8 |
-
" ?[^\\s\\p{L}\\p{N}]+[\\r\\n]*",
|
| 9 |
-
"\\s*[\\r\\n]+",
|
| 10 |
-
"\\s+(?!\\S)",
|
| 11 |
-
"\\s+",
|
| 12 |
-
].join("|"),
|
| 13 |
-
"gu",
|
| 14 |
-
);
|
| 15 |
-
|
| 16 |
-
const MAX_WORD_CACHE = 8192;
|
| 17 |
-
|
| 18 |
-
function buildByteTables() {
|
| 19 |
-
const kept = [];
|
| 20 |
-
for (let b = 33; b <= 126; b += 1) kept.push(b);
|
| 21 |
-
for (let b = 161; b <= 172; b += 1) kept.push(b);
|
| 22 |
-
for (let b = 174; b <= 255; b += 1) kept.push(b);
|
| 23 |
-
|
| 24 |
-
const keptSet = new Set(kept);
|
| 25 |
-
const chars = kept.slice();
|
| 26 |
-
let extra = 0;
|
| 27 |
-
for (let b = 0; b < 256; b += 1) {
|
| 28 |
-
if (!keptSet.has(b)) {
|
| 29 |
-
kept.push(b);
|
| 30 |
-
chars.push(256 + extra);
|
| 31 |
-
extra += 1;
|
| 32 |
-
}
|
| 33 |
-
}
|
| 34 |
-
const byteToChar = new Array(256);
|
| 35 |
-
const charToByte = new Map();
|
| 36 |
-
for (let i = 0; i < 256; i += 1) {
|
| 37 |
-
const ch = String.fromCodePoint(chars[i]);
|
| 38 |
-
byteToChar[kept[i]] = ch;
|
| 39 |
-
charToByte.set(ch, kept[i]);
|
| 40 |
-
}
|
| 41 |
-
return { byteToChar, charToByte };
|
| 42 |
-
}
|
| 43 |
-
|
| 44 |
-
const { byteToChar, charToByte } = buildByteTables();
|
| 45 |
-
const utf8Encoder = new TextEncoder();
|
| 46 |
-
|
| 47 |
-
function escapeRegex(literal) {
|
| 48 |
-
return literal.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
| 49 |
-
}
|
| 50 |
-
|
| 51 |
-
function findLowestRankPair(symbols, mergeRank) {
|
| 52 |
-
let bestRank = Infinity;
|
| 53 |
-
let bestKey = null;
|
| 54 |
-
for (let i = 0; i < symbols.length - 1; i += 1) {
|
| 55 |
-
const key = `${symbols[i]} ${symbols[i + 1]}`;
|
| 56 |
-
const rank = mergeRank.get(key);
|
| 57 |
-
if (rank !== undefined && rank < bestRank) {
|
| 58 |
-
bestRank = rank;
|
| 59 |
-
bestKey = key;
|
| 60 |
-
}
|
| 61 |
-
}
|
| 62 |
-
return bestKey;
|
| 63 |
-
}
|
| 64 |
-
|
| 65 |
-
function mergePairOnce(symbols, first, second) {
|
| 66 |
-
const next = [];
|
| 67 |
-
let i = 0;
|
| 68 |
-
while (i < symbols.length) {
|
| 69 |
-
if (i < symbols.length - 1 && symbols[i] === first && symbols[i + 1] === second) {
|
| 70 |
-
next.push(first + second);
|
| 71 |
-
i += 2;
|
| 72 |
-
} else {
|
| 73 |
-
next.push(symbols[i]);
|
| 74 |
-
i += 1;
|
| 75 |
-
}
|
| 76 |
-
}
|
| 77 |
-
return next;
|
| 78 |
-
}
|
| 79 |
-
|
| 80 |
-
function applyBpe(wordSymbols, mergeRank) {
|
| 81 |
-
let symbols = wordSymbols;
|
| 82 |
-
while (symbols.length > 1) {
|
| 83 |
-
const pairKey = findLowestRankPair(symbols, mergeRank);
|
| 84 |
-
if (pairKey === null) {
|
| 85 |
-
break;
|
| 86 |
-
}
|
| 87 |
-
const splitAt = pairKey.indexOf(" ");
|
| 88 |
-
symbols = mergePairOnce(symbols, pairKey.slice(0, splitAt), pairKey.slice(splitAt + 1));
|
| 89 |
-
}
|
| 90 |
-
return symbols;
|
| 91 |
-
}
|
| 92 |
-
|
| 93 |
-
export class BonsaiTokenizer {
|
| 94 |
-
constructor(compact) {
|
| 95 |
-
this.vocab = compact.vocab;
|
| 96 |
-
this.eosTokenId = compact.eos_token_id ?? 151645;
|
| 97 |
-
this.padTokenId = compact.pad_token_id ?? 151643;
|
| 98 |
-
this.addedTokens = new Map(Object.entries(compact.added_tokens ?? {}));
|
| 99 |
-
this.specialIds = new Set(this.addedTokens.values());
|
| 100 |
-
|
| 101 |
-
this.vocabToId = new Map();
|
| 102 |
-
this.idToBytes = new Array(this.vocab.length);
|
| 103 |
-
for (let id = 0; id < this.vocab.length; id += 1) {
|
| 104 |
-
const token = this.vocab[id];
|
| 105 |
-
if (!token) continue;
|
| 106 |
-
this.vocabToId.set(token, id);
|
| 107 |
-
if (!this.specialIds.has(id)) {
|
| 108 |
-
const bytes = new Uint8Array(token.length);
|
| 109 |
-
let len = 0;
|
| 110 |
-
for (const ch of token) {
|
| 111 |
-
const b = charToByte.get(ch);
|
| 112 |
-
if (b !== undefined) {
|
| 113 |
-
bytes[len] = b;
|
| 114 |
-
len += 1;
|
| 115 |
-
}
|
| 116 |
-
}
|
| 117 |
-
this.idToBytes[id] = bytes.subarray(0, len);
|
| 118 |
-
}
|
| 119 |
-
}
|
| 120 |
-
|
| 121 |
-
this.mergeRank = new Map();
|
| 122 |
-
const mergeLines = compact.merges.split("\n");
|
| 123 |
-
for (let i = 0; i < mergeLines.length; i += 1) {
|
| 124 |
-
if (mergeLines[i]) {
|
| 125 |
-
this.mergeRank.set(mergeLines[i], i);
|
| 126 |
-
}
|
| 127 |
-
}
|
| 128 |
-
|
| 129 |
-
const specials = [...this.addedTokens.keys()].sort((a, b) => b.length - a.length);
|
| 130 |
-
this.specialSplitRe = specials.length
|
| 131 |
-
? new RegExp(`(${specials.map(escapeRegex).join("|")})`, "g")
|
| 132 |
-
: null;
|
| 133 |
-
this.wordCache = new Map();
|
| 134 |
-
}
|
| 135 |
-
|
| 136 |
-
encodePiece(piece) {
|
| 137 |
-
const cached = this.wordCache.get(piece);
|
| 138 |
-
if (cached !== undefined) {
|
| 139 |
-
return cached;
|
| 140 |
-
}
|
| 141 |
-
const bytes = utf8Encoder.encode(piece);
|
| 142 |
-
const initial = new Array(bytes.length);
|
| 143 |
-
for (let i = 0; i < bytes.length; i += 1) {
|
| 144 |
-
initial[i] = byteToChar[bytes[i]];
|
| 145 |
-
}
|
| 146 |
-
const merged = applyBpe(initial, this.mergeRank);
|
| 147 |
-
const ids = [];
|
| 148 |
-
for (let i = 0; i < merged.length; i += 1) {
|
| 149 |
-
const id = this.vocabToId.get(merged[i]);
|
| 150 |
-
if (id !== undefined) {
|
| 151 |
-
ids.push(id);
|
| 152 |
-
}
|
| 153 |
-
}
|
| 154 |
-
if (this.wordCache.size >= MAX_WORD_CACHE) {
|
| 155 |
-
this.wordCache.clear();
|
| 156 |
-
}
|
| 157 |
-
this.wordCache.set(piece, ids);
|
| 158 |
-
return ids;
|
| 159 |
-
}
|
| 160 |
-
|
| 161 |
-
encode(text) {
|
| 162 |
-
const parts = this.specialSplitRe ? text.split(this.specialSplitRe) : [text];
|
| 163 |
-
const tokenIds = [];
|
| 164 |
-
for (const part of parts) {
|
| 165 |
-
if (!part) continue;
|
| 166 |
-
const specialId = this.addedTokens.get(part);
|
| 167 |
-
if (specialId !== undefined) {
|
| 168 |
-
tokenIds.push(specialId);
|
| 169 |
-
continue;
|
| 170 |
-
}
|
| 171 |
-
const normalized = part.normalize("NFC");
|
| 172 |
-
const matches = normalized.match(PRE_TOKENIZE_RE);
|
| 173 |
-
if (!matches) continue;
|
| 174 |
-
for (const piece of matches) {
|
| 175 |
-
const pieceIds = this.encodePiece(piece);
|
| 176 |
-
for (let i = 0; i < pieceIds.length; i += 1) {
|
| 177 |
-
tokenIds.push(pieceIds[i]);
|
| 178 |
-
}
|
| 179 |
-
}
|
| 180 |
-
}
|
| 181 |
-
return tokenIds;
|
| 182 |
-
}
|
| 183 |
-
|
| 184 |
-
createStreamDecoder() {
|
| 185 |
-
const decoder = new TextDecoder("utf-8", { fatal: false });
|
| 186 |
-
return {
|
| 187 |
-
push: (tokenId) => {
|
| 188 |
-
if (this.specialIds.has(tokenId)) {
|
| 189 |
-
return "";
|
| 190 |
-
}
|
| 191 |
-
const bytes = this.idToBytes[tokenId];
|
| 192 |
-
return bytes ? decoder.decode(bytes, { stream: true }) : "";
|
| 193 |
-
},
|
| 194 |
-
flush: () => decoder.decode(),
|
| 195 |
-
};
|
| 196 |
-
}
|
| 197 |
-
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
web/src/bonsai-webgpu.js
DELETED
|
@@ -1,605 +0,0 @@
|
|
| 1 |
-
// WebGPU runtime for Bonsai 1.7B 2-bit packed ternary Qwen3ForCausalLM.
|
| 2 |
-
import {
|
| 3 |
-
COPY_LAST_ROW_WGSL,
|
| 4 |
-
EMBED_WGSL,
|
| 5 |
-
GQA_ATTN_WGSL,
|
| 6 |
-
QK_ROPE_KV_WGSL,
|
| 7 |
-
RMSNORM_WGSL,
|
| 8 |
-
SILU_MUL_WGSL,
|
| 9 |
-
TERNARY_GEMM_WGSL,
|
| 10 |
-
TERNARY_GEMV_WGSL,
|
| 11 |
-
TOPK_BLOCKS_WGSL,
|
| 12 |
-
} from "./bonsai-shaders.js";
|
| 13 |
-
import { BonsaiTokenizer } from "./bonsai-tokenizer.js";
|
| 14 |
-
|
| 15 |
-
export const DEFAULT_MODEL_BASE =
|
| 16 |
-
"https://huggingface.co/brianbellx/codehygiene-bonsai-1.7b/resolve/main/webgpu";
|
| 17 |
-
|
| 18 |
-
const CACHE_NAME = "codehygiene-bonsai-17b-v1";
|
| 19 |
-
const NUM_LAYERS = 28;
|
| 20 |
-
const LAYERS_PER_SHARD = 7;
|
| 21 |
-
const MAX_SEQ_LEN = 4096;
|
| 22 |
-
const PREFILL_BATCH = 16;
|
| 23 |
-
const UNIFORM_ALIGN = 256;
|
| 24 |
-
const TOPK_BLOCKS = 64;
|
| 25 |
-
|
| 26 |
-
const LAYER_TRIT_STRIDE = 3_145_728;
|
| 27 |
-
const LAYER_SCALE_STRIDE = 786_432;
|
| 28 |
-
const LAYER_NORM_STRIDE = 4_352;
|
| 29 |
-
const FINAL_NORM_OFFSET = NUM_LAYERS * LAYER_NORM_STRIDE;
|
| 30 |
-
const COS_TABLE_OFFSET = FINAL_NORM_OFFSET + 2048;
|
| 31 |
-
const SIN_TABLE_OFFSET = COS_TABLE_OFFSET + MAX_SEQ_LEN * 64;
|
| 32 |
-
|
| 33 |
-
const PROJ_SPECS = [
|
| 34 |
-
{ name: "q", rows: 2048, inGroups: 64, tritOff: 0, residual: 0 },
|
| 35 |
-
{ name: "k", rows: 1024, inGroups: 64, tritOff: 262_144, residual: 0 },
|
| 36 |
-
{ name: "v", rows: 1024, inGroups: 64, tritOff: 393_216, residual: 0 },
|
| 37 |
-
{ name: "o", rows: 2048, inGroups: 64, tritOff: 524_288, residual: 1 },
|
| 38 |
-
{ name: "gate", rows: 6144, inGroups: 64, tritOff: 786_432, residual: 0 },
|
| 39 |
-
{ name: "up", rows: 6144, inGroups: 64, tritOff: 1_572_864, residual: 0 },
|
| 40 |
-
{ name: "down", rows: 2048, inGroups: 192, tritOff: 2_359_296, residual: 1 },
|
| 41 |
-
];
|
| 42 |
-
|
| 43 |
-
async function openModelCache() {
|
| 44 |
-
if (!globalThis.caches) return null;
|
| 45 |
-
try {
|
| 46 |
-
return await globalThis.caches.open(CACHE_NAME);
|
| 47 |
-
} catch (err) {
|
| 48 |
-
console.warn("Cache storage unavailable:", err);
|
| 49 |
-
return null;
|
| 50 |
-
}
|
| 51 |
-
}
|
| 52 |
-
|
| 53 |
-
async function fetchCachedResponse(url, signal) {
|
| 54 |
-
const cacheStore = await openModelCache();
|
| 55 |
-
if (cacheStore) {
|
| 56 |
-
const hit = await cacheStore.match(url);
|
| 57 |
-
if (hit) return hit;
|
| 58 |
-
}
|
| 59 |
-
const response = await fetch(url, { signal, mode: "cors" });
|
| 60 |
-
if (!response.ok) {
|
| 61 |
-
throw new Error(`Shard download failed (${response.status}): ${url}`);
|
| 62 |
-
}
|
| 63 |
-
if (cacheStore) {
|
| 64 |
-
try {
|
| 65 |
-
await cacheStore.put(url, response.clone());
|
| 66 |
-
} catch (err) {
|
| 67 |
-
console.warn("Cache write skipped:", err);
|
| 68 |
-
}
|
| 69 |
-
}
|
| 70 |
-
return response;
|
| 71 |
-
}
|
| 72 |
-
|
| 73 |
-
async function gunzipStreamToBytes(response, expectedBytes, onChunkBytes) {
|
| 74 |
-
const stream = response.body.pipeThrough(new DecompressionStream("gzip"));
|
| 75 |
-
const reader = stream.getReader();
|
| 76 |
-
const out = new Uint8Array(expectedBytes);
|
| 77 |
-
let offset = 0;
|
| 78 |
-
for (;;) {
|
| 79 |
-
const { value, done } = await reader.read();
|
| 80 |
-
if (done) break;
|
| 81 |
-
out.set(value, offset);
|
| 82 |
-
offset += value.byteLength;
|
| 83 |
-
onChunkBytes(value.byteLength);
|
| 84 |
-
}
|
| 85 |
-
if (offset !== expectedBytes) {
|
| 86 |
-
throw new Error(`Shard length mismatch: got ${offset}, expected ${expectedBytes}`);
|
| 87 |
-
}
|
| 88 |
-
return out;
|
| 89 |
-
}
|
| 90 |
-
|
| 91 |
-
function makeStorageBuffer(device, byteLength, extraUsage = 0) {
|
| 92 |
-
const aligned = Math.ceil(byteLength / 4) * 4;
|
| 93 |
-
return device.createBuffer({
|
| 94 |
-
size: aligned,
|
| 95 |
-
usage: GPUBufferUsage.STORAGE | extraUsage,
|
| 96 |
-
});
|
| 97 |
-
}
|
| 98 |
-
|
| 99 |
-
function sampleTopCandidates(stagingBytes, temperature, topP) {
|
| 100 |
-
const view = new DataView(stagingBytes);
|
| 101 |
-
let bestVal = -Infinity;
|
| 102 |
-
let bestId = 0;
|
| 103 |
-
const items = new Array(TOPK_BLOCKS);
|
| 104 |
-
for (let i = 0; i < TOPK_BLOCKS; i += 1) {
|
| 105 |
-
const logit = view.getFloat32(i * 8, true);
|
| 106 |
-
const tokenId = view.getUint32(i * 8 + 4, true);
|
| 107 |
-
items[i] = { logit, tokenId };
|
| 108 |
-
if (logit > bestVal) {
|
| 109 |
-
bestVal = logit;
|
| 110 |
-
bestId = tokenId;
|
| 111 |
-
}
|
| 112 |
-
}
|
| 113 |
-
if (!(temperature > 0)) {
|
| 114 |
-
return bestId;
|
| 115 |
-
}
|
| 116 |
-
items.sort((a, b) => b.logit - a.logit);
|
| 117 |
-
const invTemp = 1.0 / temperature;
|
| 118 |
-
const maxScaled = items[0].logit * invTemp;
|
| 119 |
-
let sumExp = 0;
|
| 120 |
-
const probs = new Float64Array(TOPK_BLOCKS);
|
| 121 |
-
for (let i = 0; i < TOPK_BLOCKS; i += 1) {
|
| 122 |
-
const p = Math.exp(items[i].logit * invTemp - maxScaled);
|
| 123 |
-
probs[i] = p;
|
| 124 |
-
sumExp += p;
|
| 125 |
-
}
|
| 126 |
-
let cum = 0;
|
| 127 |
-
let cutoff = TOPK_BLOCKS;
|
| 128 |
-
for (let i = 0; i < TOPK_BLOCKS; i += 1) {
|
| 129 |
-
probs[i] /= sumExp;
|
| 130 |
-
cum += probs[i];
|
| 131 |
-
if (cum >= topP) {
|
| 132 |
-
cutoff = i + 1;
|
| 133 |
-
break;
|
| 134 |
-
}
|
| 135 |
-
}
|
| 136 |
-
const target = Math.random() * cum;
|
| 137 |
-
let running = 0;
|
| 138 |
-
for (let i = 0; i < cutoff; i += 1) {
|
| 139 |
-
running += probs[i];
|
| 140 |
-
if (target <= running) {
|
| 141 |
-
return items[i].tokenId;
|
| 142 |
-
}
|
| 143 |
-
}
|
| 144 |
-
return items[0].tokenId;
|
| 145 |
-
}
|
| 146 |
-
|
| 147 |
-
function sharedPrefixLength(left, right) {
|
| 148 |
-
const limit = Math.min(left.length, right.length);
|
| 149 |
-
let idx = 0;
|
| 150 |
-
while (idx < limit && left[idx] === right[idx]) idx += 1;
|
| 151 |
-
return idx;
|
| 152 |
-
}
|
| 153 |
-
|
| 154 |
-
const SOFTWARE_ADAPTER_RE = /swiftshader|basic render|llvmpipe|lavapipe|warp|software/i;
|
| 155 |
-
const MAX_STEP_MS = 250;
|
| 156 |
-
|
| 157 |
-
function isUsableHardwareAdapter(adapter) {
|
| 158 |
-
if (!adapter || adapter.info?.isFallbackAdapter) return false;
|
| 159 |
-
const info = adapter.info ?? {};
|
| 160 |
-
const label = `${info.vendor ?? ""} ${info.architecture ?? ""} ${info.device ?? ""} ${info.description ?? ""}`;
|
| 161 |
-
return !SOFTWARE_ADAPTER_RE.test(label);
|
| 162 |
-
}
|
| 163 |
-
|
| 164 |
-
export async function probeWebGpuSupport() {
|
| 165 |
-
if (!globalThis.navigator?.gpu) return false;
|
| 166 |
-
try {
|
| 167 |
-
const adapter = await navigator.gpu.requestAdapter();
|
| 168 |
-
return isUsableHardwareAdapter(adapter);
|
| 169 |
-
} catch {
|
| 170 |
-
return false;
|
| 171 |
-
}
|
| 172 |
-
}
|
| 173 |
-
|
| 174 |
-
export class BonsaiWebGpuEngine {
|
| 175 |
-
constructor(device, tokenizer, buffers, pipelines, ops) {
|
| 176 |
-
this.device = device;
|
| 177 |
-
this.tokenizer = tokenizer;
|
| 178 |
-
this.buffers = buffers;
|
| 179 |
-
this.pipelines = pipelines;
|
| 180 |
-
this.ops = ops;
|
| 181 |
-
this.cachedTokens = [];
|
| 182 |
-
}
|
| 183 |
-
|
| 184 |
-
static async load({
|
| 185 |
-
baseUrl = DEFAULT_MODEL_BASE,
|
| 186 |
-
skipSpeedGate = false,
|
| 187 |
-
signal,
|
| 188 |
-
onProgress = () => {},
|
| 189 |
-
} = {}) {
|
| 190 |
-
if (!globalThis.navigator?.gpu) {
|
| 191 |
-
throw new Error("WebGPU is unavailable in this browser.");
|
| 192 |
-
}
|
| 193 |
-
const adapter = await navigator.gpu.requestAdapter();
|
| 194 |
-
if (!isUsableHardwareAdapter(adapter)) {
|
| 195 |
-
throw new Error("Hardware WebGPU adapter is unavailable.");
|
| 196 |
-
}
|
| 197 |
-
const device = await adapter.requestDevice();
|
| 198 |
-
|
| 199 |
-
const manifestRes = await fetchCachedResponse(`${baseUrl}/manifest.json`, signal);
|
| 200 |
-
const manifest = await manifestRes.json();
|
| 201 |
-
const shards = manifest.shards;
|
| 202 |
-
|
| 203 |
-
const shardOrder = [
|
| 204 |
-
["tokenizer", shards.tokenizer],
|
| 205 |
-
["norms", shards.norms],
|
| 206 |
-
["embed_trits", shards.embed_trits],
|
| 207 |
-
["embed_scales", shards.embed_scales],
|
| 208 |
-
["layer_scales", shards.layer_scales],
|
| 209 |
-
["layers_0", shards.layers_0],
|
| 210 |
-
["layers_1", shards.layers_1],
|
| 211 |
-
["layers_2", shards.layers_2],
|
| 212 |
-
["layers_3", shards.layers_3],
|
| 213 |
-
];
|
| 214 |
-
const totalRawBytes = shardOrder.reduce((sum, [, info]) => sum + info.raw_bytes, 0);
|
| 215 |
-
let loadedRawBytes = 0;
|
| 216 |
-
const reportBytes = (delta) => {
|
| 217 |
-
loadedRawBytes += delta;
|
| 218 |
-
onProgress(Math.min(loadedRawBytes, totalRawBytes), totalRawBytes);
|
| 219 |
-
};
|
| 220 |
-
|
| 221 |
-
const tokBytes = await gunzipStreamToBytes(
|
| 222 |
-
await fetchCachedResponse(`${baseUrl}/${shards.tokenizer.file}`, signal),
|
| 223 |
-
shards.tokenizer.raw_bytes,
|
| 224 |
-
reportBytes,
|
| 225 |
-
);
|
| 226 |
-
const tokenizer = new BonsaiTokenizer(JSON.parse(new TextDecoder().decode(tokBytes)));
|
| 227 |
-
|
| 228 |
-
const gpuWeights = {};
|
| 229 |
-
for (const [key, info] of shardOrder.slice(1)) {
|
| 230 |
-
const rawBytes = await gunzipStreamToBytes(
|
| 231 |
-
await fetchCachedResponse(`${baseUrl}/${info.file}`, signal),
|
| 232 |
-
info.raw_bytes,
|
| 233 |
-
reportBytes,
|
| 234 |
-
);
|
| 235 |
-
const buf = makeStorageBuffer(device, info.raw_bytes, GPUBufferUsage.COPY_DST);
|
| 236 |
-
device.queue.writeBuffer(buf, 0, rawBytes);
|
| 237 |
-
gpuWeights[key] = buf;
|
| 238 |
-
}
|
| 239 |
-
|
| 240 |
-
const kvHalfBytes = 14 * 8 * MAX_SEQ_LEN * 64 * 4;
|
| 241 |
-
const buffers = {
|
| 242 |
-
...gpuWeights,
|
| 243 |
-
stepUniform: device.createBuffer({
|
| 244 |
-
size: 16,
|
| 245 |
-
usage: GPUBufferUsage.UNIFORM | GPUBufferUsage.COPY_DST,
|
| 246 |
-
}),
|
| 247 |
-
tokenIds: makeStorageBuffer(device, PREFILL_BATCH * 4, GPUBufferUsage.COPY_DST),
|
| 248 |
-
x: makeStorageBuffer(device, PREFILL_BATCH * 2048 * 4),
|
| 249 |
-
lastX: makeStorageBuffer(device, 2048 * 4),
|
| 250 |
-
normed: makeStorageBuffer(device, PREFILL_BATCH * 2048 * 4),
|
| 251 |
-
q: makeStorageBuffer(device, PREFILL_BATCH * 2048 * 4),
|
| 252 |
-
k: makeStorageBuffer(device, PREFILL_BATCH * 1024 * 4),
|
| 253 |
-
v: makeStorageBuffer(device, PREFILL_BATCH * 1024 * 4),
|
| 254 |
-
attnOut: makeStorageBuffer(device, PREFILL_BATCH * 2048 * 4),
|
| 255 |
-
gate: makeStorageBuffer(device, PREFILL_BATCH * 6144 * 4),
|
| 256 |
-
up: makeStorageBuffer(device, PREFILL_BATCH * 6144 * 4),
|
| 257 |
-
mlpMid: makeStorageBuffer(device, PREFILL_BATCH * 6144 * 4),
|
| 258 |
-
logits: makeStorageBuffer(device, 151669 * 4),
|
| 259 |
-
topOut: makeStorageBuffer(device, TOPK_BLOCKS * 8, GPUBufferUsage.COPY_SRC),
|
| 260 |
-
topRead: device.createBuffer({
|
| 261 |
-
size: TOPK_BLOCKS * 8,
|
| 262 |
-
usage: GPUBufferUsage.COPY_DST | GPUBufferUsage.MAP_READ,
|
| 263 |
-
}),
|
| 264 |
-
kCache0: makeStorageBuffer(device, kvHalfBytes),
|
| 265 |
-
vCache0: makeStorageBuffer(device, kvHalfBytes),
|
| 266 |
-
kCache1: makeStorageBuffer(device, kvHalfBytes),
|
| 267 |
-
vCache1: makeStorageBuffer(device, kvHalfBytes),
|
| 268 |
-
};
|
| 269 |
-
|
| 270 |
-
const compile = (code) =>
|
| 271 |
-
device.createComputePipelineAsync({
|
| 272 |
-
layout: "auto",
|
| 273 |
-
compute: { module: device.createShaderModule({ code }), entryPoint: "main" },
|
| 274 |
-
});
|
| 275 |
-
|
| 276 |
-
const [embed, rmsNorm, gemv, gemm, qkRopeKv, gqaAttn, siluMul, copyLast, topk] =
|
| 277 |
-
await Promise.all([
|
| 278 |
-
compile(EMBED_WGSL),
|
| 279 |
-
compile(RMSNORM_WGSL),
|
| 280 |
-
compile(TERNARY_GEMV_WGSL),
|
| 281 |
-
compile(TERNARY_GEMM_WGSL),
|
| 282 |
-
compile(QK_ROPE_KV_WGSL),
|
| 283 |
-
compile(GQA_ATTN_WGSL),
|
| 284 |
-
compile(SILU_MUL_WGSL),
|
| 285 |
-
compile(COPY_LAST_ROW_WGSL),
|
| 286 |
-
compile(TOPK_BLOCKS_WGSL),
|
| 287 |
-
]);
|
| 288 |
-
const pipelines = { embed, rmsNorm, gemv, gemm, qkRopeKv, gqaAttn, siluMul, copyLast, topk };
|
| 289 |
-
const ops = BonsaiWebGpuEngine.buildLayerOps(device, buffers, pipelines);
|
| 290 |
-
await device.queue.onSubmittedWorkDone();
|
| 291 |
-
const engine = new BonsaiWebGpuEngine(device, tokenizer, buffers, pipelines, ops);
|
| 292 |
-
if (!skipSpeedGate) {
|
| 293 |
-
await engine.verifyRealtimeThroughput();
|
| 294 |
-
}
|
| 295 |
-
return engine;
|
| 296 |
-
}
|
| 297 |
-
|
| 298 |
-
async verifyRealtimeThroughput() {
|
| 299 |
-
this.encodeForwardStep([151644, 8948], 0, { computeLogits: true });
|
| 300 |
-
await this.readNextToken(0.0, 1.0);
|
| 301 |
-
this.encodeForwardStep([198], 2, { computeLogits: true });
|
| 302 |
-
await this.readNextToken(0.0, 1.0);
|
| 303 |
-
const t0 = performance.now();
|
| 304 |
-
this.encodeForwardStep([151644], 3, { computeLogits: true });
|
| 305 |
-
await this.readNextToken(0.0, 1.0);
|
| 306 |
-
const stepMs = performance.now() - t0;
|
| 307 |
-
this.cachedTokens = [];
|
| 308 |
-
if (stepMs > MAX_STEP_MS) {
|
| 309 |
-
throw new Error(`WebGPU decode step too slow (${Math.round(stepMs)} ms); using ZeroGPU.`);
|
| 310 |
-
}
|
| 311 |
-
}
|
| 312 |
-
|
| 313 |
-
static buildLayerOps(device, b, p) {
|
| 314 |
-
const totalSlots = NUM_LAYERS * 11 + 4;
|
| 315 |
-
const staticBytes = new ArrayBuffer(totalSlots * UNIFORM_ALIGN);
|
| 316 |
-
const staticU32 = new Uint32Array(staticBytes);
|
| 317 |
-
let nextSlot = 0;
|
| 318 |
-
const allocSlot = (vals) => {
|
| 319 |
-
const slot = nextSlot;
|
| 320 |
-
nextSlot += 1;
|
| 321 |
-
staticU32.set(vals, (slot * UNIFORM_ALIGN) / 4);
|
| 322 |
-
return slot * UNIFORM_ALIGN;
|
| 323 |
-
};
|
| 324 |
-
|
| 325 |
-
const staticBuf = device.createBuffer({
|
| 326 |
-
size: staticBytes.byteLength,
|
| 327 |
-
usage: GPUBufferUsage.UNIFORM | GPUBufferUsage.COPY_DST,
|
| 328 |
-
});
|
| 329 |
-
|
| 330 |
-
const mkBg = (pipeline, entries) =>
|
| 331 |
-
device.createBindGroup({
|
| 332 |
-
layout: pipeline.getBindGroupLayout(0),
|
| 333 |
-
entries: entries.map((resource, binding) => ({ binding, resource })),
|
| 334 |
-
});
|
| 335 |
-
const uSlice = (offset, size = 32) => ({ buffer: staticBuf, offset, size });
|
| 336 |
-
const ref = (buf) => ({ buffer: buf });
|
| 337 |
-
const stepRef = ref(b.stepUniform);
|
| 338 |
-
|
| 339 |
-
const embedBg = mkBg(p.embed, [
|
| 340 |
-
stepRef,
|
| 341 |
-
ref(b.tokenIds),
|
| 342 |
-
ref(b.embed_trits),
|
| 343 |
-
ref(b.embed_scales),
|
| 344 |
-
ref(b.x),
|
| 345 |
-
]);
|
| 346 |
-
const siluBg = mkBg(p.siluMul, [stepRef, ref(b.gate), ref(b.up), ref(b.mlpMid)]);
|
| 347 |
-
const copyLastBg = mkBg(p.copyLast, [stepRef, ref(b.x), ref(b.lastX)]);
|
| 348 |
-
|
| 349 |
-
const layerShards = [b.layers_0, b.layers_1, b.layers_2, b.layers_3];
|
| 350 |
-
const layers = [];
|
| 351 |
-
|
| 352 |
-
for (let l = 0; l < NUM_LAYERS; l += 1) {
|
| 353 |
-
const shardBuf = layerShards[Math.floor(l / LAYERS_PER_SHARD)];
|
| 354 |
-
const lInShard = l % LAYERS_PER_SHARD;
|
| 355 |
-
const kCache = l < 14 ? b.kCache0 : b.kCache1;
|
| 356 |
-
const vCache = l < 14 ? b.vCache0 : b.vCache1;
|
| 357 |
-
const localLayer = l < 14 ? l : l - 14;
|
| 358 |
-
|
| 359 |
-
const inNormOff = allocSlot([l * LAYER_NORM_STRIDE, 2048, 0, 0]);
|
| 360 |
-
const postNormOff = allocSlot([l * LAYER_NORM_STRIDE + 2048, 2048, 0, 0]);
|
| 361 |
-
const attnOff = allocSlot([
|
| 362 |
-
localLayer,
|
| 363 |
-
l * LAYER_NORM_STRIDE + 4096,
|
| 364 |
-
l * LAYER_NORM_STRIDE + 4224,
|
| 365 |
-
COS_TABLE_OFFSET,
|
| 366 |
-
SIN_TABLE_OFFSET,
|
| 367 |
-
0, 0, 0,
|
| 368 |
-
]);
|
| 369 |
-
|
| 370 |
-
const projIO = {
|
| 371 |
-
q: [b.normed, b.q],
|
| 372 |
-
k: [b.normed, b.k],
|
| 373 |
-
v: [b.normed, b.v],
|
| 374 |
-
o: [b.attnOut, b.x],
|
| 375 |
-
gate: [b.normed, b.gate],
|
| 376 |
-
up: [b.normed, b.up],
|
| 377 |
-
down: [b.mlpMid, b.x],
|
| 378 |
-
};
|
| 379 |
-
const projs = {};
|
| 380 |
-
for (const spec of PROJ_SPECS) {
|
| 381 |
-
const tritOff = lInShard * LAYER_TRIT_STRIDE + spec.tritOff;
|
| 382 |
-
const scaleOff = l * LAYER_SCALE_STRIDE + spec.tritOff / 4;
|
| 383 |
-
const off = allocSlot([spec.rows, spec.inGroups, tritOff, scaleOff, spec.residual, 0, 0, 0]);
|
| 384 |
-
const [inBuf, outBuf] = projIO[spec.name];
|
| 385 |
-
projs[spec.name] = {
|
| 386 |
-
rows: spec.rows,
|
| 387 |
-
gemvBg: mkBg(p.gemv, [
|
| 388 |
-
uSlice(off),
|
| 389 |
-
ref(inBuf),
|
| 390 |
-
ref(shardBuf),
|
| 391 |
-
ref(b.layer_scales),
|
| 392 |
-
ref(outBuf),
|
| 393 |
-
]),
|
| 394 |
-
gemmBg: mkBg(p.gemm, [
|
| 395 |
-
stepRef,
|
| 396 |
-
uSlice(off),
|
| 397 |
-
ref(inBuf),
|
| 398 |
-
ref(shardBuf),
|
| 399 |
-
ref(b.layer_scales),
|
| 400 |
-
ref(outBuf),
|
| 401 |
-
]),
|
| 402 |
-
};
|
| 403 |
-
}
|
| 404 |
-
|
| 405 |
-
layers.push({
|
| 406 |
-
inNormBg: mkBg(p.rmsNorm, [
|
| 407 |
-
stepRef, uSlice(inNormOff, 16), ref(b.x), ref(b.norms), ref(b.normed),
|
| 408 |
-
]),
|
| 409 |
-
postNormBg: mkBg(p.rmsNorm, [
|
| 410 |
-
stepRef, uSlice(postNormOff, 16), ref(b.x), ref(b.norms), ref(b.normed),
|
| 411 |
-
]),
|
| 412 |
-
qkRopeBg: mkBg(p.qkRopeKv, [
|
| 413 |
-
stepRef,
|
| 414 |
-
uSlice(attnOff),
|
| 415 |
-
ref(b.norms),
|
| 416 |
-
ref(b.q),
|
| 417 |
-
ref(b.k),
|
| 418 |
-
ref(b.v),
|
| 419 |
-
ref(kCache),
|
| 420 |
-
ref(vCache),
|
| 421 |
-
]),
|
| 422 |
-
gqaBg: mkBg(p.gqaAttn, [
|
| 423 |
-
stepRef,
|
| 424 |
-
uSlice(attnOff),
|
| 425 |
-
ref(b.q),
|
| 426 |
-
ref(kCache),
|
| 427 |
-
ref(vCache),
|
| 428 |
-
ref(b.attnOut),
|
| 429 |
-
]),
|
| 430 |
-
projs,
|
| 431 |
-
});
|
| 432 |
-
}
|
| 433 |
-
|
| 434 |
-
const finalNormStepBuf = device.createBuffer({
|
| 435 |
-
size: 16,
|
| 436 |
-
usage: GPUBufferUsage.UNIFORM | GPUBufferUsage.COPY_DST,
|
| 437 |
-
});
|
| 438 |
-
device.queue.writeBuffer(finalNormStepBuf, 0, new Uint32Array([1, 0, MAX_SEQ_LEN, 0]));
|
| 439 |
-
const finalNormOff = allocSlot([FINAL_NORM_OFFSET, 2048, 0, 0]);
|
| 440 |
-
const finalNormBg = mkBg(p.rmsNorm, [
|
| 441 |
-
ref(finalNormStepBuf),
|
| 442 |
-
uSlice(finalNormOff, 16),
|
| 443 |
-
ref(b.lastX),
|
| 444 |
-
ref(b.norms),
|
| 445 |
-
ref(b.normed),
|
| 446 |
-
]);
|
| 447 |
-
|
| 448 |
-
const lmHeadOff = allocSlot([151669, 64, 0, 0, 0, 0, 0, 0]);
|
| 449 |
-
const lmHeadBg = mkBg(p.gemv, [
|
| 450 |
-
uSlice(lmHeadOff),
|
| 451 |
-
ref(b.normed),
|
| 452 |
-
ref(b.embed_trits),
|
| 453 |
-
ref(b.embed_scales),
|
| 454 |
-
ref(b.logits),
|
| 455 |
-
]);
|
| 456 |
-
const topkBg = mkBg(p.topk, [ref(b.logits), ref(b.topOut)]);
|
| 457 |
-
|
| 458 |
-
device.queue.writeBuffer(staticBuf, 0, staticBytes);
|
| 459 |
-
return { embedBg, siluBg, copyLastBg, layers, finalNormBg, lmHeadBg, topkBg };
|
| 460 |
-
}
|
| 461 |
-
|
| 462 |
-
encodeForwardStep(batchTokens, startPos, { computeLogits }) {
|
| 463 |
-
const batchLen = batchTokens.length;
|
| 464 |
-
const { device, buffers: b, pipelines: p, ops } = this;
|
| 465 |
-
const tokenArr = new Uint32Array(PREFILL_BATCH);
|
| 466 |
-
tokenArr.set(batchTokens);
|
| 467 |
-
device.queue.writeBuffer(b.tokenIds, 0, tokenArr);
|
| 468 |
-
device.queue.writeBuffer(b.stepUniform, 0, new Uint32Array([batchLen, startPos, MAX_SEQ_LEN, 0]));
|
| 469 |
-
|
| 470 |
-
const enc = device.createCommandEncoder();
|
| 471 |
-
const pass = enc.beginComputePass();
|
| 472 |
-
const dispatchOp = (pipeline, bindGroup, wgX, wgY = 1) => {
|
| 473 |
-
pass.setPipeline(pipeline);
|
| 474 |
-
pass.setBindGroup(0, bindGroup);
|
| 475 |
-
pass.dispatchWorkgroups(wgX, wgY, 1);
|
| 476 |
-
};
|
| 477 |
-
const runMat = (proj) => {
|
| 478 |
-
if (batchLen === 1) {
|
| 479 |
-
dispatchOp(p.gemv, proj.gemvBg, Math.ceil(proj.rows / 4));
|
| 480 |
-
} else {
|
| 481 |
-
dispatchOp(p.gemm, proj.gemmBg, Math.ceil(proj.rows / 16), Math.ceil(batchLen / 16));
|
| 482 |
-
}
|
| 483 |
-
};
|
| 484 |
-
|
| 485 |
-
dispatchOp(p.embed, ops.embedBg, batchLen, 8);
|
| 486 |
-
|
| 487 |
-
for (let l = 0; l < NUM_LAYERS; l += 1) {
|
| 488 |
-
const layer = ops.layers[l];
|
| 489 |
-
dispatchOp(p.rmsNorm, layer.inNormBg, batchLen);
|
| 490 |
-
runMat(layer.projs.q);
|
| 491 |
-
runMat(layer.projs.k);
|
| 492 |
-
runMat(layer.projs.v);
|
| 493 |
-
dispatchOp(p.qkRopeKv, layer.qkRopeBg, batchLen, 24);
|
| 494 |
-
dispatchOp(p.gqaAttn, layer.gqaBg, batchLen, 16);
|
| 495 |
-
runMat(layer.projs.o);
|
| 496 |
-
dispatchOp(p.rmsNorm, layer.postNormBg, batchLen);
|
| 497 |
-
runMat(layer.projs.gate);
|
| 498 |
-
runMat(layer.projs.up);
|
| 499 |
-
dispatchOp(p.siluMul, ops.siluBg, Math.ceil((batchLen * 6144) / 256));
|
| 500 |
-
runMat(layer.projs.down);
|
| 501 |
-
}
|
| 502 |
-
|
| 503 |
-
if (computeLogits) {
|
| 504 |
-
const headSteps = [
|
| 505 |
-
[p.copyLast, ops.copyLastBg, 8],
|
| 506 |
-
[p.rmsNorm, ops.finalNormBg, 1],
|
| 507 |
-
[p.gemv, ops.lmHeadBg, Math.ceil(151669 / 4)],
|
| 508 |
-
[p.topk, ops.topkBg, TOPK_BLOCKS],
|
| 509 |
-
];
|
| 510 |
-
for (const [pl, bg, wg] of headSteps) {
|
| 511 |
-
dispatchOp(pl, bg, wg);
|
| 512 |
-
}
|
| 513 |
-
}
|
| 514 |
-
|
| 515 |
-
pass.end();
|
| 516 |
-
if (computeLogits) {
|
| 517 |
-
enc.copyBufferToBuffer(b.topOut, 0, b.topRead, 0, TOPK_BLOCKS * 8);
|
| 518 |
-
}
|
| 519 |
-
device.queue.submit([enc.finish()]);
|
| 520 |
-
}
|
| 521 |
-
|
| 522 |
-
async readNextToken(temperature, topP) {
|
| 523 |
-
const readBuf = this.buffers.topRead;
|
| 524 |
-
await readBuf.mapAsync(GPUMapMode.READ);
|
| 525 |
-
try {
|
| 526 |
-
const copy = readBuf.getMappedRange().slice(0);
|
| 527 |
-
return sampleTopCandidates(copy, temperature, topP);
|
| 528 |
-
} finally {
|
| 529 |
-
readBuf.unmap();
|
| 530 |
-
}
|
| 531 |
-
}
|
| 532 |
-
|
| 533 |
-
promptFits(promptTokens, maxNewTokens = 1024) {
|
| 534 |
-
return promptTokens.length + maxNewTokens <= MAX_SEQ_LEN;
|
| 535 |
-
}
|
| 536 |
-
|
| 537 |
-
async streamCompletion(prompt, {
|
| 538 |
-
maxNewTokens = 1024,
|
| 539 |
-
temperature = 0.0,
|
| 540 |
-
topP = 1.0,
|
| 541 |
-
shouldCancel = () => false,
|
| 542 |
-
onProgress = () => {},
|
| 543 |
-
onToken = () => {},
|
| 544 |
-
} = {}) {
|
| 545 |
-
const promptTokens = this.tokenizer.encode(prompt);
|
| 546 |
-
const totalPrompt = promptTokens.length;
|
| 547 |
-
if (!this.promptFits(promptTokens, 256)) {
|
| 548 |
-
throw new Error("Prompt exceeds browser WebGPU context window.");
|
| 549 |
-
}
|
| 550 |
-
|
| 551 |
-
const reuse = Math.min(sharedPrefixLength(this.cachedTokens, promptTokens), totalPrompt - 1);
|
| 552 |
-
onProgress(reuse, totalPrompt);
|
| 553 |
-
|
| 554 |
-
let cursor = reuse;
|
| 555 |
-
while (cursor < totalPrompt) {
|
| 556 |
-
if (shouldCancel()) return "";
|
| 557 |
-
const remaining = totalPrompt - cursor;
|
| 558 |
-
const chunkLen = Math.min(PREFILL_BATCH, remaining);
|
| 559 |
-
const isFinalChunk = cursor + chunkLen === totalPrompt;
|
| 560 |
-
const batch = promptTokens.slice(cursor, cursor + chunkLen);
|
| 561 |
-
this.encodeForwardStep(batch, cursor, { computeLogits: isFinalChunk });
|
| 562 |
-
cursor += chunkLen;
|
| 563 |
-
if (!isFinalChunk) {
|
| 564 |
-
await this.device.queue.onSubmittedWorkDone();
|
| 565 |
-
}
|
| 566 |
-
onProgress(cursor, totalPrompt);
|
| 567 |
-
}
|
| 568 |
-
|
| 569 |
-
const history = promptTokens.slice();
|
| 570 |
-
const decoder = this.tokenizer.createStreamDecoder();
|
| 571 |
-
const tokenLimit = Math.min(maxNewTokens, MAX_SEQ_LEN - totalPrompt);
|
| 572 |
-
let fullReply = "";
|
| 573 |
-
|
| 574 |
-
for (let step = 0; step < tokenLimit; step += 1) {
|
| 575 |
-
if (shouldCancel()) {
|
| 576 |
-
this.cachedTokens = history;
|
| 577 |
-
return fullReply;
|
| 578 |
-
}
|
| 579 |
-
const nextId = await this.readNextToken(temperature, topP);
|
| 580 |
-
if (nextId === this.tokenizer.eosTokenId || nextId === this.tokenizer.padTokenId) {
|
| 581 |
-
break;
|
| 582 |
-
}
|
| 583 |
-
const piece = decoder.push(nextId);
|
| 584 |
-
if (piece) {
|
| 585 |
-
fullReply += piece;
|
| 586 |
-
onToken(piece);
|
| 587 |
-
if (fullReply.includes("<|im_end|>")) {
|
| 588 |
-
break;
|
| 589 |
-
}
|
| 590 |
-
}
|
| 591 |
-
const pos = history.length;
|
| 592 |
-
history.push(nextId);
|
| 593 |
-
if (step + 1 < tokenLimit && pos < MAX_SEQ_LEN) {
|
| 594 |
-
this.encodeForwardStep([nextId], pos, { computeLogits: true });
|
| 595 |
-
}
|
| 596 |
-
}
|
| 597 |
-
const tail = decoder.flush();
|
| 598 |
-
if (tail) {
|
| 599 |
-
fullReply += tail;
|
| 600 |
-
onToken(tail);
|
| 601 |
-
}
|
| 602 |
-
this.cachedTokens = history;
|
| 603 |
-
return fullReply.split("<|im_end|>")[0];
|
| 604 |
-
}
|
| 605 |
-
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
web/src/bonsai-window.js
DELETED
|
@@ -1,337 +0,0 @@
|
|
| 1 |
-
// Declaration-aligned multi-chunk squiggle partitioning and verification matching server/windowing.py.
|
| 2 |
-
import {
|
| 3 |
-
applyChanges,
|
| 4 |
-
checkBasicSyntax,
|
| 5 |
-
checkReferenceIntegrity,
|
| 6 |
-
isBloated,
|
| 7 |
-
isGutted,
|
| 8 |
-
previewContent,
|
| 9 |
-
scoreHygieneDict,
|
| 10 |
-
verifyPatch,
|
| 11 |
-
} from "./bonsai-guards.js";
|
| 12 |
-
|
| 13 |
-
export const CHUNK_LINE_THRESHOLD = 160;
|
| 14 |
-
export const CHUNK_CHAR_THRESHOLD = 8500;
|
| 15 |
-
export const MAX_WINDOW_LINES = 84;
|
| 16 |
-
export const MAX_CHUNK_CHARS = 7500;
|
| 17 |
-
const MAX_PROMPT_SQUIGGLES = 16;
|
| 18 |
-
const MAX_CHUNKS_PER_FILE = 12;
|
| 19 |
-
|
| 20 |
-
const COL0_DECL_PREFIXES = [
|
| 21 |
-
"def ",
|
| 22 |
-
"async def ",
|
| 23 |
-
"class ",
|
| 24 |
-
"export ",
|
| 25 |
-
"function ",
|
| 26 |
-
"async function ",
|
| 27 |
-
"pub fn ",
|
| 28 |
-
"pub async fn ",
|
| 29 |
-
"pub struct ",
|
| 30 |
-
"impl ",
|
| 31 |
-
"fn ",
|
| 32 |
-
"func ",
|
| 33 |
-
"static ",
|
| 34 |
-
"void ",
|
| 35 |
-
"int ",
|
| 36 |
-
"bool ",
|
| 37 |
-
];
|
| 38 |
-
const METHOD_DECL_RE = new RegExp(
|
| 39 |
-
"^[ \\t]{2,8}(?:async\\s+def|def|pub\\s+(?:async\\s+)?fn|fn|" +
|
| 40 |
-
"(?:public|private|protected|static|async)\\s+[A-Za-z_$])\\b",
|
| 41 |
-
);
|
| 42 |
-
const LOCAL_ASSIGN_RE =
|
| 43 |
-
/^\s*(?:(?:let|const|var)\s+([A-Za-z_$][A-Za-z0-9_$]*)\b|([A-Za-z_][A-Za-z0-9_]*)\s*:=)/gm;
|
| 44 |
-
|
| 45 |
-
export function shouldChunkFile(content) {
|
| 46 |
-
if (content.length > CHUNK_CHAR_THRESHOLD) return true;
|
| 47 |
-
return content.split("\n").length > CHUNK_LINE_THRESHOLD;
|
| 48 |
-
}
|
| 49 |
-
|
| 50 |
-
function isCol0Decl(line) {
|
| 51 |
-
return COL0_DECL_PREFIXES.some((prefix) => line.startsWith(prefix));
|
| 52 |
-
}
|
| 53 |
-
|
| 54 |
-
function isDecoratorOrAttr(line) {
|
| 55 |
-
const stripped = line.trim();
|
| 56 |
-
return stripped.startsWith("@") || stripped.startsWith("#[");
|
| 57 |
-
}
|
| 58 |
-
|
| 59 |
-
function snapChunkStart(srcLines, clusterLo, pad) {
|
| 60 |
-
const floor = Math.max(0, clusterLo - 55);
|
| 61 |
-
for (let scan = clusterLo; scan >= floor; scan -= 1) {
|
| 62 |
-
if (isCol0Decl(srcLines[scan])) {
|
| 63 |
-
while (scan > 0 && isDecoratorOrAttr(srcLines[scan - 1])) scan -= 1;
|
| 64 |
-
return scan;
|
| 65 |
-
}
|
| 66 |
-
}
|
| 67 |
-
for (let scan = clusterLo; scan >= floor; scan -= 1) {
|
| 68 |
-
if (METHOD_DECL_RE.test(srcLines[scan])) {
|
| 69 |
-
while (scan > 0 && isDecoratorOrAttr(srcLines[scan - 1])) scan -= 1;
|
| 70 |
-
return scan;
|
| 71 |
-
}
|
| 72 |
-
}
|
| 73 |
-
return Math.max(0, clusterLo - pad);
|
| 74 |
-
}
|
| 75 |
-
|
| 76 |
-
function snapChunkEnd(srcLines, clusterHi, pad, useMethods) {
|
| 77 |
-
const total = srcLines.length;
|
| 78 |
-
const limit = Math.min(total, clusterHi + 55);
|
| 79 |
-
let end = Math.min(total, clusterHi + pad + 1);
|
| 80 |
-
for (let scan = clusterHi + 1; scan < limit; scan += 1) {
|
| 81 |
-
const hit = isCol0Decl(srcLines[scan]) || (useMethods && METHOD_DECL_RE.test(srcLines[scan]));
|
| 82 |
-
if (hit) {
|
| 83 |
-
end = scan;
|
| 84 |
-
while (end > clusterHi + 1 && isDecoratorOrAttr(srcLines[end - 1])) end -= 1;
|
| 85 |
-
break;
|
| 86 |
-
}
|
| 87 |
-
}
|
| 88 |
-
while (end > clusterHi + 1 && !srcLines[end - 1].trim()) end -= 1;
|
| 89 |
-
return end;
|
| 90 |
-
}
|
| 91 |
-
|
| 92 |
-
function clampChunkChars(srcLines, initLo, initHi, cLo, cHi, maxChars = MAX_CHUNK_CHARS) {
|
| 93 |
-
let wLo = initLo;
|
| 94 |
-
let wHi = initHi;
|
| 95 |
-
let charTotal = 0;
|
| 96 |
-
for (let i = wLo; i < wHi; i += 1) charTotal += srcLines[i].length + 1;
|
| 97 |
-
while (wHi - wLo > 12 && charTotal > maxChars) {
|
| 98 |
-
if (wHi - 1 > cHi && wHi - cHi >= cLo - wLo) {
|
| 99 |
-
wHi -= 1;
|
| 100 |
-
charTotal -= srcLines[wHi].length + 1;
|
| 101 |
-
} else if (wLo < cLo) {
|
| 102 |
-
charTotal -= srcLines[wLo].length + 1;
|
| 103 |
-
wLo += 1;
|
| 104 |
-
} else {
|
| 105 |
-
wHi -= 1;
|
| 106 |
-
charTotal -= srcLines[wHi].length + 1;
|
| 107 |
-
}
|
| 108 |
-
}
|
| 109 |
-
return [wLo, wHi];
|
| 110 |
-
}
|
| 111 |
-
|
| 112 |
-
export function extractSquiggleWindows(
|
| 113 |
-
content,
|
| 114 |
-
squiggles,
|
| 115 |
-
maxGap = 16,
|
| 116 |
-
pad = 14,
|
| 117 |
-
maxWindowLines = MAX_WINDOW_LINES,
|
| 118 |
-
) {
|
| 119 |
-
const srcLines = content.replace(/\r\n/g, "\n").split("\n");
|
| 120 |
-
const nLines = srcLines.length;
|
| 121 |
-
const uniq = new Set();
|
| 122 |
-
for (const sq of squiggles ?? []) {
|
| 123 |
-
if (sq.line !== undefined) {
|
| 124 |
-
uniq.add(Math.max(0, Math.min(nLines - 1, Number(sq.line) - 1)));
|
| 125 |
-
}
|
| 126 |
-
}
|
| 127 |
-
const sqIndices = [...uniq].sort((a, b) => a - b);
|
| 128 |
-
if (sqIndices.length === 0) return [];
|
| 129 |
-
|
| 130 |
-
const clusters = [];
|
| 131 |
-
for (const idx of sqIndices) {
|
| 132 |
-
const prev = clusters[clusters.length - 1];
|
| 133 |
-
if (prev && idx - prev[1] <= maxGap && idx - prev[0] <= maxWindowLines - 16) {
|
| 134 |
-
prev[1] = idx;
|
| 135 |
-
} else {
|
| 136 |
-
clusters.push([idx, idx]);
|
| 137 |
-
}
|
| 138 |
-
}
|
| 139 |
-
|
| 140 |
-
const windows = [];
|
| 141 |
-
for (const [cLo, cHi] of clusters) {
|
| 142 |
-
const rawLo = snapChunkStart(srcLines, cLo, pad);
|
| 143 |
-
const startedOnMethod = METHOD_DECL_RE.test(srcLines[rawLo]);
|
| 144 |
-
const rawHi = snapChunkEnd(srcLines, cHi, pad, startedOnMethod);
|
| 145 |
-
const [wLo, wHi] = clampChunkChars(srcLines, rawLo, rawHi, cLo, cHi);
|
| 146 |
-
|
| 147 |
-
const last = windows[windows.length - 1];
|
| 148 |
-
if (last && wLo <= last[1]) {
|
| 149 |
-
const mergedHi = Math.max(last[1], wHi);
|
| 150 |
-
let mergedChars = 0;
|
| 151 |
-
for (let i = last[0]; i < mergedHi; i += 1) mergedChars += srcLines[i].length + 1;
|
| 152 |
-
if (mergedHi - last[0] <= maxWindowLines && mergedChars <= MAX_CHUNK_CHARS) {
|
| 153 |
-
last[1] = mergedHi;
|
| 154 |
-
} else {
|
| 155 |
-
const splitLo = Math.max(last[1], Math.max(0, cLo - 4));
|
| 156 |
-
if (splitLo < wHi) windows.push([splitLo, wHi]);
|
| 157 |
-
}
|
| 158 |
-
} else {
|
| 159 |
-
windows.push([wLo, wHi]);
|
| 160 |
-
}
|
| 161 |
-
}
|
| 162 |
-
|
| 163 |
-
if (windows.length <= MAX_CHUNKS_PER_FILE) return windows;
|
| 164 |
-
const ranked = [...windows].sort((left, right) => {
|
| 165 |
-
const countIn = ([lo, hi]) => sqIndices.filter((idx) => idx >= lo && idx < hi).length;
|
| 166 |
-
return countIn(right) - countIn(left);
|
| 167 |
-
});
|
| 168 |
-
return ranked.slice(0, MAX_CHUNKS_PER_FILE).sort((a, b) => a[0] - b[0]);
|
| 169 |
-
}
|
| 170 |
-
|
| 171 |
-
function makeWindowObject(fullContent, excerpt, promptHygiene, startLine, endLine, isSliced) {
|
| 172 |
-
return {
|
| 173 |
-
fullContent,
|
| 174 |
-
excerpt,
|
| 175 |
-
promptHygiene,
|
| 176 |
-
startLine,
|
| 177 |
-
endLine,
|
| 178 |
-
isSliced,
|
| 179 |
-
splice(updatedExcerpt) {
|
| 180 |
-
if (!isSliced) return updatedExcerpt;
|
| 181 |
-
const lines = fullContent.split("\n");
|
| 182 |
-
return [...lines.slice(0, startLine), ...updatedExcerpt.split("\n"), ...lines.slice(endLine)].join("\n");
|
| 183 |
-
},
|
| 184 |
-
};
|
| 185 |
-
}
|
| 186 |
-
|
| 187 |
-
export function buildChunkWindow(fullContent, fullHygiene, wLo, wHi, weights, threshold = 0.55) {
|
| 188 |
-
const srcLines = fullContent.split("\n");
|
| 189 |
-
const nLines = srcLines.length;
|
| 190 |
-
const isSliced = wLo > 0 || wHi < nLines;
|
| 191 |
-
if (!isSliced) {
|
| 192 |
-
const capped = (fullHygiene.squiggles ?? []).slice(0, MAX_PROMPT_SQUIGGLES);
|
| 193 |
-
return makeWindowObject(
|
| 194 |
-
fullContent,
|
| 195 |
-
fullContent,
|
| 196 |
-
{ ...fullHygiene, squiggles: capped },
|
| 197 |
-
0,
|
| 198 |
-
nLines,
|
| 199 |
-
false,
|
| 200 |
-
);
|
| 201 |
-
}
|
| 202 |
-
const excerpt = srcLines.slice(wLo, wHi).join("\n");
|
| 203 |
-
const winRes = scoreHygieneDict(excerpt, weights, threshold);
|
| 204 |
-
const shifted = (fullHygiene.squiggles ?? [])
|
| 205 |
-
.filter((sq) => Number(sq.line) > wLo && Number(sq.line) <= wHi)
|
| 206 |
-
.slice(0, MAX_PROMPT_SQUIGGLES)
|
| 207 |
-
.map((sq) => ({ ...sq, line: Number(sq.line) - wLo }));
|
| 208 |
-
winRes.squiggles = shifted.length > 0 ? shifted : (winRes.squiggles ?? []).slice(0, MAX_PROMPT_SQUIGGLES);
|
| 209 |
-
return makeWindowObject(fullContent, excerpt, winRes, wLo, wHi, true);
|
| 210 |
-
}
|
| 211 |
-
|
| 212 |
-
export function selectCodeWindow(content, hygiene, weights, threshold = 0.55) {
|
| 213 |
-
const squiggles = hygiene.squiggles ?? [];
|
| 214 |
-
const totalLines = content.split("\n").length;
|
| 215 |
-
if (!shouldChunkFile(content) || squiggles.length === 0) {
|
| 216 |
-
const capped = { ...hygiene, squiggles: squiggles.slice(0, MAX_PROMPT_SQUIGGLES) };
|
| 217 |
-
return makeWindowObject(content, content, capped, 0, totalLines, false);
|
| 218 |
-
}
|
| 219 |
-
const windows = extractSquiggleWindows(content, squiggles);
|
| 220 |
-
if (windows.length === 0) {
|
| 221 |
-
return makeWindowObject(content, content, hygiene, 0, totalLines, false);
|
| 222 |
-
}
|
| 223 |
-
const [wLo, wHi] = windows[0];
|
| 224 |
-
return buildChunkWindow(content, hygiene, wLo, wHi, weights, threshold);
|
| 225 |
-
}
|
| 226 |
-
|
| 227 |
-
function collectAssignedVars(code) {
|
| 228 |
-
const found = new Set();
|
| 229 |
-
LOCAL_ASSIGN_RE.lastIndex = 0;
|
| 230 |
-
let m = LOCAL_ASSIGN_RE.exec(code);
|
| 231 |
-
while (m !== null) {
|
| 232 |
-
const name = m[1] || m[2] || "";
|
| 233 |
-
if (name.length > 1) found.add(name);
|
| 234 |
-
m = LOCAL_ASSIGN_RE.exec(code);
|
| 235 |
-
}
|
| 236 |
-
return found;
|
| 237 |
-
}
|
| 238 |
-
|
| 239 |
-
function localVarsIntactAcrossBoundary(fullLines, wLo, wHi, chunkBefore, chunkAfter) {
|
| 240 |
-
const beforeVars = collectAssignedVars(chunkBefore);
|
| 241 |
-
const afterVars = collectAssignedVars(chunkAfter);
|
| 242 |
-
const dropped = [...beforeVars].filter((name) => !afterVars.has(name));
|
| 243 |
-
if (dropped.length === 0) return true;
|
| 244 |
-
const neighborText = [
|
| 245 |
-
...fullLines.slice(Math.max(0, wLo - 40), wLo),
|
| 246 |
-
...fullLines.slice(wHi, Math.min(fullLines.length, wHi + 50)),
|
| 247 |
-
].join("\n");
|
| 248 |
-
for (const name of dropped) {
|
| 249 |
-
const escaped = name.replace(/\$/g, "\\$");
|
| 250 |
-
if (new RegExp(`\\b${escaped}\\b`).test(neighborText)) return false;
|
| 251 |
-
}
|
| 252 |
-
return true;
|
| 253 |
-
}
|
| 254 |
-
|
| 255 |
-
export function previewWindowContent(codeWindow, raw) {
|
| 256 |
-
const found = previewContent(codeWindow.excerpt, raw);
|
| 257 |
-
if (!found) return null;
|
| 258 |
-
return { content: codeWindow.splice(found.content), applied: found.applied };
|
| 259 |
-
}
|
| 260 |
-
|
| 261 |
-
export function verifyWindowPatch(
|
| 262 |
-
codeWindow,
|
| 263 |
-
reply,
|
| 264 |
-
filename,
|
| 265 |
-
threshold,
|
| 266 |
-
beforeHygiene,
|
| 267 |
-
weights,
|
| 268 |
-
originalFullContent = null,
|
| 269 |
-
) {
|
| 270 |
-
if (!codeWindow.isSliced) {
|
| 271 |
-
return verifyPatch(codeWindow.fullContent, reply, filename, threshold, beforeHygiene, weights);
|
| 272 |
-
}
|
| 273 |
-
|
| 274 |
-
const curFull = codeWindow.fullContent;
|
| 275 |
-
const rootFull = originalFullContent ?? curFull;
|
| 276 |
-
const squigglesBefore = beforeHygiene.squiggles.length;
|
| 277 |
-
const reject = (reason, applied = 0) => ({
|
| 278 |
-
accepted: false,
|
| 279 |
-
reason,
|
| 280 |
-
content: curFull,
|
| 281 |
-
appliedBlocks: applied,
|
| 282 |
-
squigglesBefore,
|
| 283 |
-
squigglesAfter: null,
|
| 284 |
-
hygiene: {},
|
| 285 |
-
scoreAfter: null,
|
| 286 |
-
reply: "",
|
| 287 |
-
});
|
| 288 |
-
|
| 289 |
-
if (!reply.trim()) return reject("empty_output");
|
| 290 |
-
let patched;
|
| 291 |
-
try {
|
| 292 |
-
patched = applyChanges(codeWindow.excerpt, reply);
|
| 293 |
-
} catch {
|
| 294 |
-
return reject("search_miss");
|
| 295 |
-
}
|
| 296 |
-
if (patched.applied === 0 || patched.content === codeWindow.excerpt) {
|
| 297 |
-
return reject("no_blocks_applied", patched.applied);
|
| 298 |
-
}
|
| 299 |
-
if (isGutted(codeWindow.excerpt, patched.content)) return reject("gutted_file", patched.applied);
|
| 300 |
-
if (isBloated(codeWindow.excerpt, patched.content)) return reject("line_bloat", patched.applied);
|
| 301 |
-
|
| 302 |
-
const curLines = curFull.split("\n");
|
| 303 |
-
const boundaryOk = localVarsIntactAcrossBoundary(
|
| 304 |
-
curLines,
|
| 305 |
-
codeWindow.startLine,
|
| 306 |
-
codeWindow.endLine,
|
| 307 |
-
codeWindow.excerpt,
|
| 308 |
-
patched.content,
|
| 309 |
-
);
|
| 310 |
-
if (!boundaryOk) return reject("invalid_syntax", patched.applied);
|
| 311 |
-
|
| 312 |
-
const updatedFull = codeWindow.splice(patched.content);
|
| 313 |
-
if (filename) {
|
| 314 |
-
const syntaxOk = checkBasicSyntax(updatedFull, filename);
|
| 315 |
-
const curRefsOk = syntaxOk && checkReferenceIntegrity(curFull, updatedFull, filename);
|
| 316 |
-
const rootRefsOk =
|
| 317 |
-
curRefsOk && (rootFull === curFull || checkReferenceIntegrity(rootFull, updatedFull, filename));
|
| 318 |
-
if (!rootRefsOk) return reject("invalid_syntax", patched.applied);
|
| 319 |
-
}
|
| 320 |
-
if (isGutted(rootFull, updatedFull)) return reject("gutted_file", patched.applied);
|
| 321 |
-
if (isBloated(rootFull, updatedFull)) return reject("line_bloat", patched.applied);
|
| 322 |
-
|
| 323 |
-
const after = scoreHygieneDict(updatedFull, weights, threshold);
|
| 324 |
-
const squigglesAfter = after.squiggles.length;
|
| 325 |
-
const accepted = squigglesAfter < squigglesBefore;
|
| 326 |
-
return {
|
| 327 |
-
accepted,
|
| 328 |
-
reason: accepted ? null : "no_improvement",
|
| 329 |
-
content: accepted ? updatedFull : curFull,
|
| 330 |
-
appliedBlocks: patched.applied,
|
| 331 |
-
squigglesBefore,
|
| 332 |
-
squigglesAfter,
|
| 333 |
-
hygiene: after,
|
| 334 |
-
scoreAfter: after.score ?? null,
|
| 335 |
-
reply,
|
| 336 |
-
};
|
| 337 |
-
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
web/src/bonsai-worker.js
DELETED
|
@@ -1,485 +0,0 @@
|
|
| 1 |
-
// Worker executing multi-chunk Bonsai 1.7B cleanup via ZeroGPU SSE (or opt-in Browser WebGPU).
|
| 2 |
-
import { loadWeights } from "shared:engine";
|
| 3 |
-
import { buildPrompt, scoreHygieneDict } from "./bonsai-guards.js";
|
| 4 |
-
import { BonsaiWebGpuEngine, DEFAULT_MODEL_BASE } from "./bonsai-webgpu.js";
|
| 5 |
-
import {
|
| 6 |
-
buildChunkWindow,
|
| 7 |
-
extractSquiggleWindows,
|
| 8 |
-
previewWindowContent,
|
| 9 |
-
selectCodeWindow,
|
| 10 |
-
shouldChunkFile,
|
| 11 |
-
verifyWindowPatch,
|
| 12 |
-
} from "./bonsai-window.js";
|
| 13 |
-
|
| 14 |
-
const DEFAULT_ZEROGPU_BASE = "https://brianbellx-codehygiene-zerogpu.hf.space";
|
| 15 |
-
const STOP_MARKER = "<|im_end|>";
|
| 16 |
-
const PIECE_PATTERN = new RegExp(" ?\\w{1,6}| ?[^\\w\\s]{1,3}|\\s+", "g");
|
| 17 |
-
const PREVIEW_INTERVAL_MS = 85;
|
| 18 |
-
const PROGRESS_INTERVAL_MS = 200;
|
| 19 |
-
const GREEDY = { temperature: 0.0, topP: 1.0 };
|
| 20 |
-
const SAMPLED = { temperature: 0.45, topP: 0.92 };
|
| 21 |
-
|
| 22 |
-
const ASSET_QUERY = new URL(import.meta.url).search;
|
| 23 |
-
const weightsReady = loadWeights(new URL(`./weights.npz${ASSET_QUERY}`, import.meta.url).href);
|
| 24 |
-
let enginePromise = null;
|
| 25 |
-
let loadedBaseUrl = null;
|
| 26 |
-
const cancelledIds = new Map();
|
| 27 |
-
|
| 28 |
-
function getWebGpuEngine(baseUrl, onLoadProgress) {
|
| 29 |
-
const resolvedBase = baseUrl || DEFAULT_MODEL_BASE;
|
| 30 |
-
if (!enginePromise || loadedBaseUrl !== resolvedBase) {
|
| 31 |
-
loadedBaseUrl = resolvedBase;
|
| 32 |
-
enginePromise = BonsaiWebGpuEngine.load({
|
| 33 |
-
baseUrl: resolvedBase,
|
| 34 |
-
skipSpeedGate: true,
|
| 35 |
-
onProgress: onLoadProgress,
|
| 36 |
-
}).catch((err) => {
|
| 37 |
-
enginePromise = null;
|
| 38 |
-
throw err;
|
| 39 |
-
});
|
| 40 |
-
}
|
| 41 |
-
return enginePromise;
|
| 42 |
-
}
|
| 43 |
-
|
| 44 |
-
async function streamZeroGpuCompletion(prompt, {
|
| 45 |
-
zeroGpuUrl = DEFAULT_ZEROGPU_BASE,
|
| 46 |
-
maxNewTokens = 1280,
|
| 47 |
-
temperature = 0.0,
|
| 48 |
-
topP = 1.0,
|
| 49 |
-
signal,
|
| 50 |
-
shouldCancel,
|
| 51 |
-
onProgress,
|
| 52 |
-
onToken,
|
| 53 |
-
}) {
|
| 54 |
-
const base = zeroGpuUrl.replace(/\/$/, "");
|
| 55 |
-
const totalEst = Math.max(1, Math.floor(prompt.length / 3));
|
| 56 |
-
onProgress(0, totalEst);
|
| 57 |
-
const startRes = await fetch(`${base}/gradio_api/call/stream_patch`, {
|
| 58 |
-
method: "POST",
|
| 59 |
-
headers: { "Content-Type": "application/json" },
|
| 60 |
-
body: JSON.stringify({ data: [prompt, maxNewTokens, temperature, topP] }),
|
| 61 |
-
signal,
|
| 62 |
-
});
|
| 63 |
-
if (!startRes.ok) {
|
| 64 |
-
throw new Error(`ZeroGPU start failed (${startRes.status})`);
|
| 65 |
-
}
|
| 66 |
-
const { event_id: eventId } = await startRes.json();
|
| 67 |
-
const sseRes = await fetch(`${base}/gradio_api/call/stream_patch/${eventId}`, {
|
| 68 |
-
headers: { Accept: "text/event-stream" },
|
| 69 |
-
signal,
|
| 70 |
-
});
|
| 71 |
-
if (!sseRes.ok || !sseRes.body) {
|
| 72 |
-
throw new Error(`ZeroGPU stream failed (${sseRes.status})`);
|
| 73 |
-
}
|
| 74 |
-
|
| 75 |
-
const reader = sseRes.body.pipeThrough(new TextDecoderStream()).getReader();
|
| 76 |
-
let sseBuffer = "";
|
| 77 |
-
let sent = 0;
|
| 78 |
-
let fullReply = "";
|
| 79 |
-
let progressDone = false;
|
| 80 |
-
|
| 81 |
-
for (;;) {
|
| 82 |
-
if (shouldCancel()) {
|
| 83 |
-
await reader.cancel();
|
| 84 |
-
return fullReply;
|
| 85 |
-
}
|
| 86 |
-
const { value, done } = await reader.read();
|
| 87 |
-
if (done) break;
|
| 88 |
-
sseBuffer = (sseBuffer + value).replace(/\r\n?/g, "\n");
|
| 89 |
-
let boundary = sseBuffer.indexOf("\n\n");
|
| 90 |
-
while (boundary >= 0) {
|
| 91 |
-
const frame = sseBuffer.slice(0, boundary);
|
| 92 |
-
sseBuffer = sseBuffer.slice(boundary + 2);
|
| 93 |
-
boundary = sseBuffer.indexOf("\n\n");
|
| 94 |
-
|
| 95 |
-
let eventName = "";
|
| 96 |
-
let payloadText = "";
|
| 97 |
-
for (const rawLine of frame.split("\n")) {
|
| 98 |
-
if (rawLine.startsWith("event:")) eventName = rawLine.slice(6).trim();
|
| 99 |
-
if (rawLine.startsWith("data:")) payloadText = rawLine.slice(5).trim();
|
| 100 |
-
}
|
| 101 |
-
if (eventName === "error") {
|
| 102 |
-
throw new Error(`ZeroGPU error: ${payloadText}`);
|
| 103 |
-
}
|
| 104 |
-
if (!payloadText || payloadText === "null") continue;
|
| 105 |
-
const parsed = JSON.parse(payloadText);
|
| 106 |
-
const cumulative = Array.isArray(parsed) && parsed.length > 0 ? String(parsed[0] || "") : "";
|
| 107 |
-
if (!progressDone) {
|
| 108 |
-
onProgress(totalEst, totalEst);
|
| 109 |
-
progressDone = true;
|
| 110 |
-
}
|
| 111 |
-
if (cumulative.length > sent) {
|
| 112 |
-
const delta = cumulative.slice(sent).split(STOP_MARKER)[0];
|
| 113 |
-
sent = cumulative.length;
|
| 114 |
-
const pieces = delta.match(PIECE_PATTERN) ?? (delta ? [delta] : []);
|
| 115 |
-
for (const piece of pieces) {
|
| 116 |
-
fullReply += piece;
|
| 117 |
-
onToken(piece);
|
| 118 |
-
}
|
| 119 |
-
}
|
| 120 |
-
if (eventName === "complete") {
|
| 121 |
-
await reader.cancel();
|
| 122 |
-
return fullReply;
|
| 123 |
-
}
|
| 124 |
-
}
|
| 125 |
-
}
|
| 126 |
-
return fullReply;
|
| 127 |
-
}
|
| 128 |
-
|
| 129 |
-
async function runWindowAttempt({
|
| 130 |
-
runner,
|
| 131 |
-
weights,
|
| 132 |
-
attemptNo,
|
| 133 |
-
passNo,
|
| 134 |
-
codeWindow,
|
| 135 |
-
curHygiene,
|
| 136 |
-
originalFullContent,
|
| 137 |
-
filename,
|
| 138 |
-
threshold,
|
| 139 |
-
sampled,
|
| 140 |
-
isCancelled,
|
| 141 |
-
emit,
|
| 142 |
-
}) {
|
| 143 |
-
if (isCancelled()) return { cancelled: true };
|
| 144 |
-
emit("attempt", { n: attemptNo, pass: passNo, sampled });
|
| 145 |
-
|
| 146 |
-
const prompt = buildPrompt(codeWindow.excerpt, codeWindow.promptHygiene, filename);
|
| 147 |
-
const sampling = sampled ? SAMPLED : GREEDY;
|
| 148 |
-
let raw = "";
|
| 149 |
-
let tokens = 0;
|
| 150 |
-
let lastPreview = codeWindow.fullContent;
|
| 151 |
-
let lastPreviewAt = 0;
|
| 152 |
-
let lastProgressAt = -PROGRESS_INTERVAL_MS;
|
| 153 |
-
let readFinished = false;
|
| 154 |
-
|
| 155 |
-
const flushPreview = () => {
|
| 156 |
-
lastPreviewAt = performance.now();
|
| 157 |
-
const found = previewWindowContent(codeWindow, raw);
|
| 158 |
-
if (found && found.content !== lastPreview) {
|
| 159 |
-
lastPreview = found.content;
|
| 160 |
-
emit("preview", { content: lastPreview, applied_blocks: found.applied });
|
| 161 |
-
}
|
| 162 |
-
};
|
| 163 |
-
|
| 164 |
-
const reply = await runner(prompt, {
|
| 165 |
-
maxNewTokens: 1280,
|
| 166 |
-
temperature: sampling.temperature,
|
| 167 |
-
topP: sampling.topP,
|
| 168 |
-
shouldCancel: isCancelled,
|
| 169 |
-
onProgress: (done, total) => {
|
| 170 |
-
if (tokens > 0 || readFinished) return;
|
| 171 |
-
const now = performance.now();
|
| 172 |
-
const finished = done >= total;
|
| 173 |
-
if (!finished && now - lastProgressAt < PROGRESS_INTERVAL_MS) return;
|
| 174 |
-
lastProgressAt = now;
|
| 175 |
-
readFinished = finished;
|
| 176 |
-
emit("progress", { phase: "read", done: Math.min(done, total), total });
|
| 177 |
-
},
|
| 178 |
-
onToken: (piece) => {
|
| 179 |
-
raw += piece;
|
| 180 |
-
tokens += 1;
|
| 181 |
-
emit("token", { text: piece });
|
| 182 |
-
if (performance.now() - lastPreviewAt >= PREVIEW_INTERVAL_MS) {
|
| 183 |
-
flushPreview();
|
| 184 |
-
}
|
| 185 |
-
},
|
| 186 |
-
});
|
| 187 |
-
|
| 188 |
-
if (isCancelled()) return { cancelled: true };
|
| 189 |
-
flushPreview();
|
| 190 |
-
|
| 191 |
-
const verdict = verifyWindowPatch(
|
| 192 |
-
codeWindow,
|
| 193 |
-
reply,
|
| 194 |
-
filename,
|
| 195 |
-
threshold,
|
| 196 |
-
curHygiene,
|
| 197 |
-
weights,
|
| 198 |
-
originalFullContent,
|
| 199 |
-
);
|
| 200 |
-
emit("verdict", {
|
| 201 |
-
n: attemptNo,
|
| 202 |
-
accepted: verdict.accepted,
|
| 203 |
-
reason: verdict.reason,
|
| 204 |
-
squiggles_before: verdict.squigglesBefore,
|
| 205 |
-
squiggles_after: verdict.squigglesAfter,
|
| 206 |
-
score_after: verdict.scoreAfter,
|
| 207 |
-
applied_blocks: verdict.appliedBlocks,
|
| 208 |
-
});
|
| 209 |
-
return { cancelled: false, verdict, tokens };
|
| 210 |
-
}
|
| 211 |
-
|
| 212 |
-
function countNewlines(text) {
|
| 213 |
-
let count = 0;
|
| 214 |
-
for (let i = 0; i < text.length; i += 1) {
|
| 215 |
-
if (text.charCodeAt(i) === 10) count += 1;
|
| 216 |
-
}
|
| 217 |
-
return count;
|
| 218 |
-
}
|
| 219 |
-
|
| 220 |
-
async function runChunkedLoop({
|
| 221 |
-
content,
|
| 222 |
-
before,
|
| 223 |
-
filename,
|
| 224 |
-
threshold,
|
| 225 |
-
maxPasses,
|
| 226 |
-
weights,
|
| 227 |
-
nextAttempt,
|
| 228 |
-
}) {
|
| 229 |
-
const windows = extractSquiggleWindows(content, before.squiggles);
|
| 230 |
-
let curCode = content;
|
| 231 |
-
let curHyg = before;
|
| 232 |
-
let lineShift = 0;
|
| 233 |
-
const acceptedList = [];
|
| 234 |
-
let passesUsed = 0;
|
| 235 |
-
let lastReason = null;
|
| 236 |
-
let totalTokens = 0;
|
| 237 |
-
const allowSecondPass = maxPasses >= 2 && windows.length <= 4;
|
| 238 |
-
|
| 239 |
-
for (const [origLo, origHi] of windows) {
|
| 240 |
-
const totalLines = countNewlines(curCode) + 1;
|
| 241 |
-
const wLo = Math.max(0, Math.min(totalLines, origLo + lineShift));
|
| 242 |
-
const wHi = Math.max(wLo, Math.min(totalLines, origHi + lineShift));
|
| 243 |
-
const hasSq = (curHyg.squiggles ?? []).some((sq) => Number(sq.line) > wLo && Number(sq.line) <= wHi);
|
| 244 |
-
if (!hasSq || wLo >= wHi) continue;
|
| 245 |
-
|
| 246 |
-
const win1 = buildChunkWindow(curCode, curHyg, wLo, wHi, weights, threshold);
|
| 247 |
-
passesUsed = Math.max(passesUsed, 1);
|
| 248 |
-
const first = await nextAttempt(1, win1, curHyg, false);
|
| 249 |
-
if (first.cancelled) return { cancelled: true };
|
| 250 |
-
totalTokens += first.tokens;
|
| 251 |
-
|
| 252 |
-
if (first.verdict.accepted) {
|
| 253 |
-
const delta = countNewlines(first.verdict.content) - countNewlines(curCode);
|
| 254 |
-
const newHi = wHi + delta;
|
| 255 |
-
lineShift += delta;
|
| 256 |
-
curCode = first.verdict.content;
|
| 257 |
-
curHyg = first.verdict.hygiene;
|
| 258 |
-
acceptedList.push(first.verdict);
|
| 259 |
-
const stillFlagged = (curHyg.squiggles ?? []).some(
|
| 260 |
-
(sq) => Number(sq.line) > wLo && Number(sq.line) <= newHi,
|
| 261 |
-
);
|
| 262 |
-
if (allowSecondPass && stillFlagged) {
|
| 263 |
-
passesUsed = 2;
|
| 264 |
-
const win2 = buildChunkWindow(curCode, curHyg, wLo, newHi, weights, threshold);
|
| 265 |
-
const second = await nextAttempt(2, win2, curHyg, false);
|
| 266 |
-
if (second.cancelled) return { cancelled: true };
|
| 267 |
-
totalTokens += second.tokens;
|
| 268 |
-
if (second.verdict.accepted) {
|
| 269 |
-
lineShift += countNewlines(second.verdict.content) - countNewlines(curCode);
|
| 270 |
-
curCode = second.verdict.content;
|
| 271 |
-
curHyg = second.verdict.hygiene;
|
| 272 |
-
acceptedList.push(second.verdict);
|
| 273 |
-
}
|
| 274 |
-
}
|
| 275 |
-
continue;
|
| 276 |
-
}
|
| 277 |
-
|
| 278 |
-
lastReason = first.verdict.reason;
|
| 279 |
-
if (allowSecondPass && first.verdict.reason !== "too_long") {
|
| 280 |
-
passesUsed = 2;
|
| 281 |
-
const retry = await nextAttempt(2, win1, curHyg, true);
|
| 282 |
-
if (retry.cancelled) return { cancelled: true };
|
| 283 |
-
totalTokens += retry.tokens;
|
| 284 |
-
if (retry.verdict.accepted) {
|
| 285 |
-
lineShift += countNewlines(retry.verdict.content) - countNewlines(curCode);
|
| 286 |
-
curCode = retry.verdict.content;
|
| 287 |
-
curHyg = retry.verdict.hygiene;
|
| 288 |
-
acceptedList.push(retry.verdict);
|
| 289 |
-
} else {
|
| 290 |
-
lastReason = retry.verdict.reason;
|
| 291 |
-
}
|
| 292 |
-
}
|
| 293 |
-
}
|
| 294 |
-
return {
|
| 295 |
-
cancelled: false,
|
| 296 |
-
acceptedList,
|
| 297 |
-
passesUsed,
|
| 298 |
-
rejectReason: acceptedList.length > 0 ? null : lastReason,
|
| 299 |
-
totalTokens,
|
| 300 |
-
};
|
| 301 |
-
}
|
| 302 |
-
|
| 303 |
-
async function runFixJob(id, body, options) {
|
| 304 |
-
const aborter = new AbortController();
|
| 305 |
-
cancelledIds.set(id, aborter);
|
| 306 |
-
const isCancelled = () => aborter.signal.aborted;
|
| 307 |
-
const emit = (name, data) => {
|
| 308 |
-
if (!isCancelled()) {
|
| 309 |
-
globalThis.postMessage({ id, event: name, data });
|
| 310 |
-
}
|
| 311 |
-
};
|
| 312 |
-
|
| 313 |
-
const startedAt = performance.now();
|
| 314 |
-
const weights = await weightsReady;
|
| 315 |
-
const threshold = body.threshold ?? 0.55;
|
| 316 |
-
const maxPasses = body.max_passes ?? 2;
|
| 317 |
-
const filename = body.filename ?? "";
|
| 318 |
-
const content = body.content ?? "";
|
| 319 |
-
|
| 320 |
-
const before = scoreHygieneDict(content, weights, threshold);
|
| 321 |
-
const modelTag = options.backend === "webgpu" ? "bonsai-1.7b-webgpu" : "brianbellx/codehygiene-bonsai-1.7b";
|
| 322 |
-
if (!before.squiggles || before.squiggles.length === 0) {
|
| 323 |
-
emit("start", { before_hygiene: before, model_id: modelTag, cached: false });
|
| 324 |
-
emit("done", {
|
| 325 |
-
filename,
|
| 326 |
-
changes: "",
|
| 327 |
-
applied_blocks: 0,
|
| 328 |
-
updated_content: content,
|
| 329 |
-
before_hygiene: before,
|
| 330 |
-
after_hygiene: before,
|
| 331 |
-
passes_used: 0,
|
| 332 |
-
latency_ms: Math.round(performance.now() - startedAt),
|
| 333 |
-
accepted: false,
|
| 334 |
-
reason: null,
|
| 335 |
-
cached: false,
|
| 336 |
-
});
|
| 337 |
-
return;
|
| 338 |
-
}
|
| 339 |
-
|
| 340 |
-
let runner;
|
| 341 |
-
if (options.backend === "webgpu") {
|
| 342 |
-
let lastLoadEmit = 0;
|
| 343 |
-
const engine = await getWebGpuEngine(options.modelBaseUrl, (done, total) => {
|
| 344 |
-
const now = performance.now();
|
| 345 |
-
if (done < total && now - lastLoadEmit < PROGRESS_INTERVAL_MS) return;
|
| 346 |
-
lastLoadEmit = now;
|
| 347 |
-
emit("progress", { phase: "read", done, total });
|
| 348 |
-
});
|
| 349 |
-
runner = (prompt, opts) => engine.streamCompletion(prompt, { ...opts, maxNewTokens: 1024 });
|
| 350 |
-
} else {
|
| 351 |
-
runner = (prompt, opts) =>
|
| 352 |
-
streamZeroGpuCompletion(prompt, {
|
| 353 |
-
...opts,
|
| 354 |
-
zeroGpuUrl: options.zeroGpuUrl || DEFAULT_ZEROGPU_BASE,
|
| 355 |
-
signal: aborter.signal,
|
| 356 |
-
});
|
| 357 |
-
}
|
| 358 |
-
|
| 359 |
-
emit("start", { before_hygiene: before, model_id: modelTag, cached: false });
|
| 360 |
-
|
| 361 |
-
let attemptCount = 0;
|
| 362 |
-
const nextAttempt = (passNo, codeWindow, curHygiene, sampled) => {
|
| 363 |
-
attemptCount += 1;
|
| 364 |
-
return runWindowAttempt({
|
| 365 |
-
runner,
|
| 366 |
-
weights,
|
| 367 |
-
attemptNo: attemptCount,
|
| 368 |
-
passNo,
|
| 369 |
-
codeWindow,
|
| 370 |
-
curHygiene,
|
| 371 |
-
originalFullContent: content,
|
| 372 |
-
filename,
|
| 373 |
-
threshold,
|
| 374 |
-
sampled,
|
| 375 |
-
isCancelled,
|
| 376 |
-
emit,
|
| 377 |
-
});
|
| 378 |
-
};
|
| 379 |
-
|
| 380 |
-
let acceptedList = [];
|
| 381 |
-
let passesUsed = 1;
|
| 382 |
-
let rejectReason = null;
|
| 383 |
-
let totalTokens = 0;
|
| 384 |
-
|
| 385 |
-
if (shouldChunkFile(content)) {
|
| 386 |
-
const chunked = await runChunkedLoop({
|
| 387 |
-
content,
|
| 388 |
-
before,
|
| 389 |
-
filename,
|
| 390 |
-
threshold,
|
| 391 |
-
maxPasses,
|
| 392 |
-
weights,
|
| 393 |
-
nextAttempt,
|
| 394 |
-
});
|
| 395 |
-
if (chunked.cancelled) return;
|
| 396 |
-
({ acceptedList, passesUsed, rejectReason, totalTokens } = chunked);
|
| 397 |
-
} else {
|
| 398 |
-
const win1 = selectCodeWindow(content, before, weights, threshold);
|
| 399 |
-
const first = await nextAttempt(1, win1, before, false);
|
| 400 |
-
if (first.cancelled) return;
|
| 401 |
-
totalTokens += first.tokens;
|
| 402 |
-
if (first.verdict.accepted) {
|
| 403 |
-
acceptedList.push(first.verdict);
|
| 404 |
-
}
|
| 405 |
-
if (maxPasses >= 2) {
|
| 406 |
-
if (first.verdict.accepted && first.verdict.squigglesAfter > 0) {
|
| 407 |
-
passesUsed = 2;
|
| 408 |
-
const win2 = selectCodeWindow(first.verdict.content, first.verdict.hygiene, weights, threshold);
|
| 409 |
-
const second = await nextAttempt(2, win2, first.verdict.hygiene, false);
|
| 410 |
-
if (second.cancelled) return;
|
| 411 |
-
totalTokens += second.tokens;
|
| 412 |
-
if (second.verdict.accepted) acceptedList.push(second.verdict);
|
| 413 |
-
} else if (!first.verdict.accepted) {
|
| 414 |
-
passesUsed = 2;
|
| 415 |
-
const retries = [];
|
| 416 |
-
for (let r = 0; r < 2; r += 1) {
|
| 417 |
-
const res = await nextAttempt(2, win1, before, true);
|
| 418 |
-
if (res.cancelled) return;
|
| 419 |
-
totalTokens += res.tokens;
|
| 420 |
-
retries.push(res.verdict);
|
| 421 |
-
}
|
| 422 |
-
const valid = retries.filter((v) => v.accepted);
|
| 423 |
-
if (valid.length > 0) {
|
| 424 |
-
valid.sort((a, b) => {
|
| 425 |
-
const sqDiff = a.squigglesAfter - b.squigglesAfter;
|
| 426 |
-
return sqDiff !== 0 ? sqDiff : (b.scoreAfter ?? 0) - (a.scoreAfter ?? 0);
|
| 427 |
-
});
|
| 428 |
-
acceptedList.push(valid[0]);
|
| 429 |
-
} else {
|
| 430 |
-
rejectReason = retries[retries.length - 1].reason;
|
| 431 |
-
}
|
| 432 |
-
}
|
| 433 |
-
}
|
| 434 |
-
if (acceptedList.length === 0 && rejectReason === null) {
|
| 435 |
-
rejectReason = first.verdict.reason;
|
| 436 |
-
}
|
| 437 |
-
}
|
| 438 |
-
|
| 439 |
-
const finalVerdict = acceptedList.length > 0 ? acceptedList[acceptedList.length - 1] : null;
|
| 440 |
-
emit("done", {
|
| 441 |
-
filename,
|
| 442 |
-
changes: acceptedList.map((v) => v.reply).join("\n\n"),
|
| 443 |
-
applied_blocks: acceptedList.reduce((s, v) => s + v.appliedBlocks, 0),
|
| 444 |
-
updated_content: finalVerdict ? finalVerdict.content : content,
|
| 445 |
-
before_hygiene: before,
|
| 446 |
-
after_hygiene: finalVerdict ? finalVerdict.hygiene : before,
|
| 447 |
-
passes_used: passesUsed,
|
| 448 |
-
latency_ms: Math.round(performance.now() - startedAt),
|
| 449 |
-
accepted: finalVerdict !== null,
|
| 450 |
-
reason: finalVerdict ? null : rejectReason,
|
| 451 |
-
cached: false,
|
| 452 |
-
tokens: totalTokens,
|
| 453 |
-
});
|
| 454 |
-
}
|
| 455 |
-
|
| 456 |
-
let queueChain = Promise.resolve();
|
| 457 |
-
|
| 458 |
-
globalThis.addEventListener("message", (messageEvent) => {
|
| 459 |
-
const { type, id, body, options } = messageEvent.data;
|
| 460 |
-
if (type === "cancel") {
|
| 461 |
-
const ctrl = cancelledIds.get(id);
|
| 462 |
-
if (ctrl) ctrl.abort();
|
| 463 |
-
else cancelledIds.set(id, { signal: { aborted: true }, abort() {} });
|
| 464 |
-
return;
|
| 465 |
-
}
|
| 466 |
-
if (type === "fix") {
|
| 467 |
-
queueChain = queueChain.then(async () => {
|
| 468 |
-
const preCancelled = cancelledIds.get(id);
|
| 469 |
-
if (preCancelled?.signal?.aborted) {
|
| 470 |
-
cancelledIds.delete(id);
|
| 471 |
-
globalThis.postMessage({ id, done: true });
|
| 472 |
-
return;
|
| 473 |
-
}
|
| 474 |
-
try {
|
| 475 |
-
await runFixJob(id, body, options ?? {});
|
| 476 |
-
globalThis.postMessage({ id, done: true });
|
| 477 |
-
} catch (err) {
|
| 478 |
-
const message = err && err.message ? err.message : String(err);
|
| 479 |
-
globalThis.postMessage({ id, fallback: true, error: message });
|
| 480 |
-
} finally {
|
| 481 |
-
cancelledIds.delete(id);
|
| 482 |
-
}
|
| 483 |
-
});
|
| 484 |
-
}
|
| 485 |
-
});
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
web/src/browser-refiner.js
DELETED
|
@@ -1,153 +0,0 @@
|
|
| 1 |
-
// ZeroGPU-first cleanup router with per-visitor CORS ZeroGPU fallback (and opt-in ?refiner=browser).
|
| 2 |
-
import { probeWebGpuSupport } from "./bonsai-webgpu.js";
|
| 3 |
-
import { streamFix } from "./refiner.js";
|
| 4 |
-
|
| 5 |
-
const ASSET_QUERY = new URL(import.meta.url).search;
|
| 6 |
-
const BONSAI_WORKER_URL = new URL(`./bonsai-worker.js${ASSET_QUERY}`, import.meta.url);
|
| 7 |
-
|
| 8 |
-
let sharedWorker = null;
|
| 9 |
-
let serverBackend = "zerogpu";
|
| 10 |
-
let serverQuotaExhausted = false;
|
| 11 |
-
let nextRequestId = 1;
|
| 12 |
-
const pendingJobs = new Map();
|
| 13 |
-
|
| 14 |
-
export function setServerBackend(backend) {
|
| 15 |
-
serverBackend = backend || "zerogpu";
|
| 16 |
-
}
|
| 17 |
-
|
| 18 |
-
function queryParam(name) {
|
| 19 |
-
try {
|
| 20 |
-
return new URLSearchParams(globalThis.location?.search ?? "").get(name) || "";
|
| 21 |
-
} catch {
|
| 22 |
-
return "";
|
| 23 |
-
}
|
| 24 |
-
}
|
| 25 |
-
|
| 26 |
-
function configuredMode() {
|
| 27 |
-
return queryParam("refiner").toLowerCase() || "zerogpu";
|
| 28 |
-
}
|
| 29 |
-
|
| 30 |
-
function isQuotaErrorMessage(text) {
|
| 31 |
-
return /zerogpu runs limit|quota exceeded/i.test(String(text || ""));
|
| 32 |
-
}
|
| 33 |
-
|
| 34 |
-
function getBonsaiWorker() {
|
| 35 |
-
if (!sharedWorker) {
|
| 36 |
-
sharedWorker = new Worker(BONSAI_WORKER_URL, { type: "module" });
|
| 37 |
-
sharedWorker.addEventListener("message", (event) => {
|
| 38 |
-
const { id, event: eventName, data, done, fallback, error } = event.data;
|
| 39 |
-
const job = pendingJobs.get(id);
|
| 40 |
-
if (!job) return;
|
| 41 |
-
if (eventName) {
|
| 42 |
-
job.onEvent(eventName, data);
|
| 43 |
-
return;
|
| 44 |
-
}
|
| 45 |
-
pendingJobs.delete(id);
|
| 46 |
-
if (fallback) {
|
| 47 |
-
job.resolve({ ok: false, error });
|
| 48 |
-
} else if (done) {
|
| 49 |
-
job.resolve({ ok: true });
|
| 50 |
-
}
|
| 51 |
-
});
|
| 52 |
-
sharedWorker.addEventListener("error", () => {
|
| 53 |
-
for (const [id, job] of pendingJobs.entries()) {
|
| 54 |
-
pendingJobs.delete(id);
|
| 55 |
-
job.resolve({ ok: false, error: "worker_crash" });
|
| 56 |
-
}
|
| 57 |
-
sharedWorker = null;
|
| 58 |
-
});
|
| 59 |
-
}
|
| 60 |
-
return sharedWorker;
|
| 61 |
-
}
|
| 62 |
-
|
| 63 |
-
function runWorkerCleanup(body, options, { signal, onEvent }) {
|
| 64 |
-
return new Promise((resolve) => {
|
| 65 |
-
const id = nextRequestId;
|
| 66 |
-
nextRequestId += 1;
|
| 67 |
-
const worker = getBonsaiWorker();
|
| 68 |
-
pendingJobs.set(id, { onEvent, resolve });
|
| 69 |
-
|
| 70 |
-
const onAbort = () => {
|
| 71 |
-
pendingJobs.delete(id);
|
| 72 |
-
worker.postMessage({ type: "cancel", id });
|
| 73 |
-
resolve({ ok: true, aborted: true });
|
| 74 |
-
};
|
| 75 |
-
if (signal?.aborted) {
|
| 76 |
-
onAbort();
|
| 77 |
-
return;
|
| 78 |
-
}
|
| 79 |
-
signal?.addEventListener("abort", onAbort, { once: true });
|
| 80 |
-
worker.postMessage({ type: "fix", id, body, options });
|
| 81 |
-
});
|
| 82 |
-
}
|
| 83 |
-
|
| 84 |
-
async function tryServerZeroGpu(body, { signal, onEvent }) {
|
| 85 |
-
const buffered = [];
|
| 86 |
-
let committed = false;
|
| 87 |
-
let unavailable = false;
|
| 88 |
-
let errorMessage = "";
|
| 89 |
-
|
| 90 |
-
const flushBuffered = () => {
|
| 91 |
-
if (committed) return;
|
| 92 |
-
committed = true;
|
| 93 |
-
while (buffered.length > 0) {
|
| 94 |
-
const [evName, evData] = buffered.shift();
|
| 95 |
-
onEvent(evName, evData);
|
| 96 |
-
}
|
| 97 |
-
};
|
| 98 |
-
|
| 99 |
-
await streamFix(body, {
|
| 100 |
-
signal,
|
| 101 |
-
onEvent: (name, data) => {
|
| 102 |
-
if (!committed && (name === "start" || name === "attempt" || name === "progress" || name === "queue")) {
|
| 103 |
-
buffered.push([name, data]);
|
| 104 |
-
return;
|
| 105 |
-
}
|
| 106 |
-
if (!committed && name === "error" && data?.code === "model_unavailable") {
|
| 107 |
-
unavailable = true;
|
| 108 |
-
errorMessage = data?.message || "";
|
| 109 |
-
return;
|
| 110 |
-
}
|
| 111 |
-
flushBuffered();
|
| 112 |
-
onEvent(name, data);
|
| 113 |
-
},
|
| 114 |
-
});
|
| 115 |
-
if (!committed && !unavailable && buffered.length > 0) {
|
| 116 |
-
flushBuffered();
|
| 117 |
-
}
|
| 118 |
-
return { unavailable, errorMessage };
|
| 119 |
-
}
|
| 120 |
-
|
| 121 |
-
export async function streamCleanup(body, { signal, onEvent }) {
|
| 122 |
-
const mode = configuredMode();
|
| 123 |
-
if (mode === "browser") {
|
| 124 |
-
if (await probeWebGpuSupport()) {
|
| 125 |
-
const modelBaseUrl = globalThis.__CODEHYGIENE_BONSAI_URL || queryParam("bonsai_url");
|
| 126 |
-
const res = await runWorkerCleanup(body, { backend: "webgpu", modelBaseUrl }, { signal, onEvent });
|
| 127 |
-
if (res.ok || signal?.aborted) return;
|
| 128 |
-
}
|
| 129 |
-
onEvent("error", { code: "webgpu_failed", message: "WebGPU unavailable" });
|
| 130 |
-
return;
|
| 131 |
-
}
|
| 132 |
-
|
| 133 |
-
if (serverBackend === "mock" || mode === "server") {
|
| 134 |
-
await streamFix(body, { signal, onEvent });
|
| 135 |
-
return;
|
| 136 |
-
}
|
| 137 |
-
|
| 138 |
-
if (mode !== "direct" && !serverQuotaExhausted) {
|
| 139 |
-
const serverRes = await tryServerZeroGpu(body, { signal, onEvent });
|
| 140 |
-
if (!serverRes.unavailable || signal?.aborted) {
|
| 141 |
-
return;
|
| 142 |
-
}
|
| 143 |
-
serverQuotaExhausted = true;
|
| 144 |
-
}
|
| 145 |
-
|
| 146 |
-
const zeroGpuUrl = queryParam("zerogpu_url");
|
| 147 |
-
const direct = await runWorkerCleanup(body, { backend: "zerogpu", zeroGpuUrl }, { signal, onEvent });
|
| 148 |
-
if (!direct.ok && !signal?.aborted) {
|
| 149 |
-
const msg = direct.error || "ZeroGPU unavailable";
|
| 150 |
-
const code = isQuotaErrorMessage(msg) ? "quota_exceeded" : "model_unavailable";
|
| 151 |
-
onEvent("error", { code, message: msg });
|
| 152 |
-
}
|
| 153 |
-
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
web/src/diff.js
DELETED
|
@@ -1,90 +0,0 @@
|
|
| 1 |
-
const MAX_CELLS = 4_000_000;
|
| 2 |
-
|
| 3 |
-
function trimCommon(before, after) {
|
| 4 |
-
let head = 0;
|
| 5 |
-
while (head < before.length && head < after.length && before[head] === after[head]) {
|
| 6 |
-
head += 1;
|
| 7 |
-
}
|
| 8 |
-
let tail = 0;
|
| 9 |
-
while (
|
| 10 |
-
tail < before.length - head
|
| 11 |
-
&& tail < after.length - head
|
| 12 |
-
&& before[before.length - 1 - tail] === after[after.length - 1 - tail]
|
| 13 |
-
) {
|
| 14 |
-
tail += 1;
|
| 15 |
-
}
|
| 16 |
-
return { head, tail };
|
| 17 |
-
}
|
| 18 |
-
|
| 19 |
-
function lcsTable(before, after) {
|
| 20 |
-
const width = after.length + 1;
|
| 21 |
-
const table = new Uint32Array((before.length + 1) * width);
|
| 22 |
-
for (let row = before.length - 1; row >= 0; row -= 1) {
|
| 23 |
-
for (let col = after.length - 1; col >= 0; col -= 1) {
|
| 24 |
-
const cell = row * width + col;
|
| 25 |
-
table[cell] = before[row] === after[col]
|
| 26 |
-
? table[cell + width + 1] + 1
|
| 27 |
-
: Math.max(table[cell + width], table[cell + 1]);
|
| 28 |
-
}
|
| 29 |
-
}
|
| 30 |
-
return table;
|
| 31 |
-
}
|
| 32 |
-
|
| 33 |
-
function walkMiddle(before, after) {
|
| 34 |
-
if (before.length * after.length > MAX_CELLS) {
|
| 35 |
-
return [
|
| 36 |
-
...before.map((text, index) => ({ kind: "removed", text, oldIndex: index })),
|
| 37 |
-
...after.map((text, index) => ({ kind: "added", text, newIndex: index })),
|
| 38 |
-
];
|
| 39 |
-
}
|
| 40 |
-
const table = lcsTable(before, after);
|
| 41 |
-
const width = after.length + 1;
|
| 42 |
-
const rows = [];
|
| 43 |
-
let row = 0;
|
| 44 |
-
let col = 0;
|
| 45 |
-
while (row < before.length || col < after.length) {
|
| 46 |
-
if (row < before.length && col < after.length && before[row] === after[col]) {
|
| 47 |
-
rows.push({ kind: "same", text: after[col], oldIndex: row, newIndex: col });
|
| 48 |
-
row += 1;
|
| 49 |
-
col += 1;
|
| 50 |
-
continue;
|
| 51 |
-
}
|
| 52 |
-
const lcsIfRemoved = table[(row + 1) * width + col];
|
| 53 |
-
const lcsIfAdded = table[row * width + col + 1];
|
| 54 |
-
if (row < before.length && (col === after.length || lcsIfRemoved >= lcsIfAdded)) {
|
| 55 |
-
rows.push({ kind: "removed", text: before[row], oldIndex: row });
|
| 56 |
-
row += 1;
|
| 57 |
-
} else {
|
| 58 |
-
rows.push({ kind: "added", text: after[col], newIndex: col });
|
| 59 |
-
col += 1;
|
| 60 |
-
}
|
| 61 |
-
}
|
| 62 |
-
return rows;
|
| 63 |
-
}
|
| 64 |
-
|
| 65 |
-
function shift(rows, offset) {
|
| 66 |
-
return rows.map((entry) => ({
|
| 67 |
-
...entry,
|
| 68 |
-
oldIndex: entry.oldIndex === undefined ? undefined : entry.oldIndex + offset,
|
| 69 |
-
newIndex: entry.newIndex === undefined ? undefined : entry.newIndex + offset,
|
| 70 |
-
}));
|
| 71 |
-
}
|
| 72 |
-
|
| 73 |
-
export function diffLines(beforeText, afterText) {
|
| 74 |
-
const before = beforeText.split("\n");
|
| 75 |
-
const after = afterText.split("\n");
|
| 76 |
-
const { head, tail } = trimCommon(before, after);
|
| 77 |
-
const sameRows = (start, end, oldOffset) => after.slice(start, end).map((text, index) => ({
|
| 78 |
-
kind: "same",
|
| 79 |
-
text,
|
| 80 |
-
oldIndex: oldOffset + start + index,
|
| 81 |
-
newIndex: start + index,
|
| 82 |
-
}));
|
| 83 |
-
const changedBefore = before.slice(head, before.length - tail);
|
| 84 |
-
const changedAfter = after.slice(head, after.length - tail);
|
| 85 |
-
return [
|
| 86 |
-
...sameRows(0, head, 0),
|
| 87 |
-
...shift(walkMiddle(changedBefore, changedAfter), head),
|
| 88 |
-
...sameRows(after.length - tail, after.length, before.length - after.length),
|
| 89 |
-
];
|
| 90 |
-
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
web/src/fix-run.js
DELETED
|
@@ -1,172 +0,0 @@
|
|
| 1 |
-
import { streamCleanup } from "./browser-refiner.js";
|
| 2 |
-
import { STATUS } from "./labels.js";
|
| 3 |
-
import { THRESHOLD } from "./scoring.js";
|
| 4 |
-
|
| 5 |
-
const SEARCH_OPEN = "<<<<<<< SEARCH\n";
|
| 6 |
-
const DIVIDER = "=======";
|
| 7 |
-
|
| 8 |
-
function searchFocusRange(rawText, docText) {
|
| 9 |
-
const openAt = rawText.lastIndexOf(SEARCH_OPEN);
|
| 10 |
-
if (openAt < 0) {
|
| 11 |
-
return null;
|
| 12 |
-
}
|
| 13 |
-
const completeLines = rawText.slice(openAt + SEARCH_OPEN.length).split("\n").slice(0, -1);
|
| 14 |
-
if (completeLines.length === 0 || completeLines.includes(DIVIDER)) {
|
| 15 |
-
return null;
|
| 16 |
-
}
|
| 17 |
-
const docLines = docText.split("\n");
|
| 18 |
-
const lastStart = docLines.length - completeLines.length;
|
| 19 |
-
for (let start = 0; start <= lastStart; start += 1) {
|
| 20 |
-
if (completeLines.every((line, offset) => docLines[start + offset] === line)) {
|
| 21 |
-
return { first: start + 1, last: start + completeLines.length };
|
| 22 |
-
}
|
| 23 |
-
}
|
| 24 |
-
return null;
|
| 25 |
-
}
|
| 26 |
-
|
| 27 |
-
function squiggleCount(hygiene) {
|
| 28 |
-
return hygiene?.squiggles?.length ?? 0;
|
| 29 |
-
}
|
| 30 |
-
|
| 31 |
-
export class FixSession {
|
| 32 |
-
constructor({ file, sourceKind, noCache, hooks }) {
|
| 33 |
-
this.file = file;
|
| 34 |
-
this.sourceKind = sourceKind;
|
| 35 |
-
this.noCache = noCache;
|
| 36 |
-
this.hooks = hooks;
|
| 37 |
-
this.baseline = file.content;
|
| 38 |
-
this.lastAccepted = this.baseline;
|
| 39 |
-
this.attemptBase = this.baseline;
|
| 40 |
-
this.preview = this.baseline;
|
| 41 |
-
this.raw = "";
|
| 42 |
-
this.phase = "reading";
|
| 43 |
-
this.readPercent = null;
|
| 44 |
-
this.outcome = { kind: "running" };
|
| 45 |
-
this.controller = new AbortController();
|
| 46 |
-
}
|
| 47 |
-
|
| 48 |
-
get running() {
|
| 49 |
-
return this.outcome.kind === "running";
|
| 50 |
-
}
|
| 51 |
-
|
| 52 |
-
get statusText() {
|
| 53 |
-
if (this.phase === "waiting") {
|
| 54 |
-
return STATUS.waiting;
|
| 55 |
-
}
|
| 56 |
-
if (this.phase === "writing") {
|
| 57 |
-
return STATUS.rewriting;
|
| 58 |
-
}
|
| 59 |
-
return this.readPercent === null ? STATUS.reading : `${STATUS.reading} ${this.readPercent}%`;
|
| 60 |
-
}
|
| 61 |
-
|
| 62 |
-
get reading() {
|
| 63 |
-
return this.phase === "writing" ? searchFocusRange(this.raw, this.preview) : null;
|
| 64 |
-
}
|
| 65 |
-
|
| 66 |
-
async start() {
|
| 67 |
-
const body = {
|
| 68 |
-
filename: this.file.path,
|
| 69 |
-
content: this.baseline,
|
| 70 |
-
threshold: THRESHOLD,
|
| 71 |
-
max_passes: 2,
|
| 72 |
-
source_kind: this.sourceKind,
|
| 73 |
-
};
|
| 74 |
-
if (this.noCache) {
|
| 75 |
-
body.no_cache = true;
|
| 76 |
-
}
|
| 77 |
-
try {
|
| 78 |
-
await streamCleanup(body, {
|
| 79 |
-
signal: this.controller.signal,
|
| 80 |
-
onEvent: (name, data) => this.handle(name, data),
|
| 81 |
-
});
|
| 82 |
-
if (this.running) {
|
| 83 |
-
this.settle({ kind: "unreachable" });
|
| 84 |
-
}
|
| 85 |
-
} catch (error) {
|
| 86 |
-
if (this.running) {
|
| 87 |
-
this.settle({ kind: error.name === "AbortError" ? "cancelled" : "unreachable" });
|
| 88 |
-
}
|
| 89 |
-
}
|
| 90 |
-
}
|
| 91 |
-
|
| 92 |
-
cancel() {
|
| 93 |
-
if (this.running) {
|
| 94 |
-
this.controller.abort();
|
| 95 |
-
this.settle({ kind: "cancelled" });
|
| 96 |
-
}
|
| 97 |
-
}
|
| 98 |
-
|
| 99 |
-
handle(name, data) {
|
| 100 |
-
const handler = this.handlers[name];
|
| 101 |
-
if (this.running && handler) {
|
| 102 |
-
handler.call(this, data);
|
| 103 |
-
}
|
| 104 |
-
}
|
| 105 |
-
|
| 106 |
-
settle(outcome) {
|
| 107 |
-
this.outcome = outcome;
|
| 108 |
-
this.hooks.onFinish(this);
|
| 109 |
-
}
|
| 110 |
-
}
|
| 111 |
-
|
| 112 |
-
FixSession.prototype.handlers = {
|
| 113 |
-
queue() {
|
| 114 |
-
this.phase = "waiting";
|
| 115 |
-
this.hooks.onStatus(this);
|
| 116 |
-
},
|
| 117 |
-
attempt() {
|
| 118 |
-
this.attemptBase = this.lastAccepted;
|
| 119 |
-
this.preview = this.attemptBase;
|
| 120 |
-
this.raw = "";
|
| 121 |
-
this.phase = "reading";
|
| 122 |
-
this.readPercent = null;
|
| 123 |
-
this.hooks.onAttempt(this);
|
| 124 |
-
},
|
| 125 |
-
progress(data) {
|
| 126 |
-
if (data.phase === "read" && data.total > 0) {
|
| 127 |
-
this.readPercent = Math.round((data.done / data.total) * 100);
|
| 128 |
-
this.hooks.onStatus(this);
|
| 129 |
-
}
|
| 130 |
-
},
|
| 131 |
-
token(data) {
|
| 132 |
-
const firstToken = this.phase !== "writing";
|
| 133 |
-
this.raw += data.text;
|
| 134 |
-
this.phase = "writing";
|
| 135 |
-
if (firstToken) {
|
| 136 |
-
this.hooks.onStatus(this);
|
| 137 |
-
}
|
| 138 |
-
this.hooks.onPreview(this);
|
| 139 |
-
},
|
| 140 |
-
preview(data) {
|
| 141 |
-
this.preview = data.content;
|
| 142 |
-
this.hooks.onPreview(this);
|
| 143 |
-
},
|
| 144 |
-
verdict(data) {
|
| 145 |
-
if (data.accepted) {
|
| 146 |
-
this.lastAccepted = this.preview;
|
| 147 |
-
} else {
|
| 148 |
-
this.preview = this.lastAccepted;
|
| 149 |
-
}
|
| 150 |
-
this.hooks.onVerdict(this, data);
|
| 151 |
-
},
|
| 152 |
-
done(data) {
|
| 153 |
-
const changed = data.updated_content !== this.baseline;
|
| 154 |
-
if (!data.accepted || !changed) {
|
| 155 |
-
this.settle({ kind: "rejected" });
|
| 156 |
-
return;
|
| 157 |
-
}
|
| 158 |
-
this.settle({
|
| 159 |
-
kind: "kept",
|
| 160 |
-
content: data.updated_content,
|
| 161 |
-
before: squiggleCount(data.before_hygiene),
|
| 162 |
-
after: squiggleCount(data.after_hygiene),
|
| 163 |
-
});
|
| 164 |
-
},
|
| 165 |
-
error(data) {
|
| 166 |
-
if (data?.code === "quota_exceeded") {
|
| 167 |
-
this.settle({ kind: "quota_exceeded" });
|
| 168 |
-
return;
|
| 169 |
-
}
|
| 170 |
-
this.settle({ kind: data?.code === "model_unavailable" ? "unreachable" : "rejected" });
|
| 171 |
-
},
|
| 172 |
-
};
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
web/src/github.js
CHANGED
|
@@ -101,7 +101,8 @@ export function parseRepoSpec(rawInput) {
|
|
| 101 |
const pathText = REPO_HOSTS.has(host) || host === RAW_HOST
|
| 102 |
? withoutScheme.slice(firstSlash + 1)
|
| 103 |
: withoutScheme;
|
| 104 |
-
const
|
|
|
|
| 105 |
if (segments.length < 2) {
|
| 106 |
return null;
|
| 107 |
}
|
|
@@ -271,52 +272,36 @@ export function isReadableText(text) {
|
|
| 271 |
return Boolean(text) && text.trim().length > 0 && !text.includes("\0");
|
| 272 |
}
|
| 273 |
|
| 274 |
-
|
| 275 |
-
|
| 276 |
-
|
| 277 |
-
|
| 278 |
-
const ref = encodeURIComponent(spec.ref || "HEAD");
|
| 279 |
-
const url = `https://api.github.com/repos/${spec.slug}/commits/${ref}`;
|
| 280 |
-
const response = await guardedFetch(url, signal, { Accept: "application/vnd.github.sha" });
|
| 281 |
-
if (!response.ok) {
|
| 282 |
-
return null;
|
| 283 |
-
}
|
| 284 |
-
const sha = (await response.text()).trim();
|
| 285 |
-
return COMMIT_SHA.test(sha) ? sha : null;
|
| 286 |
-
}
|
| 287 |
-
|
| 288 |
-
export async function fetchRepoFiles(spec, { signal, focusPath, onProgress, pinCommit = false }) {
|
| 289 |
-
const commitSha = pinCommit ? await resolveCommit(spec, signal) : null;
|
| 290 |
-
const pinned = commitSha ? { ...spec, ref: commitSha } : spec;
|
| 291 |
-
const entries = await discoverTree(pinned, signal);
|
| 292 |
-
const selected = selectEntries(entries, spec, focusPath || spec.blobPath);
|
| 293 |
-
if (selected.length === 0) {
|
| 294 |
throw new RepoError("no_files", `No code files in ${spec.slug}`);
|
| 295 |
}
|
| 296 |
|
| 297 |
-
const contents = new Array(
|
| 298 |
let cursor = 0;
|
| 299 |
let fetched = 0;
|
| 300 |
|
| 301 |
async function drain() {
|
| 302 |
-
while (cursor <
|
| 303 |
const index = cursor;
|
| 304 |
cursor += 1;
|
| 305 |
-
const text = await fetchFileText(
|
| 306 |
contents[index] = isReadableText(text) ? text : null;
|
| 307 |
fetched += 1;
|
| 308 |
-
onProgress(fetched,
|
| 309 |
}
|
| 310 |
}
|
| 311 |
|
| 312 |
-
const lanes = Math.min(FETCH_CONCURRENCY,
|
| 313 |
await Promise.all(Array.from({ length: lanes }, drain));
|
| 314 |
|
| 315 |
-
const files =
|
| 316 |
.map((entry, index) => ({ path: entry.path, content: contents[index] }))
|
| 317 |
.filter((file) => file.content !== null);
|
| 318 |
-
if (files.length
|
| 319 |
throw new RepoError("no_files", `No readable files in ${spec.slug}`);
|
| 320 |
}
|
| 321 |
-
return { files
|
| 322 |
}
|
|
|
|
| 101 |
const pathText = REPO_HOSTS.has(host) || host === RAW_HOST
|
| 102 |
? withoutScheme.slice(firstSlash + 1)
|
| 103 |
: withoutScheme;
|
| 104 |
+
const rawParts = cleanPath(pathText).split("/").filter(Boolean);
|
| 105 |
+
const segments = rawParts.map(decodeURIComponent);
|
| 106 |
if (segments.length < 2) {
|
| 107 |
return null;
|
| 108 |
}
|
|
|
|
| 272 |
return Boolean(text) && text.trim().length > 0 && !text.includes("\0");
|
| 273 |
}
|
| 274 |
|
| 275 |
+
export async function fetchRepoFiles(spec, { signal, focusPath, onProgress }) {
|
| 276 |
+
const tree = await discoverTree(spec, signal);
|
| 277 |
+
const targets = selectEntries(tree, spec, focusPath || spec.blobPath);
|
| 278 |
+
if (!targets.length) {
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 279 |
throw new RepoError("no_files", `No code files in ${spec.slug}`);
|
| 280 |
}
|
| 281 |
|
| 282 |
+
const contents = new Array(targets.length).fill(null);
|
| 283 |
let cursor = 0;
|
| 284 |
let fetched = 0;
|
| 285 |
|
| 286 |
async function drain() {
|
| 287 |
+
while (cursor < targets.length) {
|
| 288 |
const index = cursor;
|
| 289 |
cursor += 1;
|
| 290 |
+
const text = await fetchFileText(spec, targets[index].path, signal);
|
| 291 |
contents[index] = isReadableText(text) ? text : null;
|
| 292 |
fetched += 1;
|
| 293 |
+
onProgress(fetched, targets.length);
|
| 294 |
}
|
| 295 |
}
|
| 296 |
|
| 297 |
+
const lanes = Math.min(FETCH_CONCURRENCY, targets.length);
|
| 298 |
await Promise.all(Array.from({ length: lanes }, drain));
|
| 299 |
|
| 300 |
+
const files = targets
|
| 301 |
.map((entry, index) => ({ path: entry.path, content: contents[index] }))
|
| 302 |
.filter((file) => file.content !== null);
|
| 303 |
+
if (!files.length) {
|
| 304 |
throw new RepoError("no_files", `No readable files in ${spec.slug}`);
|
| 305 |
}
|
| 306 |
+
return { files };
|
| 307 |
}
|
web/src/index.html
CHANGED
|
@@ -3,7 +3,7 @@
|
|
| 3 |
<head>
|
| 4 |
<meta charset="utf-8">
|
| 5 |
<meta name="viewport" content="width=device-width, initial-scale=1">
|
| 6 |
-
<meta name="description" content="Paste a GitHub link to find hard-to-read code
|
| 7 |
<title>codehygiene</title>
|
| 8 |
<link rel="stylesheet" href="styles.css">
|
| 9 |
<script type="module" src="app.js"></script>
|
|
@@ -21,16 +21,14 @@
|
|
| 21 |
<symbol id="i-upload" viewBox="0 0 24 24"><path d="M20 20a2 2 0 0 0 2-2V8a2 2 0 0 0-2-2h-7.9a2 2 0 0 1-1.69-.9L9.6 3.9A2 2 0 0 0 7.93 3H4a2 2 0 0 0-2 2v13a2 2 0 0 0 2 2Z"/><path d="M12 10v6"/><path d="m9 13 3-3 3 3"/></symbol>
|
| 22 |
<symbol id="i-sidebar" viewBox="0 0 24 24"><rect x="3" y="4" width="18" height="16" rx="2"/><path d="M9 4v16"/></symbol>
|
| 23 |
<symbol id="i-search" viewBox="0 0 24 24"><circle cx="11" cy="11" r="7"/><path d="m20 20-3.5-3.5"/></symbol>
|
| 24 |
-
<symbol id="i-download" viewBox="0 0 24 24"><path d="M12 4v11"/><path d="m7 10 5 5 5-5"/><path d="M5 20h14"/></symbol>
|
| 25 |
-
<symbol id="i-pr" viewBox="0 0 24 24"><circle cx="6" cy="6" r="2.5"/><circle cx="18" cy="18" r="2.5"/><path d="M6 8.5V21"/><path d="M13 6h3a2 2 0 0 1 2 2v7.5"/><path d="m15.5 3.5-2.5 2.5 2.5 2.5"/></symbol>
|
| 26 |
<symbol id="i-trophy" viewBox="0 0 24 24"><path d="M6 9H4.5a2.5 2.5 0 0 1 0-5H6"/><path d="M18 9h1.5a2.5 2.5 0 0 0 0-5H18"/><path d="M4 22h16"/><path d="M10 14.66V17c0 .55-.47.98-.97 1.21C7.85 18.75 7 20.24 7 22"/><path d="M14 14.66V17c0 .55.47.98.97 1.21C16.15 18.75 17 20.24 17 22"/><path d="M18 2H6v7a6 6 0 0 0 12 0V2Z"/></symbol>
|
| 27 |
<symbol id="i-shield" viewBox="0 0 24 24"><path d="M20 13c0 5-3.5 7.5-7.66 8.95a1 1 0 0 1-.67-.01C7.5 20.5 4 18 4 13V6a1 1 0 0 1 1-1c2 0 4.5-1.2 6.24-2.72a1.17 1.17 0 0 1 1.52 0C14.51 3.81 17 5 19 5a1 1 0 0 1 1 1z"/><path d="m9 12 2 2 4-4"/></symbol>
|
| 28 |
</svg>
|
| 29 |
<aside class="banner" id="banner" aria-label="Live stats">
|
| 30 |
<div class="banner-inner">
|
| 31 |
<a class="banner-stats" id="banner-stats" href="leaderboard/#stats" target="_blank" rel="noopener" title="Stats" hidden>
|
| 32 |
-
<span class="banner-stat" id="banner-
|
| 33 |
-
<span class="banner-stat" id="banner-
|
| 34 |
<span class="banner-stat" id="banner-speed" hidden><b id="banner-speed-value"></b> per file</span>
|
| 35 |
</a>
|
| 36 |
<nav class="banner-links" aria-label="More">
|
|
@@ -79,13 +77,13 @@
|
|
| 79 |
<div class="sample" id="sample">
|
| 80 |
<header class="sample-head">
|
| 81 |
<p class="stage-meta">
|
| 82 |
-
<span class="stage-status">
|
| 83 |
<span class="stage-score"><span class="stage-score-value" id="sample-value"></span><span class="stage-score-scale">/100</span> <span class="stage-score-word" id="sample-word"></span></span>
|
| 84 |
</p>
|
| 85 |
<span class="stage-path">send_receipt.py</span>
|
| 86 |
</header>
|
| 87 |
<div class="stage-code sample-code" id="sample-code"></div>
|
| 88 |
-
<button class="sample-run" id="sample-run" type="button" aria-label="
|
| 89 |
</div>
|
| 90 |
|
| 91 |
<article class="stage" id="stage" hidden>
|
|
@@ -117,7 +115,6 @@
|
|
| 117 |
<svg class="icon" aria-hidden="true"><use href="#i-chevron"/></svg>
|
| 118 |
</button>
|
| 119 |
</div>
|
| 120 |
-
<nav class="tabs" id="tabs" aria-label="Files being cleaned"></nav>
|
| 121 |
<div class="head-actions">
|
| 122 |
<button class="icon-button is-plain" id="rank" type="button" aria-label="Rank this repo" title="Rank this repo" hidden>
|
| 123 |
<svg class="icon" aria-hidden="true"><use href="#i-trophy"/></svg>
|
|
@@ -129,26 +126,10 @@
|
|
| 129 |
<button class="icon-button is-plain" id="copy" type="button" aria-label="Copy file" title="Copy file" hidden>
|
| 130 |
<svg class="icon" aria-hidden="true"><use id="copy-icon" href="#i-copy"/></svg>
|
| 131 |
</button>
|
| 132 |
-
<button class="icon-button is-plain" id="patch" type="button" aria-label="Download patch" title="Download patch" hidden>
|
| 133 |
-
<svg class="icon" aria-hidden="true"><use href="#i-download"/></svg>
|
| 134 |
-
</button>
|
| 135 |
-
<button class="icon-button is-plain" id="pr" type="button" aria-label="Create PR" title="Create PR" hidden>
|
| 136 |
-
<svg class="icon" aria-hidden="true"><use href="#i-pr"/></svg>
|
| 137 |
-
</button>
|
| 138 |
-
<a class="icon-button is-plain is-done" id="pr-open" target="_blank" rel="noopener" aria-label="Open PR" title="Open PR" hidden>
|
| 139 |
-
<svg class="icon" aria-hidden="true"><use href="#i-pr"/></svg>
|
| 140 |
-
</a>
|
| 141 |
</div>
|
| 142 |
</header>
|
| 143 |
<div class="stage-code" id="stage-code" role="region" tabindex="0" aria-label="File contents"></div>
|
| 144 |
</div>
|
| 145 |
-
<footer class="tally" id="tally">
|
| 146 |
-
<p class="tally-line">
|
| 147 |
-
<span class="tally-label" id="tally-label">Issues fixed</span>
|
| 148 |
-
<span class="tally-count"><span id="tally-fixed">0</span> / <span id="tally-total">0</span></span>
|
| 149 |
-
</p>
|
| 150 |
-
<div class="tally-bar" aria-hidden="true"><span class="tally-fill" id="tally-fill"></span></div>
|
| 151 |
-
</footer>
|
| 152 |
</article>
|
| 153 |
|
| 154 |
</section>
|
|
@@ -161,7 +142,7 @@
|
|
| 161 |
<svg class="icon" aria-hidden="true"><use id="command-copy-icon" href="#i-copy"/></svg>
|
| 162 |
</button>
|
| 163 |
</div>
|
| 164 |
-
<p>
|
| 165 |
</footer>
|
| 166 |
</div>
|
| 167 |
</body>
|
|
|
|
| 3 |
<head>
|
| 4 |
<meta charset="utf-8">
|
| 5 |
<meta name="viewport" content="width=device-width, initial-scale=1">
|
| 6 |
+
<meta name="description" content="Paste a GitHub link to find hard-to-read code.">
|
| 7 |
<title>codehygiene</title>
|
| 8 |
<link rel="stylesheet" href="styles.css">
|
| 9 |
<script type="module" src="app.js"></script>
|
|
|
|
| 21 |
<symbol id="i-upload" viewBox="0 0 24 24"><path d="M20 20a2 2 0 0 0 2-2V8a2 2 0 0 0-2-2h-7.9a2 2 0 0 1-1.69-.9L9.6 3.9A2 2 0 0 0 7.93 3H4a2 2 0 0 0-2 2v13a2 2 0 0 0 2 2Z"/><path d="M12 10v6"/><path d="m9 13 3-3 3 3"/></symbol>
|
| 22 |
<symbol id="i-sidebar" viewBox="0 0 24 24"><rect x="3" y="4" width="18" height="16" rx="2"/><path d="M9 4v16"/></symbol>
|
| 23 |
<symbol id="i-search" viewBox="0 0 24 24"><circle cx="11" cy="11" r="7"/><path d="m20 20-3.5-3.5"/></symbol>
|
|
|
|
|
|
|
| 24 |
<symbol id="i-trophy" viewBox="0 0 24 24"><path d="M6 9H4.5a2.5 2.5 0 0 1 0-5H6"/><path d="M18 9h1.5a2.5 2.5 0 0 0 0-5H18"/><path d="M4 22h16"/><path d="M10 14.66V17c0 .55-.47.98-.97 1.21C7.85 18.75 7 20.24 7 22"/><path d="M14 14.66V17c0 .55.47.98.97 1.21C16.15 18.75 17 20.24 17 22"/><path d="M18 2H6v7a6 6 0 0 0 12 0V2Z"/></symbol>
|
| 25 |
<symbol id="i-shield" viewBox="0 0 24 24"><path d="M20 13c0 5-3.5 7.5-7.66 8.95a1 1 0 0 1-.67-.01C7.5 20.5 4 18 4 13V6a1 1 0 0 1 1-1c2 0 4.5-1.2 6.24-2.72a1.17 1.17 0 0 1 1.52 0C14.51 3.81 17 5 19 5a1 1 0 0 1 1 1z"/><path d="m9 12 2 2 4-4"/></symbol>
|
| 26 |
</svg>
|
| 27 |
<aside class="banner" id="banner" aria-label="Live stats">
|
| 28 |
<div class="banner-inner">
|
| 29 |
<a class="banner-stats" id="banner-stats" href="leaderboard/#stats" target="_blank" rel="noopener" title="Stats" hidden>
|
| 30 |
+
<span class="banner-stat" id="banner-scored" hidden><b id="banner-scored-value"></b> files scored</span>
|
| 31 |
+
<span class="banner-stat" id="banner-issues" hidden><b id="banner-issues-value"></b> issues flagged</span>
|
| 32 |
<span class="banner-stat" id="banner-speed" hidden><b id="banner-speed-value"></b> per file</span>
|
| 33 |
</a>
|
| 34 |
<nav class="banner-links" aria-label="More">
|
|
|
|
| 77 |
<div class="sample" id="sample">
|
| 78 |
<header class="sample-head">
|
| 79 |
<p class="stage-meta">
|
| 80 |
+
<span class="stage-status" id="sample-status"></span>
|
| 81 |
<span class="stage-score"><span class="stage-score-value" id="sample-value"></span><span class="stage-score-scale">/100</span> <span class="stage-score-word" id="sample-word"></span></span>
|
| 82 |
</p>
|
| 83 |
<span class="stage-path">send_receipt.py</span>
|
| 84 |
</header>
|
| 85 |
<div class="stage-code sample-code" id="sample-code"></div>
|
| 86 |
+
<button class="sample-run" id="sample-run" type="button" aria-label="Inspect examples" title="Inspect examples"></button>
|
| 87 |
</div>
|
| 88 |
|
| 89 |
<article class="stage" id="stage" hidden>
|
|
|
|
| 115 |
<svg class="icon" aria-hidden="true"><use href="#i-chevron"/></svg>
|
| 116 |
</button>
|
| 117 |
</div>
|
|
|
|
| 118 |
<div class="head-actions">
|
| 119 |
<button class="icon-button is-plain" id="rank" type="button" aria-label="Rank this repo" title="Rank this repo" hidden>
|
| 120 |
<svg class="icon" aria-hidden="true"><use href="#i-trophy"/></svg>
|
|
|
|
| 126 |
<button class="icon-button is-plain" id="copy" type="button" aria-label="Copy file" title="Copy file" hidden>
|
| 127 |
<svg class="icon" aria-hidden="true"><use id="copy-icon" href="#i-copy"/></svg>
|
| 128 |
</button>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 129 |
</div>
|
| 130 |
</header>
|
| 131 |
<div class="stage-code" id="stage-code" role="region" tabindex="0" aria-label="File contents"></div>
|
| 132 |
</div>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 133 |
</article>
|
| 134 |
|
| 135 |
</section>
|
|
|
|
| 142 |
<svg class="icon" aria-hidden="true"><use id="command-copy-icon" href="#i-copy"/></svg>
|
| 143 |
</button>
|
| 144 |
</div>
|
| 145 |
+
<p>23.84 KiB neural readability scorer · runs in your browser.</p>
|
| 146 |
</footer>
|
| 147 |
</div>
|
| 148 |
</body>
|
web/src/labels.js
CHANGED
|
@@ -17,21 +17,6 @@ export const SOURCE_ERRORS = {
|
|
| 17 |
offline: "Offline",
|
| 18 |
};
|
| 19 |
|
| 20 |
-
export const PR_ERRORS = {
|
| 21 |
-
popup_blocked: "Pop-up blocked",
|
| 22 |
-
closed: "Sign-in closed",
|
| 23 |
-
login_required: "Sign-in expired",
|
| 24 |
-
login_failed: "Sign-in failed",
|
| 25 |
-
no_push: "No write access",
|
| 26 |
-
not_installed: "App not installed",
|
| 27 |
-
not_found: "Repo not found",
|
| 28 |
-
bad_base: "Base not found",
|
| 29 |
-
bad_path: "File not in base",
|
| 30 |
-
not_cleaner: "Nothing to commit",
|
| 31 |
-
busy: "Busy, try later",
|
| 32 |
-
failed: "PR failed",
|
| 33 |
-
};
|
| 34 |
-
|
| 35 |
export const RANK_ERRORS = {
|
| 36 |
too_small: "Too small to rank",
|
| 37 |
too_large: "Too large to rank",
|
|
@@ -74,20 +59,6 @@ export function sentenceCase(text) {
|
|
| 74 |
return text.charAt(0).toUpperCase() + text.slice(1);
|
| 75 |
}
|
| 76 |
|
| 77 |
-
export const STATUS = {
|
| 78 |
-
waiting: "Waiting",
|
| 79 |
-
queued: "Queued",
|
| 80 |
-
reading: "Reading",
|
| 81 |
-
rewriting: "Rewriting",
|
| 82 |
-
cleaned: "Cleaned",
|
| 83 |
-
skipped: "Skipped",
|
| 84 |
-
offline: "Offline",
|
| 85 |
-
quotaExceeded: "GPU quota reached",
|
| 86 |
-
tooLong: "Too long",
|
| 87 |
-
filesTooLong: "Files too long",
|
| 88 |
-
noIssues: "No issues found",
|
| 89 |
-
};
|
| 90 |
-
|
| 91 |
export function plural(count, noun) {
|
| 92 |
return count === 1 ? `1 ${noun}` : `${count} ${noun}s`;
|
| 93 |
}
|
|
@@ -96,10 +67,6 @@ export function issuesText(count) {
|
|
| 96 |
return count === 0 ? "No issues" : plural(count, "issue");
|
| 97 |
}
|
| 98 |
|
| 99 |
-
export function fixedText(before, after) {
|
| 100 |
-
return after === 0 ? STATUS.cleaned : `${before - after} of ${before} fixed`;
|
| 101 |
-
}
|
| 102 |
-
|
| 103 |
export function toHundred(score) {
|
| 104 |
return Math.round(score * 100);
|
| 105 |
}
|
|
|
|
| 17 |
offline: "Offline",
|
| 18 |
};
|
| 19 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 20 |
export const RANK_ERRORS = {
|
| 21 |
too_small: "Too small to rank",
|
| 22 |
too_large: "Too large to rank",
|
|
|
|
| 59 |
return text.charAt(0).toUpperCase() + text.slice(1);
|
| 60 |
}
|
| 61 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 62 |
export function plural(count, noun) {
|
| 63 |
return count === 1 ? `1 ${noun}` : `${count} ${noun}s`;
|
| 64 |
}
|
|
|
|
| 67 |
return count === 0 ? "No issues" : plural(count, "issue");
|
| 68 |
}
|
| 69 |
|
|
|
|
|
|
|
|
|
|
|
|
|
| 70 |
export function toHundred(score) {
|
| 71 |
return Math.round(score * 100);
|
| 72 |
}
|
web/src/main.js
CHANGED
|
@@ -1,180 +1,50 @@
|
|
| 1 |
-
import {
|
| 2 |
-
import { byId, copyWithIcon, prefersReducedMotion, tweenNumber } from "./dom.js";
|
| 3 |
-
import { FixSession } from "./fix-run.js";
|
| 4 |
import { isFullRepoUrl, parseRepoSpec, specLabel } from "./github.js";
|
| 5 |
-
import {
|
| 6 |
-
import { buildPatch, downloadText } from "./patch.js";
|
| 7 |
-
import { openPullRequest } from "./pull-request.js";
|
| 8 |
-
import { DEFAULT_MAX_FILE_CHARS, fetchHealth, fetchStats } from "./refiner.js";
|
| 9 |
-
import { createScorer } from "./scoring.js";
|
| 10 |
-
import {
|
| 11 |
-
RUN_SIZE,
|
| 12 |
-
eligibleFiles,
|
| 13 |
-
isSloppy,
|
| 14 |
-
loadExamples,
|
| 15 |
-
loadFolder,
|
| 16 |
-
loadRepo,
|
| 17 |
-
loadSnippet,
|
| 18 |
-
} from "./sources.js";
|
| 19 |
-
import { createStage } from "./stage.js";
|
| 20 |
import { createNavigator } from "./navigator.js";
|
| 21 |
-
import { createTabs } from "./tabs.js";
|
| 22 |
-
import { paintSample } from "./sample.js";
|
| 23 |
import { boardLink, rankErrorText, rankLabel, requestRank } from "./ranking.js";
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 24 |
import { hideBannerStats, renderBanner, reportScan } from "./telemetry.js";
|
| 25 |
|
| 26 |
-
const TIMING = {
|
| 27 |
-
firstStart: 800,
|
| 28 |
-
beforeFix: 1400,
|
| 29 |
-
afterFix: 2200,
|
| 30 |
-
retry: 10000,
|
| 31 |
-
tallyTween: 600,
|
| 32 |
-
};
|
| 33 |
-
|
| 34 |
const nodes = {
|
| 35 |
sample: byId("sample"),
|
| 36 |
stage: byId("stage"),
|
| 37 |
input: byId("source-input"),
|
| 38 |
error: byId("source-error"),
|
| 39 |
progress: byId("source-progress"),
|
| 40 |
-
|
| 41 |
-
tally: byId("tally"),
|
| 42 |
-
tallyLabel: byId("tally-label"),
|
| 43 |
-
tallyFixed: byId("tally-fixed"),
|
| 44 |
-
tallyTotal: byId("tally-total"),
|
| 45 |
-
tallyFill: byId("tally-fill"),
|
| 46 |
-
|
| 47 |
copy: byId("copy"),
|
| 48 |
copyIcon: byId("copy-icon"),
|
| 49 |
-
patch: byId("patch"),
|
| 50 |
-
pr: byId("pr"),
|
| 51 |
-
prOpen: byId("pr-open"),
|
| 52 |
-
|
| 53 |
rank: byId("rank"),
|
| 54 |
rankOpen: byId("rank-open"),
|
| 55 |
rankText: byId("rank-text"),
|
| 56 |
};
|
| 57 |
|
| 58 |
-
const PATCH_SOURCES = new Set(["repo", "folder"]);
|
| 59 |
-
|
| 60 |
const narrowScreen = window.matchMedia("(max-width: 640px)");
|
| 61 |
const scorer = createScorer();
|
| 62 |
const stage = createStage();
|
| 63 |
const fileNav = createNavigator(selectFile);
|
| 64 |
-
const fileTabs = createTabs(selectFile);
|
| 65 |
|
| 66 |
const app = {
|
| 67 |
repoSpec: null,
|
| 68 |
sourceKind: "examples",
|
| 69 |
files: [],
|
| 70 |
-
queue: [],
|
| 71 |
-
cursor: -1,
|
| 72 |
shown: null,
|
| 73 |
-
following: true,
|
| 74 |
-
idle: true,
|
| 75 |
-
activeFile: null,
|
| 76 |
-
picked: new Set(),
|
| 77 |
-
session: null,
|
| 78 |
-
timers: new Set(),
|
| 79 |
-
initialIssues: 0,
|
| 80 |
-
shownFixed: null,
|
| 81 |
ingest: null,
|
| 82 |
-
maxChars: DEFAULT_MAX_FILE_CHARS,
|
| 83 |
-
healthReady: Promise.resolve(),
|
| 84 |
-
prEnabled: false,
|
| 85 |
-
commitSha: null,
|
| 86 |
-
prUrl: null,
|
| 87 |
standing: null,
|
| 88 |
-
|
| 89 |
-
followChange: false,
|
| 90 |
-
ingestController: null,
|
| 91 |
};
|
| 92 |
|
| 93 |
-
function
|
| 94 |
-
const timer = setTimeout(() => {
|
| 95 |
-
app.timers.delete(timer);
|
| 96 |
-
action();
|
| 97 |
-
}, ms);
|
| 98 |
-
app.timers.add(timer);
|
| 99 |
-
}
|
| 100 |
-
|
| 101 |
-
function clearTimers() {
|
| 102 |
-
app.timers.forEach(clearTimeout);
|
| 103 |
-
app.timers.clear();
|
| 104 |
-
}
|
| 105 |
-
|
| 106 |
-
function issueCount(files) {
|
| 107 |
-
return files.reduce((sum, file) => sum + file.result.squiggles.length, 0);
|
| 108 |
-
}
|
| 109 |
-
|
| 110 |
-
function tallyHeading(total) {
|
| 111 |
-
if (total === 0) {
|
| 112 |
-
return STATUS.noIssues;
|
| 113 |
-
}
|
| 114 |
-
const unstarted = app.idle && app.queue.length === 0 && !app.files.some((file) => file.attempted);
|
| 115 |
-
return unstarted ? STATUS.filesTooLong : "Issues fixed";
|
| 116 |
-
}
|
| 117 |
-
|
| 118 |
-
function renderTally() {
|
| 119 |
nodes.progress.hidden = !app.ingest;
|
| 120 |
nodes.progress.textContent = app.ingest ?? "";
|
| 121 |
-
const total = app.initialIssues;
|
| 122 |
-
const fixed = Math.max(0, total - issueCount(app.files));
|
| 123 |
-
nodes.tally.classList.toggle("is-empty", total === 0);
|
| 124 |
-
nodes.tallyLabel.textContent = tallyHeading(total);
|
| 125 |
-
nodes.tallyTotal.textContent = String(total);
|
| 126 |
-
tweenNumber(nodes.tallyFixed, app.shownFixed ?? fixed, fixed, TIMING.tallyTween);
|
| 127 |
-
app.shownFixed = fixed;
|
| 128 |
-
nodes.tallyFill.style.width = `${total === 0 ? 0 : (fixed / total) * 100}%`;
|
| 129 |
-
}
|
| 130 |
-
|
| 131 |
-
function tabFiles(current) {
|
| 132 |
-
const working = new Set([...app.queue, ...app.picked]);
|
| 133 |
-
if (current) {
|
| 134 |
-
working.add(current);
|
| 135 |
-
}
|
| 136 |
-
return [...working];
|
| 137 |
-
}
|
| 138 |
-
|
| 139 |
-
function paintNav(current, { fresh = false } = {}) {
|
| 140 |
-
if (fresh) {
|
| 141 |
-
fileNav.render(app.files, current);
|
| 142 |
-
} else {
|
| 143 |
-
fileNav.update(app.files, current);
|
| 144 |
-
}
|
| 145 |
-
fileTabs.render(tabFiles(current), current, app.session?.file ?? null);
|
| 146 |
-
}
|
| 147 |
-
function fileStatus(file) {
|
| 148 |
-
if (app.session?.file === file) {
|
| 149 |
-
return app.session.statusText;
|
| 150 |
-
}
|
| 151 |
-
if (app.picked.has(file) && !file.attempted && app.activeFile !== file) {
|
| 152 |
-
return STATUS.queued;
|
| 153 |
-
}
|
| 154 |
-
if (file.kept) {
|
| 155 |
-
return fixedText(file.originalResult.squiggles.length, file.result.squiggles.length);
|
| 156 |
-
}
|
| 157 |
-
if (file.lastRejected && file.attempted) {
|
| 158 |
-
return STATUS.skipped;
|
| 159 |
-
}
|
| 160 |
-
if (isSloppy(file) && !file.kept && file.content.length > app.maxChars) {
|
| 161 |
-
return STATUS.tooLong;
|
| 162 |
-
}
|
| 163 |
-
return issuesText(file.result.squiggles.length);
|
| 164 |
-
}
|
| 165 |
-
|
| 166 |
-
function keptFiles() {
|
| 167 |
-
return app.files.filter((file) => file.kept && file.content !== file.original);
|
| 168 |
}
|
| 169 |
|
| 170 |
function renderActions(file) {
|
| 171 |
-
nodes.copy.hidden = !file
|
| 172 |
-
const anyKept = keptFiles().length > 0;
|
| 173 |
-
const canPr = anyKept && app.prEnabled && app.sourceKind === "repo" && Boolean(app.commitSha);
|
| 174 |
-
nodes.patch.hidden = !anyKept || !PATCH_SOURCES.has(app.sourceKind);
|
| 175 |
-
nodes.pr.hidden = !canPr || Boolean(app.prUrl);
|
| 176 |
-
nodes.prOpen.hidden = !app.prUrl;
|
| 177 |
-
nodes.prOpen.href = app.prUrl ?? "";
|
| 178 |
const rankable = app.sourceKind === "repo" && Boolean(app.repoSpec);
|
| 179 |
nodes.rank.hidden = !rankable || Boolean(app.standing);
|
| 180 |
nodes.rankOpen.hidden = !rankable || !app.standing;
|
|
@@ -187,6 +57,17 @@ function paintStanding(standing) {
|
|
| 187 |
nodes.rankOpen.title = `${standing.repo}: ${points}/100, #${standing.rank} of ${standing.total}`;
|
| 188 |
}
|
| 189 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 190 |
async function rankRepo() {
|
| 191 |
const spec = app.repoSpec;
|
| 192 |
showSourceError("");
|
|
@@ -205,269 +86,36 @@ async function rankRepo() {
|
|
| 205 |
}
|
| 206 |
}
|
| 207 |
|
| 208 |
-
function
|
| 209 |
-
return app.repoSpec ? `codehygiene-${app.repoSpec.repo}.patch` : "codehygiene.patch";
|
| 210 |
-
}
|
| 211 |
-
|
| 212 |
-
function downloadPatch() {
|
| 213 |
-
downloadText(buildPatch(keptFiles()), patchName());
|
| 214 |
-
}
|
| 215 |
-
|
| 216 |
-
function setBusy(button, busy) {
|
| 217 |
-
button.disabled = busy;
|
| 218 |
-
button.classList.toggle("is-busy", busy);
|
| 219 |
-
button.setAttribute("aria-busy", String(busy));
|
| 220 |
-
}
|
| 221 |
-
|
| 222 |
-
async function createPr() {
|
| 223 |
-
showSourceError("");
|
| 224 |
-
setBusy(nodes.pr, true);
|
| 225 |
-
try {
|
| 226 |
-
const opened = await openPullRequest({
|
| 227 |
-
spec: app.repoSpec,
|
| 228 |
-
commitSha: app.commitSha,
|
| 229 |
-
files: keptFiles(),
|
| 230 |
-
});
|
| 231 |
-
app.prUrl = opened.url;
|
| 232 |
-
} catch (error) {
|
| 233 |
-
showSourceError(PR_ERRORS[error.code] ?? PR_ERRORS.failed);
|
| 234 |
-
} finally {
|
| 235 |
-
setBusy(nodes.pr, false);
|
| 236 |
-
renderActions(app.shown);
|
| 237 |
-
}
|
| 238 |
-
}
|
| 239 |
-
|
| 240 |
-
function renderRestingDiff(file, text, followChange) {
|
| 241 |
-
stage.renderDiff({
|
| 242 |
-
base: file.original,
|
| 243 |
-
next: text,
|
| 244 |
-
filename: file.path,
|
| 245 |
-
followChange,
|
| 246 |
-
squiggles: file.result.squiggles,
|
| 247 |
-
squigglesOn: "next",
|
| 248 |
-
});
|
| 249 |
-
}
|
| 250 |
-
|
| 251 |
-
function renderShown({ enter = false, animateScore = false } = {}) {
|
| 252 |
-
const file = app.shown;
|
| 253 |
-
stage.setHead({
|
| 254 |
-
filename: file.path,
|
| 255 |
-
statusText: fileStatus(file),
|
| 256 |
-
points: toHundred(file.result.score),
|
| 257 |
-
animate: animateScore,
|
| 258 |
-
});
|
| 259 |
-
if (file.kept) {
|
| 260 |
-
renderRestingDiff(file, file.content, enter);
|
| 261 |
-
} else {
|
| 262 |
-
stage.renderPlain({ text: file.content, filename: file.path, squiggles: file.result.squiggles, enter });
|
| 263 |
-
}
|
| 264 |
-
renderActions(file);
|
| 265 |
-
}
|
| 266 |
-
|
| 267 |
-
function showFile(file, { enter = true } = {}) {
|
| 268 |
app.shown = file;
|
| 269 |
stage.resetScore();
|
| 270 |
-
|
| 271 |
-
|
| 272 |
-
|
| 273 |
-
|
| 274 |
-
return;
|
| 275 |
}
|
| 276 |
-
renderShown({ enter });
|
| 277 |
-
}
|
| 278 |
-
|
| 279 |
-
function renderSessionDiff(session, followChange) {
|
| 280 |
-
const scoredBase = session.attemptBase === (session.file.scoredText ?? session.file.content);
|
| 281 |
stage.setHead({
|
| 282 |
-
filename:
|
| 283 |
-
statusText:
|
| 284 |
-
points: toHundred(
|
| 285 |
-
});
|
| 286 |
-
stage.renderDiff({
|
| 287 |
-
base: session.attemptBase,
|
| 288 |
-
next: session.preview,
|
| 289 |
-
filename: session.file.path,
|
| 290 |
-
reading: session.reading,
|
| 291 |
-
followChange,
|
| 292 |
-
squiggles: scoredBase ? session.file.result.squiggles : [],
|
| 293 |
-
});
|
| 294 |
-
}
|
| 295 |
-
|
| 296 |
-
function queueSessionRender(session, followChange) {
|
| 297 |
-
app.followChange ||= followChange;
|
| 298 |
-
if (app.renderQueued || app.shown !== session.file) {
|
| 299 |
-
return;
|
| 300 |
-
}
|
| 301 |
-
app.renderQueued = true;
|
| 302 |
-
requestAnimationFrame(() => {
|
| 303 |
-
const follow = app.followChange;
|
| 304 |
-
Object.assign(app, { renderQueued: false, followChange: false });
|
| 305 |
-
if (app.session === session && app.shown === session.file) {
|
| 306 |
-
renderSessionDiff(session, follow);
|
| 307 |
-
}
|
| 308 |
});
|
| 309 |
-
|
| 310 |
-
|
| 311 |
-
|
| 312 |
-
|
| 313 |
-
|
| 314 |
-
|
| 315 |
-
function sessionMovedOn(file, acceptedText) {
|
| 316 |
-
return app.session?.file === file && app.session.preview !== acceptedText;
|
| 317 |
-
}
|
| 318 |
-
|
| 319 |
-
async function handleVerdict(session, verdict) {
|
| 320 |
-
const file = session.file;
|
| 321 |
-
if (!verdict.accepted) {
|
| 322 |
-
if (app.shown === file) {
|
| 323 |
-
stage.setStatus(STATUS.skipped);
|
| 324 |
-
renderFileText(file, session.preview);
|
| 325 |
-
}
|
| 326 |
-
return;
|
| 327 |
-
}
|
| 328 |
-
const acceptedText = session.preview;
|
| 329 |
-
if (app.shown === file) {
|
| 330 |
-
stage.setStatus(fixedText(file.originalResult.squiggles.length, verdict.squiggles_after));
|
| 331 |
-
}
|
| 332 |
-
const result = await scorer.scoreOne(acceptedText);
|
| 333 |
-
if (session.outcome.kind === "cancelled") {
|
| 334 |
-
return;
|
| 335 |
-
}
|
| 336 |
-
file.result = result;
|
| 337 |
-
file.scoredText = acceptedText;
|
| 338 |
-
paintNav(app.shown);
|
| 339 |
-
renderTally();
|
| 340 |
-
if (app.shown !== file || sessionMovedOn(file, acceptedText)) {
|
| 341 |
-
return;
|
| 342 |
-
}
|
| 343 |
-
file.restingText = acceptedText;
|
| 344 |
-
stage.setScore(toHundred(file.result.score), true);
|
| 345 |
-
renderRestingDiff(file, acceptedText, true);
|
| 346 |
-
}
|
| 347 |
-
|
| 348 |
-
async function restoreScore(file) {
|
| 349 |
-
file.result = await scorer.scoreOne(file.content);
|
| 350 |
-
file.scoredText = file.content;
|
| 351 |
-
paintNav(app.shown);
|
| 352 |
-
renderTally();
|
| 353 |
-
}
|
| 354 |
-
|
| 355 |
-
async function finishSession(session) {
|
| 356 |
-
const file = session.file;
|
| 357 |
-
const { outcome } = session;
|
| 358 |
-
if (app.session === session) {
|
| 359 |
-
app.session = null;
|
| 360 |
-
}
|
| 361 |
-
if (outcome.kind === "cancelled") {
|
| 362 |
-
if (file.scoredText !== file.content) {
|
| 363 |
-
await restoreScore(file);
|
| 364 |
-
}
|
| 365 |
-
return;
|
| 366 |
-
}
|
| 367 |
-
if (outcome.kind === "quota_exceeded") {
|
| 368 |
-
app.activeFile = null;
|
| 369 |
-
app.idle = true;
|
| 370 |
-
app.queue = [];
|
| 371 |
-
if (app.shown === file) {
|
| 372 |
-
renderShown();
|
| 373 |
-
stage.setStatus(STATUS.quotaExceeded);
|
| 374 |
-
}
|
| 375 |
-
paintNav(app.shown);
|
| 376 |
-
return;
|
| 377 |
-
}
|
| 378 |
-
if (outcome.kind === "unreachable") {
|
| 379 |
-
if (app.shown === file) {
|
| 380 |
-
renderShown();
|
| 381 |
-
stage.setStatus(STATUS.offline);
|
| 382 |
-
}
|
| 383 |
-
later(TIMING.retry, () => fixFile(file));
|
| 384 |
-
return;
|
| 385 |
-
}
|
| 386 |
-
app.activeFile = null;
|
| 387 |
-
file.attempted = true;
|
| 388 |
-
file.lastRejected = outcome.kind === "rejected";
|
| 389 |
-
if (outcome.kind === "kept") {
|
| 390 |
-
file.content = outcome.content;
|
| 391 |
-
file.result = await scorer.scoreOne(outcome.content);
|
| 392 |
-
file.scoredText = outcome.content;
|
| 393 |
-
file.kept = true;
|
| 394 |
-
file.cleaned = file.result.squiggles.length === 0;
|
| 395 |
-
}
|
| 396 |
-
paintNav(app.shown);
|
| 397 |
-
renderTally();
|
| 398 |
-
renderActions(app.shown);
|
| 399 |
-
const resting = file.kept && file.restingText === file.content;
|
| 400 |
-
file.restingText = null;
|
| 401 |
-
if (app.shown === file && resting) {
|
| 402 |
-
stage.setStatus(fileStatus(file));
|
| 403 |
-
renderActions(file);
|
| 404 |
-
} else if (app.shown === file) {
|
| 405 |
-
renderShown();
|
| 406 |
-
}
|
| 407 |
-
later(TIMING.afterFix, advance);
|
| 408 |
-
}
|
| 409 |
-
|
| 410 |
-
function fixFile(file) {
|
| 411 |
-
if (app.following && app.shown !== file) {
|
| 412 |
-
showFile(file, { enter: false });
|
| 413 |
-
}
|
| 414 |
-
const session = new FixSession({
|
| 415 |
-
file,
|
| 416 |
-
sourceKind: app.sourceKind,
|
| 417 |
-
noCache: file.lastRejected,
|
| 418 |
-
hooks: {
|
| 419 |
-
onStatus: (current) => app.shown === file && stage.setStatus(current.statusText),
|
| 420 |
-
onAttempt: (current) => queueSessionRender(current, true),
|
| 421 |
-
onPreview: (current) => queueSessionRender(current, false),
|
| 422 |
-
onVerdict: handleVerdict,
|
| 423 |
-
onFinish: finishSession,
|
| 424 |
-
},
|
| 425 |
});
|
| 426 |
-
|
| 427 |
-
paintNav(app.shown);
|
| 428 |
-
renderActions(app.shown);
|
| 429 |
-
session.start();
|
| 430 |
-
}
|
| 431 |
-
|
| 432 |
-
function openAndFix(index, { reshow = true } = {}) {
|
| 433 |
-
app.cursor = index;
|
| 434 |
-
const file = app.queue[index];
|
| 435 |
-
app.activeFile = file;
|
| 436 |
-
if (app.following && (reshow || app.shown !== file)) {
|
| 437 |
-
showFile(file);
|
| 438 |
-
}
|
| 439 |
-
later(TIMING.beforeFix, () => fixFile(file));
|
| 440 |
-
}
|
| 441 |
-
|
| 442 |
-
function advance() {
|
| 443 |
-
if (app.cursor + 1 < app.queue.length) {
|
| 444 |
-
openAndFix(app.cursor + 1);
|
| 445 |
-
return;
|
| 446 |
-
}
|
| 447 |
-
app.idle = true;
|
| 448 |
-
}
|
| 449 |
-
|
| 450 |
-
function stopEverything() {
|
| 451 |
-
clearTimers();
|
| 452 |
-
app.session?.cancel();
|
| 453 |
-
app.session = null;
|
| 454 |
-
app.activeFile = null;
|
| 455 |
}
|
| 456 |
|
| 457 |
-
function
|
| 458 |
-
|
| 459 |
-
Object.assign(app, { queue, cursor: -1, following: true, idle: queue.length === 0, picked: new Set() });
|
| 460 |
-
renderTally();
|
| 461 |
-
paintNav(queue[0] ?? app.files[0], { fresh: true });
|
| 462 |
-
showFile(queue[0] ?? app.files[0]);
|
| 463 |
-
if (queue.length > 0) {
|
| 464 |
-
later(startDelay, () => openAndFix(0, { reshow: false }));
|
| 465 |
-
}
|
| 466 |
}
|
| 467 |
|
| 468 |
function bringStageIntoView() {
|
| 469 |
if (narrowScreen.matches) {
|
| 470 |
-
|
|
|
|
| 471 |
}
|
| 472 |
}
|
| 473 |
|
|
@@ -489,31 +137,6 @@ function revealStage(update) {
|
|
| 489 |
document.startViewTransition(swap).finished.then(bringStageIntoView);
|
| 490 |
}
|
| 491 |
|
| 492 |
-
function startFiles({ files, sourceKind, repoSpec = null, commitSha = null, queue, startDelay }) {
|
| 493 |
-
Object.assign(app, { files, sourceKind, repoSpec, commitSha });
|
| 494 |
-
Object.assign(app, { prUrl: null, standing: null, ingest: null, shownFixed: null });
|
| 495 |
-
app.initialIssues = issueCount(files);
|
| 496 |
-
syncUrl();
|
| 497 |
-
reportScan(sourceKind, files);
|
| 498 |
-
revealStage(() => beginRun(queue, startDelay));
|
| 499 |
-
}
|
| 500 |
-
|
| 501 |
-
async function startExamples() {
|
| 502 |
-
showSourceError("");
|
| 503 |
-
stopEverything();
|
| 504 |
-
const files = await loadExamples(scorer.scoreAll);
|
| 505 |
-
startFiles({
|
| 506 |
-
files,
|
| 507 |
-
sourceKind: "examples",
|
| 508 |
-
queue: files.filter(isSloppy),
|
| 509 |
-
startDelay: TIMING.firstStart,
|
| 510 |
-
});
|
| 511 |
-
}
|
| 512 |
-
|
| 513 |
-
function repoQueue(files) {
|
| 514 |
-
return eligibleFiles(files, app.maxChars).slice(0, RUN_SIZE);
|
| 515 |
-
}
|
| 516 |
-
|
| 517 |
function syncUrl() {
|
| 518 |
const url = new URL(window.location.href);
|
| 519 |
url.searchParams.delete("repo");
|
|
@@ -524,76 +147,75 @@ function syncUrl() {
|
|
| 524 |
window.history.replaceState(null, "", url);
|
| 525 |
}
|
| 526 |
|
| 527 |
-
function
|
| 528 |
-
|
| 529 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 530 |
}
|
| 531 |
|
| 532 |
async function startRepo(spec) {
|
| 533 |
showSourceError("");
|
| 534 |
-
|
| 535 |
-
|
| 536 |
-
const controller = new AbortController();
|
| 537 |
app.ingest = "Fetching";
|
| 538 |
-
app.
|
| 539 |
-
|
| 540 |
try {
|
| 541 |
-
await
|
| 542 |
-
|
| 543 |
-
signal: controller.signal,
|
| 544 |
scoreAll: scorer.scoreAll,
|
| 545 |
-
pinCommit: app.prEnabled,
|
| 546 |
onProgress: (text) => {
|
| 547 |
app.ingest = text;
|
| 548 |
-
|
| 549 |
},
|
| 550 |
});
|
| 551 |
-
startFiles({
|
| 552 |
-
files,
|
| 553 |
-
sourceKind: "repo",
|
| 554 |
-
repoSpec: spec,
|
| 555 |
-
commitSha,
|
| 556 |
-
queue: repoQueue(files),
|
| 557 |
-
startDelay: 0,
|
| 558 |
-
});
|
| 559 |
} catch (error) {
|
| 560 |
if (error.name === "AbortError") {
|
| 561 |
return;
|
| 562 |
}
|
| 563 |
app.ingest = null;
|
| 564 |
-
|
| 565 |
showSourceError(SOURCE_ERRORS[error.code] ?? SOURCE_ERRORS.offline);
|
| 566 |
}
|
| 567 |
}
|
| 568 |
|
| 569 |
async function startFolder(fileList) {
|
| 570 |
showSourceError("");
|
| 571 |
-
|
| 572 |
-
app.ingestController?.abort();
|
| 573 |
app.ingest = "Reading";
|
| 574 |
-
|
| 575 |
try {
|
| 576 |
const files = await loadFolder(fileList, {
|
| 577 |
scoreAll: scorer.scoreAll,
|
| 578 |
onProgress: (text) => {
|
| 579 |
app.ingest = text;
|
| 580 |
-
|
| 581 |
},
|
| 582 |
});
|
| 583 |
nodes.input.value = "";
|
| 584 |
-
startFiles({ files, sourceKind: "folder"
|
| 585 |
} catch (error) {
|
| 586 |
app.ingest = null;
|
| 587 |
-
|
| 588 |
showSourceError(SOURCE_ERRORS[error.code] ?? SOURCE_ERRORS.no_files);
|
| 589 |
}
|
| 590 |
}
|
| 591 |
|
| 592 |
async function startSnippet(text) {
|
| 593 |
showSourceError("");
|
|
|
|
| 594 |
const files = await loadSnippet(text, scorer.scoreAll);
|
| 595 |
-
|
| 596 |
-
startFiles({ files, sourceKind: "snippet", queue, startDelay: 0 });
|
| 597 |
}
|
| 598 |
|
| 599 |
function submitLink(text) {
|
|
@@ -606,44 +228,9 @@ function submitLink(text) {
|
|
| 606 |
startRepo(spec);
|
| 607 |
}
|
| 608 |
|
| 609 |
-
function canClean(file) {
|
| 610 |
-
const position = app.queue.indexOf(file);
|
| 611 |
-
const started = position !== -1 && position <= app.cursor;
|
| 612 |
-
return isSloppy(file) && !file.attempted && !started && file.content.length <= app.maxChars;
|
| 613 |
-
}
|
| 614 |
-
|
| 615 |
-
function cleanNext(file) {
|
| 616 |
-
if (!canClean(file)) {
|
| 617 |
-
return;
|
| 618 |
-
}
|
| 619 |
-
app.picked.add(file);
|
| 620 |
-
app.queue = app.queue.filter((queued) => queued !== file);
|
| 621 |
-
const autoplayBusy = app.activeFile && !app.picked.has(app.activeFile);
|
| 622 |
-
if (autoplayBusy) {
|
| 623 |
-
const cursor = app.cursor;
|
| 624 |
-
stopEverything();
|
| 625 |
-
app.queue.splice(cursor, 0, file);
|
| 626 |
-
openAndFix(cursor);
|
| 627 |
-
return;
|
| 628 |
-
}
|
| 629 |
-
app.queue.splice(app.cursor + 1, 0, file);
|
| 630 |
-
stage.setStatus(fileStatus(file));
|
| 631 |
-
if (app.idle) {
|
| 632 |
-
app.idle = false;
|
| 633 |
-
advance();
|
| 634 |
-
}
|
| 635 |
-
}
|
| 636 |
-
|
| 637 |
-
function selectFile(file) {
|
| 638 |
-
app.following = false;
|
| 639 |
-
showFile(file);
|
| 640 |
-
cleanNext(file);
|
| 641 |
-
}
|
| 642 |
-
|
| 643 |
function handlePaste(event) {
|
| 644 |
const text = event.clipboardData?.getData("text/plain") ?? "";
|
| 645 |
-
|
| 646 |
-
if (multiLine) {
|
| 647 |
event.preventDefault();
|
| 648 |
startSnippet(text);
|
| 649 |
return;
|
|
@@ -655,25 +242,12 @@ function handlePaste(event) {
|
|
| 655 |
}
|
| 656 |
}
|
| 657 |
|
| 658 |
-
function applyHealth(health) {
|
| 659 |
-
setServerBackend(health.backend);
|
| 660 |
-
app.maxChars = health.max_file_chars || DEFAULT_MAX_FILE_CHARS;
|
| 661 |
-
app.prEnabled = health.github_pr_enabled === true;
|
| 662 |
-
byId("foot-mode").hidden = health.backend !== "mock";
|
| 663 |
-
}
|
| 664 |
-
|
| 665 |
-
function keepDefaultLimits() {
|
| 666 |
-
app.maxChars = DEFAULT_MAX_FILE_CHARS;
|
| 667 |
-
}
|
| 668 |
-
|
| 669 |
function bindControls() {
|
| 670 |
byId("source").addEventListener("submit", (event) => {
|
| 671 |
event.preventDefault();
|
| 672 |
submitLink(nodes.input.value);
|
| 673 |
});
|
| 674 |
nodes.copy.addEventListener("click", () => copyWithIcon(nodes.copy, nodes.copyIcon, app.shown.content));
|
| 675 |
-
nodes.patch.addEventListener("click", downloadPatch);
|
| 676 |
-
nodes.pr.addEventListener("click", createPr);
|
| 677 |
nodes.rank.addEventListener("click", rankRepo);
|
| 678 |
byId("command-copy").addEventListener("click", () => copyWithIcon(
|
| 679 |
byId("command-copy"),
|
|
@@ -694,7 +268,6 @@ function bindControls() {
|
|
| 694 |
|
| 695 |
function boot() {
|
| 696 |
bindControls();
|
| 697 |
-
app.healthReady = fetchHealth().then(applyHealth, keepDefaultLimits);
|
| 698 |
fetchStats().then(renderBanner, hideBannerStats);
|
| 699 |
paintSample(scorer.scoreAll);
|
| 700 |
const repoParam = new URLSearchParams(window.location.search).get("repo");
|
|
|
|
| 1 |
+
import { byId, copyWithIcon, prefersReducedMotion } from "./dom.js";
|
|
|
|
|
|
|
| 2 |
import { isFullRepoUrl, parseRepoSpec, specLabel } from "./github.js";
|
| 3 |
+
import { SOURCE_ERRORS, issuesText, toHundred } from "./labels.js";
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 4 |
import { createNavigator } from "./navigator.js";
|
|
|
|
|
|
|
| 5 |
import { boardLink, rankErrorText, rankLabel, requestRank } from "./ranking.js";
|
| 6 |
+
import { fetchStats } from "./refiner.js";
|
| 7 |
+
import { paintSample } from "./sample.js";
|
| 8 |
+
import { createScorer } from "./scoring.js";
|
| 9 |
+
import { loadExamples, loadFolder, loadRepo, loadSnippet } from "./sources.js";
|
| 10 |
+
import { createStage } from "./stage.js";
|
| 11 |
import { hideBannerStats, renderBanner, reportScan } from "./telemetry.js";
|
| 12 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 13 |
const nodes = {
|
| 14 |
sample: byId("sample"),
|
| 15 |
stage: byId("stage"),
|
| 16 |
input: byId("source-input"),
|
| 17 |
error: byId("source-error"),
|
| 18 |
progress: byId("source-progress"),
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 19 |
copy: byId("copy"),
|
| 20 |
copyIcon: byId("copy-icon"),
|
|
|
|
|
|
|
|
|
|
|
|
|
| 21 |
rank: byId("rank"),
|
| 22 |
rankOpen: byId("rank-open"),
|
| 23 |
rankText: byId("rank-text"),
|
| 24 |
};
|
| 25 |
|
|
|
|
|
|
|
| 26 |
const narrowScreen = window.matchMedia("(max-width: 640px)");
|
| 27 |
const scorer = createScorer();
|
| 28 |
const stage = createStage();
|
| 29 |
const fileNav = createNavigator(selectFile);
|
|
|
|
| 30 |
|
| 31 |
const app = {
|
| 32 |
repoSpec: null,
|
| 33 |
sourceKind: "examples",
|
| 34 |
files: [],
|
|
|
|
|
|
|
| 35 |
shown: null,
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 36 |
ingest: null,
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 37 |
standing: null,
|
| 38 |
+
ingestAbort: null,
|
|
|
|
|
|
|
| 39 |
};
|
| 40 |
|
| 41 |
+
function renderProgress() {
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 42 |
nodes.progress.hidden = !app.ingest;
|
| 43 |
nodes.progress.textContent = app.ingest ?? "";
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 44 |
}
|
| 45 |
|
| 46 |
function renderActions(file) {
|
| 47 |
+
nodes.copy.hidden = !file;
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 48 |
const rankable = app.sourceKind === "repo" && Boolean(app.repoSpec);
|
| 49 |
nodes.rank.hidden = !rankable || Boolean(app.standing);
|
| 50 |
nodes.rankOpen.hidden = !rankable || !app.standing;
|
|
|
|
| 57 |
nodes.rankOpen.title = `${standing.repo}: ${points}/100, #${standing.rank} of ${standing.total}`;
|
| 58 |
}
|
| 59 |
|
| 60 |
+
function setBusy(button, busy) {
|
| 61 |
+
button.disabled = busy;
|
| 62 |
+
button.classList.toggle("is-busy", busy);
|
| 63 |
+
button.setAttribute("aria-busy", String(busy));
|
| 64 |
+
}
|
| 65 |
+
|
| 66 |
+
function showSourceError(message) {
|
| 67 |
+
nodes.error.hidden = !message;
|
| 68 |
+
nodes.error.textContent = message;
|
| 69 |
+
}
|
| 70 |
+
|
| 71 |
async function rankRepo() {
|
| 72 |
const spec = app.repoSpec;
|
| 73 |
showSourceError("");
|
|
|
|
| 86 |
}
|
| 87 |
}
|
| 88 |
|
| 89 |
+
function showFile(file, { enter = true, freshTree = false } = {}) {
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 90 |
app.shown = file;
|
| 91 |
stage.resetScore();
|
| 92 |
+
if (freshTree) {
|
| 93 |
+
fileNav.render(app.files, file);
|
| 94 |
+
} else {
|
| 95 |
+
fileNav.update(app.files, file);
|
|
|
|
| 96 |
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 97 |
stage.setHead({
|
| 98 |
+
filename: file.path,
|
| 99 |
+
statusText: issuesText(file.result.squiggles.length),
|
| 100 |
+
points: toHundred(file.result.score),
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 101 |
});
|
| 102 |
+
stage.renderPlain({
|
| 103 |
+
text: file.content,
|
| 104 |
+
filename: file.path,
|
| 105 |
+
squiggles: file.result.squiggles,
|
| 106 |
+
enter,
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 107 |
});
|
| 108 |
+
renderActions(file);
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 109 |
}
|
| 110 |
|
| 111 |
+
function selectFile(file) {
|
| 112 |
+
showFile(file);
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 113 |
}
|
| 114 |
|
| 115 |
function bringStageIntoView() {
|
| 116 |
if (narrowScreen.matches) {
|
| 117 |
+
const behavior = prefersReducedMotion() ? "auto" : "smooth";
|
| 118 |
+
nodes.stage.scrollIntoView({ behavior, block: "start" });
|
| 119 |
}
|
| 120 |
}
|
| 121 |
|
|
|
|
| 137 |
document.startViewTransition(swap).finished.then(bringStageIntoView);
|
| 138 |
}
|
| 139 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 140 |
function syncUrl() {
|
| 141 |
const url = new URL(window.location.href);
|
| 142 |
url.searchParams.delete("repo");
|
|
|
|
| 147 |
window.history.replaceState(null, "", url);
|
| 148 |
}
|
| 149 |
|
| 150 |
+
function startFiles({ files, sourceKind, repoSpec = null }) {
|
| 151 |
+
Object.assign(app, { files, sourceKind, repoSpec, standing: null, ingest: null });
|
| 152 |
+
syncUrl();
|
| 153 |
+
renderProgress();
|
| 154 |
+
reportScan(sourceKind, files);
|
| 155 |
+
revealStage(() => showFile(files[0], { enter: true, freshTree: true }));
|
| 156 |
+
}
|
| 157 |
+
|
| 158 |
+
async function startExamples() {
|
| 159 |
+
showSourceError("");
|
| 160 |
+
app.ingestAbort?.abort();
|
| 161 |
+
const files = await loadExamples(scorer.scoreAll);
|
| 162 |
+
startFiles({ files, sourceKind: "examples" });
|
| 163 |
}
|
| 164 |
|
| 165 |
async function startRepo(spec) {
|
| 166 |
showSourceError("");
|
| 167 |
+
app.ingestAbort?.abort();
|
| 168 |
+
const aborter = new AbortController();
|
|
|
|
| 169 |
app.ingest = "Fetching";
|
| 170 |
+
app.ingestAbort = aborter;
|
| 171 |
+
renderProgress();
|
| 172 |
try {
|
| 173 |
+
const files = await loadRepo(spec, {
|
| 174 |
+
signal: aborter.signal,
|
|
|
|
| 175 |
scoreAll: scorer.scoreAll,
|
|
|
|
| 176 |
onProgress: (text) => {
|
| 177 |
app.ingest = text;
|
| 178 |
+
renderProgress();
|
| 179 |
},
|
| 180 |
});
|
| 181 |
+
startFiles({ files, sourceKind: "repo", repoSpec: spec });
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 182 |
} catch (error) {
|
| 183 |
if (error.name === "AbortError") {
|
| 184 |
return;
|
| 185 |
}
|
| 186 |
app.ingest = null;
|
| 187 |
+
renderProgress();
|
| 188 |
showSourceError(SOURCE_ERRORS[error.code] ?? SOURCE_ERRORS.offline);
|
| 189 |
}
|
| 190 |
}
|
| 191 |
|
| 192 |
async function startFolder(fileList) {
|
| 193 |
showSourceError("");
|
| 194 |
+
app.ingestAbort?.abort();
|
|
|
|
| 195 |
app.ingest = "Reading";
|
| 196 |
+
renderProgress();
|
| 197 |
try {
|
| 198 |
const files = await loadFolder(fileList, {
|
| 199 |
scoreAll: scorer.scoreAll,
|
| 200 |
onProgress: (text) => {
|
| 201 |
app.ingest = text;
|
| 202 |
+
renderProgress();
|
| 203 |
},
|
| 204 |
});
|
| 205 |
nodes.input.value = "";
|
| 206 |
+
startFiles({ files, sourceKind: "folder" });
|
| 207 |
} catch (error) {
|
| 208 |
app.ingest = null;
|
| 209 |
+
renderProgress();
|
| 210 |
showSourceError(SOURCE_ERRORS[error.code] ?? SOURCE_ERRORS.no_files);
|
| 211 |
}
|
| 212 |
}
|
| 213 |
|
| 214 |
async function startSnippet(text) {
|
| 215 |
showSourceError("");
|
| 216 |
+
app.ingestAbort?.abort();
|
| 217 |
const files = await loadSnippet(text, scorer.scoreAll);
|
| 218 |
+
startFiles({ files, sourceKind: "snippet" });
|
|
|
|
| 219 |
}
|
| 220 |
|
| 221 |
function submitLink(text) {
|
|
|
|
| 228 |
startRepo(spec);
|
| 229 |
}
|
| 230 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 231 |
function handlePaste(event) {
|
| 232 |
const text = event.clipboardData?.getData("text/plain") ?? "";
|
| 233 |
+
if (text.trim().includes("\n")) {
|
|
|
|
| 234 |
event.preventDefault();
|
| 235 |
startSnippet(text);
|
| 236 |
return;
|
|
|
|
| 242 |
}
|
| 243 |
}
|
| 244 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 245 |
function bindControls() {
|
| 246 |
byId("source").addEventListener("submit", (event) => {
|
| 247 |
event.preventDefault();
|
| 248 |
submitLink(nodes.input.value);
|
| 249 |
});
|
| 250 |
nodes.copy.addEventListener("click", () => copyWithIcon(nodes.copy, nodes.copyIcon, app.shown.content));
|
|
|
|
|
|
|
| 251 |
nodes.rank.addEventListener("click", rankRepo);
|
| 252 |
byId("command-copy").addEventListener("click", () => copyWithIcon(
|
| 253 |
byId("command-copy"),
|
|
|
|
| 268 |
|
| 269 |
function boot() {
|
| 270 |
bindControls();
|
|
|
|
| 271 |
fetchStats().then(renderBanner, hideBannerStats);
|
| 272 |
paintSample(scorer.scoreAll);
|
| 273 |
const repoParam = new URLSearchParams(window.location.search).get("repo");
|
web/src/navigator.js
CHANGED
|
@@ -39,18 +39,11 @@ function flatten(folder) {
|
|
| 39 |
}
|
| 40 |
|
| 41 |
export function fileState(file) {
|
| 42 |
-
|
| 43 |
-
return "has-issues";
|
| 44 |
-
}
|
| 45 |
-
return file.cleaned ? "is-cleaned" : "";
|
| 46 |
}
|
| 47 |
|
| 48 |
function folderState(folder) {
|
| 49 |
-
|
| 50 |
-
if (states.includes("has-issues")) {
|
| 51 |
-
return "has-issues";
|
| 52 |
-
}
|
| 53 |
-
return states.includes("is-cleaned") ? "is-cleaned" : "";
|
| 54 |
}
|
| 55 |
|
| 56 |
function ancestors(path) {
|
|
@@ -79,10 +72,7 @@ function icon(href, className = "icon") {
|
|
| 79 |
|
| 80 |
function fileBadge(file) {
|
| 81 |
const count = file.result.squiggles.length;
|
| 82 |
-
|
| 83 |
-
return el("span", { class: "tree-count", text: String(count) });
|
| 84 |
-
}
|
| 85 |
-
return file.cleaned ? el("span", { class: "tree-mark" }) : "";
|
| 86 |
}
|
| 87 |
|
| 88 |
function rowClass(state, selected) {
|
|
|
|
| 39 |
}
|
| 40 |
|
| 41 |
export function fileState(file) {
|
| 42 |
+
return file.result.squiggles.length > 0 ? "has-issues" : "";
|
|
|
|
|
|
|
|
|
|
| 43 |
}
|
| 44 |
|
| 45 |
function folderState(folder) {
|
| 46 |
+
return flatten(folder).some((file) => fileState(file) === "has-issues") ? "has-issues" : "";
|
|
|
|
|
|
|
|
|
|
|
|
|
| 47 |
}
|
| 48 |
|
| 49 |
function ancestors(path) {
|
|
|
|
| 72 |
|
| 73 |
function fileBadge(file) {
|
| 74 |
const count = file.result.squiggles.length;
|
| 75 |
+
return count > 0 ? el("span", { class: "tree-count", text: String(count) }) : "";
|
|
|
|
|
|
|
|
|
|
| 76 |
}
|
| 77 |
|
| 78 |
function rowClass(state, selected) {
|
web/src/patch.js
DELETED
|
@@ -1,78 +0,0 @@
|
|
| 1 |
-
import { diffLines } from "./diff.js";
|
| 2 |
-
import { el } from "./dom.js";
|
| 3 |
-
|
| 4 |
-
const CONTEXT = 3;
|
| 5 |
-
const NO_NEWLINE = "\0";
|
| 6 |
-
|
| 7 |
-
// One entry per line; a last line without a trailing newline carries a marker so the diff sees it.
|
| 8 |
-
function lineEntries(text) {
|
| 9 |
-
const lines = text.split("\n");
|
| 10 |
-
if (lines[lines.length - 1] === "") {
|
| 11 |
-
lines.pop();
|
| 12 |
-
} else {
|
| 13 |
-
lines[lines.length - 1] += NO_NEWLINE;
|
| 14 |
-
}
|
| 15 |
-
return lines;
|
| 16 |
-
}
|
| 17 |
-
|
| 18 |
-
function hunkRanges(rows) {
|
| 19 |
-
const ranges = [];
|
| 20 |
-
rows.forEach((row, index) => {
|
| 21 |
-
if (row.kind === "same") {
|
| 22 |
-
return;
|
| 23 |
-
}
|
| 24 |
-
const start = Math.max(0, index - CONTEXT);
|
| 25 |
-
const end = Math.min(rows.length, index + CONTEXT + 1);
|
| 26 |
-
const last = ranges[ranges.length - 1];
|
| 27 |
-
if (last && start <= last.end) {
|
| 28 |
-
last.end = Math.max(last.end, end);
|
| 29 |
-
} else {
|
| 30 |
-
ranges.push({ start, end });
|
| 31 |
-
}
|
| 32 |
-
});
|
| 33 |
-
return ranges;
|
| 34 |
-
}
|
| 35 |
-
|
| 36 |
-
function patchLine(prefix, text) {
|
| 37 |
-
const missingNewline = text.endsWith(NO_NEWLINE);
|
| 38 |
-
const body = missingNewline ? text.slice(0, -1) : text;
|
| 39 |
-
return missingNewline ? `${prefix}${body}\n\\n` : `${prefix}${body}\n`;
|
| 40 |
-
}
|
| 41 |
-
|
| 42 |
-
const PREFIX = { same: " ", removed: "-", added: "+" };
|
| 43 |
-
|
| 44 |
-
function rangeStart(count, linesBefore) {
|
| 45 |
-
return count === 0 ? linesBefore : linesBefore + 1;
|
| 46 |
-
}
|
| 47 |
-
|
| 48 |
-
function renderHunk(rows, { start, end }) {
|
| 49 |
-
const slice = rows.slice(start, end);
|
| 50 |
-
const oldBefore = rows.slice(0, start).filter((row) => row.kind !== "added").length;
|
| 51 |
-
const newBefore = rows.slice(0, start).filter((row) => row.kind !== "removed").length;
|
| 52 |
-
const oldCount = slice.filter((row) => row.kind !== "added").length;
|
| 53 |
-
const newCount = slice.filter((row) => row.kind !== "removed").length;
|
| 54 |
-
const oldRange = `${rangeStart(oldCount, oldBefore)},${oldCount}`;
|
| 55 |
-
const newRange = `${rangeStart(newCount, newBefore)},${newCount}`;
|
| 56 |
-
const lines = slice.map((row) => patchLine(PREFIX[row.kind], row.text)).join("");
|
| 57 |
-
return `@@ -${oldRange} +${newRange} @@\n${lines}`;
|
| 58 |
-
}
|
| 59 |
-
|
| 60 |
-
export function filePatch(path, before, after) {
|
| 61 |
-
const rows = diffLines(lineEntries(before).join("\n"), lineEntries(after).join("\n"));
|
| 62 |
-
const ranges = hunkRanges(rows);
|
| 63 |
-
if (ranges.length === 0) {
|
| 64 |
-
return "";
|
| 65 |
-
}
|
| 66 |
-
const header = `diff --git a/${path} b/${path}\n--- a/${path}\n+++ b/${path}\n`;
|
| 67 |
-
return header + ranges.map((range) => renderHunk(rows, range)).join("");
|
| 68 |
-
}
|
| 69 |
-
|
| 70 |
-
export function buildPatch(files) {
|
| 71 |
-
return files.map((file) => filePatch(file.path, file.original, file.content)).join("");
|
| 72 |
-
}
|
| 73 |
-
|
| 74 |
-
export function downloadText(text, filename) {
|
| 75 |
-
const url = URL.createObjectURL(new Blob([text], { type: "text/x-diff" }));
|
| 76 |
-
el("a", { href: url, download: filename }).click();
|
| 77 |
-
setTimeout(() => URL.revokeObjectURL(url), 0);
|
| 78 |
-
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
web/src/privacy.html
CHANGED
|
@@ -1,76 +1,68 @@
|
|
| 1 |
-
<!doctype html>
|
| 2 |
-
<html lang="en" data-root="../">
|
| 3 |
-
<head>
|
| 4 |
-
<meta charset="utf-8">
|
| 5 |
-
<meta name="viewport" content="width=device-width, initial-scale=1">
|
| 6 |
-
<meta name="description" content="CodeHygiene keeps no code. This is everything it keeps.">
|
| 7 |
-
<title>Zero data retention · codehygiene</title>
|
| 8 |
-
<link rel="stylesheet" href="../styles.css">
|
| 9 |
-
<link rel="stylesheet" href="../board.css">
|
| 10 |
-
<link rel="stylesheet" href="../privacy.css">
|
| 11 |
-
</head>
|
| 12 |
-
<body class="board-page">
|
| 13 |
-
<svg class="sprite" aria-hidden="true">
|
| 14 |
-
<symbol id="i-back" viewBox="0 0 24 24"><path d="M19 12H5"/><path d="m11 18-6-6 6-6"/></symbol>
|
| 15 |
-
<symbol id="i-external" viewBox="5 5 14 14"><path d="M7 17 17 7"/><path d="M8 7h9v9"/></symbol>
|
| 16 |
-
</svg>
|
| 17 |
-
|
| 18 |
-
<div class="board privacy-page">
|
| 19 |
-
<header class="board-head">
|
| 20 |
-
<a class="board-home" href="../" title="Back to CodeHygiene">
|
| 21 |
-
<svg class="icon" aria-hidden="true"><use href="#i-back"/></svg>
|
| 22 |
-
<span>CodeHygiene</span>
|
| 23 |
-
</a>
|
| 24 |
-
<p class="board-updated privacy-updated">Updated November 2026</p>
|
| 25 |
-
</header>
|
| 26 |
-
|
| 27 |
-
<main>
|
| 28 |
-
<h1 class="privacy-title">Zero data retention</h1>
|
| 29 |
-
<p class="privacy-lede">Your code is never stored. This is everything that is.</p>
|
| 30 |
-
|
| 31 |
-
<figure class="receipt">
|
| 32 |
-
<div class="receipt-head">
|
| 33 |
-
<span class="stage-path"><span class="path-dir">app/</span>send_receipt.py</span>
|
| 34 |
-
<span class="receipt-meta"><span class="stage-status">Kept</span> <b>1</b> of 6 lines</span>
|
| 35 |
-
</div>
|
| 36 |
-
<div class="stage-code receipt-code">
|
| 37 |
-
<div class="line is-removed"><span class="ln"></span><span class="src"><span>path</span> app/send_receipt.py</span></div>
|
| 38 |
-
<div class="line is-removed"><span class="ln"></span><span class="src"><span>repo</span> acme/payments</span></div>
|
| 39 |
-
<div class="line is-removed"><span class="ln"></span><span class="src"><span>code</span> def send_receipt(order): …</span></div>
|
| 40 |
-
<div class="line is-removed"><span class="ln"></span><span class="src"><span>user</span> octocat</span></div>
|
| 41 |
-
<div class="line is-removed"><span class="ln"></span><span class="src"><span>ip </span> 203.0.113.7</span></div>
|
| 42 |
-
<div class="line is-added"><span class="ln">1</span><span class="src"><span class="receipt-key">kept</span> py · 100–300 lines · score 29 · 3 issues · 14 ms</span></div>
|
| 43 |
-
</div>
|
| 44 |
-
<figcaption class="receipt-caption">Even that line is not saved as a row. It is added to <a href="../leaderboard/#stats">public totals</a>, and a one-way fingerprint held only in memory keeps a file from counting twice.</figcaption>
|
| 45 |
-
</figure>
|
| 46 |
-
|
| 47 |
-
<dl class="facts">
|
| 48 |
-
<div class="fact">
|
| 49 |
-
<dt>Scoring</dt>
|
| 50 |
-
<dd>Runs in your browser. Code is
|
| 51 |
-
</div>
|
| 52 |
-
<div class="fact">
|
| 53 |
-
<dt>
|
| 54 |
-
<dd>A
|
| 55 |
-
</div>
|
| 56 |
-
<div class="fact">
|
| 57 |
-
<dt>
|
| 58 |
-
<dd>
|
| 59 |
-
</div>
|
| 60 |
-
<div class="fact">
|
| 61 |
-
<dt>
|
| 62 |
-
<dd>
|
| 63 |
-
</div>
|
| 64 |
-
|
| 65 |
-
|
| 66 |
-
|
| 67 |
-
|
| 68 |
-
|
| 69 |
-
<dt>Hosting</dt>
|
| 70 |
-
<dd>Hugging Face Spaces, under the <a href="https://huggingface.co/privacy" target="_blank" rel="noopener">Hugging Face privacy policy</a>.</dd>
|
| 71 |
-
</div>
|
| 72 |
-
</dl>
|
| 73 |
-
</main>
|
| 74 |
-
</div>
|
| 75 |
-
</body>
|
| 76 |
-
</html>
|
|
|
|
| 1 |
+
<!doctype html>
|
| 2 |
+
<html lang="en" data-root="../">
|
| 3 |
+
<head>
|
| 4 |
+
<meta charset="utf-8">
|
| 5 |
+
<meta name="viewport" content="width=device-width, initial-scale=1">
|
| 6 |
+
<meta name="description" content="CodeHygiene keeps no code. This is everything it keeps.">
|
| 7 |
+
<title>Zero data retention · codehygiene</title>
|
| 8 |
+
<link rel="stylesheet" href="../styles.css">
|
| 9 |
+
<link rel="stylesheet" href="../board.css">
|
| 10 |
+
<link rel="stylesheet" href="../privacy.css">
|
| 11 |
+
</head>
|
| 12 |
+
<body class="board-page">
|
| 13 |
+
<svg class="sprite" aria-hidden="true">
|
| 14 |
+
<symbol id="i-back" viewBox="0 0 24 24"><path d="M19 12H5"/><path d="m11 18-6-6 6-6"/></symbol>
|
| 15 |
+
<symbol id="i-external" viewBox="5 5 14 14"><path d="M7 17 17 7"/><path d="M8 7h9v9"/></symbol>
|
| 16 |
+
</svg>
|
| 17 |
+
|
| 18 |
+
<div class="board privacy-page">
|
| 19 |
+
<header class="board-head">
|
| 20 |
+
<a class="board-home" href="../" title="Back to CodeHygiene">
|
| 21 |
+
<svg class="icon" aria-hidden="true"><use href="#i-back"/></svg>
|
| 22 |
+
<span>CodeHygiene</span>
|
| 23 |
+
</a>
|
| 24 |
+
<p class="board-updated privacy-updated">Updated November 2026</p>
|
| 25 |
+
</header>
|
| 26 |
+
|
| 27 |
+
<main>
|
| 28 |
+
<h1 class="privacy-title">Zero data retention</h1>
|
| 29 |
+
<p class="privacy-lede">Your code is never stored. This is everything that is.</p>
|
| 30 |
+
|
| 31 |
+
<figure class="receipt">
|
| 32 |
+
<div class="receipt-head">
|
| 33 |
+
<span class="stage-path"><span class="path-dir">app/</span>send_receipt.py</span>
|
| 34 |
+
<span class="receipt-meta"><span class="stage-status">Kept</span> <b>1</b> of 6 lines</span>
|
| 35 |
+
</div>
|
| 36 |
+
<div class="stage-code receipt-code">
|
| 37 |
+
<div class="line is-removed"><span class="ln"></span><span class="src"><span>path</span> app/send_receipt.py</span></div>
|
| 38 |
+
<div class="line is-removed"><span class="ln"></span><span class="src"><span>repo</span> acme/payments</span></div>
|
| 39 |
+
<div class="line is-removed"><span class="ln"></span><span class="src"><span>code</span> def send_receipt(order): …</span></div>
|
| 40 |
+
<div class="line is-removed"><span class="ln"></span><span class="src"><span>user</span> octocat</span></div>
|
| 41 |
+
<div class="line is-removed"><span class="ln"></span><span class="src"><span>ip </span> 203.0.113.7</span></div>
|
| 42 |
+
<div class="line is-added"><span class="ln">1</span><span class="src"><span class="receipt-key">kept</span> py · 100–300 lines · score 29 · 3 issues · 14 ms</span></div>
|
| 43 |
+
</div>
|
| 44 |
+
<figcaption class="receipt-caption">Even that line is not saved as a row. It is added to <a href="../leaderboard/#stats">public totals</a>, and a one-way fingerprint held only in memory keeps a file from counting twice.</figcaption>
|
| 45 |
+
</figure>
|
| 46 |
+
|
| 47 |
+
<dl class="facts">
|
| 48 |
+
<div class="fact">
|
| 49 |
+
<dt>Scoring</dt>
|
| 50 |
+
<dd>Runs locally in your browser. Code is never uploaded to score it.</dd>
|
| 51 |
+
</div>
|
| 52 |
+
<div class="fact">
|
| 53 |
+
<dt>Leaderboard</dt>
|
| 54 |
+
<dd>A public repo is listed only when someone asks to rank it.</dd>
|
| 55 |
+
</div>
|
| 56 |
+
<div class="fact">
|
| 57 |
+
<dt>Cookies</dt>
|
| 58 |
+
<dd>CodeHygiene sets none. No accounts and no third‑party scripts.</dd>
|
| 59 |
+
</div>
|
| 60 |
+
<div class="fact">
|
| 61 |
+
<dt>Hosting</dt>
|
| 62 |
+
<dd>Hugging Face Spaces, under the <a href="https://huggingface.co/privacy" target="_blank" rel="noopener">Hugging Face privacy policy</a>.</dd>
|
| 63 |
+
</div>
|
| 64 |
+
</dl>
|
| 65 |
+
</main>
|
| 66 |
+
</div>
|
| 67 |
+
</body>
|
| 68 |
+
</html>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
web/src/pull-request.js
DELETED
|
@@ -1,99 +0,0 @@
|
|
| 1 |
-
import { toHundred } from "./labels.js";
|
| 2 |
-
import { apiUrl } from "./refiner.js";
|
| 3 |
-
|
| 4 |
-
const MESSAGE_TYPE = "codehygiene:github";
|
| 5 |
-
const SIGN_IN_TIMEOUT_MS = 5 * 60 * 1000;
|
| 6 |
-
const POPUP_CHECK_MS = 500;
|
| 7 |
-
const POPUP_FEATURES = "popup,width=720,height=800";
|
| 8 |
-
const MAX_PR_FILES = 20;
|
| 9 |
-
|
| 10 |
-
export class PrFailure extends Error {
|
| 11 |
-
constructor(code) {
|
| 12 |
-
super(code);
|
| 13 |
-
this.code = code;
|
| 14 |
-
}
|
| 15 |
-
}
|
| 16 |
-
|
| 17 |
-
// The sign-in pop-up posts a one-time ticket back to this page only (same origin, same window).
|
| 18 |
-
function waitForTicket(popup) {
|
| 19 |
-
return new Promise((resolve, reject) => {
|
| 20 |
-
const startedAt = Date.now();
|
| 21 |
-
let timer = 0;
|
| 22 |
-
const finish = (settle, value) => {
|
| 23 |
-
clearInterval(timer);
|
| 24 |
-
window.removeEventListener("message", onMessage);
|
| 25 |
-
settle(value);
|
| 26 |
-
};
|
| 27 |
-
function onMessage(event) {
|
| 28 |
-
const fromPopup = event.source === popup && event.origin === window.location.origin;
|
| 29 |
-
if (!fromPopup || event.data?.type !== MESSAGE_TYPE) {
|
| 30 |
-
return;
|
| 31 |
-
}
|
| 32 |
-
if (event.data.ticket) {
|
| 33 |
-
finish(resolve, event.data.ticket);
|
| 34 |
-
} else {
|
| 35 |
-
finish(reject, new PrFailure(event.data.error ?? "failed"));
|
| 36 |
-
}
|
| 37 |
-
}
|
| 38 |
-
timer = setInterval(() => {
|
| 39 |
-
if (popup.closed) {
|
| 40 |
-
finish(reject, new PrFailure("closed"));
|
| 41 |
-
} else if (Date.now() - startedAt > SIGN_IN_TIMEOUT_MS) {
|
| 42 |
-
popup.close();
|
| 43 |
-
finish(reject, new PrFailure("login_required"));
|
| 44 |
-
}
|
| 45 |
-
}, POPUP_CHECK_MS);
|
| 46 |
-
window.addEventListener("message", onMessage);
|
| 47 |
-
});
|
| 48 |
-
}
|
| 49 |
-
|
| 50 |
-
function prFiles(files) {
|
| 51 |
-
return files.slice(0, MAX_PR_FILES).map((file) => ({
|
| 52 |
-
path: file.path,
|
| 53 |
-
content_after: file.content,
|
| 54 |
-
score_before: toHundred(file.originalResult.score),
|
| 55 |
-
score_after: toHundred(file.result.score),
|
| 56 |
-
squiggles_fixed: Math.max(0, file.originalResult.squiggles.length - file.result.squiggles.length),
|
| 57 |
-
}));
|
| 58 |
-
}
|
| 59 |
-
|
| 60 |
-
async function readReply(response) {
|
| 61 |
-
const text = await response.text();
|
| 62 |
-
try {
|
| 63 |
-
return JSON.parse(text);
|
| 64 |
-
} catch {
|
| 65 |
-
return { code: "failed" };
|
| 66 |
-
}
|
| 67 |
-
}
|
| 68 |
-
|
| 69 |
-
// Must be called straight from a click so the pop-up is not blocked.
|
| 70 |
-
export async function openPullRequest({ spec, commitSha, files }) {
|
| 71 |
-
const popup = window.open(apiUrl("github/login"), "codehygiene-github", POPUP_FEATURES);
|
| 72 |
-
if (!popup) {
|
| 73 |
-
throw new PrFailure("popup_blocked");
|
| 74 |
-
}
|
| 75 |
-
const ticket = await waitForTicket(popup);
|
| 76 |
-
const response = await fetch(apiUrl("github/create-pr"), {
|
| 77 |
-
method: "POST",
|
| 78 |
-
headers: { "Content-Type": "application/json" },
|
| 79 |
-
body: JSON.stringify({
|
| 80 |
-
ticket,
|
| 81 |
-
owner: spec.owner,
|
| 82 |
-
repo: spec.repo,
|
| 83 |
-
base_branch: spec.ref,
|
| 84 |
-
base_commit_sha: commitSha,
|
| 85 |
-
files: prFiles(files),
|
| 86 |
-
}),
|
| 87 |
-
});
|
| 88 |
-
const reply = await readReply(response);
|
| 89 |
-
const next = response.ok ? reply.url : reply.install_url;
|
| 90 |
-
if (next) {
|
| 91 |
-
popup.location.href = next;
|
| 92 |
-
} else {
|
| 93 |
-
popup.close();
|
| 94 |
-
}
|
| 95 |
-
if (!response.ok) {
|
| 96 |
-
throw new PrFailure(reply.code ?? "failed");
|
| 97 |
-
}
|
| 98 |
-
return reply;
|
| 99 |
-
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
web/src/refiner.js
CHANGED
|
@@ -1,5 +1,3 @@
|
|
| 1 |
-
export const DEFAULT_MAX_FILE_CHARS = 131072;
|
| 2 |
-
|
| 3 |
// Pages below the site root (the leaderboard) set data-root="../" so API calls still reach /api.
|
| 4 |
const SITE_ROOT = new URL(document.documentElement.dataset.root ?? "./", document.baseURI);
|
| 5 |
|
|
@@ -15,10 +13,6 @@ async function getJson(path, label) {
|
|
| 15 |
return response.json();
|
| 16 |
}
|
| 17 |
|
| 18 |
-
export function fetchHealth() {
|
| 19 |
-
return getJson("health", "Health check");
|
| 20 |
-
}
|
| 21 |
-
|
| 22 |
export function fetchStats() {
|
| 23 |
return getJson("stats", "Stats request");
|
| 24 |
}
|
|
@@ -26,67 +20,3 @@ export function fetchStats() {
|
|
| 26 |
export function fetchLeaderboard() {
|
| 27 |
return getJson("leaderboard", "Leaderboard request");
|
| 28 |
}
|
| 29 |
-
|
| 30 |
-
function parseFrame(frame) {
|
| 31 |
-
let name = "message";
|
| 32 |
-
const dataLines = [];
|
| 33 |
-
for (const line of frame.split("\n")) {
|
| 34 |
-
if (line.startsWith(":")) {
|
| 35 |
-
continue;
|
| 36 |
-
}
|
| 37 |
-
const colon = line.indexOf(":");
|
| 38 |
-
const field = colon < 0 ? line : line.slice(0, colon);
|
| 39 |
-
const value = colon < 0 ? "" : line.slice(colon + 1).replace(/^ /, "");
|
| 40 |
-
if (field === "event") {
|
| 41 |
-
name = value;
|
| 42 |
-
} else if (field === "data") {
|
| 43 |
-
dataLines.push(value);
|
| 44 |
-
}
|
| 45 |
-
}
|
| 46 |
-
if (dataLines.length === 0) {
|
| 47 |
-
return null;
|
| 48 |
-
}
|
| 49 |
-
return { name, data: JSON.parse(dataLines.join("\n")) };
|
| 50 |
-
}
|
| 51 |
-
|
| 52 |
-
async function responseError(response) {
|
| 53 |
-
const body = await response.text();
|
| 54 |
-
try {
|
| 55 |
-
const parsed = JSON.parse(body);
|
| 56 |
-
return parsed.detail || parsed.message || `Request failed (${response.status})`;
|
| 57 |
-
} catch {
|
| 58 |
-
return body || `Request failed (${response.status})`;
|
| 59 |
-
}
|
| 60 |
-
}
|
| 61 |
-
|
| 62 |
-
export async function streamFix(body, { signal, onEvent }) {
|
| 63 |
-
const response = await fetch(apiUrl("fix/stream"), {
|
| 64 |
-
method: "POST",
|
| 65 |
-
headers: { "Content-Type": "application/json", Accept: "text/event-stream" },
|
| 66 |
-
body: JSON.stringify(body),
|
| 67 |
-
signal,
|
| 68 |
-
});
|
| 69 |
-
if (!response.ok || !response.body) {
|
| 70 |
-
onEvent("error", { code: "http", message: await responseError(response) });
|
| 71 |
-
return;
|
| 72 |
-
}
|
| 73 |
-
|
| 74 |
-
const reader = response.body.pipeThrough(new TextDecoderStream()).getReader();
|
| 75 |
-
let buffer = "";
|
| 76 |
-
for (;;) {
|
| 77 |
-
const { value, done } = await reader.read();
|
| 78 |
-
if (done) {
|
| 79 |
-
break;
|
| 80 |
-
}
|
| 81 |
-
buffer = (buffer + value).replace(/\r\n?/g, "\n");
|
| 82 |
-
let boundary = buffer.indexOf("\n\n");
|
| 83 |
-
while (boundary >= 0) {
|
| 84 |
-
const event = parseFrame(buffer.slice(0, boundary));
|
| 85 |
-
buffer = buffer.slice(boundary + 2);
|
| 86 |
-
if (event) {
|
| 87 |
-
onEvent(event.name, event.data);
|
| 88 |
-
}
|
| 89 |
-
boundary = buffer.indexOf("\n\n");
|
| 90 |
-
}
|
| 91 |
-
}
|
| 92 |
-
}
|
|
|
|
|
|
|
|
|
|
| 1 |
// Pages below the site root (the leaderboard) set data-root="../" so API calls still reach /api.
|
| 2 |
const SITE_ROOT = new URL(document.documentElement.dataset.root ?? "./", document.baseURI);
|
| 3 |
|
|
|
|
| 13 |
return response.json();
|
| 14 |
}
|
| 15 |
|
|
|
|
|
|
|
|
|
|
|
|
|
| 16 |
export function fetchStats() {
|
| 17 |
return getJson("stats", "Stats request");
|
| 18 |
}
|
|
|
|
| 20 |
export function fetchLeaderboard() {
|
| 21 |
return getJson("leaderboard", "Leaderboard request");
|
| 22 |
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
web/src/sample.js
CHANGED
|
@@ -1,32 +1,23 @@
|
|
| 1 |
import presets from "shared:presets";
|
| 2 |
import { byId } from "./dom.js";
|
| 3 |
-
import { scoreWord, toHundred } from "./labels.js";
|
| 4 |
-
import {
|
| 5 |
|
| 6 |
const FILENAME = "send_receipt.py";
|
| 7 |
-
const CLEANED = `import logging
|
| 8 |
-
|
| 9 |
-
logger = logging.getLogger(__name__)
|
| 10 |
-
|
| 11 |
-
|
| 12 |
-
def send_receipt(order: Order) -> None:
|
| 13 |
-
if order is None or not order.paid or order.refunded:
|
| 14 |
-
return
|
| 15 |
-
email = order.email
|
| 16 |
-
if not email or not email.strip():
|
| 17 |
-
return
|
| 18 |
-
try:
|
| 19 |
-
mailer.send(email, order.id)
|
| 20 |
-
except Exception:
|
| 21 |
-
logger.exception("Could not send receipt for order %s", order.id)`;
|
| 22 |
|
| 23 |
export async function paintSample(scoreAll) {
|
| 24 |
const original = presets.find((preset) => preset.filename === FILENAME).before;
|
| 25 |
-
|
| 26 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 27 |
const points = toHundred(result.score);
|
| 28 |
const { word, className } = scoreWord(points);
|
| 29 |
const wordNode = byId("sample-word");
|
|
|
|
| 30 |
byId("sample-value").textContent = String(points);
|
| 31 |
wordNode.textContent = word;
|
| 32 |
wordNode.className = `stage-score-word ${className}`.trim();
|
|
|
|
| 1 |
import presets from "shared:presets";
|
| 2 |
import { byId } from "./dom.js";
|
| 3 |
+
import { issuesText, scoreWord, toHundred } from "./labels.js";
|
| 4 |
+
import { paintLines } from "./stage.js";
|
| 5 |
|
| 6 |
const FILENAME = "send_receipt.py";
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 7 |
|
| 8 |
export async function paintSample(scoreAll) {
|
| 9 |
const original = presets.find((preset) => preset.filename === FILENAME).before;
|
| 10 |
+
const [result] = await scoreAll([original]);
|
| 11 |
+
paintLines(byId("sample-code"), {
|
| 12 |
+
text: original,
|
| 13 |
+
filename: FILENAME,
|
| 14 |
+
squiggles: result.squiggles,
|
| 15 |
+
enter: false,
|
| 16 |
+
});
|
| 17 |
const points = toHundred(result.score);
|
| 18 |
const { word, className } = scoreWord(points);
|
| 19 |
const wordNode = byId("sample-word");
|
| 20 |
+
byId("sample-status").textContent = issuesText(result.squiggles.length);
|
| 21 |
byId("sample-value").textContent = String(points);
|
| 22 |
wordNode.textContent = word;
|
| 23 |
wordNode.className = `stage-score-word ${className}`.trim();
|
web/src/sources.js
CHANGED
|
@@ -1,40 +1,17 @@
|
|
| 1 |
import presets from "shared:presets";
|
| 2 |
-
import { RepoError, fetchRepoFiles, isReadableText,
|
| 3 |
import { sniffExtension } from "./languages.js";
|
| 4 |
|
| 5 |
const EXAMPLE_ORDER = ["send_receipt.py", "webhook.go", "engine.js", "pricing.ts", "frame.rs"];
|
| 6 |
-
export const RUN_SIZE = 5;
|
| 7 |
|
| 8 |
function makeFile(path, content, result) {
|
| 9 |
-
return {
|
| 10 |
-
path,
|
| 11 |
-
original: content,
|
| 12 |
-
content,
|
| 13 |
-
originalResult: result,
|
| 14 |
-
result,
|
| 15 |
-
cleaned: false,
|
| 16 |
-
kept: false,
|
| 17 |
-
lastRejected: false,
|
| 18 |
-
attempted: false,
|
| 19 |
-
restingText: null,
|
| 20 |
-
};
|
| 21 |
-
}
|
| 22 |
-
|
| 23 |
-
export function isSloppy(file) {
|
| 24 |
-
return file.result.squiggles.length > 0;
|
| 25 |
}
|
| 26 |
|
| 27 |
function byScore(left, right) {
|
| 28 |
return left.result.score - right.result.score || left.path.localeCompare(right.path);
|
| 29 |
}
|
| 30 |
|
| 31 |
-
export function eligibleFiles(files, maxChars) {
|
| 32 |
-
const fits = (file) => !file.attempted && file.content.length <= maxChars;
|
| 33 |
-
const sloppy = files.filter((file) => isSloppy(file) && fits(file));
|
| 34 |
-
const nonTest = sloppy.filter((file) => !isTestPath(file.path));
|
| 35 |
-
return (nonTest.length > 0 ? nonTest : sloppy).sort(byScore);
|
| 36 |
-
}
|
| 37 |
-
|
| 38 |
function scoredFiles(entries, results) {
|
| 39 |
const files = entries.map((entry, index) => makeFile(entry.path, entry.content, results[index]));
|
| 40 |
return files.sort(byScore);
|
|
@@ -46,10 +23,9 @@ export async function loadExamples(scoreAll) {
|
|
| 46 |
return ordered.map((preset, index) => makeFile(preset.filename, preset.before, results[index]));
|
| 47 |
}
|
| 48 |
|
| 49 |
-
export async function loadRepo(spec, { signal, scoreAll, onProgress
|
| 50 |
const fetched = await fetchRepoFiles(spec, {
|
| 51 |
signal,
|
| 52 |
-
pinCommit,
|
| 53 |
focusPath: spec.blobPath,
|
| 54 |
onProgress: (done, total) => onProgress(`Fetching ${done} of ${total}`),
|
| 55 |
});
|
|
@@ -58,7 +34,7 @@ export async function loadRepo(spec, { signal, scoreAll, onProgress, pinCommit }
|
|
| 58 |
(done, total) => onProgress(`Scoring ${done} of ${total}`),
|
| 59 |
);
|
| 60 |
signal.throwIfAborted();
|
| 61 |
-
return
|
| 62 |
}
|
| 63 |
|
| 64 |
function relativePath(file) {
|
|
|
|
| 1 |
import presets from "shared:presets";
|
| 2 |
+
import { RepoError, fetchRepoFiles, isReadableText, selectLocalEntries } from "./github.js";
|
| 3 |
import { sniffExtension } from "./languages.js";
|
| 4 |
|
| 5 |
const EXAMPLE_ORDER = ["send_receipt.py", "webhook.go", "engine.js", "pricing.ts", "frame.rs"];
|
|
|
|
| 6 |
|
| 7 |
function makeFile(path, content, result) {
|
| 8 |
+
return { path, content, result };
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 9 |
}
|
| 10 |
|
| 11 |
function byScore(left, right) {
|
| 12 |
return left.result.score - right.result.score || left.path.localeCompare(right.path);
|
| 13 |
}
|
| 14 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 15 |
function scoredFiles(entries, results) {
|
| 16 |
const files = entries.map((entry, index) => makeFile(entry.path, entry.content, results[index]));
|
| 17 |
return files.sort(byScore);
|
|
|
|
| 23 |
return ordered.map((preset, index) => makeFile(preset.filename, preset.before, results[index]));
|
| 24 |
}
|
| 25 |
|
| 26 |
+
export async function loadRepo(spec, { signal, scoreAll, onProgress }) {
|
| 27 |
const fetched = await fetchRepoFiles(spec, {
|
| 28 |
signal,
|
|
|
|
| 29 |
focusPath: spec.blobPath,
|
| 30 |
onProgress: (done, total) => onProgress(`Fetching ${done} of ${total}`),
|
| 31 |
});
|
|
|
|
| 34 |
(done, total) => onProgress(`Scoring ${done} of ${total}`),
|
| 35 |
);
|
| 36 |
signal.throwIfAborted();
|
| 37 |
+
return scoredFiles(fetched.files, results);
|
| 38 |
}
|
| 39 |
|
| 40 |
function relativePath(file) {
|
web/src/stage.js
CHANGED
|
@@ -1,12 +1,10 @@
|
|
| 1 |
import { scorerSpanToOffsets } from "./columns.js";
|
| 2 |
-
import { diffLines } from "./diff.js";
|
| 3 |
import { byId, el, prefersReducedMotion, tweenNumber } from "./dom.js";
|
| 4 |
import { tokenizeLines } from "./highlight.js";
|
| 5 |
-
import { NOTES,
|
| 6 |
|
| 7 |
const NOTE_STAGGER_MS = 30;
|
| 8 |
const NOTE_STAGGER_STEPS = 7;
|
| 9 |
-
const BUSY_STATUSES = new Set([STATUS.reading, STATUS.rewriting]);
|
| 10 |
const SCORE_TWEEN_MS = 600;
|
| 11 |
const TAB_COLUMNS = 4;
|
| 12 |
|
|
@@ -96,7 +94,7 @@ function flaggedLines(lines, squiggles) {
|
|
| 96 |
return byLine;
|
| 97 |
}
|
| 98 |
|
| 99 |
-
function paintLines(container, { text, filename, squiggles, enter }) {
|
| 100 |
const lines = text.split("\n");
|
| 101 |
const tokens = tokenizeLines(text, filename);
|
| 102 |
const flagged = flaggedLines(lines, squiggles);
|
|
@@ -127,41 +125,6 @@ function scrollIntoStage(code, line) {
|
|
| 127 |
code.scrollTop = Math.max(0, offset - code.clientHeight / 3);
|
| 128 |
}
|
| 129 |
|
| 130 |
-
function readingClass(row, reading) {
|
| 131 |
-
if (!reading || row.kind !== "same") {
|
| 132 |
-
return "line";
|
| 133 |
-
}
|
| 134 |
-
const lineNumber = row.newIndex + 1;
|
| 135 |
-
return lineNumber >= reading.first && lineNumber <= reading.last ? "line is-reading" : "line";
|
| 136 |
-
}
|
| 137 |
-
|
| 138 |
-
export function diffNodes(options) {
|
| 139 |
-
const { base, next, filename, reading = null } = options;
|
| 140 |
-
const { squiggles = [], squigglesOn = "base" } = options;
|
| 141 |
-
const onNext = squigglesOn === "next";
|
| 142 |
-
const flagged = flaggedLines((onNext ? next : base).split("\n"), squiggles);
|
| 143 |
-
const flagFor = (row) => flagged.get((onNext ? row.newIndex : row.oldIndex) + 1);
|
| 144 |
-
const oldTokens = tokenizeLines(base, filename);
|
| 145 |
-
const newTokens = tokenizeLines(next, filename);
|
| 146 |
-
return diffLines(base, next).map((row) => {
|
| 147 |
-
if (row.kind === "removed") {
|
| 148 |
-
const tokens = oldTokens[row.oldIndex];
|
| 149 |
-
return lineNode({ number: null, text: row.text, tokens, className: "line is-removed" });
|
| 150 |
-
}
|
| 151 |
-
const added = row.kind === "added";
|
| 152 |
-
const flag = added && !onNext ? null : flagFor(row);
|
| 153 |
-
const rowClass = added ? "line is-added" : readingClass(row, reading);
|
| 154 |
-
return lineNode({
|
| 155 |
-
number: row.newIndex + 1,
|
| 156 |
-
text: row.text,
|
| 157 |
-
tokens: newTokens[row.newIndex],
|
| 158 |
-
className: flag ? `${rowClass} is-flagged` : rowClass,
|
| 159 |
-
squiggle: flag?.span,
|
| 160 |
-
note: flag?.note,
|
| 161 |
-
});
|
| 162 |
-
});
|
| 163 |
-
}
|
| 164 |
-
|
| 165 |
export function createStage() {
|
| 166 |
const path = byId("stage-path");
|
| 167 |
const status = byId("stage-status");
|
|
@@ -171,7 +134,6 @@ export function createStage() {
|
|
| 171 |
|
| 172 |
function setStatus(text) {
|
| 173 |
status.textContent = text;
|
| 174 |
-
status.toggleAttribute("data-busy", BUSY_STATUSES.has(text));
|
| 175 |
}
|
| 176 |
|
| 177 |
function setScore(points, animate) {
|
|
@@ -195,15 +157,6 @@ export function createStage() {
|
|
| 195 |
}
|
| 196 |
}
|
| 197 |
|
| 198 |
-
function renderDiff(options) {
|
| 199 |
-
code.classList.remove("is-entering");
|
| 200 |
-
code.replaceChildren(...diffNodes(options));
|
| 201 |
-
const target = options.followChange ? code.querySelector(".is-removed, .is-added, .is-reading") : null;
|
| 202 |
-
if (target) {
|
| 203 |
-
scrollIntoStage(code, target);
|
| 204 |
-
}
|
| 205 |
-
}
|
| 206 |
-
|
| 207 |
return {
|
| 208 |
setHead({ filename, statusText, points, animate = false }) {
|
| 209 |
const slash = filename.lastIndexOf("/");
|
|
@@ -215,12 +168,9 @@ export function createStage() {
|
|
| 215 |
setStatus(statusText);
|
| 216 |
setScore(points, animate);
|
| 217 |
},
|
| 218 |
-
setStatus,
|
| 219 |
-
setScore,
|
| 220 |
resetScore() {
|
| 221 |
shownPoints = null;
|
| 222 |
},
|
| 223 |
renderPlain,
|
| 224 |
-
renderDiff,
|
| 225 |
};
|
| 226 |
}
|
|
|
|
| 1 |
import { scorerSpanToOffsets } from "./columns.js";
|
|
|
|
| 2 |
import { byId, el, prefersReducedMotion, tweenNumber } from "./dom.js";
|
| 3 |
import { tokenizeLines } from "./highlight.js";
|
| 4 |
+
import { NOTES, scoreWord } from "./labels.js";
|
| 5 |
|
| 6 |
const NOTE_STAGGER_MS = 30;
|
| 7 |
const NOTE_STAGGER_STEPS = 7;
|
|
|
|
| 8 |
const SCORE_TWEEN_MS = 600;
|
| 9 |
const TAB_COLUMNS = 4;
|
| 10 |
|
|
|
|
| 94 |
return byLine;
|
| 95 |
}
|
| 96 |
|
| 97 |
+
export function paintLines(container, { text, filename, squiggles, enter }) {
|
| 98 |
const lines = text.split("\n");
|
| 99 |
const tokens = tokenizeLines(text, filename);
|
| 100 |
const flagged = flaggedLines(lines, squiggles);
|
|
|
|
| 125 |
code.scrollTop = Math.max(0, offset - code.clientHeight / 3);
|
| 126 |
}
|
| 127 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 128 |
export function createStage() {
|
| 129 |
const path = byId("stage-path");
|
| 130 |
const status = byId("stage-status");
|
|
|
|
| 134 |
|
| 135 |
function setStatus(text) {
|
| 136 |
status.textContent = text;
|
|
|
|
| 137 |
}
|
| 138 |
|
| 139 |
function setScore(points, animate) {
|
|
|
|
| 157 |
}
|
| 158 |
}
|
| 159 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 160 |
return {
|
| 161 |
setHead({ filename, statusText, points, animate = false }) {
|
| 162 |
const slash = filename.lastIndexOf("/");
|
|
|
|
| 168 |
setStatus(statusText);
|
| 169 |
setScore(points, animate);
|
| 170 |
},
|
|
|
|
|
|
|
| 171 |
resetScore() {
|
| 172 |
shownPoints = null;
|
| 173 |
},
|
| 174 |
renderPlain,
|
|
|
|
| 175 |
};
|
| 176 |
}
|
web/src/styles.css
CHANGED
|
@@ -638,7 +638,7 @@ body:has(#source-progress:not([hidden])) :is(.stage-code, .tree-list, .sample) {
|
|
| 638 |
display: grid;
|
| 639 |
flex: 1 1 0;
|
| 640 |
grid-template-columns: 240px minmax(0, 1fr);
|
| 641 |
-
grid-template-rows:
|
| 642 |
min-height: 360px;
|
| 643 |
border: 1px solid var(--line-strong);
|
| 644 |
border-radius: var(--radius-stage);
|
|
@@ -651,7 +651,7 @@ body:has(#source-progress:not([hidden])) :is(.stage-code, .tree-list, .sample) {
|
|
| 651 |
grid-template-columns: 0 minmax(0, 1fr);
|
| 652 |
}
|
| 653 |
|
| 654 |
-
.stage.is-collapsed
|
| 655 |
visibility: hidden;
|
| 656 |
}
|
| 657 |
|
|
@@ -667,7 +667,7 @@ body:has(#source-progress:not([hidden])) :is(.stage-code, .tree-list, .sample) {
|
|
| 667 |
}
|
| 668 |
|
| 669 |
.editor {
|
| 670 |
-
grid-area: 1 / 2
|
| 671 |
display: flex;
|
| 672 |
flex-direction: column;
|
| 673 |
min-width: 0;
|
|
@@ -944,63 +944,6 @@ body:has(#source-progress:not([hidden])) :is(.stage-code, .tree-list, .sample) {
|
|
| 944 |
font-size: 13px;
|
| 945 |
}
|
| 946 |
|
| 947 |
-
.tally {
|
| 948 |
-
grid-area: 2 / 1;
|
| 949 |
-
min-width: 0;
|
| 950 |
-
padding: 12px 16px 14px;
|
| 951 |
-
border-top: 1px solid var(--line);
|
| 952 |
-
border-right: 1px solid var(--line);
|
| 953 |
-
background: var(--sunken);
|
| 954 |
-
}
|
| 955 |
-
|
| 956 |
-
.tally-line {
|
| 957 |
-
display: flex;
|
| 958 |
-
align-items: baseline;
|
| 959 |
-
justify-content: space-between;
|
| 960 |
-
gap: 12px;
|
| 961 |
-
font-size: 13px;
|
| 962 |
-
line-height: 20px;
|
| 963 |
-
}
|
| 964 |
-
|
| 965 |
-
.tally-label {
|
| 966 |
-
color: var(--ink-2);
|
| 967 |
-
}
|
| 968 |
-
|
| 969 |
-
.tally-count {
|
| 970 |
-
color: var(--ink-3);
|
| 971 |
-
font-variant-numeric: tabular-nums;
|
| 972 |
-
}
|
| 973 |
-
|
| 974 |
-
#tally-fixed {
|
| 975 |
-
display: inline-block;
|
| 976 |
-
min-width: 2ch;
|
| 977 |
-
color: var(--ink);
|
| 978 |
-
font-weight: 500;
|
| 979 |
-
text-align: right;
|
| 980 |
-
}
|
| 981 |
-
|
| 982 |
-
.tally-bar {
|
| 983 |
-
height: 4px;
|
| 984 |
-
margin-top: 8px;
|
| 985 |
-
border-radius: 2px;
|
| 986 |
-
background: var(--selected);
|
| 987 |
-
overflow: hidden;
|
| 988 |
-
}
|
| 989 |
-
|
| 990 |
-
.tally-fill {
|
| 991 |
-
display: block;
|
| 992 |
-
width: 0;
|
| 993 |
-
height: 100%;
|
| 994 |
-
border-radius: inherit;
|
| 995 |
-
background: var(--green);
|
| 996 |
-
transition: width 600ms var(--ease-out);
|
| 997 |
-
}
|
| 998 |
-
|
| 999 |
-
.tally.is-empty .tally-count,
|
| 1000 |
-
.tally.is-empty .tally-bar {
|
| 1001 |
-
display: none;
|
| 1002 |
-
}
|
| 1003 |
-
|
| 1004 |
.stage-code {
|
| 1005 |
flex: 1;
|
| 1006 |
min-height: 0;
|
|
@@ -1351,29 +1294,6 @@ body:has(#source-progress:not([hidden])) :is(.stage-code, .tree-list, .sample) {
|
|
| 1351 |
grid-area: 1 / 1;
|
| 1352 |
}
|
| 1353 |
|
| 1354 |
-
.tally,
|
| 1355 |
-
.stage.is-collapsed .tally {
|
| 1356 |
-
grid-area: 2 / 1;
|
| 1357 |
-
display: flex;
|
| 1358 |
-
align-items: center;
|
| 1359 |
-
gap: 14px;
|
| 1360 |
-
padding: 10px 16px;
|
| 1361 |
-
border-right: 0;
|
| 1362 |
-
background: var(--surface);
|
| 1363 |
-
visibility: visible;
|
| 1364 |
-
}
|
| 1365 |
-
|
| 1366 |
-
.tally-line {
|
| 1367 |
-
flex: none;
|
| 1368 |
-
gap: 8px;
|
| 1369 |
-
font-size: 12.5px;
|
| 1370 |
-
}
|
| 1371 |
-
|
| 1372 |
-
.tally-bar {
|
| 1373 |
-
flex: 1;
|
| 1374 |
-
margin-top: 0;
|
| 1375 |
-
}
|
| 1376 |
-
|
| 1377 |
.stage.is-collapsed .explorer {
|
| 1378 |
display: none;
|
| 1379 |
opacity: 0;
|
|
|
|
| 638 |
display: grid;
|
| 639 |
flex: 1 1 0;
|
| 640 |
grid-template-columns: 240px minmax(0, 1fr);
|
| 641 |
+
grid-template-rows: 1fr;
|
| 642 |
min-height: 360px;
|
| 643 |
border: 1px solid var(--line-strong);
|
| 644 |
border-radius: var(--radius-stage);
|
|
|
|
| 651 |
grid-template-columns: 0 minmax(0, 1fr);
|
| 652 |
}
|
| 653 |
|
| 654 |
+
.stage.is-collapsed .explorer {
|
| 655 |
visibility: hidden;
|
| 656 |
}
|
| 657 |
|
|
|
|
| 667 |
}
|
| 668 |
|
| 669 |
.editor {
|
| 670 |
+
grid-area: 1 / 2;
|
| 671 |
display: flex;
|
| 672 |
flex-direction: column;
|
| 673 |
min-width: 0;
|
|
|
|
| 944 |
font-size: 13px;
|
| 945 |
}
|
| 946 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 947 |
.stage-code {
|
| 948 |
flex: 1;
|
| 949 |
min-height: 0;
|
|
|
|
| 1294 |
grid-area: 1 / 1;
|
| 1295 |
}
|
| 1296 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1297 |
.stage.is-collapsed .explorer {
|
| 1298 |
display: none;
|
| 1299 |
opacity: 0;
|
web/src/tabs.js
DELETED
|
@@ -1,53 +0,0 @@
|
|
| 1 |
-
import { byId, el } from "./dom.js";
|
| 2 |
-
import { baseName, fileState } from "./navigator.js";
|
| 3 |
-
|
| 4 |
-
const EDGE = 16;
|
| 5 |
-
|
| 6 |
-
function badge(file) {
|
| 7 |
-
const count = file.result.squiggles.length;
|
| 8 |
-
if (count > 0) {
|
| 9 |
-
return el("span", { class: "tab-count", text: String(count) });
|
| 10 |
-
}
|
| 11 |
-
return file.cleaned ? el("span", { class: "tree-mark" }) : "";
|
| 12 |
-
}
|
| 13 |
-
|
| 14 |
-
function keepTabVisible(bar, tab) {
|
| 15 |
-
const left = tab.offsetLeft - EDGE;
|
| 16 |
-
const right = tab.offsetLeft + tab.offsetWidth + EDGE - bar.clientWidth;
|
| 17 |
-
if (bar.scrollLeft > left) {
|
| 18 |
-
bar.scrollTo({ left, behavior: "smooth" });
|
| 19 |
-
} else if (bar.scrollLeft < right) {
|
| 20 |
-
bar.scrollTo({ left: right, behavior: "smooth" });
|
| 21 |
-
}
|
| 22 |
-
}
|
| 23 |
-
|
| 24 |
-
export function createTabs(onSelect) {
|
| 25 |
-
const bar = byId("tabs");
|
| 26 |
-
|
| 27 |
-
function tab(file, selected, busy) {
|
| 28 |
-
const classes = ["tab", fileState(file)];
|
| 29 |
-
if (selected) {
|
| 30 |
-
classes.push("is-current");
|
| 31 |
-
}
|
| 32 |
-
if (busy) {
|
| 33 |
-
classes.push("is-busy");
|
| 34 |
-
}
|
| 35 |
-
return el("button", {
|
| 36 |
-
class: classes.join(" "),
|
| 37 |
-
type: "button",
|
| 38 |
-
title: file.path,
|
| 39 |
-
"aria-current": selected ? "true" : null,
|
| 40 |
-
onclick: () => onSelect(file),
|
| 41 |
-
}, [el("span", { class: "tab-name", text: baseName(file.path) }), badge(file)]);
|
| 42 |
-
}
|
| 43 |
-
|
| 44 |
-
return {
|
| 45 |
-
render(files, current, busyFile) {
|
| 46 |
-
bar.replaceChildren(...files.map((file) => tab(file, file === current, file === busyFile)));
|
| 47 |
-
const active = bar.querySelector(".is-current");
|
| 48 |
-
if (active) {
|
| 49 |
-
keepTabVisible(bar, active);
|
| 50 |
-
}
|
| 51 |
-
},
|
| 52 |
-
};
|
| 53 |
-
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
web/src/telemetry.js
CHANGED
|
@@ -1,11 +1,10 @@
|
|
| 1 |
-
// Aggregate, code-free
|
| 2 |
import { byId } from "./dom.js";
|
| 3 |
import { NOTES, toHundred } from "./labels.js";
|
| 4 |
import { extensionOf } from "./languages.js";
|
| 5 |
import { apiUrl } from "./refiner.js";
|
| 6 |
|
| 7 |
-
export const
|
| 8 |
-
export const MIN_VISIBLE_PRS = 20;
|
| 9 |
export const MIN_VISIBLE_TIMED = 1000;
|
| 10 |
export const MIN_BOARD_ENTRIES = 5;
|
| 11 |
const MAX_SCAN_FILES = 200;
|
|
@@ -47,12 +46,12 @@ function scoreMs(result) {
|
|
| 47 |
async function scanEntry(file) {
|
| 48 |
const ext = baseExtension(file.path);
|
| 49 |
return {
|
| 50 |
-
digest16: await digest16(ext, file.
|
| 51 |
ext,
|
| 52 |
-
loc_bucket: locBucket(file.
|
| 53 |
-
|
| 54 |
-
squiggles: squiggleCounts(file.
|
| 55 |
-
score_ms: scoreMs(file.
|
| 56 |
};
|
| 57 |
}
|
| 58 |
|
|
@@ -67,7 +66,6 @@ export async function reportScan(sourceKind, files) {
|
|
| 67 |
body: JSON.stringify({ source_kind: sourceKind, files: entries }),
|
| 68 |
keepalive: true,
|
| 69 |
});
|
| 70 |
-
// Reporting is best effort: a failed report must never interrupt the run.
|
| 71 |
return request.then((response) => response.ok, () => false);
|
| 72 |
}
|
| 73 |
|
|
@@ -90,11 +88,14 @@ function bannerStat(id, value) {
|
|
| 90 |
}
|
| 91 |
|
| 92 |
export function renderBanner(stats) {
|
| 93 |
-
const
|
| 94 |
-
const
|
|
|
|
|
|
|
|
|
|
| 95 |
const timed = stats.score_ms_samples >= MIN_VISIBLE_TIMED ? formatMs(stats.score_ms_median) : null;
|
| 96 |
-
const ids = ["banner-
|
| 97 |
-
const shown = [
|
| 98 |
const visibleIds = ids.filter((_, index) => shown[index]);
|
| 99 |
for (const id of ids) {
|
| 100 |
byId(id).classList.toggle("is-extra", visibleIds.indexOf(id) >= PHONE_BANNER_STATS);
|
|
|
|
| 1 |
+
// Aggregate, code-free scoring telemetry. Every field sent here is listed in space/TELEMETRY.md.
|
| 2 |
import { byId } from "./dom.js";
|
| 3 |
import { NOTES, toHundred } from "./labels.js";
|
| 4 |
import { extensionOf } from "./languages.js";
|
| 5 |
import { apiUrl } from "./refiner.js";
|
| 6 |
|
| 7 |
+
export const MIN_VISIBLE_SCORED = 1000;
|
|
|
|
| 8 |
export const MIN_VISIBLE_TIMED = 1000;
|
| 9 |
export const MIN_BOARD_ENTRIES = 5;
|
| 10 |
const MAX_SCAN_FILES = 200;
|
|
|
|
| 46 |
async function scanEntry(file) {
|
| 47 |
const ext = baseExtension(file.path);
|
| 48 |
return {
|
| 49 |
+
digest16: await digest16(ext, file.content),
|
| 50 |
ext,
|
| 51 |
+
loc_bucket: locBucket(file.content),
|
| 52 |
+
score: toHundred(file.result.score),
|
| 53 |
+
squiggles: squiggleCounts(file.result.squiggles),
|
| 54 |
+
score_ms: scoreMs(file.result),
|
| 55 |
};
|
| 56 |
}
|
| 57 |
|
|
|
|
| 66 |
body: JSON.stringify({ source_kind: sourceKind, files: entries }),
|
| 67 |
keepalive: true,
|
| 68 |
});
|
|
|
|
| 69 |
return request.then((response) => response.ok, () => false);
|
| 70 |
}
|
| 71 |
|
|
|
|
| 88 |
}
|
| 89 |
|
| 90 |
export function renderBanner(stats) {
|
| 91 |
+
const scored = stats.files_scored >= MIN_VISIBLE_SCORED ? formatCount(stats.files_scored) : null;
|
| 92 |
+
const flagged =
|
| 93 |
+
stats.files_scored >= MIN_VISIBLE_SCORED && stats.issues_found > 0
|
| 94 |
+
? formatCount(stats.issues_found)
|
| 95 |
+
: null;
|
| 96 |
const timed = stats.score_ms_samples >= MIN_VISIBLE_TIMED ? formatMs(stats.score_ms_median) : null;
|
| 97 |
+
const ids = ["banner-scored", "banner-issues", "banner-speed"];
|
| 98 |
+
const shown = [scored, flagged, timed].map((value, index) => bannerStat(ids[index], value));
|
| 99 |
const visibleIds = ids.filter((_, index) => shown[index]);
|
| 100 |
for (const id of ids) {
|
| 101 |
byId(id).classList.toggle("is-extra", visibleIds.indexOf(id) >= PHONE_BANNER_STATS);
|