Text Classification
Transformers
Safetensors
deberta-v2
agents
agent-safety
tool-use
prompt-injection
deberta-v3
text-embeddings-inference
Instructions to use kontext-security/Merlin with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- Transformers
How to use kontext-security/Merlin with Transformers:
# Use a pipeline as a high-level helper from transformers import pipeline pipe = pipeline("text-classification", model="kontext-security/Merlin")# pip install -U transformers accelerate # Load model directly from transformers import AutoTokenizer, AutoModelForSequenceClassification tokenizer = AutoTokenizer.from_pretrained("kontext-security/Merlin") model = AutoModelForSequenceClassification.from_pretrained("kontext-security/Merlin", device_map="auto") - Notebooks
- Google Colab
- Kaggle
Fix native replay numeric fidelity and retire unused startup dependency
Browse files
v3-ungated/README.md
CHANGED
|
@@ -47,9 +47,14 @@ have an `error_code` and no score. Do not treat those as safe classifications.
|
|
| 47 |
| Cohort | Assessed / available | Detections or alerts |
|
| 48 |
|---|---:|---|
|
| 49 |
| AgentDojo-Traj | 1,159 / 1,220 | 85/352 positives detected (24.1% recall); 20/868 benign cases flagged (2.3% FPR) |
|
| 50 |
-
| Private historical
|
| 51 |
| Private later workload sample | 600 / 1,207 | 3 clear false alerts, 1 uncertain low-impact alert |
|
| 52 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 53 |
The original collected false-positive subset has 539 saved cases, of which 409
|
| 54 |
were reconstructed and assessed: Claude Code 0/128 alerts; Codex 1/281. The expanded
|
| 55 |
historical panel above includes these cases. These cohorts are not additive.
|
|
|
|
| 47 |
| Cohort | Assessed / available | Detections or alerts |
|
| 48 |
|---|---:|---|
|
| 49 |
| AgentDojo-Traj | 1,159 / 1,220 | 85/352 positives detected (24.1% recall); 20/868 benign cases flagged (2.3% FPR) |
|
| 50 |
+
| Private historical replay cohort | 415 / 415 | 1 false alert |
|
| 51 |
| Private later workload sample | 600 / 1,207 | 3 clear false alerts, 1 uncertain low-impact alert |
|
| 52 |
|
| 53 |
+
The historical source pool contains 610 unique saved calls. Context was reliably
|
| 54 |
+
reconstructed for 415, forming the replay cohort above; the other 195 were
|
| 55 |
+
excluded before scoring because their context remains unresolved. They are not
|
| 56 |
+
runtime skips or successful safe predictions.
|
| 57 |
+
|
| 58 |
The original collected false-positive subset has 539 saved cases, of which 409
|
| 59 |
were reconstructed and assessed: Claude Code 0/128 alerts; Codex 1/281. The expanded
|
| 60 |
historical panel above includes these cases. These cohorts are not additive.
|
v3-ungated/evaluation.json
CHANGED
|
@@ -21,7 +21,10 @@
|
|
| 21 |
"total": 415,
|
| 22 |
"assessed": 415,
|
| 23 |
"skipped": 0,
|
| 24 |
-
"alerts": 1
|
|
|
|
|
|
|
|
|
|
| 25 |
},
|
| 26 |
"private_original_flagged_no_risk": {
|
| 27 |
"saved": 539,
|
|
|
|
| 21 |
"total": 415,
|
| 22 |
"assessed": 415,
|
| 23 |
"skipped": 0,
|
| 24 |
+
"alerts": 1,
|
| 25 |
+
"source_pool_total": 610,
|
| 26 |
+
"excluded_before_replay": 195,
|
| 27 |
+
"exclusion_reason": "original context could not be reliably reconstructed"
|
| 28 |
},
|
| 29 |
"private_original_flagged_no_risk": {
|
| 30 |
"saved": 539,
|
v3-ungated/native/cmd/merlin/main.go
CHANGED
|
@@ -2,15 +2,39 @@ package main
|
|
| 2 |
|
| 3 |
import (
|
| 4 |
"bufio"
|
|
|
|
| 5 |
"context"
|
| 6 |
"encoding/json"
|
| 7 |
"fmt"
|
|
|
|
| 8 |
"os"
|
| 9 |
"time"
|
| 10 |
|
| 11 |
"github.com/kontext-security/merlin/native/stepsafety"
|
| 12 |
)
|
| 13 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 14 |
// Merlin only evaluates these inputs; it never executes the supplied tool calls.
|
| 15 |
func main() {
|
| 16 |
ctx := context.Background()
|
|
@@ -24,11 +48,8 @@ func main() {
|
|
| 24 |
scan.Buffer(make([]byte, 65536), 4*1024*1024)
|
| 25 |
output := json.NewEncoder(os.Stdout)
|
| 26 |
for scan.Scan() {
|
| 27 |
-
|
| 28 |
-
|
| 29 |
-
Input stepsafety.Input `json:"input"`
|
| 30 |
-
}
|
| 31 |
-
if err := json.Unmarshal(scan.Bytes(), &row); err != nil {
|
| 32 |
fmt.Fprintln(os.Stderr, "Invalid input JSON:", err)
|
| 33 |
os.Exit(1)
|
| 34 |
}
|
|
|
|
| 2 |
|
| 3 |
import (
|
| 4 |
"bufio"
|
| 5 |
+
"bytes"
|
| 6 |
"context"
|
| 7 |
"encoding/json"
|
| 8 |
"fmt"
|
| 9 |
+
"io"
|
| 10 |
"os"
|
| 11 |
"time"
|
| 12 |
|
| 13 |
"github.com/kontext-security/merlin/native/stepsafety"
|
| 14 |
)
|
| 15 |
|
| 16 |
+
type replayRow struct {
|
| 17 |
+
ID string `json:"id"`
|
| 18 |
+
Input stepsafety.Input `json:"input"`
|
| 19 |
+
}
|
| 20 |
+
|
| 21 |
+
func decodeRow(data []byte) (replayRow, error) {
|
| 22 |
+
var row replayRow
|
| 23 |
+
decoder := json.NewDecoder(bytes.NewReader(data))
|
| 24 |
+
decoder.UseNumber()
|
| 25 |
+
if err := decoder.Decode(&row); err != nil {
|
| 26 |
+
return row, err
|
| 27 |
+
}
|
| 28 |
+
var trailing any
|
| 29 |
+
if err := decoder.Decode(&trailing); err != io.EOF {
|
| 30 |
+
if err == nil {
|
| 31 |
+
err = fmt.Errorf("multiple JSON values on one input line")
|
| 32 |
+
}
|
| 33 |
+
return row, err
|
| 34 |
+
}
|
| 35 |
+
return row, nil
|
| 36 |
+
}
|
| 37 |
+
|
| 38 |
// Merlin only evaluates these inputs; it never executes the supplied tool calls.
|
| 39 |
func main() {
|
| 40 |
ctx := context.Background()
|
|
|
|
| 48 |
scan.Buffer(make([]byte, 65536), 4*1024*1024)
|
| 49 |
output := json.NewEncoder(os.Stdout)
|
| 50 |
for scan.Scan() {
|
| 51 |
+
row, err := decodeRow(scan.Bytes())
|
| 52 |
+
if err != nil {
|
|
|
|
|
|
|
|
|
|
| 53 |
fmt.Fprintln(os.Stderr, "Invalid input JSON:", err)
|
| 54 |
os.Exit(1)
|
| 55 |
}
|
v3-ungated/native/stepsafety/agreement.go
CHANGED
|
@@ -10,6 +10,7 @@ import (
|
|
| 10 |
"golang.org/x/text/cases"
|
| 11 |
"golang.org/x/text/language"
|
| 12 |
"io"
|
|
|
|
| 13 |
"math"
|
| 14 |
"strings"
|
| 15 |
"unicode"
|
|
@@ -29,8 +30,8 @@ var agreementModel struct {
|
|
| 29 |
SparseThreshold float64 `json:"sparse_threshold"`
|
| 30 |
}
|
| 31 |
|
| 32 |
-
func loadAgreementAssets() error {
|
| 33 |
-
f, err :=
|
| 34 |
if err != nil {
|
| 35 |
return err
|
| 36 |
}
|
|
|
|
| 10 |
"golang.org/x/text/cases"
|
| 11 |
"golang.org/x/text/language"
|
| 12 |
"io"
|
| 13 |
+
"io/fs"
|
| 14 |
"math"
|
| 15 |
"strings"
|
| 16 |
"unicode"
|
|
|
|
| 30 |
SparseThreshold float64 `json:"sparse_threshold"`
|
| 31 |
}
|
| 32 |
|
| 33 |
+
func loadAgreementAssets(assets fs.FS) error {
|
| 34 |
+
f, err := assets.Open("model/native/agreement.json.gz")
|
| 35 |
if err != nil {
|
| 36 |
return err
|
| 37 |
}
|
v3-ungated/native/stepsafety/native_backend.go
CHANGED
|
@@ -36,13 +36,7 @@ func newNativeBackend(ctx context.Context, cfg Config) (*nativeBackend, error) {
|
|
| 36 |
nativeOnce.Do(func() {
|
| 37 |
go func() {
|
| 38 |
defer close(nativeReady)
|
| 39 |
-
nativeShared.model, nativeShared.tokenizer, nativeShared.err =
|
| 40 |
-
if nativeShared.err == nil {
|
| 41 |
-
nativeShared.err = loadAgreementAssets()
|
| 42 |
-
if nativeShared.err == nil {
|
| 43 |
-
nativeShared.err = loadRequestGateAssets()
|
| 44 |
-
}
|
| 45 |
-
}
|
| 46 |
}()
|
| 47 |
})
|
| 48 |
select {
|
|
@@ -56,14 +50,27 @@ func newNativeBackend(ctx context.Context, cfg Config) (*nativeBackend, error) {
|
|
| 56 |
}
|
| 57 |
}
|
| 58 |
|
| 59 |
-
|
| 60 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 61 |
if err != nil || len(names) != 5 {
|
| 62 |
return nil, nil, errors.New("missing embedded native weights")
|
| 63 |
}
|
| 64 |
readers := make([]io.Reader, 0, len(names))
|
| 65 |
for _, name := range names {
|
| 66 |
-
f, err :=
|
| 67 |
if err != nil {
|
| 68 |
return nil, nil, err
|
| 69 |
}
|
|
@@ -83,7 +90,7 @@ func loadNativeAssets(ctx context.Context) (*nativeModel, *tokenizer, error) {
|
|
| 83 |
if err != nil {
|
| 84 |
return nil, nil, err
|
| 85 |
}
|
| 86 |
-
f, err :=
|
| 87 |
if err != nil {
|
| 88 |
return nil, nil, err
|
| 89 |
}
|
|
|
|
| 36 |
nativeOnce.Do(func() {
|
| 37 |
go func() {
|
| 38 |
defer close(nativeReady)
|
| 39 |
+
nativeShared.model, nativeShared.tokenizer, nativeShared.err = loadNativeRuntime(context.Background(), nativeAssets)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 40 |
}()
|
| 41 |
})
|
| 42 |
select {
|
|
|
|
| 50 |
}
|
| 51 |
}
|
| 52 |
|
| 53 |
+
// loadNativeRuntime loads only assets used by ungated inference. The retired
|
| 54 |
+
// request-intent asset is retained for historical provenance, never startup.
|
| 55 |
+
func loadNativeRuntime(ctx context.Context, assets fs.FS) (*nativeModel, *tokenizer, error) {
|
| 56 |
+
model, tok, err := loadNativeAssets(ctx, assets)
|
| 57 |
+
if err != nil {
|
| 58 |
+
return nil, nil, err
|
| 59 |
+
}
|
| 60 |
+
if err := loadAgreementAssets(assets); err != nil {
|
| 61 |
+
return nil, nil, err
|
| 62 |
+
}
|
| 63 |
+
return model, tok, nil
|
| 64 |
+
}
|
| 65 |
+
|
| 66 |
+
func loadNativeAssets(ctx context.Context, assets fs.FS) (*nativeModel, *tokenizer, error) {
|
| 67 |
+
names, err := fs.Glob(assets, "model/native/weights.*.gz")
|
| 68 |
if err != nil || len(names) != 5 {
|
| 69 |
return nil, nil, errors.New("missing embedded native weights")
|
| 70 |
}
|
| 71 |
readers := make([]io.Reader, 0, len(names))
|
| 72 |
for _, name := range names {
|
| 73 |
+
f, err := assets.Open(name)
|
| 74 |
if err != nil {
|
| 75 |
return nil, nil, err
|
| 76 |
}
|
|
|
|
| 90 |
if err != nil {
|
| 91 |
return nil, nil, err
|
| 92 |
}
|
| 93 |
+
f, err := assets.Open("model/native/tokenizer.json.gz")
|
| 94 |
if err != nil {
|
| 95 |
return nil, nil, err
|
| 96 |
}
|
v3-ungated/release_manifest.json
CHANGED
|
@@ -10,16 +10,16 @@
|
|
| 10 |
"size": 271
|
| 11 |
},
|
| 12 |
"README.md": {
|
| 13 |
-
"sha256": "
|
| 14 |
-
"size":
|
| 15 |
},
|
| 16 |
"evaluation.json": {
|
| 17 |
-
"sha256": "
|
| 18 |
-
"size":
|
| 19 |
},
|
| 20 |
"native/cmd/merlin/main.go": {
|
| 21 |
-
"sha256": "
|
| 22 |
-
"size":
|
| 23 |
},
|
| 24 |
"native/go.mod": {
|
| 25 |
"sha256": "97ffbb6542f80e9cbc14bf46354388a7ba0ce18a1ad4822b64c750b7eda7abe4",
|
|
@@ -30,8 +30,8 @@
|
|
| 30 |
"size": 312
|
| 31 |
},
|
| 32 |
"native/stepsafety/agreement.go": {
|
| 33 |
-
"sha256": "
|
| 34 |
-
"size":
|
| 35 |
},
|
| 36 |
"native/stepsafety/context.go": {
|
| 37 |
"sha256": "616f619ec4ff787d810085e1566836d13e5106ec6ee42a086411999329dd1563",
|
|
@@ -106,8 +106,8 @@
|
|
| 106 |
"size": 454
|
| 107 |
},
|
| 108 |
"native/stepsafety/native_backend.go": {
|
| 109 |
-
"sha256": "
|
| 110 |
-
"size":
|
| 111 |
},
|
| 112 |
"native/stepsafety/native_linear_arm64.go": {
|
| 113 |
"sha256": "2a510eb9fc849bdf59d4e13fc9288dc14bca41034fb307f76f84ab30d49cb53b",
|
|
|
|
| 10 |
"size": 271
|
| 11 |
},
|
| 12 |
"README.md": {
|
| 13 |
+
"sha256": "ddf197582c7f06225cec0e4e171898449e5040b40e524b87addbf53f3c38ae29",
|
| 14 |
+
"size": 4709
|
| 15 |
},
|
| 16 |
"evaluation.json": {
|
| 17 |
+
"sha256": "697e6e9e0b5dbdad78a002bfea07c0771a02a46195994f687a76c7d59750454d",
|
| 18 |
+
"size": 1662
|
| 19 |
},
|
| 20 |
"native/cmd/merlin/main.go": {
|
| 21 |
+
"sha256": "2771e870f89ddbebcdd6e1ef8a8b9020392907f7ce18808e51d64014ea9b1ee4",
|
| 22 |
+
"size": 1828
|
| 23 |
},
|
| 24 |
"native/go.mod": {
|
| 25 |
"sha256": "97ffbb6542f80e9cbc14bf46354388a7ba0ce18a1ad4822b64c750b7eda7abe4",
|
|
|
|
| 30 |
"size": 312
|
| 31 |
},
|
| 32 |
"native/stepsafety/agreement.go": {
|
| 33 |
+
"sha256": "b93a6d46e014178b2389511e4cb3c2b7bf404edbd9dc257dc186b608659f69ac",
|
| 34 |
+
"size": 3813
|
| 35 |
},
|
| 36 |
"native/stepsafety/context.go": {
|
| 37 |
"sha256": "616f619ec4ff787d810085e1566836d13e5106ec6ee42a086411999329dd1563",
|
|
|
|
| 106 |
"size": 454
|
| 107 |
},
|
| 108 |
"native/stepsafety/native_backend.go": {
|
| 109 |
+
"sha256": "95ba17ffef0d6f59bb04a8e650230ffab72fca3687ab52ab9ee03d459d879607",
|
| 110 |
+
"size": 3902
|
| 111 |
},
|
| 112 |
"native/stepsafety/native_linear_arm64.go": {
|
| 113 |
"sha256": "2a510eb9fc849bdf59d4e13fc9288dc14bca41034fb307f76f84ab30d49cb53b",
|