hasandmrkrn commited on
Commit
c028026
·
verified ·
1 Parent(s): 9e8e328

Fix native replay numeric fidelity and retire unused startup dependency

Browse files
v3-ungated/README.md CHANGED
@@ -47,9 +47,14 @@ have an `error_code` and no score. Do not treat those as safe classifications.
47
  | Cohort | Assessed / available | Detections or alerts |
48
  |---|---:|---|
49
  | AgentDojo-Traj | 1,159 / 1,220 | 85/352 positives detected (24.1% recall); 20/868 benign cases flagged (2.3% FPR) |
50
- | Private historical regression | 415 / 610 | 1 false alert |
51
  | Private later workload sample | 600 / 1,207 | 3 clear false alerts, 1 uncertain low-impact alert |
52
 
 
 
 
 
 
53
  The original collected false-positive subset has 539 saved cases, of which 409
54
  were reconstructed and assessed: Claude Code 0/128 alerts; Codex 1/281. The expanded
55
  historical panel above includes these cases. These cohorts are not additive.
 
47
  | Cohort | Assessed / available | Detections or alerts |
48
  |---|---:|---|
49
  | AgentDojo-Traj | 1,159 / 1,220 | 85/352 positives detected (24.1% recall); 20/868 benign cases flagged (2.3% FPR) |
50
+ | Private historical replay cohort | 415 / 415 | 1 false alert |
51
  | Private later workload sample | 600 / 1,207 | 3 clear false alerts, 1 uncertain low-impact alert |
52
 
53
+ The historical source pool contains 610 unique saved calls. Context was reliably
54
+ reconstructed for 415, forming the replay cohort above; the other 195 were
55
+ excluded before scoring because their context remains unresolved. They are not
56
+ runtime skips or successful safe predictions.
57
+
58
  The original collected false-positive subset has 539 saved cases, of which 409
59
  were reconstructed and assessed: Claude Code 0/128 alerts; Codex 1/281. The expanded
60
  historical panel above includes these cases. These cohorts are not additive.
v3-ungated/evaluation.json CHANGED
@@ -21,7 +21,10 @@
21
  "total": 415,
22
  "assessed": 415,
23
  "skipped": 0,
24
- "alerts": 1
 
 
 
25
  },
26
  "private_original_flagged_no_risk": {
27
  "saved": 539,
 
21
  "total": 415,
22
  "assessed": 415,
23
  "skipped": 0,
24
+ "alerts": 1,
25
+ "source_pool_total": 610,
26
+ "excluded_before_replay": 195,
27
+ "exclusion_reason": "original context could not be reliably reconstructed"
28
  },
29
  "private_original_flagged_no_risk": {
30
  "saved": 539,
v3-ungated/native/cmd/merlin/main.go CHANGED
@@ -2,15 +2,39 @@ package main
2
 
3
  import (
4
  "bufio"
 
5
  "context"
6
  "encoding/json"
7
  "fmt"
 
8
  "os"
9
  "time"
10
 
11
  "github.com/kontext-security/merlin/native/stepsafety"
12
  )
13
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
14
  // Merlin only evaluates these inputs; it never executes the supplied tool calls.
15
  func main() {
16
  ctx := context.Background()
@@ -24,11 +48,8 @@ func main() {
24
  scan.Buffer(make([]byte, 65536), 4*1024*1024)
25
  output := json.NewEncoder(os.Stdout)
26
  for scan.Scan() {
27
- var row struct {
28
- ID string `json:"id"`
29
- Input stepsafety.Input `json:"input"`
30
- }
31
- if err := json.Unmarshal(scan.Bytes(), &row); err != nil {
32
  fmt.Fprintln(os.Stderr, "Invalid input JSON:", err)
33
  os.Exit(1)
34
  }
 
2
 
3
  import (
4
  "bufio"
5
+ "bytes"
6
  "context"
7
  "encoding/json"
8
  "fmt"
9
+ "io"
10
  "os"
11
  "time"
12
 
13
  "github.com/kontext-security/merlin/native/stepsafety"
14
  )
15
 
16
+ type replayRow struct {
17
+ ID string `json:"id"`
18
+ Input stepsafety.Input `json:"input"`
19
+ }
20
+
21
+ func decodeRow(data []byte) (replayRow, error) {
22
+ var row replayRow
23
+ decoder := json.NewDecoder(bytes.NewReader(data))
24
+ decoder.UseNumber()
25
+ if err := decoder.Decode(&row); err != nil {
26
+ return row, err
27
+ }
28
+ var trailing any
29
+ if err := decoder.Decode(&trailing); err != io.EOF {
30
+ if err == nil {
31
+ err = fmt.Errorf("multiple JSON values on one input line")
32
+ }
33
+ return row, err
34
+ }
35
+ return row, nil
36
+ }
37
+
38
  // Merlin only evaluates these inputs; it never executes the supplied tool calls.
39
  func main() {
40
  ctx := context.Background()
 
48
  scan.Buffer(make([]byte, 65536), 4*1024*1024)
49
  output := json.NewEncoder(os.Stdout)
50
  for scan.Scan() {
51
+ row, err := decodeRow(scan.Bytes())
52
+ if err != nil {
 
 
 
53
  fmt.Fprintln(os.Stderr, "Invalid input JSON:", err)
54
  os.Exit(1)
55
  }
v3-ungated/native/stepsafety/agreement.go CHANGED
@@ -10,6 +10,7 @@ import (
10
  "golang.org/x/text/cases"
11
  "golang.org/x/text/language"
12
  "io"
 
13
  "math"
14
  "strings"
15
  "unicode"
@@ -29,8 +30,8 @@ var agreementModel struct {
29
  SparseThreshold float64 `json:"sparse_threshold"`
30
  }
31
 
32
- func loadAgreementAssets() error {
33
- f, err := nativeAssets.Open("model/native/agreement.json.gz")
34
  if err != nil {
35
  return err
36
  }
 
10
  "golang.org/x/text/cases"
11
  "golang.org/x/text/language"
12
  "io"
13
+ "io/fs"
14
  "math"
15
  "strings"
16
  "unicode"
 
30
  SparseThreshold float64 `json:"sparse_threshold"`
31
  }
32
 
33
+ func loadAgreementAssets(assets fs.FS) error {
34
+ f, err := assets.Open("model/native/agreement.json.gz")
35
  if err != nil {
36
  return err
37
  }
v3-ungated/native/stepsafety/native_backend.go CHANGED
@@ -36,13 +36,7 @@ func newNativeBackend(ctx context.Context, cfg Config) (*nativeBackend, error) {
36
  nativeOnce.Do(func() {
37
  go func() {
38
  defer close(nativeReady)
39
- nativeShared.model, nativeShared.tokenizer, nativeShared.err = loadNativeAssets(context.Background())
40
- if nativeShared.err == nil {
41
- nativeShared.err = loadAgreementAssets()
42
- if nativeShared.err == nil {
43
- nativeShared.err = loadRequestGateAssets()
44
- }
45
- }
46
  }()
47
  })
48
  select {
@@ -56,14 +50,27 @@ func newNativeBackend(ctx context.Context, cfg Config) (*nativeBackend, error) {
56
  }
57
  }
58
 
59
- func loadNativeAssets(ctx context.Context) (*nativeModel, *tokenizer, error) {
60
- names, err := fs.Glob(nativeAssets, "model/native/weights.*.gz")
 
 
 
 
 
 
 
 
 
 
 
 
 
61
  if err != nil || len(names) != 5 {
62
  return nil, nil, errors.New("missing embedded native weights")
63
  }
64
  readers := make([]io.Reader, 0, len(names))
65
  for _, name := range names {
66
- f, err := nativeAssets.Open(name)
67
  if err != nil {
68
  return nil, nil, err
69
  }
@@ -83,7 +90,7 @@ func loadNativeAssets(ctx context.Context) (*nativeModel, *tokenizer, error) {
83
  if err != nil {
84
  return nil, nil, err
85
  }
86
- f, err := nativeAssets.Open("model/native/tokenizer.json.gz")
87
  if err != nil {
88
  return nil, nil, err
89
  }
 
36
  nativeOnce.Do(func() {
37
  go func() {
38
  defer close(nativeReady)
39
+ nativeShared.model, nativeShared.tokenizer, nativeShared.err = loadNativeRuntime(context.Background(), nativeAssets)
 
 
 
 
 
 
40
  }()
41
  })
42
  select {
 
50
  }
51
  }
52
 
53
+ // loadNativeRuntime loads only assets used by ungated inference. The retired
54
+ // request-intent asset is retained for historical provenance, never startup.
55
+ func loadNativeRuntime(ctx context.Context, assets fs.FS) (*nativeModel, *tokenizer, error) {
56
+ model, tok, err := loadNativeAssets(ctx, assets)
57
+ if err != nil {
58
+ return nil, nil, err
59
+ }
60
+ if err := loadAgreementAssets(assets); err != nil {
61
+ return nil, nil, err
62
+ }
63
+ return model, tok, nil
64
+ }
65
+
66
+ func loadNativeAssets(ctx context.Context, assets fs.FS) (*nativeModel, *tokenizer, error) {
67
+ names, err := fs.Glob(assets, "model/native/weights.*.gz")
68
  if err != nil || len(names) != 5 {
69
  return nil, nil, errors.New("missing embedded native weights")
70
  }
71
  readers := make([]io.Reader, 0, len(names))
72
  for _, name := range names {
73
+ f, err := assets.Open(name)
74
  if err != nil {
75
  return nil, nil, err
76
  }
 
90
  if err != nil {
91
  return nil, nil, err
92
  }
93
+ f, err := assets.Open("model/native/tokenizer.json.gz")
94
  if err != nil {
95
  return nil, nil, err
96
  }
v3-ungated/release_manifest.json CHANGED
@@ -10,16 +10,16 @@
10
  "size": 271
11
  },
12
  "README.md": {
13
- "sha256": "5d4285e2dd08af2d9a449889780ab2cf55c8ba13c1d8154aae04c734b2b04128",
14
- "size": 4424
15
  },
16
  "evaluation.json": {
17
- "sha256": "186ec26dd500b5f8791283a619200450c0f10e523ce3faca92a296f6acec1562",
18
- "size": 1517
19
  },
20
  "native/cmd/merlin/main.go": {
21
- "sha256": "0a9bbc83520ef46b1b3fc031f16fb2315f7a94d73e8dfef0d78374e64e466376",
22
- "size": 1414
23
  },
24
  "native/go.mod": {
25
  "sha256": "97ffbb6542f80e9cbc14bf46354388a7ba0ce18a1ad4822b64c750b7eda7abe4",
@@ -30,8 +30,8 @@
30
  "size": 312
31
  },
32
  "native/stepsafety/agreement.go": {
33
- "sha256": "a19ce6cba08771b455741ba7d2622334636c50b7ffde44fc98341c767f1d2363",
34
- "size": 3798
35
  },
36
  "native/stepsafety/context.go": {
37
  "sha256": "616f619ec4ff787d810085e1566836d13e5106ec6ee42a086411999329dd1563",
@@ -106,8 +106,8 @@
106
  "size": 454
107
  },
108
  "native/stepsafety/native_backend.go": {
109
- "sha256": "e94731e7b57cd0bdac1b045dd76f10fcf8690a5bec67ef2b5f1b97d7141f97b1",
110
- "size": 3611
111
  },
112
  "native/stepsafety/native_linear_arm64.go": {
113
  "sha256": "2a510eb9fc849bdf59d4e13fc9288dc14bca41034fb307f76f84ab30d49cb53b",
 
10
  "size": 271
11
  },
12
  "README.md": {
13
+ "sha256": "ddf197582c7f06225cec0e4e171898449e5040b40e524b87addbf53f3c38ae29",
14
+ "size": 4709
15
  },
16
  "evaluation.json": {
17
+ "sha256": "697e6e9e0b5dbdad78a002bfea07c0771a02a46195994f687a76c7d59750454d",
18
+ "size": 1662
19
  },
20
  "native/cmd/merlin/main.go": {
21
+ "sha256": "2771e870f89ddbebcdd6e1ef8a8b9020392907f7ce18808e51d64014ea9b1ee4",
22
+ "size": 1828
23
  },
24
  "native/go.mod": {
25
  "sha256": "97ffbb6542f80e9cbc14bf46354388a7ba0ce18a1ad4822b64c750b7eda7abe4",
 
30
  "size": 312
31
  },
32
  "native/stepsafety/agreement.go": {
33
+ "sha256": "b93a6d46e014178b2389511e4cb3c2b7bf404edbd9dc257dc186b608659f69ac",
34
+ "size": 3813
35
  },
36
  "native/stepsafety/context.go": {
37
  "sha256": "616f619ec4ff787d810085e1566836d13e5106ec6ee42a086411999329dd1563",
 
106
  "size": 454
107
  },
108
  "native/stepsafety/native_backend.go": {
109
+ "sha256": "95ba17ffef0d6f59bb04a8e650230ffab72fca3687ab52ab9ee03d459d879607",
110
+ "size": 3902
111
  },
112
  "native/stepsafety/native_linear_arm64.go": {
113
  "sha256": "2a510eb9fc849bdf59d4e13fc9288dc14bca41034fb307f76f84ab30d49cb53b",