Buckets:

download
raw
475 kB
[ { "slug": "authentication_pwd__Clinical-Genomics_statina", "id": "pwd-based/statina/ClaudeCodeopus4.7", "pattern": "authentication_pwd", "repo": "Clinical-Genomics_statina", "error": null, "metrics": [ { "key": "level1", "level": "Level 1 \u2014 Role Detection", "formula": "agreement-rate (present/external/missing)", "available": true, "headlineLabel": "Roles agreed", "score": 0.8571428571428571, "drift": 0.1428571428571429, "detail": { "agreedRoles": 6, "totalRoles": 7 }, "perRole": [ { "patternRole": "Enforcer", "groundTruth": "present", "recoveryResult": "present", "groundTruthDetected": true, "recoveryResultDetected": true, "outcome": "agreed" }, { "patternRole": "Verification manager", "groundTruth": "present", "recoveryResult": "present", "groundTruthDetected": true, "recoveryResultDetected": true, "outcome": "agreed" }, { "patternRole": "Password store", "groundTruth": "present", "recoveryResult": "present", "groundTruthDetected": true, "recoveryResultDetected": true, "outcome": "agreed" }, { "patternRole": "Hasher", "groundTruth": "external", "recoveryResult": "external", "groundTruthDetected": true, "recoveryResultDetected": true, "outcome": "agreed" }, { "patternRole": "Pepper store", "groundTruth": "missing", "recoveryResult": "missing", "groundTruthDetected": false, "recoveryResultDetected": false, "outcome": "agreed" }, { "patternRole": "Encrypter", "groundTruth": "external", "recoveryResult": "missing", "groundTruthDetected": true, "recoveryResultDetected": false, "outcome": "mismatched" }, { "patternRole": "Comparator", "groundTruth": "external", "recoveryResult": "external", "groundTruthDetected": true, "recoveryResultDetected": true, "outcome": "agreed" } ], "notes": [] }, { "key": "level2", "level": "Level 2 \u2014 Role Grounding", "formula": "path-length", "available": true, "headlineLabel": "Mean grounding score", "score": 0.9375, "drift": 0.0625, "detail": { "aggregate": 0.9375, "scoredRoles": 4, "skippedRoles": 8, "treeHeight": 8 }, "perRole": [ { "patternRole": "Comparator", "score": 1.0, "distance": 0, "groundTruth": { "entity": "verify", "loc": "statina/API/external/api/deps.py:19" }, "recoveryResult": { "entity": "verify", "loc": "statina/API/external/api/deps.py:16" }, "note": "path-length (d=0)" }, { "patternRole": "Enforcer", "score": 0.875, "distance": 2, "groundTruth": { "entity": "login_for_access_token", "loc": "statina/API/v2/endpoints/user.py:85\u2013104" }, "recoveryResult": { "entity": "get_current_user", "loc": "statina/API/v2/endpoints/user.py:48" }, "note": "path-length (d=2)" }, { "patternRole": "Hasher", "score": 0.875, "distance": 2, "groundTruth": { "entity": "verify", "loc": "statina/API/external/api/deps.py:19" }, "recoveryResult": { "entity": "CryptContext", "loc": "statina/API/external/api/deps.py:14" }, "note": "path-length (d=2)" }, { "patternRole": "Verification manager", "score": 1.0, "distance": 0, "groundTruth": { "entity": "authenticate_user", "loc": "statina/API/external/api/deps.py:26" }, "recoveryResult": { "entity": "authenticate_user", "loc": "statina/API/external/api/deps.py:22" }, "note": "path-length (d=0)" } ], "notes": [ "Encrypter: absent from the recovery result (Level 1 false negative)", "Hash manager: absent from the ground truth (Level 1 negative)", "Password policy: absent from the ground truth (Level 1 negative)", "Password store: ground truth cites no entity", "Pepper store: absent from the ground truth (Level 1 negative)", "Registrar: absent from the ground truth (Level 1 negative)", "Resetter: absent from the ground truth (Level 1 negative)", "SRNG: absent from the ground truth (Level 1 negative)" ] }, { "key": "level3", "level": "Level 3 \u2014 Interaction Order", "formula": "jaccard \u00d7 kendall-tau (shared interactions)", "available": true, "headlineLabel": "Interaction agreement", "score": 1.0, "drift": 0.0, "detail": { "agreedRoles": [ "Comparator", "Enforcer", "Hasher", "Verification manager" ], "excludedRoles": [ "Encrypter", "Hash manager", "Password policy", "Password store", "Pepper store", "Registrar", "Resetter", "SRNG" ], "interactionsGroundTruth": 3, "interactionsRecoveryResult": 3, "sharedInteractions": 3, "unionSize": 3, "entryRoleGroundTruth": "Enforcer", "entryRoleRecoveryResult": "Enforcer", "entryRuleGroundTruth": "behaviour", "entryRuleRecoveryResult": "stored", "onlyGroundTruth": 0, "onlyRecoveryResult": 0, "interactionOverlap": 1.0, "orderAgreement": 1.0, "rankedInteractions": 3, "totalPairs": 3, "concordantPairs": 3, "discordantPairs": 0, "kendallTau": 1.0, "prescribedActionSteps": 6, "prescribedStepsCovered": 3 }, "perRole": [ { "interaction": "Enforcer \u2192 Verification manager", "from": "Enforcer", "to": "Verification manager", "groundTruthRank": 1, "recoveryResultRank": 1, "outcome": "agreed", "discordantWith": 0, "evidence": { "groundTruth": { "file": "statina/API/v2/endpoints/user.py", "line": 89, "via": [] }, "recoveryResult": { "file": "statina/API/v2/endpoints/user.py", "line": 70, "via": [] } } }, { "interaction": "Verification manager \u2192 Hasher", "from": "Verification manager", "to": "Hasher", "groundTruthRank": 2, "recoveryResultRank": 2, "outcome": "agreed", "discordantWith": 0, "evidence": { "groundTruth": { "file": "statina/API/external/api/deps.py", "line": 19, "via": [ "statina/API/external/api/deps.py:<module>.verify_password" ] }, "recoveryResult": { "file": "statina/API/external/api/deps.py", "line": 16, "via": [] } } }, { "interaction": "Verification manager \u2192 Comparator", "from": "Verification manager", "to": "Comparator", "groundTruthRank": 3, "recoveryResultRank": 3, "outcome": "agreed", "discordantWith": 0, "evidence": { "groundTruth": { "file": "statina/API/external/api/deps.py", "line": 19, "via": [ "statina/API/external/api/deps.py:<module>.verify_password" ] }, "recoveryResult": { "file": "statina/API/external/api/deps.py", "line": 16, "via": [] } } } ], "notes": [ "Score = overlap \u00d7 order = 1 \u00d7 1. Overlap: 3 of 3 interactions were found by both sides (0 only in the ground truth, 0 only in the recovery result), |shared| / |union|.", "Order: Kendall's Tau over the 3 shared interactions, in each side's own relative order \u2014 0 of 3 pairs run in opposite orders. Interactions only one side found take part in no pair (Fagin, Kumar & Sivakumar's p = 0, the induced sub-order); the overlap factor is what charges for them.", "Scored over the 4 roles both sides detected (Comparator, Enforcer, Hasher, Verification manager). An interaction counts only when both of its endpoint roles were detected on both sides \u2014 whether a role was found at all is Level 1's measurement, and charging for it again here would say the same thing twice.", "The interaction order is derived statically from the code \u2014 no one annotated it. Each role's cited entity is located in the AST index and control flow is followed from it, through as many non-role helper functions as it takes to reach the next role. See secpat_scoring/sequence/JOERN.md.", "Left out \u2014 only one side detected them: Encrypter, Hash manager, Password policy, Password store, Pepper store, Registrar, Resetter, SRNG. 8 roles were excluded from both walks, so neither side's ordering mentions them. The walk still passes through the code behind such a role, attributing what it finds there to the nearest agreed role.", "Ground truth: the walk started at Enforcer \u2014 the earliest role the reference pattern has speaking that this side grounded in walkable code.", "Recovery result: the walk started at Enforcer \u2014 stored.", "An entry point the pattern did not name means its prescribed speakers were either ungrounded on that side or not where this code begins \u2014 a role something else reaches cannot open the flow \u2014 so the call graph had to stand in. Treat the ordering below as weaker evidence: it starts wherever the code happens to start.", "Ground truth: Comparator, Hasher cite entities that are not symbols in this repository (a library or framework call). They can receive an interaction, but the walk cannot continue through them." ] } ], "ground_truth_source": "annotation", "vocabulary": [ "Enforcer", "Verification manager", "Password store", "Hasher", "Pepper store", "Encrypter", "Comparator" ], "role_scope": "required", "excluded_roles": [ "Subject", "System" ], "dropped_ground_truth_rows": [ "Hash manager", "Password policy", "Registrar", "Resetter", "SRNG", "Subject", "System" ], "dropped_recovery_rows": [ "Subject", "System" ], "gt_roles": 7, "rr_roles": 7, "struct_files": 76, "annotated_at": "2026-08-05 17:07:39.992000", "recovery_order": "stored" }, { "slug": "authentication_pwd__aspuru-guzik-group_molar", "id": "pwd-based/molar/ClaudeCodeopus4.7", "pattern": "authentication_pwd", "repo": "aspuru-guzik-group_molar", "error": null, "metrics": [ { "key": "level1", "level": "Level 1 \u2014 Role Detection", "formula": "agreement-rate (present/external/missing)", "available": true, "headlineLabel": "Roles agreed", "score": 0.7142857142857143, "drift": 0.2857142857142857, "detail": { "agreedRoles": 5, "totalRoles": 7 }, "perRole": [ { "patternRole": "Enforcer", "groundTruth": "present", "recoveryResult": "present", "groundTruthDetected": true, "recoveryResultDetected": true, "outcome": "agreed" }, { "patternRole": "Verification manager", "groundTruth": "present", "recoveryResult": "present", "groundTruthDetected": true, "recoveryResultDetected": true, "outcome": "agreed" }, { "patternRole": "Password store", "groundTruth": "present", "recoveryResult": "present", "groundTruthDetected": true, "recoveryResultDetected": true, "outcome": "agreed" }, { "patternRole": "Hasher", "groundTruth": "external", "recoveryResult": "present", "groundTruthDetected": true, "recoveryResultDetected": true, "outcome": "mismatched" }, { "patternRole": "Pepper store", "groundTruth": "missing", "recoveryResult": "missing", "groundTruthDetected": false, "recoveryResultDetected": false, "outcome": "agreed" }, { "patternRole": "Encrypter", "groundTruth": "missing", "recoveryResult": "missing", "groundTruthDetected": false, "recoveryResultDetected": false, "outcome": "agreed" }, { "patternRole": "Comparator", "groundTruth": "external", "recoveryResult": "present", "groundTruthDetected": true, "recoveryResultDetected": true, "outcome": "mismatched" } ], "notes": [] }, { "key": "level2", "level": "Level 2 \u2014 Role Grounding", "formula": "path-length", "available": true, "headlineLabel": "Mean grounding score", "score": 0.9428, "drift": 0.0572, "detail": { "aggregate": 0.9428, "scoredRoles": 5, "skippedRoles": 7, "treeHeight": 7 }, "perRole": [ { "patternRole": "Comparator", "score": 0.8571, "distance": 2, "groundTruth": { "entity": "verify", "loc": "molar/backend/core/security.py:32" }, "recoveryResult": { "entity": "verify_password", "loc": "molar/backend/core/security.py:29" }, "note": "path-length (d=2)" }, { "patternRole": "Enforcer", "score": 1.0, "distance": 0, "groundTruth": { "entity": "login_access_token", "loc": "molar/backend/api/api_v1/endpoints/login.py:27" }, "recoveryResult": { "entity": "login_access_token", "loc": "molar/backend/api/api_v1/endpoints/login.py:29" }, "note": "path-length (d=0)" }, { "patternRole": "Hasher", "score": 0.8571, "distance": 2, "groundTruth": { "entity": "verify", "loc": "molar/backend/core/security.py:32" }, "recoveryResult": { "entity": "pwd_context", "loc": "molar/backend/core/security.py:10" }, "note": "path-length (d=2)" }, { "patternRole": "Password store", "score": 1.0, "distance": 0, "groundTruth": { "entity": "get_by_email", "loc": "molar/backend/crud/crud_user.py:15\u201316" }, "recoveryResult": { "entity": "get_by_email", "loc": "molar/backend/crud/crud_user.py:14" }, "note": "path-length (d=0)" }, { "patternRole": "Verification manager", "score": 1.0, "distance": 0, "groundTruth": { "entity": "authenticate", "loc": "molar/backend/crud/crud_user.py:55" }, "recoveryResult": { "entity": "authenticate", "loc": "molar/backend/crud/crud_user.py:68" }, "note": "path-length (d=0)" } ], "notes": [ "Encrypter: absent from the ground truth (Level 1 negative)", "Hash manager: absent from the ground truth (Level 1 negative)", "Password policy: absent from the ground truth (Level 1 negative)", "Pepper store: absent from the ground truth (Level 1 negative)", "Registrar: absent from the ground truth (Level 1 negative)", "Resetter: absent from the ground truth (Level 1 negative)", "SRNG: absent from the ground truth (Level 1 negative)" ] }, { "key": "level3", "level": "Level 3 \u2014 Interaction Order", "formula": "jaccard \u00d7 kendall-tau (shared interactions)", "available": true, "headlineLabel": "Interaction agreement", "score": 0.6, "drift": 0.4, "detail": { "agreedRoles": [ "Comparator", "Enforcer", "Hasher", "Password store", "Verification manager" ], "excludedRoles": [ "Encrypter", "Hash manager", "Password policy", "Pepper store", "Registrar", "Resetter", "SRNG" ], "interactionsGroundTruth": 4, "interactionsRecoveryResult": 4, "sharedInteractions": 3, "unionSize": 5, "entryRoleGroundTruth": "Enforcer", "entryRoleRecoveryResult": "Enforcer", "entryRuleGroundTruth": "behaviour", "entryRuleRecoveryResult": "stored", "onlyGroundTruth": 1, "onlyRecoveryResult": 1, "interactionOverlap": 0.6, "orderAgreement": 1.0, "rankedInteractions": 3, "totalPairs": 3, "concordantPairs": 3, "discordantPairs": 0, "kendallTau": 1.0, "prescribedActionSteps": 6, "prescribedStepsCovered": 4 }, "perRole": [ { "interaction": "Enforcer \u2192 Verification manager", "from": "Enforcer", "to": "Verification manager", "groundTruthRank": 1, "recoveryResultRank": 1, "outcome": "agreed", "discordantWith": 0, "evidence": { "groundTruth": { "file": "molar/backend/api/api_v1/endpoints/login.py", "line": 41, "via": [] }, "recoveryResult": { "file": "molar/backend/api/api_v1/endpoints/login.py", "line": 42, "via": [] } } }, { "interaction": "Verification manager \u2192 Password store", "from": "Verification manager", "to": "Password store", "groundTruthRank": 2, "recoveryResultRank": 2, "outcome": "agreed", "discordantWith": 0, "evidence": { "groundTruth": { "file": "molar/backend/crud/crud_user.py", "line": 58, "via": [] }, "recoveryResult": { "file": "molar/backend/crud/crud_user.py", "line": 70, "via": [] } } }, { "interaction": "Verification manager \u2192 Hasher", "from": "Verification manager", "to": "Hasher", "groundTruthRank": 3, "recoveryResultRank": null, "outcome": "onlyGroundTruth", "discordantWith": null, "evidence": { "groundTruth": { "file": "molar/backend/core/security.py", "line": 32, "via": [ "molar/backend/core/security.py:<module>.verify_password" ] }, "recoveryResult": null } }, { "interaction": "Verification manager \u2192 Comparator", "from": "Verification manager", "to": "Comparator", "groundTruthRank": 4, "recoveryResultRank": 3, "outcome": "agreed", "discordantWith": 0, "evidence": { "groundTruth": { "file": "molar/backend/core/security.py", "line": 32, "via": [ "molar/backend/core/security.py:<module>.verify_password" ] }, "recoveryResult": { "file": "molar/backend/crud/crud_user.py", "line": 73, "via": [] } } }, { "interaction": "Comparator \u2192 Hasher", "from": "Comparator", "to": "Hasher", "groundTruthRank": null, "recoveryResultRank": 4, "outcome": "onlyRecoveryResult", "discordantWith": null, "evidence": { "groundTruth": null, "recoveryResult": { "file": "molar/backend/core/security.py", "line": 30, "via": [] } } } ], "notes": [ "Score = overlap \u00d7 order = 0.6 \u00d7 1. Overlap: 3 of 5 interactions were found by both sides (1 only in the ground truth, 1 only in the recovery result), |shared| / |union|.", "Order: Kendall's Tau over the 3 shared interactions, in each side's own relative order \u2014 0 of 3 pairs run in opposite orders. Interactions only one side found take part in no pair (Fagin, Kumar & Sivakumar's p = 0, the induced sub-order); the overlap factor is what charges for them.", "Scored over the 5 roles both sides detected (Comparator, Enforcer, Hasher, Password store, Verification manager). An interaction counts only when both of its endpoint roles were detected on both sides \u2014 whether a role was found at all is Level 1's measurement, and charging for it again here would say the same thing twice.", "The interaction order is derived statically from the code \u2014 no one annotated it. Each role's cited entity is located in the AST index and control flow is followed from it, through as many non-role helper functions as it takes to reach the next role. See secpat_scoring/sequence/JOERN.md.", "Left out \u2014 only one side detected them: Encrypter, Hash manager, Password policy, Pepper store, Registrar, Resetter, SRNG. 7 roles were excluded from both walks, so neither side's ordering mentions them. The walk still passes through the code behind such a role, attributing what it finds there to the nearest agreed role.", "Ground truth: the walk started at Enforcer \u2014 the earliest role the reference pattern has speaking that this side grounded in walkable code.", "Recovery result: the walk started at Enforcer \u2014 stored.", "An entry point the pattern did not name means its prescribed speakers were either ungrounded on that side or not where this code begins \u2014 a role something else reaches cannot open the flow \u2014 so the call graph had to stand in. Treat the ordering below as weaker evidence: it starts wherever the code happens to start.", "Ground truth: Comparator, Hasher cite entities that are not symbols in this repository (a library or framework call). They can receive an interaction, but the walk cannot continue through them." ] } ], "ground_truth_source": "annotation", "vocabulary": [ "Enforcer", "Verification manager", "Password store", "Hasher", "Pepper store", "Encrypter", "Comparator" ], "role_scope": "required", "excluded_roles": [ "Subject", "System" ], "dropped_ground_truth_rows": [ "Hash manager", "Password policy", "Registrar", "Resetter", "SRNG", "Subject", "System" ], "dropped_recovery_rows": [ "Subject", "System" ], "gt_roles": 7, "rr_roles": 7, "struct_files": 77, "annotated_at": "2026-08-12 16:14:39.955000", "recovery_order": "stored" }, { "slug": "authentication_pwd__cxmjg_yaml-to-backend", "id": "pwd-based/yaml-to-backend/ClaudeCodeopus4.7", "pattern": "authentication_pwd", "repo": "cxmjg_yaml-to-backend", "error": null, "metrics": [ { "key": "level1", "level": "Level 1 \u2014 Role Detection", "formula": "agreement-rate (present/external/missing)", "available": true, "headlineLabel": "Roles agreed", "score": 0.8571428571428571, "drift": 0.1428571428571429, "detail": { "agreedRoles": 6, "totalRoles": 7 }, "perRole": [ { "patternRole": "Enforcer", "groundTruth": "present", "recoveryResult": "present", "groundTruthDetected": true, "recoveryResultDetected": true, "outcome": "agreed" }, { "patternRole": "Verification manager", "groundTruth": "present", "recoveryResult": "present", "groundTruthDetected": true, "recoveryResultDetected": true, "outcome": "agreed" }, { "patternRole": "Password store", "groundTruth": "present", "recoveryResult": "present", "groundTruthDetected": true, "recoveryResultDetected": true, "outcome": "agreed" }, { "patternRole": "Hasher", "groundTruth": "external", "recoveryResult": "external", "groundTruthDetected": true, "recoveryResultDetected": true, "outcome": "agreed" }, { "patternRole": "Pepper store", "groundTruth": "missing", "recoveryResult": "missing", "groundTruthDetected": false, "recoveryResultDetected": false, "outcome": "agreed" }, { "patternRole": "Encrypter", "groundTruth": "missing", "recoveryResult": "missing", "groundTruthDetected": false, "recoveryResultDetected": false, "outcome": "agreed" }, { "patternRole": "Comparator", "groundTruth": "external", "recoveryResult": "present", "groundTruthDetected": true, "recoveryResultDetected": true, "outcome": "mismatched" } ], "notes": [] }, { "key": "level2", "level": "Level 2 \u2014 Role Grounding", "formula": "path-length", "available": true, "headlineLabel": "Mean grounding score", "score": 0.725, "drift": 0.275, "detail": { "aggregate": 0.725, "scoredRoles": 5, "skippedRoles": 7, "treeHeight": 4 }, "perRole": [ { "patternRole": "Comparator", "score": 0.625, "distance": 3, "groundTruth": { "entity": "verify", "loc": "yaml_to_backend/security/auth.py:30" }, "recoveryResult": { "entity": "verify_password", "loc": "yaml_to_backend/security/auth.py:23" }, "note": "path-length (d=3)" }, { "patternRole": "Enforcer", "score": 1.0, "distance": 0, "groundTruth": { "entity": "login", "loc": "yaml_to_backend/api/auth_routes.py:34" }, "recoveryResult": { "entity": "login", "loc": "yaml_to_backend/api/auth_routes.py:35" }, "note": "path-length (d=0)" }, { "patternRole": "Hasher", "score": 0.75, "distance": 2, "groundTruth": { "entity": "verify", "loc": "yaml_to_backend/security/auth.py:30" }, "recoveryResult": { "entity": "pwd_context", "loc": "yaml_to_backend/security/auth.py:15" }, "note": "path-length (d=2)" }, { "patternRole": "Password store", "score": 0.25, "distance": 6, "groundTruth": { "entity": "session", "loc": "yaml_to_backend/security/auth.py:73" }, "recoveryResult": { "entity": "Usuario", "loc": "yaml_to_backend/db/models.py:101" }, "note": "path-length (d=6)" }, { "patternRole": "Verification manager", "score": 1.0, "distance": 0, "groundTruth": { "entity": "authenticate_user", "loc": "yaml_to_backend/security/auth.py:57" }, "recoveryResult": { "entity": "authenticate_user", "loc": "yaml_to_backend/security/auth.py:57" }, "note": "path-length (d=0)" } ], "notes": [ "Encrypter: absent from the ground truth (Level 1 negative)", "Hash manager: absent from the ground truth (Level 1 negative)", "Password policy: absent from the ground truth (Level 1 negative)", "Pepper store: absent from the ground truth (Level 1 negative)", "Registrar: absent from the ground truth (Level 1 negative)", "Resetter: absent from the ground truth (Level 1 negative)", "SRNG: absent from the ground truth (Level 1 negative)" ] }, { "key": "level3", "level": "Level 3 \u2014 Interaction Order", "formula": "jaccard \u00d7 kendall-tau (shared interactions)", "available": true, "headlineLabel": "Interaction agreement", "score": 0.6, "drift": 0.4, "detail": { "agreedRoles": [ "Comparator", "Enforcer", "Hasher", "Password store", "Verification manager" ], "excludedRoles": [ "Encrypter", "Hash manager", "Password policy", "Pepper store", "Registrar", "Resetter", "SRNG" ], "interactionsGroundTruth": 4, "interactionsRecoveryResult": 4, "sharedInteractions": 3, "unionSize": 5, "entryRoleGroundTruth": "Enforcer", "entryRoleRecoveryResult": "Enforcer", "entryRuleGroundTruth": "behaviour", "entryRuleRecoveryResult": "stored", "onlyGroundTruth": 1, "onlyRecoveryResult": 1, "interactionOverlap": 0.6, "orderAgreement": 1.0, "rankedInteractions": 3, "totalPairs": 3, "concordantPairs": 3, "discordantPairs": 0, "kendallTau": 1.0, "prescribedActionSteps": 6, "prescribedStepsCovered": 4 }, "perRole": [ { "interaction": "Enforcer \u2192 Verification manager", "from": "Enforcer", "to": "Verification manager", "groundTruthRank": 1, "recoveryResultRank": 1, "outcome": "agreed", "discordantWith": 0, "evidence": { "groundTruth": { "file": "yaml_to_backend/api/auth_routes.py", "line": 45, "via": [] }, "recoveryResult": { "file": "yaml_to_backend/api/auth_routes.py", "line": 50, "via": [] } } }, { "interaction": "Verification manager \u2192 Password store", "from": "Verification manager", "to": "Password store", "groundTruthRank": 2, "recoveryResultRank": 2, "outcome": "agreed", "discordantWith": 0, "evidence": { "groundTruth": { "file": "yaml_to_backend/security/auth.py", "line": 73, "via": [] }, "recoveryResult": { "file": "yaml_to_backend/security/auth.py", "line": 79, "via": [] } } }, { "interaction": "Verification manager \u2192 Hasher", "from": "Verification manager", "to": "Hasher", "groundTruthRank": 3, "recoveryResultRank": null, "outcome": "onlyGroundTruth", "discordantWith": null, "evidence": { "groundTruth": { "file": "yaml_to_backend/security/auth.py", "line": 30, "via": [ "yaml_to_backend/security/auth.py:<module>.AuthManager.verify_password" ] }, "recoveryResult": null } }, { "interaction": "Verification manager \u2192 Comparator", "from": "Verification manager", "to": "Comparator", "groundTruthRank": 4, "recoveryResultRank": 3, "outcome": "agreed", "discordantWith": 0, "evidence": { "groundTruth": { "file": "yaml_to_backend/security/auth.py", "line": 30, "via": [ "yaml_to_backend/security/auth.py:<module>.AuthManager.verify_password" ] }, "recoveryResult": { "file": "yaml_to_backend/security/auth.py", "line": 84, "via": [] } } }, { "interaction": "Comparator \u2192 Hasher", "from": "Comparator", "to": "Hasher", "groundTruthRank": null, "recoveryResultRank": 4, "outcome": "onlyRecoveryResult", "discordantWith": null, "evidence": { "groundTruth": null, "recoveryResult": { "file": "yaml_to_backend/security/auth.py", "line": 24, "via": [] } } } ], "notes": [ "Score = overlap \u00d7 order = 0.6 \u00d7 1. Overlap: 3 of 5 interactions were found by both sides (1 only in the ground truth, 1 only in the recovery result), |shared| / |union|.", "Order: Kendall's Tau over the 3 shared interactions, in each side's own relative order \u2014 0 of 3 pairs run in opposite orders. Interactions only one side found take part in no pair (Fagin, Kumar & Sivakumar's p = 0, the induced sub-order); the overlap factor is what charges for them.", "Scored over the 5 roles both sides detected (Comparator, Enforcer, Hasher, Password store, Verification manager). An interaction counts only when both of its endpoint roles were detected on both sides \u2014 whether a role was found at all is Level 1's measurement, and charging for it again here would say the same thing twice.", "The interaction order is derived statically from the code \u2014 no one annotated it. Each role's cited entity is located in the AST index and control flow is followed from it, through as many non-role helper functions as it takes to reach the next role. See secpat_scoring/sequence/JOERN.md.", "Left out \u2014 only one side detected them: Encrypter, Hash manager, Password policy, Pepper store, Registrar, Resetter, SRNG. 7 roles were excluded from both walks, so neither side's ordering mentions them. The walk still passes through the code behind such a role, attributing what it finds there to the nearest agreed role.", "Ground truth: the walk started at Enforcer \u2014 the earliest role the reference pattern has speaking that this side grounded in walkable code.", "Recovery result: the walk started at Enforcer \u2014 stored.", "An entry point the pattern did not name means its prescribed speakers were either ungrounded on that side or not where this code begins \u2014 a role something else reaches cannot open the flow \u2014 so the call graph had to stand in. Treat the ordering below as weaker evidence: it starts wherever the code happens to start.", "Ground truth: Comparator, Hasher cite entities that are not symbols in this repository (a library or framework call). They can receive an interaction, but the walk cannot continue through them." ] } ], "ground_truth_source": "annotation", "vocabulary": [ "Enforcer", "Verification manager", "Password store", "Hasher", "Pepper store", "Encrypter", "Comparator" ], "role_scope": "required", "excluded_roles": [ "Subject", "System" ], "dropped_ground_truth_rows": [ "Hash manager", "Password policy", "Registrar", "Resetter", "SRNG", "Subject", "System" ], "dropped_recovery_rows": [ "Subject", "System" ], "gt_roles": 7, "rr_roles": 7, "struct_files": 16, "annotated_at": "2026-08-12 16:55:50.859000", "recovery_order": "stored" }, { "slug": "authentication_pwd__Aquiles-ai_Aquiles-RAG", "id": "pwd-based/aquiles-rag/ClaudeCodeopus4.7", "pattern": "authentication_pwd", "repo": "Aquiles-ai_Aquiles-RAG", "error": null, "metrics": [ { "key": "level1", "level": "Level 1 \u2014 Role Detection", "formula": "agreement-rate (present/external/missing)", "available": true, "headlineLabel": "Roles agreed", "score": 1.0, "drift": 0.0, "detail": { "agreedRoles": 7, "totalRoles": 7 }, "perRole": [ { "patternRole": "Enforcer", "groundTruth": "present", "recoveryResult": "present", "groundTruthDetected": true, "recoveryResultDetected": true, "outcome": "agreed" }, { "patternRole": "Verification manager", "groundTruth": "present", "recoveryResult": "present", "groundTruthDetected": true, "recoveryResultDetected": true, "outcome": "agreed" }, { "patternRole": "Password store", "groundTruth": "present", "recoveryResult": "present", "groundTruthDetected": true, "recoveryResultDetected": true, "outcome": "agreed" }, { "patternRole": "Hasher", "groundTruth": "missing", "recoveryResult": "missing", "groundTruthDetected": false, "recoveryResultDetected": false, "outcome": "agreed" }, { "patternRole": "Pepper store", "groundTruth": "missing", "recoveryResult": "missing", "groundTruthDetected": false, "recoveryResultDetected": false, "outcome": "agreed" }, { "patternRole": "Encrypter", "groundTruth": "missing", "recoveryResult": "missing", "groundTruthDetected": false, "recoveryResultDetected": false, "outcome": "agreed" }, { "patternRole": "Comparator", "groundTruth": "present", "recoveryResult": "present", "groundTruthDetected": true, "recoveryResultDetected": true, "outcome": "agreed" } ], "notes": [] }, { "key": "level2", "level": "Level 2 \u2014 Role Grounding", "formula": "path-length", "available": true, "headlineLabel": "Mean grounding score", "score": 0.8, "drift": 0.2, "detail": { "aggregate": 0.8, "scoredRoles": 3, "skippedRoles": 9, "treeHeight": 5 }, "perRole": [ { "patternRole": "Enforcer", "score": 1.0, "distance": 0, "groundTruth": { "entity": "login_for_access_token", "loc": "aquiles/main.py:309" }, "recoveryResult": { "entity": "login_for_access_token", "loc": "aquiles/main.py:292" }, "note": "path-length (d=0)" }, { "patternRole": "Password store", "score": 0.4, "distance": 6, "groundTruth": { "entity": "users_db", "loc": "aquiles/auth/jwt_aq.py:29" }, "recoveryResult": { "entity": "AllowedUser", "loc": "aquiles/configs/configs.py:18" }, "note": "path-length (d=6)" }, { "patternRole": "Verification manager", "score": 1.0, "distance": 0, "groundTruth": { "entity": "authenticate_user", "loc": "aquiles/auth/jwt_aq.py:27" }, "recoveryResult": { "entity": "authenticate_user", "loc": "aquiles/auth/jwt_aq.py:27" }, "note": "path-length (d=0)" } ], "notes": [ "Comparator: ground truth cites no entity", "Encrypter: absent from the ground truth (Level 1 negative)", "Hash manager: absent from the ground truth (Level 1 negative)", "Hasher: absent from the ground truth (Level 1 negative)", "Password policy: absent from the ground truth (Level 1 negative)", "Pepper store: absent from the ground truth (Level 1 negative)", "Registrar: absent from the ground truth (Level 1 negative)", "Resetter: absent from the ground truth (Level 1 negative)", "SRNG: absent from the ground truth (Level 1 negative)" ] }, { "key": "level3", "level": "Level 3 \u2014 Interaction Order", "formula": "jaccard \u00d7 kendall-tau (shared interactions)", "available": true, "headlineLabel": "Interaction agreement", "score": 1.0, "drift": 0.0, "detail": { "agreedRoles": [ "Enforcer", "Password store", "Verification manager" ], "excludedRoles": [ "Comparator", "Encrypter", "Hash manager", "Hasher", "Password policy", "Pepper store", "Registrar", "Resetter", "SRNG" ], "interactionsGroundTruth": 2, "interactionsRecoveryResult": 2, "sharedInteractions": 2, "unionSize": 2, "entryRoleGroundTruth": "Enforcer", "entryRoleRecoveryResult": "Enforcer", "entryRuleGroundTruth": "behaviour", "entryRuleRecoveryResult": "stored", "onlyGroundTruth": 0, "onlyRecoveryResult": 0, "interactionOverlap": 1.0, "orderAgreement": 1.0, "rankedInteractions": 2, "totalPairs": 1, "concordantPairs": 1, "discordantPairs": 0, "kendallTau": 1.0, "prescribedActionSteps": 6, "prescribedStepsCovered": 2 }, "perRole": [ { "interaction": "Enforcer \u2192 Verification manager", "from": "Enforcer", "to": "Verification manager", "groundTruthRank": 1, "recoveryResultRank": 1, "outcome": "agreed", "discordantWith": 0, "evidence": { "groundTruth": { "file": "aquiles/main.py", "line": 310, "via": [] }, "recoveryResult": { "file": "aquiles/main.py", "line": 293, "via": [] } } }, { "interaction": "Verification manager \u2192 Password store", "from": "Verification manager", "to": "Password store", "groundTruthRank": 2, "recoveryResultRank": 2, "outcome": "agreed", "discordantWith": 0, "evidence": { "groundTruth": { "file": "aquiles/auth/jwt_aq.py", "line": 29, "via": [] }, "recoveryResult": { "file": "aquiles/auth/jwt_aq.py", "line": 29, "via": [] } } } ], "notes": [ "Score = overlap \u00d7 order = 1 \u00d7 1. Overlap: 2 of 2 interactions were found by both sides (0 only in the ground truth, 0 only in the recovery result), |shared| / |union|.", "Order: Kendall's Tau over the 2 shared interactions, in each side's own relative order \u2014 0 of 1 pairs run in opposite orders. Interactions only one side found take part in no pair (Fagin, Kumar & Sivakumar's p = 0, the induced sub-order); the overlap factor is what charges for them.", "Only 1 ranked pair \u2014 the order factor can land on just a few values here and one disagreement swings it fully.", "Scored over the 3 roles both sides detected (Enforcer, Password store, Verification manager). An interaction counts only when both of its endpoint roles were detected on both sides \u2014 whether a role was found at all is Level 1's measurement, and charging for it again here would say the same thing twice.", "The interaction order is derived statically from the code \u2014 no one annotated it. Each role's cited entity is located in the AST index and control flow is followed from it, through as many non-role helper functions as it takes to reach the next role. See secpat_scoring/sequence/JOERN.md.", "Left out \u2014 only one side detected them: Comparator, Encrypter, Hash manager, Hasher, Password policy, Pepper store, Registrar, Resetter, SRNG. 9 roles were excluded from both walks, so neither side's ordering mentions them. The walk still passes through the code behind such a role, attributing what it finds there to the nearest agreed role.", "Ground truth: the walk started at Enforcer \u2014 the earliest role the reference pattern has speaking that this side grounded in walkable code.", "Recovery result: the walk started at Enforcer \u2014 stored.", "An entry point the pattern did not name means its prescribed speakers were either ungrounded on that side or not where this code begins \u2014 a role something else reaches cannot open the flow \u2014 so the call graph had to stand in. Treat the ordering below as weaker evidence: it starts wherever the code happens to start." ] } ], "ground_truth_source": "annotation", "vocabulary": [ "Enforcer", "Verification manager", "Password store", "Hasher", "Pepper store", "Encrypter", "Comparator" ], "role_scope": "required", "excluded_roles": [ "Subject", "System" ], "dropped_ground_truth_rows": [ "Hash manager", "Password policy", "Registrar", "Resetter", "SRNG", "Subject", "System" ], "dropped_recovery_rows": [ "Subject", "System" ], "gt_roles": 7, "rr_roles": 7, "struct_files": 46, "annotated_at": "2026-08-05 09:22:18.029000", "recovery_order": "stored" }, { "slug": "authentication_pwd__AmberLee2427_nancy-brain", "id": "pwd-based/nancy-brain/ClaudeCodeopus4.7", "pattern": "authentication_pwd", "repo": "AmberLee2427_nancy-brain", "error": null, "metrics": [ { "key": "level1", "level": "Level 1 \u2014 Role Detection", "formula": "agreement-rate (present/external/missing)", "available": true, "headlineLabel": "Roles agreed", "score": 1.0, "drift": 0.0, "detail": { "agreedRoles": 7, "totalRoles": 7 }, "perRole": [ { "patternRole": "Enforcer", "groundTruth": "present", "recoveryResult": "present", "groundTruthDetected": true, "recoveryResultDetected": true, "outcome": "agreed" }, { "patternRole": "Verification manager", "groundTruth": "present", "recoveryResult": "present", "groundTruthDetected": true, "recoveryResultDetected": true, "outcome": "agreed" }, { "patternRole": "Password store", "groundTruth": "present", "recoveryResult": "present", "groundTruthDetected": true, "recoveryResultDetected": true, "outcome": "agreed" }, { "patternRole": "Hasher", "groundTruth": "external", "recoveryResult": "external", "groundTruthDetected": true, "recoveryResultDetected": true, "outcome": "agreed" }, { "patternRole": "Pepper store", "groundTruth": "missing", "recoveryResult": "missing", "groundTruthDetected": false, "recoveryResultDetected": false, "outcome": "agreed" }, { "patternRole": "Encrypter", "groundTruth": "missing", "recoveryResult": "missing", "groundTruthDetected": false, "recoveryResultDetected": false, "outcome": "agreed" }, { "patternRole": "Comparator", "groundTruth": "external", "recoveryResult": "external", "groundTruthDetected": true, "recoveryResultDetected": true, "outcome": "agreed" } ], "notes": [] }, { "key": "level2", "level": "Level 2 \u2014 Role Grounding", "formula": "path-length", "available": true, "headlineLabel": "Mean grounding score", "score": 0.92, "drift": 0.08, "detail": { "aggregate": 0.92, "scoredRoles": 5, "skippedRoles": 7, "treeHeight": 5 }, "perRole": [ { "patternRole": "Comparator", "score": 0.8, "distance": 2, "groundTruth": { "entity": "verify", "loc": "connectors/http_api/auth.py:221" }, "recoveryResult": { "entity": "verify_password", "loc": "connectors/http_api/auth.py:313" }, "note": "path-length (d=2)" }, { "patternRole": "Enforcer", "score": 1.0, "distance": 0, "groundTruth": { "entity": "login", "loc": "connectors/http_api/app.py:323" }, "recoveryResult": { "entity": "login", "loc": "connectors/http_api/app.py:284" }, "note": "path-length (d=0)" }, { "patternRole": "Hasher", "score": 0.8, "distance": 2, "groundTruth": { "entity": "verify", "loc": "connectors/http_api/auth.py:221" }, "recoveryResult": { "entity": "pwd_context", "loc": "connectors/http_api/auth.py:20" }, "note": "path-length (d=2)" }, { "patternRole": "Password store", "score": 1.0, "distance": 0, "groundTruth": { "entity": "get_user", "loc": "connectors/http_api/auth.py:210" }, "recoveryResult": { "entity": "get_user", "loc": "connectors/http_api/auth.py:305" }, "note": "path-length (d=0)" }, { "patternRole": "Verification manager", "score": 1.0, "distance": 0, "groundTruth": { "entity": "authenticate_user", "loc": "connectors/http_api/auth.py:226" }, "recoveryResult": { "entity": "authenticate_user", "loc": "connectors/http_api/auth.py:317" }, "note": "path-length (d=0)" } ], "notes": [ "Encrypter: absent from the ground truth (Level 1 negative)", "Hash manager: absent from the ground truth (Level 1 negative)", "Password policy: absent from the ground truth (Level 1 negative)", "Pepper store: absent from the ground truth (Level 1 negative)", "Registrar: absent from the ground truth (Level 1 negative)", "Resetter: absent from the ground truth (Level 1 negative)", "SRNG: absent from the ground truth (Level 1 negative)" ] }, { "key": "level3", "level": "Level 3 \u2014 Interaction Order", "formula": "jaccard \u00d7 kendall-tau (shared interactions)", "available": true, "headlineLabel": "Interaction agreement", "score": 0.5, "drift": 0.5, "detail": { "agreedRoles": [ "Comparator", "Enforcer", "Hasher", "Password store", "Verification manager" ], "excludedRoles": [ "Encrypter", "Hash manager", "Password policy", "Pepper store", "Registrar", "Resetter", "SRNG" ], "interactionsGroundTruth": 2, "interactionsRecoveryResult": 4, "sharedInteractions": 2, "unionSize": 4, "entryRoleGroundTruth": "Enforcer", "entryRoleRecoveryResult": "Enforcer", "entryRuleGroundTruth": "behaviour", "entryRuleRecoveryResult": "stored", "onlyGroundTruth": 0, "onlyRecoveryResult": 2, "interactionOverlap": 0.5, "orderAgreement": 1.0, "rankedInteractions": 2, "totalPairs": 1, "concordantPairs": 1, "discordantPairs": 0, "kendallTau": 1.0, "prescribedActionSteps": 6, "prescribedStepsCovered": 4 }, "perRole": [ { "interaction": "Enforcer \u2192 Verification manager", "from": "Enforcer", "to": "Verification manager", "groundTruthRank": 1, "recoveryResultRank": 1, "outcome": "agreed", "discordantWith": 0, "evidence": { "groundTruth": { "file": "connectors/http_api/app.py", "line": 324, "via": [] }, "recoveryResult": { "file": "connectors/http_api/app.py", "line": 286, "via": [] } } }, { "interaction": "Verification manager \u2192 Password store", "from": "Verification manager", "to": "Password store", "groundTruthRank": 2, "recoveryResultRank": 2, "outcome": "agreed", "discordantWith": 0, "evidence": { "groundTruth": { "file": "connectors/http_api/auth.py", "line": 316, "via": [] }, "recoveryResult": { "file": "connectors/http_api/auth.py", "line": 318, "via": [] } } }, { "interaction": "Verification manager \u2192 Hasher", "from": "Verification manager", "to": "Hasher", "groundTruthRank": null, "recoveryResultRank": 3, "outcome": "onlyRecoveryResult", "discordantWith": null, "evidence": { "groundTruth": null, "recoveryResult": { "file": "connectors/http_api/auth.py", "line": 321, "via": [] } } }, { "interaction": "Verification manager \u2192 Comparator", "from": "Verification manager", "to": "Comparator", "groundTruthRank": null, "recoveryResultRank": 4, "outcome": "onlyRecoveryResult", "discordantWith": null, "evidence": { "groundTruth": null, "recoveryResult": { "file": "connectors/http_api/auth.py", "line": 315, "via": [] } } } ], "notes": [ "Score = overlap \u00d7 order = 0.5 \u00d7 1. Overlap: 2 of 4 interactions were found by both sides (0 only in the ground truth, 2 only in the recovery result), |shared| / |union|.", "Order: Kendall's Tau over the 2 shared interactions, in each side's own relative order \u2014 0 of 1 pairs run in opposite orders. Interactions only one side found take part in no pair (Fagin, Kumar & Sivakumar's p = 0, the induced sub-order); the overlap factor is what charges for them.", "Only 1 ranked pair \u2014 the order factor can land on just a few values here and one disagreement swings it fully.", "Scored over the 5 roles both sides detected (Comparator, Enforcer, Hasher, Password store, Verification manager). An interaction counts only when both of its endpoint roles were detected on both sides \u2014 whether a role was found at all is Level 1's measurement, and charging for it again here would say the same thing twice.", "The interaction order is derived statically from the code \u2014 no one annotated it. Each role's cited entity is located in the AST index and control flow is followed from it, through as many non-role helper functions as it takes to reach the next role. See secpat_scoring/sequence/JOERN.md.", "Left out \u2014 only one side detected them: Encrypter, Hash manager, Password policy, Pepper store, Registrar, Resetter, SRNG. 7 roles were excluded from both walks, so neither side's ordering mentions them. The walk still passes through the code behind such a role, attributing what it finds there to the nearest agreed role.", "Ground truth: the walk started at Enforcer \u2014 the earliest role the reference pattern has speaking that this side grounded in walkable code.", "Recovery result: the walk started at Enforcer \u2014 stored.", "An entry point the pattern did not name means its prescribed speakers were either ungrounded on that side or not where this code begins \u2014 a role something else reaches cannot open the flow \u2014 so the call graph had to stand in. Treat the ordering below as weaker evidence: it starts wherever the code happens to start.", "Ground truth: Comparator, Hasher cite entities that are not symbols in this repository (a library or framework call). They can receive an interaction, but the walk cannot continue through them." ] } ], "ground_truth_source": "annotation", "vocabulary": [ "Enforcer", "Verification manager", "Password store", "Hasher", "Pepper store", "Encrypter", "Comparator" ], "role_scope": "required", "excluded_roles": [ "Subject", "System" ], "dropped_ground_truth_rows": [ "Hash manager", "Password policy", "Registrar", "Resetter", "SRNG", "Subject", "System" ], "dropped_recovery_rows": [ "Subject", "System" ], "gt_roles": 7, "rr_roles": 7, "struct_files": 86, "annotated_at": "2026-08-12 16:16:03.607000", "recovery_order": "stored" }, { "slug": "authentication_pwd__JBEI_dnada", "id": "pwd-based/dnada/ClaudeCodeopus4.7", "pattern": "authentication_pwd", "repo": "JBEI_dnada", "error": null, "metrics": [ { "key": "level1", "level": "Level 1 \u2014 Role Detection", "formula": "agreement-rate (present/external/missing)", "available": true, "headlineLabel": "Roles agreed", "score": 0.7142857142857143, "drift": 0.2857142857142857, "detail": { "agreedRoles": 5, "totalRoles": 7 }, "perRole": [ { "patternRole": "Enforcer", "groundTruth": "present", "recoveryResult": "present", "groundTruthDetected": true, "recoveryResultDetected": true, "outcome": "agreed" }, { "patternRole": "Verification manager", "groundTruth": "present", "recoveryResult": "present", "groundTruthDetected": true, "recoveryResultDetected": true, "outcome": "agreed" }, { "patternRole": "Password store", "groundTruth": "present", "recoveryResult": "present", "groundTruthDetected": true, "recoveryResultDetected": true, "outcome": "agreed" }, { "patternRole": "Hasher", "groundTruth": "external", "recoveryResult": "present", "groundTruthDetected": true, "recoveryResultDetected": true, "outcome": "mismatched" }, { "patternRole": "Pepper store", "groundTruth": "missing", "recoveryResult": "missing", "groundTruthDetected": false, "recoveryResultDetected": false, "outcome": "agreed" }, { "patternRole": "Encrypter", "groundTruth": "missing", "recoveryResult": "missing", "groundTruthDetected": false, "recoveryResultDetected": false, "outcome": "agreed" }, { "patternRole": "Comparator", "groundTruth": "external", "recoveryResult": "present", "groundTruthDetected": true, "recoveryResultDetected": true, "outcome": "mismatched" } ], "notes": [] }, { "key": "level2", "level": "Level 2 \u2014 Role Grounding", "formula": "path-length", "available": true, "headlineLabel": "Mean grounding score", "score": 0.7571, "drift": 0.2429, "detail": { "aggregate": 0.7571, "scoredRoles": 5, "skippedRoles": 7, "treeHeight": 7 }, "perRole": [ { "patternRole": "Comparator", "score": 0.8571, "distance": 2, "groundTruth": { "entity": "verify", "loc": "backend/app/app/core/security.py:30" }, "recoveryResult": { "entity": "verify_password", "loc": "backend/app/app/core/security.py:27" }, "note": "path-length (d=2)" }, { "patternRole": "Enforcer", "score": 0.5714, "distance": 6, "groundTruth": { "entity": "login_access_token", "loc": "backend/app/app/api/api_v1/endpoints/login.py:22\u201343" }, "recoveryResult": { "entity": "get_current_user", "loc": "backend/app/app/api/deps.py:21" }, "note": "path-length (d=6)" }, { "patternRole": "Hasher", "score": 0.8571, "distance": 2, "groundTruth": { "entity": "verify", "loc": "backend/app/app/core/security.py:30" }, "recoveryResult": { "entity": "pwd_context", "loc": "backend/app/app/core/security.py:9" }, "note": "path-length (d=2)" }, { "patternRole": "Password store", "score": 0.5, "distance": 7, "groundTruth": { "entity": "get_by_email", "loc": "backend/app/app/crud/crud_user.py:15\u201316" }, "recoveryResult": { "entity": "User", "loc": "backend/app/app/models/user.py:6" }, "note": "path-length (d=7)" }, { "patternRole": "Verification manager", "score": 1.0, "distance": 0, "groundTruth": { "entity": "authenticate", "loc": "backend/app/app/crud/crud_user.py:30" }, "recoveryResult": { "entity": "authenticate", "loc": "backend/app/app/crud/crud_user.py:35" }, "note": "path-length (d=0)" } ], "notes": [ "Encrypter: absent from the ground truth (Level 1 negative)", "Hash manager: absent from the ground truth (Level 1 negative)", "Password policy: absent from the ground truth (Level 1 negative)", "Pepper store: absent from the ground truth (Level 1 negative)", "Registrar: absent from the ground truth (Level 1 negative)", "Resetter: absent from the ground truth (Level 1 negative)", "SRNG: absent from the ground truth (Level 1 negative)" ] }, { "key": "level3", "level": "Level 3 \u2014 Interaction Order", "formula": "jaccard \u00d7 kendall-tau (shared interactions)", "available": true, "headlineLabel": "Interaction agreement", "score": 0.6, "drift": 0.4, "detail": { "agreedRoles": [ "Comparator", "Enforcer", "Hasher", "Password store", "Verification manager" ], "excludedRoles": [ "Encrypter", "Hash manager", "Password policy", "Pepper store", "Registrar", "Resetter", "SRNG" ], "interactionsGroundTruth": 4, "interactionsRecoveryResult": 4, "sharedInteractions": 3, "unionSize": 5, "entryRoleGroundTruth": "Enforcer", "entryRoleRecoveryResult": "Enforcer", "entryRuleGroundTruth": "behaviour", "entryRuleRecoveryResult": "stored", "onlyGroundTruth": 1, "onlyRecoveryResult": 1, "interactionOverlap": 0.6, "orderAgreement": 1.0, "rankedInteractions": 3, "totalPairs": 3, "concordantPairs": 3, "discordantPairs": 0, "kendallTau": 1.0, "prescribedActionSteps": 6, "prescribedStepsCovered": 4 }, "perRole": [ { "interaction": "Enforcer \u2192 Verification manager", "from": "Enforcer", "to": "Verification manager", "groundTruthRank": 1, "recoveryResultRank": 1, "outcome": "agreed", "discordantWith": 0, "evidence": { "groundTruth": { "file": "backend/app/app/api/api_v1/endpoints/login.py", "line": 30, "via": [] }, "recoveryResult": { "file": "backend/app/app/api/api_v1/endpoints/login.py", "line": 29, "via": [] } } }, { "interaction": "Verification manager \u2192 Password store", "from": "Verification manager", "to": "Password store", "groundTruthRank": 2, "recoveryResultRank": 2, "outcome": "agreed", "discordantWith": 0, "evidence": { "groundTruth": { "file": "backend/app/app/crud/crud_user.py", "line": 31, "via": [] }, "recoveryResult": { "file": "backend/app/app/crud/crud_user.py", "line": 36, "via": [] } } }, { "interaction": "Verification manager \u2192 Hasher", "from": "Verification manager", "to": "Hasher", "groundTruthRank": 3, "recoveryResultRank": null, "outcome": "onlyGroundTruth", "discordantWith": null, "evidence": { "groundTruth": { "file": "backend/app/app/core/security.py", "line": 30, "via": [ "backend/app/app/core/security.py:<module>.verify_password" ] }, "recoveryResult": null } }, { "interaction": "Verification manager \u2192 Comparator", "from": "Verification manager", "to": "Comparator", "groundTruthRank": 4, "recoveryResultRank": 3, "outcome": "agreed", "discordantWith": 0, "evidence": { "groundTruth": { "file": "backend/app/app/core/security.py", "line": 30, "via": [ "backend/app/app/core/security.py:<module>.verify_password" ] }, "recoveryResult": { "file": "backend/app/app/crud/crud_user.py", "line": 38, "via": [] } } }, { "interaction": "Comparator \u2192 Hasher", "from": "Comparator", "to": "Hasher", "groundTruthRank": null, "recoveryResultRank": 4, "outcome": "onlyRecoveryResult", "discordantWith": null, "evidence": { "groundTruth": null, "recoveryResult": { "file": "backend/app/app/core/security.py", "line": 28, "via": [] } } } ], "notes": [ "Score = overlap \u00d7 order = 0.6 \u00d7 1. Overlap: 3 of 5 interactions were found by both sides (1 only in the ground truth, 1 only in the recovery result), |shared| / |union|.", "Order: Kendall's Tau over the 3 shared interactions, in each side's own relative order \u2014 0 of 3 pairs run in opposite orders. Interactions only one side found take part in no pair (Fagin, Kumar & Sivakumar's p = 0, the induced sub-order); the overlap factor is what charges for them.", "Scored over the 5 roles both sides detected (Comparator, Enforcer, Hasher, Password store, Verification manager). An interaction counts only when both of its endpoint roles were detected on both sides \u2014 whether a role was found at all is Level 1's measurement, and charging for it again here would say the same thing twice.", "The interaction order is derived statically from the code \u2014 no one annotated it. Each role's cited entity is located in the AST index and control flow is followed from it, through as many non-role helper functions as it takes to reach the next role. See secpat_scoring/sequence/JOERN.md.", "Left out \u2014 only one side detected them: Encrypter, Hash manager, Password policy, Pepper store, Registrar, Resetter, SRNG. 7 roles were excluded from both walks, so neither side's ordering mentions them. The walk still passes through the code behind such a role, attributing what it finds there to the nearest agreed role.", "Ground truth: the walk started at Enforcer \u2014 the earliest role the reference pattern has speaking that this side grounded in walkable code.", "Recovery result: the walk started at Enforcer \u2014 stored.", "An entry point the pattern did not name means its prescribed speakers were either ungrounded on that side or not where this code begins \u2014 a role something else reaches cannot open the flow \u2014 so the call graph had to stand in. Treat the ordering below as weaker evidence: it starts wherever the code happens to start.", "Ground truth: Comparator, Hasher cite entities that are not symbols in this repository (a library or framework call). They can receive an interaction, but the walk cannot continue through them." ] } ], "ground_truth_source": "annotation", "vocabulary": [ "Enforcer", "Verification manager", "Password store", "Hasher", "Pepper store", "Encrypter", "Comparator" ], "role_scope": "required", "excluded_roles": [ "Subject", "System" ], "dropped_ground_truth_rows": [ "Hash manager", "Password policy", "Registrar", "Resetter", "SRNG", "Subject", "System" ], "dropped_recovery_rows": [ "Subject", "System" ], "gt_roles": 7, "rr_roles": 7, "struct_files": 208, "annotated_at": "2026-08-13 09:17:51.425000", "recovery_order": "stored" }, { "slug": "authentication_pwd__vnpy_vnpy_webtrader", "id": "pwd-based/vnpy_webtrader/ClaudeCodeOpus5.5", "pattern": "authentication_pwd", "repo": "vnpy_vnpy_webtrader", "error": null, "metrics": [ { "key": "level1", "level": "Level 1 \u2014 Role Detection", "formula": "agreement-rate (present/external/missing)", "available": true, "headlineLabel": "Roles agreed", "score": 0.7143, "drift": 0.2857, "detail": { "agreedRoles": 5, "totalRoles": 7 }, "perRole": [ { "patternRole": "Enforcer", "groundTruth": "present", "recoveryResult": "present", "groundTruthDetected": true, "recoveryResultDetected": true, "outcome": "agreed" }, { "patternRole": "Verification manager", "groundTruth": "present", "recoveryResult": "present", "groundTruthDetected": true, "recoveryResultDetected": true, "outcome": "agreed" }, { "patternRole": "Password store", "groundTruth": "present", "recoveryResult": "present", "groundTruthDetected": true, "recoveryResultDetected": true, "outcome": "agreed" }, { "patternRole": "Hasher", "groundTruth": "external", "recoveryResult": "external", "groundTruthDetected": true, "recoveryResultDetected": true, "outcome": "agreed" }, { "patternRole": "Pepper store", "groundTruth": "missing", "recoveryResult": "missing", "groundTruthDetected": false, "recoveryResultDetected": false, "outcome": "agreed" }, { "patternRole": "Encrypter", "groundTruth": "missing", "recoveryResult": "external", "groundTruthDetected": false, "recoveryResultDetected": true, "outcome": "mismatched" }, { "patternRole": "Comparator", "groundTruth": "external", "recoveryResult": "present", "groundTruthDetected": true, "recoveryResultDetected": true, "outcome": "mismatched" } ], "notes": [] }, { "key": "level2", "level": "Level 2 \u2014 Role Grounding", "formula": "path-length", "available": true, "headlineLabel": "Mean grounding score", "score": 0.84, "drift": 0.16, "detail": { "aggregate": 0.84, "scoredRoles": 5, "skippedRoles": 2, "treeHeight": 5 }, "perRole": [ { "patternRole": "Comparator", "score": 0.8, "distance": 2, "groundTruth": { "entity": "verify", "loc": "vnpy_webtrader/web.py:89" }, "recoveryResult": { "entity": "compare_digest", "loc": "vnpy_webtrader/web.py:165" }, "note": "path-length (d=2)" }, { "patternRole": "Enforcer", "score": 0.8, "distance": 2, "groundTruth": { "entity": "login", "loc": "vnpy_webtrader/web.py:146" }, "recoveryResult": { "entity": "get_access", "loc": "vnpy_webtrader/web.py:177" }, "note": "path-length (d=2)" }, { "patternRole": "Hasher", "score": 0.8, "distance": 2, "groundTruth": { "entity": "verify", "loc": "vnpy_webtrader/web.py:89" }, "recoveryResult": { "entity": "pwd_context", "loc": "vnpy_webtrader/web.py:69" }, "note": "path-length (d=2)" }, { "patternRole": "Password store", "score": 0.8, "distance": 2, "groundTruth": { "entity": "PASSWORD", "loc": "vnpy_webtrader/web.py:43" }, "recoveryResult": { "entity": "setting", "loc": "vnpy_webtrader/web.py:50" }, "note": "path-length (d=2)" }, { "patternRole": "Verification manager", "score": 1.0, "distance": 0, "groundTruth": { "entity": "authenticate_user", "loc": "vnpy_webtrader/web.py:82" }, "recoveryResult": { "entity": "authenticate_user", "loc": "vnpy_webtrader/web.py:158" }, "note": "path-length (d=0)" } ], "notes": [ "Encrypter: absent from the ground truth (Level 1 negative)", "Pepper store: absent from the ground truth (Level 1 negative)" ] }, { "key": "level3", "level": "Level 3 \u2014 Interaction Order", "formula": "jaccard \u00d7 kendall-tau (shared interactions)", "available": true, "headlineLabel": "Interaction agreement", "score": 0.75, "drift": 0.25, "detail": { "agreedRoles": [ "Comparator", "Enforcer", "Hasher", "Password store", "Verification manager" ], "excludedRoles": [ "Encrypter", "Pepper store" ], "interactionsGroundTruth": 3, "interactionsRecoveryResult": 4, "sharedInteractions": 3, "unionSize": 4, "entryRoleGroundTruth": "Enforcer", "entryRoleRecoveryResult": "Enforcer", "entryRuleGroundTruth": "behaviour", "entryRuleRecoveryResult": "stored", "onlyGroundTruth": 0, "onlyRecoveryResult": 1, "interactionOverlap": 0.75, "orderAgreement": 1.0, "rankedInteractions": 3, "totalPairs": 3, "concordantPairs": 3, "discordantPairs": 0, "kendallTau": 1.0, "prescribedActionSteps": 6, "prescribedStepsCovered": 4 }, "perRole": [ { "interaction": "Enforcer \u2192 Verification manager", "from": "Enforcer", "to": "Verification manager", "groundTruthRank": 1, "recoveryResultRank": 1, "outcome": "agreed", "discordantWith": 0, "evidence": { "groundTruth": { "file": "vnpy_webtrader/web.py", "line": 148, "via": [] }, "recoveryResult": { "file": "vnpy_webtrader/web.py", "line": 242, "via": [] } } }, { "interaction": "Verification manager \u2192 Hasher", "from": "Verification manager", "to": "Hasher", "groundTruthRank": 2, "recoveryResultRank": 3, "outcome": "agreed", "discordantWith": 0, "evidence": { "groundTruth": { "file": "vnpy_webtrader/web.py", "line": 89, "via": [] }, "recoveryResult": { "file": "vnpy_webtrader/web.py", "line": 160, "via": [] } } }, { "interaction": "Verification manager \u2192 Comparator", "from": "Verification manager", "to": "Comparator", "groundTruthRank": 3, "recoveryResultRank": 4, "outcome": "agreed", "discordantWith": 0, "evidence": { "groundTruth": { "file": "vnpy_webtrader/web.py", "line": 89, "via": [] }, "recoveryResult": { "file": "vnpy_webtrader/web.py", "line": 163, "via": [] } } }, { "interaction": "Verification manager \u2192 Password store", "from": "Verification manager", "to": "Password store", "groundTruthRank": null, "recoveryResultRank": 2, "outcome": "onlyRecoveryResult", "discordantWith": null, "evidence": { "groundTruth": null, "recoveryResult": { "file": "vnpy_webtrader/web.py", "line": 50, "via": [] } } } ], "notes": [ "Score = overlap \u00d7 order = 0.75 \u00d7 1. Overlap: 3 of 4 interactions were found by both sides (0 only in the ground truth, 1 only in the recovery result), |shared| / |union|.", "Order: Kendall's Tau over the 3 shared interactions, in each side's own relative order \u2014 0 of 3 pairs run in opposite orders. Interactions only one side found take part in no pair (Fagin, Kumar & Sivakumar's p = 0, the induced sub-order); the overlap factor is what charges for them.", "Scored over the 5 roles both sides detected (Comparator, Enforcer, Hasher, Password store, Verification manager). An interaction counts only when both of its endpoint roles were detected on both sides \u2014 whether a role was found at all is Level 1's measurement, and charging for it again here would say the same thing twice.", "The interaction order is derived statically from the code \u2014 no one annotated it. Each role's cited entity is located in the AST index and control flow is followed from it, through as many non-role helper functions as it takes to reach the next role. See secpat_scoring/sequence/JOERN.md.", "Left out \u2014 only one side detected them: Encrypter, Pepper store. 2 roles were excluded from both walks, so neither side's ordering mentions them. The walk still passes through the code behind such a role, attributing what it finds there to the nearest agreed role.", "Ground truth: the walk started at Enforcer \u2014 the earliest role the reference pattern has speaking that this side grounded in walkable code.", "Recovery result: the walk started at Enforcer \u2014 stored.", "An entry point the pattern did not name means its prescribed speakers were either ungrounded on that side or not where this code begins \u2014 a role something else reaches cannot open the flow \u2014 so the call graph had to stand in. Treat the ordering below as weaker evidence: it starts wherever the code happens to start.", "Ground truth: Comparator, Hasher cite entities that are not symbols in this repository (a library or framework call). They can receive an interaction, but the walk cannot continue through them." ] } ], "ground_truth_source": "recovery result", "vocabulary": [ "Enforcer", "Verification manager", "Password store", "Hasher", "Pepper store", "Encrypter", "Comparator" ], "role_scope": "required", "excluded_roles": [ "Subject", "System" ], "dropped_ground_truth_rows": [ "Subject", "System" ], "dropped_recovery_rows": [ "Subject", "System" ], "gt_roles": 7, "rr_roles": 7, "struct_files": 7, "annotated_at": "2026-10-02 10:00:38.838000", "recovery_order": "stored" }, { "slug": "authentication_pwd__Nostalgia-Season-Train_DesksetBack", "id": "pwd-based/DesksetBack/ClaudeCodeOpus5.5", "pattern": "authentication_pwd", "repo": "Nostalgia-Season-Train_DesksetBack", "error": null, "metrics": [ { "key": "level1", "level": "Level 1 \u2014 Role Detection", "formula": "agreement-rate (present/external/missing)", "available": true, "headlineLabel": "Roles agreed", "score": 1.0, "drift": 0.0, "detail": { "agreedRoles": 7, "totalRoles": 7 }, "perRole": [ { "patternRole": "Enforcer", "groundTruth": "present", "recoveryResult": "present", "groundTruthDetected": true, "recoveryResultDetected": true, "outcome": "agreed" }, { "patternRole": "Verification manager", "groundTruth": "present", "recoveryResult": "present", "groundTruthDetected": true, "recoveryResultDetected": true, "outcome": "agreed" }, { "patternRole": "Password store", "groundTruth": "present", "recoveryResult": "present", "groundTruthDetected": true, "recoveryResultDetected": true, "outcome": "agreed" }, { "patternRole": "Hasher", "groundTruth": "missing", "recoveryResult": "missing", "groundTruthDetected": false, "recoveryResultDetected": false, "outcome": "agreed" }, { "patternRole": "Pepper store", "groundTruth": "missing", "recoveryResult": "missing", "groundTruthDetected": false, "recoveryResultDetected": false, "outcome": "agreed" }, { "patternRole": "Encrypter", "groundTruth": "missing", "recoveryResult": "missing", "groundTruthDetected": false, "recoveryResultDetected": false, "outcome": "agreed" }, { "patternRole": "Comparator", "groundTruth": "present", "recoveryResult": "present", "groundTruthDetected": true, "recoveryResultDetected": true, "outcome": "agreed" } ], "notes": [] }, { "key": "level2", "level": "Level 2 \u2014 Role Grounding", "formula": "path-length", "available": true, "headlineLabel": "Mean grounding score", "score": 0.9531, "drift": 0.0469, "detail": { "aggregate": 0.9531, "scoredRoles": 4, "skippedRoles": 3, "treeHeight": 8 }, "perRole": [ { "patternRole": "Comparator", "score": 1.0, "distance": 0, "groundTruth": { "entity": "login", "loc": "src/deskset/router/unify/access.py:119" }, "recoveryResult": { "entity": "login", "loc": "src/deskset/router/unify/access.py:112" }, "note": "path-length (d=0)" }, { "patternRole": "Enforcer", "score": 0.875, "distance": 2, "groundTruth": { "entity": "login", "loc": "src/deskset/router/unify/access.py:119" }, "recoveryResult": { "entity": "check_token", "loc": "src/deskset/router/unify/access.py:91" }, "note": "path-length (d=2)" }, { "patternRole": "Password store", "score": 0.9375, "distance": 1, "groundTruth": { "entity": "password", "loc": "src/deskset/core/config.py:201" }, "recoveryResult": { "entity": "Config", "loc": "src/deskset/core/config.py:88" }, "note": "path-length (d=1)" }, { "patternRole": "Verification manager", "score": 1.0, "distance": 0, "groundTruth": { "entity": "login", "loc": "src/deskset/router/unify/access.py:119" }, "recoveryResult": { "entity": "login", "loc": "src/deskset/router/unify/access.py:112" }, "note": "path-length (d=0)" } ], "notes": [ "Encrypter: absent from the ground truth (Level 1 negative)", "Hasher: absent from the ground truth (Level 1 negative)", "Pepper store: absent from the ground truth (Level 1 negative)" ] }, { "key": "level3", "level": "Level 3 \u2014 Interaction Order", "formula": "jaccard \u00d7 kendall-tau (shared interactions)", "available": true, "headlineLabel": "Interaction agreement", "score": 0.3333, "drift": 0.6667, "detail": { "agreedRoles": [ "Comparator", "Enforcer", "Password store", "Verification manager" ], "excludedRoles": [ "Encrypter", "Hasher", "Pepper store" ], "interactionsGroundTruth": 2, "interactionsRecoveryResult": 2, "sharedInteractions": 1, "unionSize": 3, "entryRoleGroundTruth": "Enforcer", "entryRoleRecoveryResult": "Verification manager", "entryRuleGroundTruth": "behaviour", "entryRuleRecoveryResult": "stored", "onlyGroundTruth": 1, "onlyRecoveryResult": 1, "interactionOverlap": 0.3333, "orderAgreement": 1.0, "rankedInteractions": 1, "totalPairs": 0, "concordantPairs": 0, "discordantPairs": 0, "kendallTau": null, "prescribedActionSteps": 6, "prescribedStepsCovered": 3 }, "perRole": [ { "interaction": "Enforcer \u2192 Verification manager", "from": "Enforcer", "to": "Verification manager", "groundTruthRank": 1, "recoveryResultRank": null, "outcome": "onlyGroundTruth", "discordantWith": null, "evidence": { "groundTruth": { "file": "src/deskset/router/unify/access.py", "line": 119, "via": [] }, "recoveryResult": null } }, { "interaction": "Verification manager \u2192 Comparator", "from": "Verification manager", "to": "Comparator", "groundTruthRank": 2, "recoveryResultRank": 2, "outcome": "agreed", "discordantWith": 0, "evidence": { "groundTruth": { "file": "src/deskset/router/unify/access.py", "line": 119, "via": [] }, "recoveryResult": { "file": "src/deskset/router/unify/access.py", "line": 134, "via": [] } } }, { "interaction": "Verification manager \u2192 Password store", "from": "Verification manager", "to": "Password store", "groundTruthRank": null, "recoveryResultRank": 1, "outcome": "onlyRecoveryResult", "discordantWith": null, "evidence": { "groundTruth": null, "recoveryResult": { "file": "src/deskset/router/unify/access.py", "line": 134, "via": [] } } } ], "notes": [ "Score = overlap \u00d7 order = 0.3333 \u00d7 1. Overlap: 1 of 3 interactions were found by both sides (1 only in the ground truth, 1 only in the recovery result), |shared| / |union|.", "Order: 1 \u2014 a single shared interaction has no pair whose relative order could differ.", "Scored over the 4 roles both sides detected (Comparator, Enforcer, Password store, Verification manager). An interaction counts only when both of its endpoint roles were detected on both sides \u2014 whether a role was found at all is Level 1's measurement, and charging for it again here would say the same thing twice.", "The interaction order is derived statically from the code \u2014 no one annotated it. Each role's cited entity is located in the AST index and control flow is followed from it, through as many non-role helper functions as it takes to reach the next role. See secpat_scoring/sequence/JOERN.md.", "Left out \u2014 only one side detected them: Encrypter, Hasher, Pepper store. 3 roles were excluded from both walks, so neither side's ordering mentions them. The walk still passes through the code behind such a role, attributing what it finds there to the nearest agreed role.", "Ground truth: the walk started at Enforcer \u2014 the earliest role the reference pattern has speaking that this side grounded in walkable code.", "Recovery result: the walk started at Verification manager \u2014 stored.", "An entry point the pattern did not name means its prescribed speakers were either ungrounded on that side or not where this code begins \u2014 a role something else reaches cannot open the flow \u2014 so the call graph had to stand in. Treat the ordering below as weaker evidence: it starts wherever the code happens to start." ] } ], "ground_truth_source": "recovery result", "vocabulary": [ "Enforcer", "Verification manager", "Password store", "Hasher", "Pepper store", "Encrypter", "Comparator" ], "role_scope": "required", "excluded_roles": [ "Subject", "System" ], "dropped_ground_truth_rows": [ "Subject", "System" ], "dropped_recovery_rows": [ "Subject", "System" ], "gt_roles": 7, "rr_roles": 7, "struct_files": 65, "annotated_at": "2026-10-02 10:00:38.838000", "recovery_order": "stored" }, { "slug": "authentication_pwd__olaflaitinen_Promethium", "id": "pwd-based/Promethium/ClaudeCodeOpus5.5", "pattern": "authentication_pwd", "repo": "olaflaitinen_Promethium", "error": null, "metrics": [ { "key": "level1", "level": "Level 1 \u2014 Role Detection", "formula": "agreement-rate (present/external/missing)", "available": true, "headlineLabel": "Roles agreed", "score": 0.7143, "drift": 0.2857, "detail": { "agreedRoles": 5, "totalRoles": 7 }, "perRole": [ { "patternRole": "Enforcer", "groundTruth": "present", "recoveryResult": "present", "groundTruthDetected": true, "recoveryResultDetected": true, "outcome": "agreed" }, { "patternRole": "Verification manager", "groundTruth": "present", "recoveryResult": "present", "groundTruthDetected": true, "recoveryResultDetected": true, "outcome": "agreed" }, { "patternRole": "Password store", "groundTruth": "present", "recoveryResult": "present", "groundTruthDetected": true, "recoveryResultDetected": true, "outcome": "agreed" }, { "patternRole": "Hasher", "groundTruth": "external", "recoveryResult": "present", "groundTruthDetected": true, "recoveryResultDetected": true, "outcome": "mismatched" }, { "patternRole": "Pepper store", "groundTruth": "missing", "recoveryResult": "missing", "groundTruthDetected": false, "recoveryResultDetected": false, "outcome": "agreed" }, { "patternRole": "Encrypter", "groundTruth": "missing", "recoveryResult": "missing", "groundTruthDetected": false, "recoveryResultDetected": false, "outcome": "agreed" }, { "patternRole": "Comparator", "groundTruth": "external", "recoveryResult": "present", "groundTruthDetected": true, "recoveryResultDetected": true, "outcome": "mismatched" } ], "notes": [] }, { "key": "level2", "level": "Level 2 \u2014 Role Grounding", "formula": "path-length", "available": true, "headlineLabel": "Mean grounding score", "score": 0.9333, "drift": 0.0667, "detail": { "aggregate": 0.9333, "scoredRoles": 5, "skippedRoles": 2, "treeHeight": 6 }, "perRole": [ { "patternRole": "Comparator", "score": 0.8333, "distance": 2, "groundTruth": { "entity": "verify", "loc": "src/promethium/core/security.py:6" }, "recoveryResult": { "entity": "verify_password", "loc": "src/promethium/core/security.py:6" }, "note": "path-length (d=2)" }, { "patternRole": "Enforcer", "score": 1.0, "distance": 0, "groundTruth": { "entity": "login_access_token", "loc": "src/promethium/api/routers/auth.py:22" }, "recoveryResult": { "entity": "login_access_token", "loc": "src/promethium/api/routers/auth.py:23" }, "note": "path-length (d=0)" }, { "patternRole": "Hasher", "score": 0.8333, "distance": 2, "groundTruth": { "entity": "verify", "loc": "src/promethium/core/security.py:6" }, "recoveryResult": { "entity": "CryptContext", "loc": "src/promethium/core/security.py:3" }, "note": "path-length (d=2)" }, { "patternRole": "Password store", "score": 1.0, "distance": 0, "groundTruth": { "entity": "User", "loc": "src/promethium/api/models/user.py:12" }, "recoveryResult": { "entity": "User", "loc": "src/promethium/api/models/user.py:19" }, "note": "path-length (d=0)" }, { "patternRole": "Verification manager", "score": 1.0, "distance": 0, "groundTruth": { "entity": "authenticate_user", "loc": "src/promethium/api/services/auth_service.py:18" }, "recoveryResult": { "entity": "authenticate_user", "loc": "src/promethium/api/services/auth_service.py:13" }, "note": "path-length (d=0)" } ], "notes": [ "Encrypter: absent from the ground truth (Level 1 negative)", "Pepper store: absent from the ground truth (Level 1 negative)" ] }, { "key": "level3", "level": "Level 3 \u2014 Interaction Order", "formula": "jaccard \u00d7 kendall-tau (shared interactions)", "available": true, "headlineLabel": "Interaction agreement", "score": 0.4, "drift": 0.6, "detail": { "agreedRoles": [ "Comparator", "Enforcer", "Hasher", "Password store", "Verification manager" ], "excludedRoles": [ "Encrypter", "Pepper store" ], "interactionsGroundTruth": 3, "interactionsRecoveryResult": 4, "sharedInteractions": 2, "unionSize": 5, "entryRoleGroundTruth": "Enforcer", "entryRoleRecoveryResult": "Enforcer", "entryRuleGroundTruth": "behaviour", "entryRuleRecoveryResult": "stored", "onlyGroundTruth": 1, "onlyRecoveryResult": 2, "interactionOverlap": 0.4, "orderAgreement": 1.0, "rankedInteractions": 2, "totalPairs": 1, "concordantPairs": 1, "discordantPairs": 0, "kendallTau": 1.0, "prescribedActionSteps": 6, "prescribedStepsCovered": 4 }, "perRole": [ { "interaction": "Enforcer \u2192 Verification manager", "from": "Enforcer", "to": "Verification manager", "groundTruthRank": 1, "recoveryResultRank": 1, "outcome": "agreed", "discordantWith": 0, "evidence": { "groundTruth": { "file": "src/promethium/api/routers/auth.py", "line": 29, "via": [] }, "recoveryResult": { "file": "src/promethium/api/routers/auth.py", "line": 33, "via": [] } } }, { "interaction": "Verification manager \u2192 Hasher", "from": "Verification manager", "to": "Hasher", "groundTruthRank": 2, "recoveryResultRank": null, "outcome": "onlyGroundTruth", "discordantWith": null, "evidence": { "groundTruth": { "file": "src/promethium/core/security.py", "line": 6, "via": [ "src/promethium/core/security.py:<module>.verify_password" ] }, "recoveryResult": null } }, { "interaction": "Verification manager \u2192 Comparator", "from": "Verification manager", "to": "Comparator", "groundTruthRank": 3, "recoveryResultRank": 3, "outcome": "agreed", "discordantWith": 0, "evidence": { "groundTruth": { "file": "src/promethium/core/security.py", "line": 6, "via": [ "src/promethium/core/security.py:<module>.verify_password" ] }, "recoveryResult": { "file": "src/promethium/api/services/auth_service.py", "line": 30, "via": [] } } }, { "interaction": "Verification manager \u2192 Password store", "from": "Verification manager", "to": "Password store", "groundTruthRank": null, "recoveryResultRank": 2, "outcome": "onlyRecoveryResult", "discordantWith": null, "evidence": { "groundTruth": null, "recoveryResult": { "file": "src/promethium/api/services/auth_service.py", "line": 22, "via": [] } } }, { "interaction": "Comparator \u2192 Hasher", "from": "Comparator", "to": "Hasher", "groundTruthRank": null, "recoveryResultRank": 4, "outcome": "onlyRecoveryResult", "discordantWith": null, "evidence": { "groundTruth": null, "recoveryResult": { "file": "src/promethium/core/security.py", "line": 7, "via": [] } } } ], "notes": [ "Score = overlap \u00d7 order = 0.4 \u00d7 1. Overlap: 2 of 5 interactions were found by both sides (1 only in the ground truth, 2 only in the recovery result), |shared| / |union|.", "Order: Kendall's Tau over the 2 shared interactions, in each side's own relative order \u2014 0 of 1 pairs run in opposite orders. Interactions only one side found take part in no pair (Fagin, Kumar & Sivakumar's p = 0, the induced sub-order); the overlap factor is what charges for them.", "Only 1 ranked pair \u2014 the order factor can land on just a few values here and one disagreement swings it fully.", "Scored over the 5 roles both sides detected (Comparator, Enforcer, Hasher, Password store, Verification manager). An interaction counts only when both of its endpoint roles were detected on both sides \u2014 whether a role was found at all is Level 1's measurement, and charging for it again here would say the same thing twice.", "The interaction order is derived statically from the code \u2014 no one annotated it. Each role's cited entity is located in the AST index and control flow is followed from it, through as many non-role helper functions as it takes to reach the next role. See secpat_scoring/sequence/JOERN.md.", "Left out \u2014 only one side detected them: Encrypter, Pepper store. 2 roles were excluded from both walks, so neither side's ordering mentions them. The walk still passes through the code behind such a role, attributing what it finds there to the nearest agreed role.", "Ground truth: the walk started at Enforcer \u2014 the earliest role the reference pattern has speaking that this side grounded in walkable code.", "Recovery result: the walk started at Enforcer \u2014 stored.", "An entry point the pattern did not name means its prescribed speakers were either ungrounded on that side or not where this code begins \u2014 a role something else reaches cannot open the flow \u2014 so the call graph had to stand in. Treat the ordering below as weaker evidence: it starts wherever the code happens to start.", "Ground truth: Comparator, Hasher cite entities that are not symbols in this repository (a library or framework call). They can receive an interaction, but the walk cannot continue through them." ] } ], "ground_truth_source": "recovery result", "vocabulary": [ "Enforcer", "Verification manager", "Password store", "Hasher", "Pepper store", "Encrypter", "Comparator" ], "role_scope": "required", "excluded_roles": [ "Subject", "System" ], "dropped_ground_truth_rows": [ "Subject", "System" ], "dropped_recovery_rows": [ "Subject", "System" ], "gt_roles": 7, "rr_roles": 7, "struct_files": 104, "annotated_at": "2026-10-02 10:00:38.838000", "recovery_order": "stored" }, { "slug": "authentication_pwd__tumma72_mlx-manager", "id": "pwd-based/tumma72_mlx-manager/ClaudeCodeOpus5.5", "pattern": "authentication_pwd", "repo": "tumma72_mlx-manager", "error": null, "metrics": [ { "key": "level1", "level": "Level 1 \u2014 Role Detection", "formula": "agreement-rate", "available": true, "headlineLabel": "Roles agreed", "score": 0.8571, "drift": 0.1429, "detail": { "agreedRoles": 6, "totalRoles": 7 }, "perRole": [ { "patternRole": "Enforcer", "groundTruth": "present", "recoveryResult": "present", "groundTruthDetected": true, "recoveryResultDetected": true, "outcome": "agreed" }, { "patternRole": "Verification manager", "groundTruth": "present", "recoveryResult": "present", "groundTruthDetected": true, "recoveryResultDetected": true, "outcome": "agreed" }, { "patternRole": "Password store", "groundTruth": "present", "recoveryResult": "present", "groundTruthDetected": true, "recoveryResultDetected": true, "outcome": "agreed" }, { "patternRole": "Hasher", "groundTruth": "external", "recoveryResult": "external", "groundTruthDetected": true, "recoveryResultDetected": true, "outcome": "agreed" }, { "patternRole": "Pepper store", "groundTruth": "missing", "recoveryResult": "missing", "groundTruthDetected": false, "recoveryResultDetected": false, "outcome": "agreed" }, { "patternRole": "Encrypter", "groundTruth": "missing", "recoveryResult": "missing", "groundTruthDetected": false, "recoveryResultDetected": false, "outcome": "agreed" }, { "patternRole": "Comparator", "groundTruth": "external", "recoveryResult": "present", "groundTruthDetected": true, "recoveryResultDetected": true, "outcome": "mismatched" } ], "notes": [] }, { "key": "level2", "level": "Level 2 \u2014 Role Grounding", "formula": "path-length", "available": true, "headlineLabel": "Mean grounding score", "score": 0.8125, "drift": 0.1875, "detail": { "aggregate": 0.8125, "scoredRoles": 5, "skippedRoles": 2, "treeHeight": 8 }, "perRole": [ { "patternRole": "Comparator", "score": 0.875, "distance": 2, "groundTruth": { "entity": "verify", "loc": "backend/mlx_manager/services/auth_service.py:27" }, "recoveryResult": { "entity": "verify_password", "loc": "backend/mlx_manager/services/auth_service.py:22" }, "note": "path-length (d=2)" }, { "patternRole": "Enforcer", "score": 0.6875, "distance": 5, "groundTruth": { "entity": "login", "loc": "backend/mlx_manager/routers/auth.py:69" }, "recoveryResult": { "entity": "get_current_user", "loc": "backend/mlx_manager/dependencies.py:42" }, "note": "path-length (d=5)" }, { "patternRole": "Hasher", "score": 0.875, "distance": 2, "groundTruth": { "entity": "verify", "loc": "backend/mlx_manager/services/auth_service.py:27" }, "recoveryResult": { "entity": "PasswordHash", "loc": "backend/mlx_manager/services/auth_service.py:14" }, "note": "path-length (d=2)" }, { "patternRole": "Password store", "score": 1.0, "distance": 0, "groundTruth": { "entity": "User", "loc": "backend/mlx_manager/models/entities.py:30" }, "recoveryResult": { "entity": "User", "loc": "backend/mlx_manager/models/entities.py:27" }, "note": "path-length (d=0)" }, { "patternRole": "Verification manager", "score": 0.625, "distance": 6, "groundTruth": { "entity": "verify_password", "loc": "backend/mlx_manager/services/auth_service.py:25" }, "recoveryResult": { "entity": "login", "loc": "backend/mlx_manager/routers/auth.py:62" }, "note": "path-length (d=6)" } ], "notes": [ "Encrypter: absent from the ground truth (Level 1 negative)", "Pepper store: absent from the ground truth (Level 1 negative)" ] }, { "key": "level3", "level": "Level 3 \u2014 Interaction Order", "formula": "jaccard \u00d7 kendall-tau (shared interactions)", "available": true, "headlineLabel": "Interaction agreement", "score": 0.1429, "drift": 0.8571, "detail": { "agreedRoles": [ "Comparator", "Enforcer", "Hasher", "Password store", "Verification manager" ], "excludedRoles": [ "Encrypter", "Pepper store" ], "interactionsGroundTruth": 3, "interactionsRecoveryResult": 5, "sharedInteractions": 1, "unionSize": 7, "entryRoleGroundTruth": "Enforcer", "entryRoleRecoveryResult": "Verification manager", "entryRuleGroundTruth": "behaviour", "entryRuleRecoveryResult": "stored", "onlyGroundTruth": 2, "onlyRecoveryResult": 4, "interactionOverlap": 0.1429, "orderAgreement": 1.0, "rankedInteractions": 1, "totalPairs": 0, "concordantPairs": 0, "discordantPairs": 0, "kendallTau": null, "prescribedActionSteps": 6, "prescribedStepsCovered": 4 }, "perRole": [ { "interaction": "Enforcer \u2192 Verification manager", "from": "Enforcer", "to": "Verification manager", "groundTruthRank": 1, "recoveryResultRank": null, "outcome": "onlyGroundTruth", "discordantWith": null, "evidence": { "groundTruth": { "file": "backend/mlx_manager/routers/auth.py", "line": 82, "via": [] }, "recoveryResult": null } }, { "interaction": "Verification manager \u2192 Hasher", "from": "Verification manager", "to": "Hasher", "groundTruthRank": 2, "recoveryResultRank": null, "outcome": "onlyGroundTruth", "discordantWith": null, "evidence": { "groundTruth": { "file": "backend/mlx_manager/services/auth_service.py", "line": 27, "via": [] }, "recoveryResult": null } }, { "interaction": "Verification manager \u2192 Comparator", "from": "Verification manager", "to": "Comparator", "groundTruthRank": 3, "recoveryResultRank": 2, "outcome": "agreed", "discordantWith": 0, "evidence": { "groundTruth": { "file": "backend/mlx_manager/services/auth_service.py", "line": 27, "via": [] }, "recoveryResult": { "file": "backend/mlx_manager/routers/auth.py", "line": 72, "via": [] } } }, { "interaction": "Verification manager \u2192 Password store", "from": "Verification manager", "to": "Password store", "groundTruthRank": null, "recoveryResultRank": 1, "outcome": "onlyRecoveryResult", "discordantWith": null, "evidence": { "groundTruth": null, "recoveryResult": { "file": "backend/mlx_manager/routers/auth.py", "line": 67, "via": [] } } }, { "interaction": "Comparator \u2192 Hasher", "from": "Comparator", "to": "Hasher", "groundTruthRank": null, "recoveryResultRank": 3, "outcome": "onlyRecoveryResult", "discordantWith": null, "evidence": { "groundTruth": null, "recoveryResult": { "file": "backend/mlx_manager/services/auth_service.py", "line": 23, "via": [] } } }, { "interaction": "Verification manager \u2192 Enforcer", "from": "Verification manager", "to": "Enforcer", "groundTruthRank": null, "recoveryResultRank": 4, "outcome": "onlyRecoveryResult", "discordantWith": null, "evidence": { "groundTruth": null, "recoveryResult": { "file": "backend/mlx_manager/routers/auth.py", "line": 90, "via": [] } } }, { "interaction": "Enforcer \u2192 Password store", "from": "Enforcer", "to": "Password store", "groundTruthRank": null, "recoveryResultRank": 5, "outcome": "onlyRecoveryResult", "discordantWith": null, "evidence": { "groundTruth": null, "recoveryResult": { "file": "backend/mlx_manager/dependencies.py", "line": 60, "via": [] } } } ], "notes": [ "Score = overlap \u00d7 order = 0.1429 \u00d7 1. Overlap: 1 of 7 interactions were found by both sides (2 only in the ground truth, 4 only in the recovery result), |shared| / |union|.", "Order: 1 \u2014 a single shared interaction has no pair whose relative order could differ.", "Scored over the 5 roles both sides detected (Comparator, Enforcer, Hasher, Password store, Verification manager). An interaction counts only when both of its endpoint roles were detected on both sides \u2014 whether a role was found at all is Level 1's measurement, and charging for it again here would say the same thing twice.", "The interaction order is derived statically from the code \u2014 no one annotated it. Each role's cited entity is located in the AST index and control flow is followed from it, through as many non-role helper functions as it takes to reach the next role. See secpat_scoring/sequence/JOERN.md.", "Left out \u2014 only one side detected them: Encrypter, Pepper store. 2 roles were excluded from both walks, so neither side's ordering mentions them. The walk still passes through the code behind such a role, attributing what it finds there to the nearest agreed role.", "Ground truth: the walk started at Enforcer \u2014 the earliest role the reference pattern has speaking that this side grounded in walkable code.", "Recovery result: the walk started at Verification manager \u2014 stored.", "An entry point the pattern did not name means its prescribed speakers were either ungrounded on that side or not where this code begins \u2014 a role something else reaches cannot open the flow \u2014 so the call graph had to stand in. Treat the ordering below as weaker evidence: it starts wherever the code happens to start.", "Ground truth: Comparator, Hasher cite entities that are not symbols in this repository (a library or framework call). They can receive an interaction, but the walk cannot continue through them." ] } ], "ground_truth_source": "recovery result", "vocabulary": [ "Enforcer", "Verification manager", "Password store", "Hasher", "Pepper store", "Encrypter", "Comparator" ], "role_scope": "required", "excluded_roles": [ "Subject", "System" ], "dropped_ground_truth_rows": [ "Subject", "System" ], "dropped_recovery_rows": [ "Subject", "System" ], "gt_roles": 7, "rr_roles": 7, "struct_files": 282, "annotated_at": "2026-10-06 09:53:49.581000", "recovery_order": "stored" }, { "slug": "obscure_token_based_access_control__Athroniaeth_keyshield", "id": "obscure-token-ac/keyshield/ClaudeCodeopus4.7", "pattern": "obscure_token_based_access_control", "repo": "Athroniaeth_keyshield", "error": null, "metrics": [ { "key": "level1", "level": "Level 1 \u2014 Role Detection", "formula": "agreement-rate (present/external/missing)", "available": true, "headlineLabel": "Roles agreed", "score": 0.75, "drift": 0.25, "detail": { "agreedRoles": 3, "totalRoles": 4 }, "perRole": [ { "patternRole": "Enforcer", "groundTruth": "present", "recoveryResult": "present", "groundTruthDetected": true, "recoveryResultDetected": true, "outcome": "agreed" }, { "patternRole": "Validator", "groundTruth": "present", "recoveryResult": "present", "groundTruthDetected": true, "recoveryResultDetected": true, "outcome": "agreed" }, { "patternRole": "Hasher", "groundTruth": "missing", "recoveryResult": "present", "groundTruthDetected": false, "recoveryResultDetected": true, "outcome": "mismatched" }, { "patternRole": "Token manager", "groundTruth": "present", "recoveryResult": "present", "groundTruthDetected": true, "recoveryResultDetected": true, "outcome": "agreed" } ], "notes": [] }, { "key": "level2", "level": "Level 2 \u2014 Role Grounding", "formula": "path-length", "available": true, "headlineLabel": "Mean grounding score", "score": 0.5667, "drift": 0.4333, "detail": { "aggregate": 0.5667, "scoredRoles": 3, "skippedRoles": 3, "treeHeight": 5 }, "perRole": [ { "patternRole": "Enforcer", "score": 0.8, "distance": 2, "groundTruth": { "entity": "_handle_verify_key", "loc": "src/keyshield/api.py:374\u2013415" }, "recoveryResult": { "entity": "create_depends_api_key", "loc": "src/keyshield/api.py:413" }, "note": "path-length (d=2)" }, { "patternRole": "Token manager", "score": 0.3, "distance": 7, "groundTruth": { "entity": "get_by_key_id", "loc": "src/keyshield/services/base.py:349\u2013355" }, "recoveryResult": { "entity": "SqlAlchemyApiKeyRepository", "loc": "src/keyshield/repositories/sql.py:172" }, "note": "path-length (d=7)" }, { "patternRole": "Validator", "score": 0.6, "distance": 4, "groundTruth": { "entity": "_verify_key", "loc": "src/keyshield/services/base.py:437\u2013443" }, "recoveryResult": { "entity": "verify_key", "loc": "src/keyshield/services/base.py:245" }, "note": "path-length (d=4)" } ], "notes": [ "Hasher: absent from the ground truth (Level 1 negative)", "Registrar: absent from the ground truth (Level 1 negative)", "Token generator: absent from the ground truth (Level 1 negative)" ] }, { "key": "level3", "level": "Level 3 \u2014 Interaction Order", "formula": "jaccard \u00d7 kendall-tau (shared interactions)", "available": true, "headlineLabel": "Interaction agreement", "score": 1.0, "drift": 0.0, "detail": { "agreedRoles": [ "Enforcer", "Token manager", "Validator" ], "excludedRoles": [ "Hasher", "Registrar", "Token generator" ], "interactionsGroundTruth": 2, "interactionsRecoveryResult": 2, "sharedInteractions": 2, "unionSize": 2, "entryRoleGroundTruth": "Enforcer", "entryRoleRecoveryResult": "Enforcer", "entryRuleGroundTruth": "behaviour", "entryRuleRecoveryResult": "stored", "onlyGroundTruth": 0, "onlyRecoveryResult": 0, "interactionOverlap": 1.0, "orderAgreement": 1.0, "rankedInteractions": 2, "totalPairs": 1, "concordantPairs": 1, "discordantPairs": 0, "kendallTau": 1.0, "prescribedActionSteps": 3, "prescribedStepsCovered": 2 }, "perRole": [ { "interaction": "Enforcer \u2192 Validator", "from": "Enforcer", "to": "Validator", "groundTruthRank": 1, "recoveryResultRank": 1, "outcome": "agreed", "discordantWith": 0, "evidence": { "groundTruth": { "file": "src/keyshield/services/base.py", "line": 242, "via": [ "src/keyshield/services/base.py:<module>.AbstractApiKeyService.verify_key" ] }, "recoveryResult": { "file": "src/keyshield/api.py", "line": 366, "via": [] } } }, { "interaction": "Validator \u2192 Token manager", "from": "Validator", "to": "Token manager", "groundTruthRank": 2, "recoveryResultRank": 2, "outcome": "agreed", "discordantWith": 0, "evidence": { "groundTruth": { "file": "src/keyshield/services/base.py", "line": 441, "via": [] }, "recoveryResult": { "file": "src/keyshield/services/base.py", "line": 471, "via": [] } } } ], "notes": [ "Score = overlap \u00d7 order = 1 \u00d7 1. Overlap: 2 of 2 interactions were found by both sides (0 only in the ground truth, 0 only in the recovery result), |shared| / |union|.", "Order: Kendall's Tau over the 2 shared interactions, in each side's own relative order \u2014 0 of 1 pairs run in opposite orders. Interactions only one side found take part in no pair (Fagin, Kumar & Sivakumar's p = 0, the induced sub-order); the overlap factor is what charges for them.", "Only 1 ranked pair \u2014 the order factor can land on just a few values here and one disagreement swings it fully.", "Scored over the 3 roles both sides detected (Enforcer, Token manager, Validator). An interaction counts only when both of its endpoint roles were detected on both sides \u2014 whether a role was found at all is Level 1's measurement, and charging for it again here would say the same thing twice.", "The interaction order is derived statically from the code \u2014 no one annotated it. Each role's cited entity is located in the AST index and control flow is followed from it, through as many non-role helper functions as it takes to reach the next role. See secpat_scoring/sequence/JOERN.md.", "Left out \u2014 only one side detected them: Hasher, Registrar, Token generator. 3 roles were excluded from both walks, so neither side's ordering mentions them. The walk still passes through the code behind such a role, attributing what it finds there to the nearest agreed role.", "Ground truth: the walk started at Enforcer \u2014 the earliest role the reference pattern has speaking that this side grounded in walkable code.", "Recovery result: the walk started at Enforcer \u2014 stored.", "An entry point the pattern did not name means its prescribed speakers were either ungrounded on that side or not where this code begins \u2014 a role something else reaches cannot open the flow \u2014 so the call graph had to stand in. Treat the ordering below as weaker evidence: it starts wherever the code happens to start." ] } ], "ground_truth_source": "annotation", "vocabulary": [ "Enforcer", "Validator", "Hasher", "Token manager" ], "role_scope": "required", "excluded_roles": [ "Subject", "System" ], "dropped_ground_truth_rows": [ "Registrar", "Subject", "System", "Token generator" ], "dropped_recovery_rows": [ "Subject", "System" ], "gt_roles": 4, "rr_roles": 4, "struct_files": 58, "annotated_at": "2026-09-26 22:01:38.718000", "recovery_order": "stored" }, { "slug": "obscure_token_based_access_control__contextfs_contextfs", "id": "obscure-token-ac/contextfs/ClaudeCodeopus4.7", "pattern": "obscure_token_based_access_control", "repo": "contextfs_contextfs", "error": null, "metrics": [ { "key": "level1", "level": "Level 1 \u2014 Role Detection", "formula": "agreement-rate (present/external/missing)", "available": true, "headlineLabel": "Roles agreed", "score": 1.0, "drift": 0.0, "detail": { "agreedRoles": 4, "totalRoles": 4 }, "perRole": [ { "patternRole": "Enforcer", "groundTruth": "present", "recoveryResult": "present", "groundTruthDetected": true, "recoveryResultDetected": true, "outcome": "agreed" }, { "patternRole": "Validator", "groundTruth": "present", "recoveryResult": "present", "groundTruthDetected": true, "recoveryResultDetected": true, "outcome": "agreed" }, { "patternRole": "Hasher", "groundTruth": "present", "recoveryResult": "present", "groundTruthDetected": true, "recoveryResultDetected": true, "outcome": "agreed" }, { "patternRole": "Token manager", "groundTruth": "present", "recoveryResult": "present", "groundTruthDetected": true, "recoveryResultDetected": true, "outcome": "agreed" } ], "notes": [] }, { "key": "level2", "level": "Level 2 \u2014 Role Grounding", "formula": "path-length", "available": true, "headlineLabel": "Mean grounding score", "score": 0.4791, "drift": 0.5209, "detail": { "aggregate": 0.4791, "scoredRoles": 4, "skippedRoles": 2, "treeHeight": 6 }, "perRole": [ { "patternRole": "Enforcer", "score": 0.3333, "distance": 8, "groundTruth": { "entity": "require_auth", "loc": "src/contextfs/auth/middleware.py:93" }, "recoveryResult": { "entity": "require_auth", "loc": "service/api/auth_middleware.py:78" }, "note": "path-length (d=8)" }, { "patternRole": "Hasher", "score": 1.0, "distance": 0, "groundTruth": { "entity": "hash_api_key", "loc": "src/contextfs/auth/api_keys.py:74\u201392" }, "recoveryResult": { "entity": "hash_api_key", "loc": "src/contextfs/auth/api_keys.py:76" }, "note": "path-length (d=0)" }, { "patternRole": "Token manager", "score": 0.3333, "distance": 8, "groundTruth": { "entity": "db", "loc": "src/contextfs/auth/api_keys.py:170" }, "recoveryResult": { "entity": "APIKeyModel", "loc": "service/db/models.py:271" }, "note": "path-length (d=8)" }, { "patternRole": "Validator", "score": 0.25, "distance": 9, "groundTruth": { "entity": "validate_key", "loc": "src/contextfs/auth/api_keys.py:152" }, "recoveryResult": { "entity": "validate_api_key", "loc": "service/api/auth_middleware.py:25" }, "note": "path-length (d=9)" } ], "notes": [ "Registrar: absent from the ground truth (Level 1 negative)", "Token generator: absent from the ground truth (Level 1 negative)" ] }, { "key": "level3", "level": "Level 3 \u2014 Interaction Order", "formula": "jaccard \u00d7 kendall-tau (shared interactions)", "available": true, "headlineLabel": "Interaction agreement", "score": 0.6667, "drift": 0.3333, "detail": { "agreedRoles": [ "Enforcer", "Hasher", "Token manager", "Validator" ], "excludedRoles": [ "Registrar", "Token generator" ], "interactionsGroundTruth": 2, "interactionsRecoveryResult": 3, "sharedInteractions": 2, "unionSize": 3, "entryRoleGroundTruth": "Enforcer", "entryRoleRecoveryResult": "Enforcer", "entryRuleGroundTruth": "behaviour", "entryRuleRecoveryResult": "stored", "onlyGroundTruth": 0, "onlyRecoveryResult": 1, "interactionOverlap": 0.6667, "orderAgreement": 1.0, "rankedInteractions": 2, "totalPairs": 1, "concordantPairs": 1, "discordantPairs": 0, "kendallTau": 1.0, "prescribedActionSteps": 3, "prescribedStepsCovered": 3 }, "perRole": [ { "interaction": "Enforcer \u2192 Validator", "from": "Enforcer", "to": "Validator", "groundTruthRank": 1, "recoveryResultRank": 1, "outcome": "agreed", "discordantWith": 0, "evidence": { "groundTruth": { "file": "src/contextfs/auth/middleware.py", "line": 41, "via": [ "src/contextfs/auth/middleware.py:<module>.get_current_user", "src/contextfs/auth/middleware.py:<module>.AuthMiddleware.authenticate" ] }, "recoveryResult": { "file": "service/api/auth_middleware.py", "line": 88, "via": [] } } }, { "interaction": "Validator \u2192 Hasher", "from": "Validator", "to": "Hasher", "groundTruthRank": 2, "recoveryResultRank": 2, "outcome": "agreed", "discordantWith": 0, "evidence": { "groundTruth": { "file": "src/contextfs/auth/api_keys.py", "line": 164, "via": [] }, "recoveryResult": { "file": "service/api/auth_middleware.py", "line": 36, "via": [] } } }, { "interaction": "Validator \u2192 Token manager", "from": "Validator", "to": "Token manager", "groundTruthRank": null, "recoveryResultRank": 3, "outcome": "onlyRecoveryResult", "discordantWith": null, "evidence": { "groundTruth": null, "recoveryResult": { "file": "service/api/auth_middleware.py", "line": 39, "via": [] } } } ], "notes": [ "Score = overlap \u00d7 order = 0.6667 \u00d7 1. Overlap: 2 of 3 interactions were found by both sides (0 only in the ground truth, 1 only in the recovery result), |shared| / |union|.", "Order: Kendall's Tau over the 2 shared interactions, in each side's own relative order \u2014 0 of 1 pairs run in opposite orders. Interactions only one side found take part in no pair (Fagin, Kumar & Sivakumar's p = 0, the induced sub-order); the overlap factor is what charges for them.", "Only 1 ranked pair \u2014 the order factor can land on just a few values here and one disagreement swings it fully.", "Scored over the 4 roles both sides detected (Enforcer, Hasher, Token manager, Validator). An interaction counts only when both of its endpoint roles were detected on both sides \u2014 whether a role was found at all is Level 1's measurement, and charging for it again here would say the same thing twice.", "The interaction order is derived statically from the code \u2014 no one annotated it. Each role's cited entity is located in the AST index and control flow is followed from it, through as many non-role helper functions as it takes to reach the next role. See secpat_scoring/sequence/JOERN.md.", "Left out \u2014 only one side detected them: Registrar, Token generator. 2 roles were excluded from both walks, so neither side's ordering mentions them. The walk still passes through the code behind such a role, attributing what it finds there to the nearest agreed role.", "Ground truth: the walk started at Enforcer \u2014 the earliest role the reference pattern has speaking that this side grounded in walkable code.", "Recovery result: the walk started at Enforcer \u2014 stored.", "An entry point the pattern did not name means its prescribed speakers were either ungrounded on that side or not where this code begins \u2014 a role something else reaches cannot open the flow \u2014 so the call graph had to stand in. Treat the ordering below as weaker evidence: it starts wherever the code happens to start." ] } ], "ground_truth_source": "annotation", "vocabulary": [ "Enforcer", "Validator", "Hasher", "Token manager" ], "role_scope": "required", "excluded_roles": [ "Subject", "System" ], "dropped_ground_truth_rows": [ "Registrar", "Subject", "System", "Token generator" ], "dropped_recovery_rows": [ "Subject", "System" ], "gt_roles": 4, "rr_roles": 4, "struct_files": 177, "annotated_at": "2026-08-10 16:32:35.439000", "recovery_order": "stored" }, { "slug": "obscure_token_based_access_control__datastax_ragstack-ai-langflow", "id": "obscure-token-ac/ragstack-ai-langflow/ClaudeCodeopus4.7", "pattern": "obscure_token_based_access_control", "repo": "datastax_ragstack-ai-langflow", "error": null, "metrics": [ { "key": "level1", "level": "Level 1 \u2014 Role Detection", "formula": "agreement-rate (present/external/missing)", "available": true, "headlineLabel": "Roles agreed", "score": 1.0, "drift": 0.0, "detail": { "agreedRoles": 4, "totalRoles": 4 }, "perRole": [ { "patternRole": "Enforcer", "groundTruth": "present", "recoveryResult": "present", "groundTruthDetected": true, "recoveryResultDetected": true, "outcome": "agreed" }, { "patternRole": "Validator", "groundTruth": "present", "recoveryResult": "present", "groundTruthDetected": true, "recoveryResultDetected": true, "outcome": "agreed" }, { "patternRole": "Hasher", "groundTruth": "missing", "recoveryResult": "missing", "groundTruthDetected": false, "recoveryResultDetected": false, "outcome": "agreed" }, { "patternRole": "Token manager", "groundTruth": "present", "recoveryResult": "present", "groundTruthDetected": true, "recoveryResultDetected": true, "outcome": "agreed" } ], "notes": [] }, { "key": "level2", "level": "Level 2 \u2014 Role Grounding", "formula": "path-length", "available": true, "headlineLabel": "Mean grounding score", "score": 0.8889, "drift": 0.1111, "detail": { "aggregate": 0.8889, "scoredRoles": 3, "skippedRoles": 3, "treeHeight": 9 }, "perRole": [ { "patternRole": "Enforcer", "score": 0.8889, "distance": 2, "groundTruth": { "entity": "get_current_user", "loc": "src/backend/base/langflow/services/auth/utils.py:71" }, "recoveryResult": { "entity": "api_key_security", "loc": "src/backend/base/langflow/services/auth/utils.py:34" }, "note": "path-length (d=2)" }, { "patternRole": "Token manager", "score": 0.7778, "distance": 4, "groundTruth": { "entity": "session", "loc": "src/backend/base/langflow/services/database/models/api_key/crud.py:49" }, "recoveryResult": { "entity": "ApiKey", "loc": "src/backend/base/langflow/services/database/models/api_key/model.py:23" }, "note": "path-length (d=4)" }, { "patternRole": "Validator", "score": 1.0, "distance": 0, "groundTruth": { "entity": "check_key", "loc": "src/backend/base/langflow/services/database/models/api_key/crud.py:46" }, "recoveryResult": { "entity": "check_key", "loc": "src/backend/base/langflow/services/database/models/api_key/crud.py:57" }, "note": "path-length (d=0)" } ], "notes": [ "Hasher: absent from the ground truth (Level 1 negative)", "Registrar: absent from the ground truth (Level 1 negative)", "Token generator: absent from the ground truth (Level 1 negative)" ] }, { "key": "level3", "level": "Level 3 \u2014 Interaction Order", "formula": "jaccard \u00d7 kendall-tau (shared interactions)", "available": true, "headlineLabel": "Interaction agreement", "score": 1.0, "drift": 0.0, "detail": { "agreedRoles": [ "Enforcer", "Token manager", "Validator" ], "excludedRoles": [ "Hasher", "Registrar", "Token generator" ], "interactionsGroundTruth": 2, "interactionsRecoveryResult": 2, "sharedInteractions": 2, "unionSize": 2, "entryRoleGroundTruth": "Enforcer", "entryRoleRecoveryResult": "Enforcer", "entryRuleGroundTruth": "behaviour", "entryRuleRecoveryResult": "stored", "onlyGroundTruth": 0, "onlyRecoveryResult": 0, "interactionOverlap": 1.0, "orderAgreement": 1.0, "rankedInteractions": 2, "totalPairs": 1, "concordantPairs": 1, "discordantPairs": 0, "kendallTau": 1.0, "prescribedActionSteps": 3, "prescribedStepsCovered": 2 }, "perRole": [ { "interaction": "Enforcer \u2192 Validator", "from": "Enforcer", "to": "Validator", "groundTruthRank": 1, "recoveryResultRank": 1, "outcome": "agreed", "discordantWith": 0, "evidence": { "groundTruth": { "file": "src/backend/base/langflow/services/auth/utils.py", "line": 55, "via": [ "src/backend/base/langflow/services/auth/utils.py:<module>.api_key_security" ] }, "recoveryResult": { "file": "src/backend/base/langflow/services/auth/utils.py", "line": 76, "via": [] } } }, { "interaction": "Validator \u2192 Token manager", "from": "Validator", "to": "Token manager", "groundTruthRank": 2, "recoveryResultRank": 2, "outcome": "agreed", "discordantWith": 0, "evidence": { "groundTruth": { "file": "src/backend/base/langflow/services/database/models/api_key/crud.py", "line": 49, "via": [] }, "recoveryResult": { "file": "src/backend/base/langflow/services/database/models/api_key/crud.py", "line": 60, "via": [] } } } ], "notes": [ "Score = overlap \u00d7 order = 1 \u00d7 1. Overlap: 2 of 2 interactions were found by both sides (0 only in the ground truth, 0 only in the recovery result), |shared| / |union|.", "Order: Kendall's Tau over the 2 shared interactions, in each side's own relative order \u2014 0 of 1 pairs run in opposite orders. Interactions only one side found take part in no pair (Fagin, Kumar & Sivakumar's p = 0, the induced sub-order); the overlap factor is what charges for them.", "Only 1 ranked pair \u2014 the order factor can land on just a few values here and one disagreement swings it fully.", "Scored over the 3 roles both sides detected (Enforcer, Token manager, Validator). An interaction counts only when both of its endpoint roles were detected on both sides \u2014 whether a role was found at all is Level 1's measurement, and charging for it again here would say the same thing twice.", "The interaction order is derived statically from the code \u2014 no one annotated it. Each role's cited entity is located in the AST index and control flow is followed from it, through as many non-role helper functions as it takes to reach the next role. See secpat_scoring/sequence/JOERN.md.", "Left out \u2014 only one side detected them: Hasher, Registrar, Token generator. 3 roles were excluded from both walks, so neither side's ordering mentions them. The walk still passes through the code behind such a role, attributing what it finds there to the nearest agreed role.", "Ground truth: the walk started at Enforcer \u2014 the earliest role the reference pattern has speaking that this side grounded in walkable code.", "Recovery result: the walk started at Enforcer \u2014 stored.", "An entry point the pattern did not name means its prescribed speakers were either ungrounded on that side or not where this code begins \u2014 a role something else reaches cannot open the flow \u2014 so the call graph had to stand in. Treat the ordering below as weaker evidence: it starts wherever the code happens to start." ] } ], "ground_truth_source": "annotation", "vocabulary": [ "Enforcer", "Validator", "Hasher", "Token manager" ], "role_scope": "required", "excluded_roles": [ "Subject", "System" ], "dropped_ground_truth_rows": [ "Registrar", "Subject", "System", "Token generator" ], "dropped_recovery_rows": [ "Subject", "System" ], "gt_roles": 4, "rr_roles": 4, "struct_files": 543, "annotated_at": "2026-08-11 07:55:18.185000", "recovery_order": "stored" }, { "slug": "obscure_token_based_access_control__elisiariocouto_leggen", "id": "obscure-token-ac/leggen/ClaudeCodeopus4.7", "pattern": "obscure_token_based_access_control", "repo": "elisiariocouto_leggen", "error": null, "metrics": [ { "key": "level1", "level": "Level 1 \u2014 Role Detection", "formula": "agreement-rate (present/external/missing)", "available": true, "headlineLabel": "Roles agreed", "score": 0.5, "drift": 0.5, "detail": { "agreedRoles": 2, "totalRoles": 4 }, "perRole": [ { "patternRole": "Enforcer", "groundTruth": "present", "recoveryResult": "present", "groundTruthDetected": true, "recoveryResultDetected": true, "outcome": "agreed" }, { "patternRole": "Validator", "groundTruth": "present", "recoveryResult": "present", "groundTruthDetected": true, "recoveryResultDetected": true, "outcome": "agreed" }, { "patternRole": "Hasher", "groundTruth": "missing", "recoveryResult": "external", "groundTruthDetected": false, "recoveryResultDetected": true, "outcome": "mismatched" }, { "patternRole": "Token manager", "groundTruth": "missing", "recoveryResult": "present", "groundTruthDetected": false, "recoveryResultDetected": true, "outcome": "mismatched" } ], "notes": [] }, { "key": "level2", "level": "Level 2 \u2014 Role Grounding", "formula": "path-length", "available": true, "headlineLabel": "Mean grounding score", "score": 0.9166, "drift": 0.0834, "detail": { "aggregate": 0.9166, "scoredRoles": 2, "skippedRoles": 4, "treeHeight": 6 }, "perRole": [ { "patternRole": "Enforcer", "score": 1.0, "distance": 0, "groundTruth": { "entity": "get_current_user", "loc": "leggen/api/dependencies/auth.py:13" }, "recoveryResult": { "entity": "get_current_user", "loc": "leggen/api/dependencies/auth.py:12" }, "note": "path-length (d=0)" }, { "patternRole": "Validator", "score": 0.8333, "distance": 2, "groundTruth": { "entity": "verify_api_key", "loc": "leggen/utils/auth.py:47\u201349" }, "recoveryResult": { "entity": "decode_access_token", "loc": "leggen/utils/auth.py:43" }, "note": "path-length (d=2)" } ], "notes": [ "Hasher: absent from the ground truth (Level 1 negative)", "Registrar: absent from the ground truth (Level 1 negative)", "Token generator: absent from the ground truth (Level 1 negative)", "Token manager: absent from the ground truth (Level 1 negative)" ] }, { "key": "level3", "level": "Level 3 \u2014 Interaction Order", "formula": "jaccard \u00d7 kendall-tau (shared interactions)", "available": true, "headlineLabel": "Interaction agreement", "score": 1.0, "drift": 0.0, "detail": { "agreedRoles": [ "Enforcer", "Validator" ], "excludedRoles": [ "Hasher", "Registrar", "Token generator", "Token manager" ], "interactionsGroundTruth": 1, "interactionsRecoveryResult": 1, "sharedInteractions": 1, "unionSize": 1, "entryRoleGroundTruth": "Enforcer", "entryRoleRecoveryResult": "Enforcer", "entryRuleGroundTruth": "behaviour", "entryRuleRecoveryResult": "stored", "onlyGroundTruth": 0, "onlyRecoveryResult": 0, "interactionOverlap": 1.0, "orderAgreement": 1.0, "rankedInteractions": 1, "totalPairs": 0, "concordantPairs": 0, "discordantPairs": 0, "kendallTau": null, "prescribedActionSteps": 3, "prescribedStepsCovered": 1 }, "perRole": [ { "interaction": "Enforcer \u2192 Validator", "from": "Enforcer", "to": "Validator", "groundTruthRank": 1, "recoveryResultRank": 1, "outcome": "agreed", "discordantWith": 0, "evidence": { "groundTruth": { "file": "leggen/api/dependencies/auth.py", "line": 30, "via": [] }, "recoveryResult": { "file": "leggen/api/dependencies/auth.py", "line": 27, "via": [] } } } ], "notes": [ "Score = overlap \u00d7 order = 1 \u00d7 1. Overlap: 1 of 1 interactions were found by both sides (0 only in the ground truth, 0 only in the recovery result), |shared| / |union|.", "Order: 1 \u2014 a single shared interaction has no pair whose relative order could differ.", "Scored over the 2 roles both sides detected (Enforcer, Validator). An interaction counts only when both of its endpoint roles were detected on both sides \u2014 whether a role was found at all is Level 1's measurement, and charging for it again here would say the same thing twice.", "The interaction order is derived statically from the code \u2014 no one annotated it. Each role's cited entity is located in the AST index and control flow is followed from it, through as many non-role helper functions as it takes to reach the next role. See secpat_scoring/sequence/JOERN.md.", "Left out \u2014 only one side detected them: Hasher, Registrar, Token generator, Token manager. 4 roles were excluded from both walks, so neither side's ordering mentions them. The walk still passes through the code behind such a role, attributing what it finds there to the nearest agreed role.", "Ground truth: the walk started at Enforcer \u2014 the earliest role the reference pattern has speaking that this side grounded in walkable code.", "Recovery result: the walk started at Enforcer \u2014 stored.", "An entry point the pattern did not name means its prescribed speakers were either ungrounded on that side or not where this code begins \u2014 a role something else reaches cannot open the flow \u2014 so the call graph had to stand in. Treat the ordering below as weaker evidence: it starts wherever the code happens to start." ] } ], "ground_truth_source": "annotation", "vocabulary": [ "Enforcer", "Validator", "Hasher", "Token manager" ], "role_scope": "required", "excluded_roles": [ "Subject", "System" ], "dropped_ground_truth_rows": [ "Registrar", "Subject", "System", "Token generator" ], "dropped_recovery_rows": [ "Subject", "System" ], "gt_roles": 4, "rr_roles": 4, "struct_files": 89, "annotated_at": "2026-08-11 08:04:34.777000", "recovery_order": "stored" }, { "slug": "obscure_token_based_access_control__bluesky_tiled", "id": "obscure-token-ac/tiled/ClaudeCodeopus4.7", "pattern": "obscure_token_based_access_control", "repo": "bluesky_tiled", "error": null, "metrics": [ { "key": "level1", "level": "Level 1 \u2014 Role Detection", "formula": "agreement-rate", "available": true, "headlineLabel": "Roles agreed", "score": 1.0, "drift": 0.0, "detail": { "agreedRoles": 4, "totalRoles": 4 }, "perRole": [ { "patternRole": "Enforcer", "groundTruth": "present", "recoveryResult": "present", "groundTruthDetected": true, "recoveryResultDetected": true, "outcome": "agreed" }, { "patternRole": "Validator", "groundTruth": "present", "recoveryResult": "present", "groundTruthDetected": true, "recoveryResultDetected": true, "outcome": "agreed" }, { "patternRole": "Hasher", "groundTruth": "present", "recoveryResult": "present", "groundTruthDetected": true, "recoveryResultDetected": true, "outcome": "agreed" }, { "patternRole": "Token manager", "groundTruth": "present", "recoveryResult": "present", "groundTruthDetected": true, "recoveryResultDetected": true, "outcome": "agreed" } ], "notes": [] }, { "key": "level2", "level": "Level 2 \u2014 Role Grounding", "formula": "path-length", "available": true, "headlineLabel": "Mean grounding score", "score": 0.8333, "drift": 0.1667, "detail": { "aggregate": 0.8333, "scoredRoles": 4, "skippedRoles": 2, "treeHeight": 6 }, "perRole": [ { "patternRole": "Enforcer", "score": 0.8333, "distance": 2, "groundTruth": { "entity": "get_current_principal", "loc": "tiled/server/authentication.py:679\u2013754" }, "recoveryResult": { "entity": "get_api_key", "loc": "tiled/server/authentication.py:111" }, "note": "path-length (d=2)" }, { "patternRole": "Hasher", "score": 0.8333, "distance": 2, "groundTruth": { "entity": "digest", "loc": "tiled/authn_database/core.py:259" }, "recoveryResult": { "entity": "sha256", "loc": "tiled/authn_database/core.py:275" }, "note": "path-length (d=2)" }, { "patternRole": "Token manager", "score": 0.6667, "distance": 4, "groundTruth": { "entity": "db", "loc": "tiled/authn_database/core.py:261" }, "recoveryResult": { "entity": "APIKey", "loc": "tiled/authn_database/orm.py:150" }, "note": "path-length (d=4)" }, { "patternRole": "Validator", "score": 1.0, "distance": 0, "groundTruth": { "entity": "get_current_principal_from_api_key", "loc": "tiled/server/authentication.py:573\u2013617" }, "recoveryResult": { "entity": "get_current_principal_from_api_key", "loc": "tiled/server/authentication.py:330" }, "note": "path-length (d=0)" } ], "notes": [ "Registrar: absent from the ground truth (Level 1 negative)", "Token generator: absent from the ground truth (Level 1 negative)" ] }, { "key": "level3", "level": "Level 3 \u2014 Interaction Order", "formula": "jaccard \u00d7 kendall-tau (shared interactions)", "available": true, "headlineLabel": "Interaction agreement", "score": 1.0, "drift": 0.0, "detail": { "agreedRoles": [ "Enforcer", "Hasher", "Token manager", "Validator" ], "excludedRoles": [ "Registrar", "Token generator" ], "interactionsGroundTruth": 3, "interactionsRecoveryResult": 3, "sharedInteractions": 3, "unionSize": 3, "entryRoleGroundTruth": "Enforcer", "entryRoleRecoveryResult": "Enforcer", "entryRuleGroundTruth": "behaviour", "entryRuleRecoveryResult": "stored", "onlyGroundTruth": 0, "onlyRecoveryResult": 0, "interactionOverlap": 1.0, "orderAgreement": 1.0, "rankedInteractions": 3, "totalPairs": 3, "concordantPairs": 3, "discordantPairs": 0, "kendallTau": 1.0, "prescribedActionSteps": 3, "prescribedStepsCovered": 3 }, "perRole": [ { "interaction": "Enforcer \u2192 Validator", "from": "Enforcer", "to": "Validator", "groundTruthRank": 1, "recoveryResultRank": 1, "outcome": "agreed", "discordantWith": 0, "evidence": { "groundTruth": { "file": "tiled/server/authentication.py", "line": 694, "via": [] }, "recoveryResult": { "file": "tiled/server/authentication.py", "line": 430, "via": [] } } }, { "interaction": "Validator \u2192 Hasher", "from": "Validator", "to": "Hasher", "groundTruthRank": 2, "recoveryResultRank": 2, "outcome": "agreed", "discordantWith": 0, "evidence": { "groundTruth": { "file": "tiled/authn_database/core.py", "line": 259, "via": [ "tiled/authn_database/core.py:<module>.lookup_valid_api_key" ] }, "recoveryResult": { "file": "tiled/authn_database/core.py", "line": 275, "via": [] } } }, { "interaction": "Validator \u2192 Token manager", "from": "Validator", "to": "Token manager", "groundTruthRank": 3, "recoveryResultRank": 3, "outcome": "agreed", "discordantWith": 0, "evidence": { "groundTruth": { "file": "tiled/authn_database/core.py", "line": 261, "via": [ "tiled/authn_database/core.py:<module>.lookup_valid_api_key" ] }, "recoveryResult": { "file": "tiled/authn_database/core.py", "line": 272, "via": [] } } } ], "notes": [ "Score = overlap \u00d7 order = 1 \u00d7 1. Overlap: 3 of 3 interactions were found by both sides (0 only in the ground truth, 0 only in the recovery result), |shared| / |union|.", "Order: Kendall's Tau over the 3 shared interactions, in each side's own relative order \u2014 0 of 3 pairs run in opposite orders. Interactions only one side found take part in no pair (Fagin, Kumar & Sivakumar's p = 0, the induced sub-order); the overlap factor is what charges for them.", "Scored over the 4 roles both sides detected (Enforcer, Hasher, Token manager, Validator). An interaction counts only when both of its endpoint roles were detected on both sides \u2014 whether a role was found at all is Level 1's measurement, and charging for it again here would say the same thing twice.", "The interaction order is derived statically from the code \u2014 no one annotated it. Each role's cited entity is located in the AST index and control flow is followed from it, through as many non-role helper functions as it takes to reach the next role. See secpat_scoring/sequence/JOERN.md.", "Left out \u2014 only one side detected them: Registrar, Token generator. 2 roles were excluded from both walks, so neither side's ordering mentions them. The walk still passes through the code behind such a role, attributing what it finds there to the nearest agreed role.", "Ground truth: the walk started at Enforcer \u2014 the earliest role the reference pattern has speaking that this side grounded in walkable code.", "Recovery result: the walk started at Enforcer \u2014 stored.", "An entry point the pattern did not name means its prescribed speakers were either ungrounded on that side or not where this code begins \u2014 a role something else reaches cannot open the flow \u2014 so the call graph had to stand in. Treat the ordering below as weaker evidence: it starts wherever the code happens to start.", "Ground truth: Hasher cites an entity that is not a symbol in this repository (a library or framework call). It can receive an interaction, but the walk cannot continue through it." ] } ], "ground_truth_source": "annotation", "vocabulary": [ "Enforcer", "Validator", "Hasher", "Token manager" ], "role_scope": "required", "excluded_roles": [ "Subject", "System" ], "dropped_ground_truth_rows": [ "Registrar", "Subject", "System", "Token generator" ], "dropped_recovery_rows": [ "Subject", "System" ], "gt_roles": 4, "rr_roles": 4, "struct_files": 308, "annotated_at": "2026-08-10 10:25:10.336000", "recovery_order": "stored" }, { "slug": "obscure_token_based_access_control__Extralit_extralit", "id": "obscure-token-ac/extralit/ClaudeCodeopus4.7", "pattern": "obscure_token_based_access_control", "repo": "Extralit_extralit", "error": null, "metrics": [ { "key": "level1", "level": "Level 1 \u2014 Role Detection", "formula": "agreement-rate", "available": true, "headlineLabel": "Roles agreed", "score": 0.75, "drift": 0.25, "detail": { "agreedRoles": 3, "totalRoles": 4 }, "perRole": [ { "patternRole": "Enforcer", "groundTruth": "present", "recoveryResult": "present", "groundTruthDetected": true, "recoveryResultDetected": true, "outcome": "agreed" }, { "patternRole": "Validator", "groundTruth": "present", "recoveryResult": "present", "groundTruthDetected": true, "recoveryResultDetected": true, "outcome": "agreed" }, { "patternRole": "Hasher", "groundTruth": "missing", "recoveryResult": "present", "groundTruthDetected": false, "recoveryResultDetected": true, "outcome": "mismatched" }, { "patternRole": "Token manager", "groundTruth": "present", "recoveryResult": "present", "groundTruthDetected": true, "recoveryResultDetected": true, "outcome": "agreed" } ], "notes": [] }, { "key": "level2", "level": "Level 2 \u2014 Role Grounding", "formula": "path-length", "available": true, "headlineLabel": "Mean grounding score", "score": 0.7424, "drift": 0.2576, "detail": { "aggregate": 0.7424, "scoredRoles": 3, "skippedRoles": 3, "treeHeight": 11 }, "perRole": [ { "patternRole": "Enforcer", "score": 0.6818, "distance": 7, "groundTruth": { "entity": "authenticate", "loc": "extralit-server/src/extralit_server/security/authentication/db/api_key_backend.py:15" }, "recoveryResult": { "entity": "get_current_user", "loc": "extralit-server/src/extralit_server/security/authentication/provider.py:65" }, "note": "path-length (d=7)" }, { "patternRole": "Token manager", "score": 0.9091, "distance": 2, "groundTruth": { "entity": "db", "loc": "extralit-server/src/extralit_server/contexts/accounts.py:91" }, "recoveryResult": { "entity": "get_user_by_api_key", "loc": "extralit-server/src/extralit_server/contexts/accounts.py:91" }, "note": "path-length (d=2)" }, { "patternRole": "Validator", "score": 0.6364, "distance": 8, "groundTruth": { "entity": "get_user_by_api_key", "loc": "extralit-server/src/extralit_server/contexts/accounts.py:90" }, "recoveryResult": { "entity": "APIKeyAuthenticationBackend", "loc": "extralit-server/src/extralit_server/security/authentication/db/api_key_backend.py:12" }, "note": "path-length (d=8)" } ], "notes": [ "Hasher: absent from the ground truth (Level 1 negative)", "Registrar: absent from the ground truth (Level 1 negative)", "Token generator: absent from the ground truth (Level 1 negative)" ] }, { "key": "level3", "level": "Level 3 \u2014 Interaction Order", "formula": "jaccard \u00d7 kendall-tau (shared interactions)", "available": true, "headlineLabel": "Interaction agreement", "score": 1.0, "drift": 0.0, "detail": { "agreedRoles": [ "Enforcer", "Token manager", "Validator" ], "excludedRoles": [ "Hasher", "Registrar", "Token generator" ], "interactionsGroundTruth": 2, "interactionsRecoveryResult": 2, "sharedInteractions": 2, "unionSize": 2, "entryRoleGroundTruth": "Enforcer", "entryRoleRecoveryResult": "Enforcer", "entryRuleGroundTruth": "behaviour", "entryRuleRecoveryResult": "stored", "onlyGroundTruth": 0, "onlyRecoveryResult": 0, "interactionOverlap": 1.0, "orderAgreement": 1.0, "rankedInteractions": 2, "totalPairs": 1, "concordantPairs": 1, "discordantPairs": 0, "kendallTau": 1.0, "prescribedActionSteps": 3, "prescribedStepsCovered": 2 }, "perRole": [ { "interaction": "Enforcer \u2192 Validator", "from": "Enforcer", "to": "Validator", "groundTruthRank": 1, "recoveryResultRank": 1, "outcome": "agreed", "discordantWith": 0, "evidence": { "groundTruth": { "file": "extralit-server/src/extralit_server/security/authentication/db/api_key_backend.py", "line": 22, "via": [] }, "recoveryResult": { "file": "extralit-server/src/extralit_server/security/authentication/provider.py", "line": 105, "via": [] } } }, { "interaction": "Validator \u2192 Token manager", "from": "Validator", "to": "Token manager", "groundTruthRank": 2, "recoveryResultRank": 2, "outcome": "agreed", "discordantWith": 0, "evidence": { "groundTruth": { "file": "extralit-server/src/extralit_server/contexts/accounts.py", "line": 91, "via": [] }, "recoveryResult": { "file": "extralit-server/src/extralit_server/security/authentication/db/api_key_backend.py", "line": 24, "via": [] } } } ], "notes": [ "Score = overlap \u00d7 order = 1 \u00d7 1. Overlap: 2 of 2 interactions were found by both sides (0 only in the ground truth, 0 only in the recovery result), |shared| / |union|.", "Order: Kendall's Tau over the 2 shared interactions, in each side's own relative order \u2014 0 of 1 pairs run in opposite orders. Interactions only one side found take part in no pair (Fagin, Kumar & Sivakumar's p = 0, the induced sub-order); the overlap factor is what charges for them.", "Only 1 ranked pair \u2014 the order factor can land on just a few values here and one disagreement swings it fully.", "Scored over the 3 roles both sides detected (Enforcer, Token manager, Validator). An interaction counts only when both of its endpoint roles were detected on both sides \u2014 whether a role was found at all is Level 1's measurement, and charging for it again here would say the same thing twice.", "The interaction order is derived statically from the code \u2014 no one annotated it. Each role's cited entity is located in the AST index and control flow is followed from it, through as many non-role helper functions as it takes to reach the next role. See secpat_scoring/sequence/JOERN.md.", "Left out \u2014 only one side detected them: Hasher, Registrar, Token generator. 3 roles were excluded from both walks, so neither side's ordering mentions them. The walk still passes through the code behind such a role, attributing what it finds there to the nearest agreed role.", "Ground truth: the walk started at Enforcer \u2014 the earliest role the reference pattern has speaking that this side grounded in walkable code.", "Recovery result: the walk started at Enforcer \u2014 stored.", "An entry point the pattern did not name means its prescribed speakers were either ungrounded on that side or not where this code begins \u2014 a role something else reaches cannot open the flow \u2014 so the call graph had to stand in. Treat the ordering below as weaker evidence: it starts wherever the code happens to start." ] } ], "ground_truth_source": "annotation", "vocabulary": [ "Enforcer", "Validator", "Hasher", "Token manager" ], "role_scope": "required", "excluded_roles": [ "Subject", "System" ], "dropped_ground_truth_rows": [ "Registrar", "Subject", "System", "Token generator" ], "dropped_recovery_rows": [ "Subject", "System" ], "gt_roles": 4, "rr_roles": 4, "struct_files": 732, "annotated_at": "2026-10-06 17:23:51.570000", "recovery_order": "stored" }, { "slug": "obscure_token_based_access_control__adrian-lorenz_privacy-guard", "id": "obscure-token-ac/privacy-guard/ClaudeCodeopus4.7", "pattern": "obscure_token_based_access_control", "repo": "adrian-lorenz_privacy-guard", "error": null, "metrics": [ { "key": "level1", "level": "Level 1 \u2014 Role Detection", "formula": "agreement-rate (present/external/missing)", "available": true, "headlineLabel": "Roles agreed", "score": 0.75, "drift": 0.25, "detail": { "agreedRoles": 3, "totalRoles": 4 }, "perRole": [ { "patternRole": "Enforcer", "groundTruth": "present", "recoveryResult": "present", "groundTruthDetected": true, "recoveryResultDetected": true, "outcome": "agreed" }, { "patternRole": "Validator", "groundTruth": "present", "recoveryResult": "present", "groundTruthDetected": true, "recoveryResultDetected": true, "outcome": "agreed" }, { "patternRole": "Hasher", "groundTruth": "present", "recoveryResult": "external", "groundTruthDetected": true, "recoveryResultDetected": true, "outcome": "mismatched" }, { "patternRole": "Token manager", "groundTruth": "present", "recoveryResult": "present", "groundTruthDetected": true, "recoveryResultDetected": true, "outcome": "agreed" } ], "notes": [] }, { "key": "level2", "level": "Level 2 \u2014 Role Grounding", "formula": "path-length", "available": true, "headlineLabel": "Mean grounding score", "score": 0.95, "drift": 0.05, "detail": { "aggregate": 0.95, "scoredRoles": 4, "skippedRoles": 2, "treeHeight": 5 }, "perRole": [ { "patternRole": "Enforcer", "score": 1.0, "distance": 0, "groundTruth": { "entity": "verify_api_key", "loc": "api/main.py:36" }, "recoveryResult": { "entity": "verify_api_key", "loc": "api/main.py:26" }, "note": "path-length (d=0)" }, { "patternRole": "Hasher", "score": 1.0, "distance": 0, "groundTruth": { "entity": "hashlib.sha256", "loc": "api/db.py:197" }, "recoveryResult": { "entity": "sha256", "loc": "api/db.py:184" }, "note": "path-length (d=0)" }, { "patternRole": "Token manager", "score": 0.8, "distance": 2, "groundTruth": { "entity": "con", "loc": "api/db.py:198" }, "recoveryResult": { "entity": "create_api_key", "loc": "api/db.py:135" }, "note": "path-length (d=2)" }, { "patternRole": "Validator", "score": 1.0, "distance": 0, "groundTruth": { "entity": "check_api_key", "loc": "api/db.py:195\u2013209" }, "recoveryResult": { "entity": "check_api_key", "loc": "api/db.py:182" }, "note": "path-length (d=0)" } ], "notes": [ "Registrar: absent from the ground truth (Level 1 negative)", "Token generator: absent from the ground truth (Level 1 negative)" ] }, { "key": "level3", "level": "Level 3 \u2014 Interaction Order", "formula": "jaccard \u00d7 kendall-tau (shared interactions)", "available": true, "headlineLabel": "Interaction agreement", "score": 0.6667, "drift": 0.3333, "detail": { "agreedRoles": [ "Enforcer", "Hasher", "Token manager", "Validator" ], "excludedRoles": [ "Registrar", "Token generator" ], "interactionsGroundTruth": 2, "interactionsRecoveryResult": 3, "sharedInteractions": 2, "unionSize": 3, "entryRoleGroundTruth": "Enforcer", "entryRoleRecoveryResult": "Enforcer", "entryRuleGroundTruth": "behaviour", "entryRuleRecoveryResult": "stored", "onlyGroundTruth": 0, "onlyRecoveryResult": 1, "interactionOverlap": 0.6667, "orderAgreement": 1.0, "rankedInteractions": 2, "totalPairs": 1, "concordantPairs": 1, "discordantPairs": 0, "kendallTau": 1.0, "prescribedActionSteps": 3, "prescribedStepsCovered": 3 }, "perRole": [ { "interaction": "Enforcer \u2192 Validator", "from": "Enforcer", "to": "Validator", "groundTruthRank": 1, "recoveryResultRank": 1, "outcome": "agreed", "discordantWith": 0, "evidence": { "groundTruth": { "file": "api/main.py", "line": 44, "via": [] }, "recoveryResult": { "file": "api/main.py", "line": 34, "via": [] } } }, { "interaction": "Validator \u2192 Hasher", "from": "Validator", "to": "Hasher", "groundTruthRank": 2, "recoveryResultRank": 2, "outcome": "agreed", "discordantWith": 0, "evidence": { "groundTruth": { "file": "api/db.py", "line": 197, "via": [] }, "recoveryResult": { "file": "api/db.py", "line": 184, "via": [] } } }, { "interaction": "Validator \u2192 Token manager", "from": "Validator", "to": "Token manager", "groundTruthRank": null, "recoveryResultRank": 3, "outcome": "onlyRecoveryResult", "discordantWith": null, "evidence": { "groundTruth": null, "recoveryResult": { "file": "api/db.py", "line": 186, "via": [] } } } ], "notes": [ "Score = overlap \u00d7 order = 0.6667 \u00d7 1. Overlap: 2 of 3 interactions were found by both sides (0 only in the ground truth, 1 only in the recovery result), |shared| / |union|.", "Order: Kendall's Tau over the 2 shared interactions, in each side's own relative order \u2014 0 of 1 pairs run in opposite orders. Interactions only one side found take part in no pair (Fagin, Kumar & Sivakumar's p = 0, the induced sub-order); the overlap factor is what charges for them.", "Only 1 ranked pair \u2014 the order factor can land on just a few values here and one disagreement swings it fully.", "Scored over the 4 roles both sides detected (Enforcer, Hasher, Token manager, Validator). An interaction counts only when both of its endpoint roles were detected on both sides \u2014 whether a role was found at all is Level 1's measurement, and charging for it again here would say the same thing twice.", "The interaction order is derived statically from the code \u2014 no one annotated it. Each role's cited entity is located in the AST index and control flow is followed from it, through as many non-role helper functions as it takes to reach the next role. See secpat_scoring/sequence/JOERN.md.", "Left out \u2014 only one side detected them: Registrar, Token generator. 2 roles were excluded from both walks, so neither side's ordering mentions them. The walk still passes through the code behind such a role, attributing what it finds there to the nearest agreed role.", "Ground truth: the walk started at Enforcer \u2014 the earliest role the reference pattern has speaking that this side grounded in walkable code.", "Recovery result: the walk started at Enforcer \u2014 stored.", "An entry point the pattern did not name means its prescribed speakers were either ungrounded on that side or not where this code begins \u2014 a role something else reaches cannot open the flow \u2014 so the call graph had to stand in. Treat the ordering below as weaker evidence: it starts wherever the code happens to start.", "Ground truth: Hasher cites an entity that is not a symbol in this repository (a library or framework call). It can receive an interaction, but the walk cannot continue through it." ] } ], "ground_truth_source": "annotation", "vocabulary": [ "Enforcer", "Validator", "Hasher", "Token manager" ], "role_scope": "required", "excluded_roles": [ "Subject", "System" ], "dropped_ground_truth_rows": [ "Registrar", "Subject", "System", "Token generator" ], "dropped_recovery_rows": [ "Subject", "System" ], "gt_roles": 4, "rr_roles": 4, "struct_files": 45, "annotated_at": "2026-08-10 09:04:47.082000", "recovery_order": "stored" }, { "slug": "obscure_token_based_access_control__pavanad_beegen", "id": "obscure-token-ac/beegen/ClaudeCodeOpus5.5", "pattern": "obscure_token_based_access_control", "repo": "pavanad_beegen", "error": null, "metrics": [ { "key": "level1", "level": "Level 1 \u2014 Role Detection", "formula": "agreement-rate (present/external/missing)", "available": true, "headlineLabel": "Roles agreed", "score": 1.0, "drift": 0.0, "detail": { "agreedRoles": 4, "totalRoles": 4 }, "perRole": [ { "patternRole": "Enforcer", "groundTruth": "present", "recoveryResult": "present", "groundTruthDetected": true, "recoveryResultDetected": true, "outcome": "agreed" }, { "patternRole": "Validator", "groundTruth": "present", "recoveryResult": "present", "groundTruthDetected": true, "recoveryResultDetected": true, "outcome": "agreed" }, { "patternRole": "Hasher", "groundTruth": "missing", "recoveryResult": "missing", "groundTruthDetected": false, "recoveryResultDetected": false, "outcome": "agreed" }, { "patternRole": "Token manager", "groundTruth": "present", "recoveryResult": "present", "groundTruthDetected": true, "recoveryResultDetected": true, "outcome": "agreed" } ], "notes": [] }, { "key": "level2", "level": "Level 2 \u2014 Role Grounding", "formula": "path-length", "available": true, "headlineLabel": "Mean grounding score", "score": 0.9722, "drift": 0.0278, "detail": { "aggregate": 0.9722, "scoredRoles": 3, "skippedRoles": 1, "treeHeight": 6 }, "perRole": [ { "patternRole": "Enforcer", "score": 1.0, "distance": 0, "groundTruth": { "entity": "route", "loc": "beegen/services/mock.py:44" }, "recoveryResult": { "entity": "route", "loc": "beegen/services/mock.py:44" }, "note": "path-length (d=0)" }, { "patternRole": "Token manager", "score": 0.9167, "distance": 1, "groundTruth": { "entity": "__create_service", "loc": "beegen/services/mock.py:18" }, "recoveryResult": { "entity": "MockService", "loc": "beegen/services/mock.py:9" }, "note": "path-length (d=1)" }, { "patternRole": "Validator", "score": 1.0, "distance": 0, "groundTruth": { "entity": "route", "loc": "beegen/services/mock.py:44" }, "recoveryResult": { "entity": "route", "loc": "beegen/services/mock.py:45" }, "note": "path-length (d=0)" } ], "notes": [ "Hasher: absent from the ground truth (Level 1 negative)" ] }, { "key": "level3", "level": "Level 3 \u2014 Interaction Order", "formula": "jaccard \u00d7 kendall-tau (shared interactions)", "available": true, "headlineLabel": "Interaction agreement", "score": 0.5, "drift": 0.5, "detail": { "agreedRoles": [ "Enforcer", "Token manager", "Validator" ], "excludedRoles": [ "Hasher" ], "interactionsGroundTruth": 1, "interactionsRecoveryResult": 2, "sharedInteractions": 1, "unionSize": 2, "entryRoleGroundTruth": "Enforcer", "entryRoleRecoveryResult": "Enforcer", "entryRuleGroundTruth": "behaviour", "entryRuleRecoveryResult": "stored", "onlyGroundTruth": 0, "onlyRecoveryResult": 1, "interactionOverlap": 0.5, "orderAgreement": 1.0, "rankedInteractions": 1, "totalPairs": 0, "concordantPairs": 0, "discordantPairs": 0, "kendallTau": null, "prescribedActionSteps": 3, "prescribedStepsCovered": 2 }, "perRole": [ { "interaction": "Enforcer \u2192 Validator", "from": "Enforcer", "to": "Validator", "groundTruthRank": 1, "recoveryResultRank": 1, "outcome": "agreed", "discordantWith": 0, "evidence": { "groundTruth": { "file": "beegen/services/mock.py", "line": 44, "via": [] }, "recoveryResult": { "file": "beegen/services/mock.py", "line": 45, "via": [] } } }, { "interaction": "Validator \u2192 Token manager", "from": "Validator", "to": "Token manager", "groundTruthRank": null, "recoveryResultRank": 2, "outcome": "onlyRecoveryResult", "discordantWith": null, "evidence": { "groundTruth": null, "recoveryResult": { "file": "beegen/services/mock.py", "line": 47, "via": [] } } } ], "notes": [ "Score = overlap \u00d7 order = 0.5 \u00d7 1. Overlap: 1 of 2 interactions were found by both sides (0 only in the ground truth, 1 only in the recovery result), |shared| / |union|.", "Order: 1 \u2014 a single shared interaction has no pair whose relative order could differ.", "Scored over the 3 roles both sides detected (Enforcer, Token manager, Validator). An interaction counts only when both of its endpoint roles were detected on both sides \u2014 whether a role was found at all is Level 1's measurement, and charging for it again here would say the same thing twice.", "The interaction order is derived statically from the code \u2014 no one annotated it. Each role's cited entity is located in the AST index and control flow is followed from it, through as many non-role helper functions as it takes to reach the next role. See secpat_scoring/sequence/JOERN.md.", "Left out \u2014 only one side detected it: Hasher. 1 role was excluded from both walks, so neither side's ordering mentions it. The walk still passes through the code behind such a role, attributing what it finds there to the nearest agreed role.", "Ground truth: the walk started at Enforcer \u2014 the earliest role the reference pattern has speaking that this side grounded in walkable code.", "Recovery result: the walk started at Enforcer \u2014 stored.", "An entry point the pattern did not name means its prescribed speakers were either ungrounded on that side or not where this code begins \u2014 a role something else reaches cannot open the flow \u2014 so the call graph had to stand in. Treat the ordering below as weaker evidence: it starts wherever the code happens to start." ] } ], "ground_truth_source": "recovery result", "vocabulary": [ "Enforcer", "Validator", "Hasher", "Token manager" ], "role_scope": "required", "excluded_roles": [ "Subject", "System" ], "dropped_ground_truth_rows": [ "Subject", "System" ], "dropped_recovery_rows": [ "Subject", "System" ], "gt_roles": 4, "rr_roles": 4, "struct_files": 42, "annotated_at": "2026-10-02 10:00:38.838000", "recovery_order": "stored" }, { "slug": "obscure_token_based_access_control__JeanExtreme002_Dokku-API", "id": "obscure-token-ac/dokku-api/ClaudeCodeopus4.7", "pattern": "obscure_token_based_access_control", "repo": "JeanExtreme002_Dokku-API", "error": null, "metrics": [ { "key": "level1", "level": "Level 1 \u2014 Role Detection", "formula": "agreement-rate (present/external/missing)", "available": true, "headlineLabel": "Roles agreed", "score": 1.0, "drift": 0.0, "detail": { "agreedRoles": 4, "totalRoles": 4 }, "perRole": [ { "patternRole": "Enforcer", "groundTruth": "present", "recoveryResult": "present", "groundTruthDetected": true, "recoveryResultDetected": true, "outcome": "agreed" }, { "patternRole": "Validator", "groundTruth": "present", "recoveryResult": "present", "groundTruthDetected": true, "recoveryResultDetected": true, "outcome": "agreed" }, { "patternRole": "Hasher", "groundTruth": "present", "recoveryResult": "present", "groundTruthDetected": true, "recoveryResultDetected": true, "outcome": "agreed" }, { "patternRole": "Token manager", "groundTruth": "present", "recoveryResult": "present", "groundTruthDetected": true, "recoveryResultDetected": true, "outcome": "agreed" } ], "notes": [] }, { "key": "level2", "level": "Level 2 \u2014 Role Grounding", "formula": "path-length", "available": true, "headlineLabel": "Mean grounding score", "score": 0.8959, "drift": 0.1041, "detail": { "aggregate": 0.8959, "scoredRoles": 4, "skippedRoles": 2, "treeHeight": 6 }, "perRole": [ { "patternRole": "Enforcer", "score": 0.9167, "distance": 1, "groundTruth": { "entity": "dispatch", "loc": "src/api/middlewares/session.py:15" }, "recoveryResult": { "entity": "UserSessionMiddleware", "loc": "src/api/middlewares/session.py:14" }, "note": "path-length (d=1)" }, { "patternRole": "Hasher", "score": 1.0, "distance": 0, "groundTruth": { "entity": "hash_access_token", "loc": "src/api/tools/token.py:4" }, "recoveryResult": { "entity": "hash_access_token", "loc": "src/api/tools/token.py:6" }, "note": "path-length (d=0)" }, { "patternRole": "Token manager", "score": 0.6667, "distance": 4, "groundTruth": { "entity": "db_session", "loc": "src/api/models/tools.py:97" }, "recoveryResult": { "entity": "User", "loc": "src/api/models/user.py:7" }, "note": "path-length (d=4)" }, { "patternRole": "Validator", "score": 1.0, "distance": 0, "groundTruth": { "entity": "get_user_by_access_token", "loc": "src/api/models/tools.py:89" }, "recoveryResult": { "entity": "get_user_by_access_token", "loc": "src/api/models/tools.py:74" }, "note": "path-length (d=0)" } ], "notes": [ "Registrar: absent from the recovery result (Level 1 false negative)", "Token generator: absent from the recovery result (Level 1 false negative)" ] }, { "key": "level3", "level": "Level 3 \u2014 Interaction Order", "formula": "jaccard \u00d7 kendall-tau (shared interactions)", "available": true, "headlineLabel": "Interaction agreement", "score": 1.0, "drift": 0.0, "detail": { "agreedRoles": [ "Enforcer", "Hasher", "Token manager", "Validator" ], "excludedRoles": [ "Registrar", "Token generator" ], "interactionsGroundTruth": 3, "interactionsRecoveryResult": 3, "sharedInteractions": 3, "unionSize": 3, "entryRoleGroundTruth": "Enforcer", "entryRoleRecoveryResult": "Enforcer", "entryRuleGroundTruth": "behaviour", "entryRuleRecoveryResult": "stored", "onlyGroundTruth": 0, "onlyRecoveryResult": 0, "interactionOverlap": 1.0, "orderAgreement": 1.0, "rankedInteractions": 3, "totalPairs": 3, "concordantPairs": 3, "discordantPairs": 0, "kendallTau": 1.0, "prescribedActionSteps": 3, "prescribedStepsCovered": 3 }, "perRole": [ { "interaction": "Enforcer \u2192 Validator", "from": "Enforcer", "to": "Validator", "groundTruthRank": 1, "recoveryResultRank": 1, "outcome": "agreed", "discordantWith": 0, "evidence": { "groundTruth": { "file": "src/api/middlewares/session.py", "line": 28, "via": [] }, "recoveryResult": { "file": "src/api/middlewares/session.py", "line": 30, "via": [] } } }, { "interaction": "Validator \u2192 Hasher", "from": "Validator", "to": "Hasher", "groundTruthRank": 2, "recoveryResultRank": 2, "outcome": "agreed", "discordantWith": 0, "evidence": { "groundTruth": { "file": "src/api/models/tools.py", "line": 95, "via": [] }, "recoveryResult": { "file": "src/api/models/tools.py", "line": 77, "via": [] } } }, { "interaction": "Validator \u2192 Token manager", "from": "Validator", "to": "Token manager", "groundTruthRank": 3, "recoveryResultRank": 3, "outcome": "agreed", "discordantWith": 0, "evidence": { "groundTruth": { "file": "src/api/models/tools.py", "line": 97, "via": [] }, "recoveryResult": { "file": "src/api/models/tools.py", "line": 79, "via": [] } } } ], "notes": [ "Score = overlap \u00d7 order = 1 \u00d7 1. Overlap: 3 of 3 interactions were found by both sides (0 only in the ground truth, 0 only in the recovery result), |shared| / |union|.", "Order: Kendall's Tau over the 3 shared interactions, in each side's own relative order \u2014 0 of 3 pairs run in opposite orders. Interactions only one side found take part in no pair (Fagin, Kumar & Sivakumar's p = 0, the induced sub-order); the overlap factor is what charges for them.", "Scored over the 4 roles both sides detected (Enforcer, Hasher, Token manager, Validator). An interaction counts only when both of its endpoint roles were detected on both sides \u2014 whether a role was found at all is Level 1's measurement, and charging for it again here would say the same thing twice.", "The interaction order is derived statically from the code \u2014 no one annotated it. Each role's cited entity is located in the AST index and control flow is followed from it, through as many non-role helper functions as it takes to reach the next role. See secpat_scoring/sequence/JOERN.md.", "Left out \u2014 only one side detected them: Registrar, Token generator. 2 roles were excluded from both walks, so neither side's ordering mentions them. The walk still passes through the code behind such a role, attributing what it finds there to the nearest agreed role.", "Ground truth: the walk started at Enforcer \u2014 the earliest role the reference pattern has speaking that this side grounded in walkable code.", "Recovery result: the walk started at Enforcer \u2014 stored.", "An entry point the pattern did not name means its prescribed speakers were either ungrounded on that side or not where this code begins \u2014 a role something else reaches cannot open the flow \u2014 so the call graph had to stand in. Treat the ordering below as weaker evidence: it starts wherever the code happens to start." ] } ], "ground_truth_source": "annotation", "vocabulary": [ "Enforcer", "Validator", "Hasher", "Token manager" ], "role_scope": "required", "excluded_roles": [ "Subject", "System" ], "dropped_ground_truth_rows": [ "Registrar", "Subject", "System", "Token generator" ], "dropped_recovery_rows": [ "Subject", "System" ], "gt_roles": 4, "rr_roles": 4, "struct_files": 85, "annotated_at": "2026-10-06 21:48:07.146000", "recovery_order": "stored" }, { "slug": "obscure_token_based_access_control__autonomi-ai_nos", "id": "obscure-token-ac/nos/ClaudeCodeOpus5.5", "pattern": "obscure_token_based_access_control", "repo": "autonomi-ai_nos", "error": null, "metrics": [ { "key": "level1", "level": "Level 1 \u2014 Role Detection", "formula": "agreement-rate (present/external/missing)", "available": true, "headlineLabel": "Roles agreed", "score": 1.0, "drift": 0.0, "detail": { "agreedRoles": 4, "totalRoles": 4 }, "perRole": [ { "patternRole": "Enforcer", "groundTruth": "present", "recoveryResult": "present", "groundTruthDetected": true, "recoveryResultDetected": true, "outcome": "agreed" }, { "patternRole": "Validator", "groundTruth": "present", "recoveryResult": "present", "groundTruthDetected": true, "recoveryResultDetected": true, "outcome": "agreed" }, { "patternRole": "Hasher", "groundTruth": "missing", "recoveryResult": "missing", "groundTruthDetected": false, "recoveryResultDetected": false, "outcome": "agreed" }, { "patternRole": "Token manager", "groundTruth": "present", "recoveryResult": "present", "groundTruthDetected": true, "recoveryResultDetected": true, "outcome": "agreed" } ], "notes": [] }, { "key": "level2", "level": "Level 2 \u2014 Role Grounding", "formula": "path-length", "available": true, "headlineLabel": "Mean grounding score", "score": 0.9524, "drift": 0.0476, "detail": { "aggregate": 0.9524, "scoredRoles": 3, "skippedRoles": 1, "treeHeight": 7 }, "perRole": [ { "patternRole": "Enforcer", "score": 0.8571, "distance": 2, "groundTruth": { "entity": "validate_m2m_key", "loc": "nos/server/http/_security.py:33" }, "recoveryResult": { "entity": "ValidMachineToMachine", "loc": "nos/server/http/_security.py:49" }, "note": "path-length (d=2)" }, { "patternRole": "Token manager", "score": 1.0, "distance": 0, "groundTruth": { "entity": "valid_m2m_keys", "loc": "nos/server/http/_security.py:9" }, "recoveryResult": { "entity": "valid_m2m_keys", "loc": "nos/server/http/_security.py:11" }, "note": "path-length (d=0)" }, { "patternRole": "Validator", "score": 1.0, "distance": 0, "groundTruth": { "entity": "validate_m2m_key", "loc": "nos/server/http/_security.py:33" }, "recoveryResult": { "entity": "validate_m2m_key", "loc": "nos/server/http/_security.py:28" }, "note": "path-length (d=0)" } ], "notes": [ "Hasher: absent from the ground truth (Level 1 negative)" ] }, { "key": "level3", "level": "Level 3 \u2014 Interaction Order", "formula": "jaccard \u00d7 kendall-tau (shared interactions)", "available": true, "headlineLabel": "Interaction agreement", "score": 0.5, "drift": 0.5, "detail": { "agreedRoles": [ "Enforcer", "Token manager", "Validator" ], "excludedRoles": [ "Hasher" ], "interactionsGroundTruth": 1, "interactionsRecoveryResult": 2, "sharedInteractions": 1, "unionSize": 2, "entryRoleGroundTruth": "Enforcer", "entryRoleRecoveryResult": "Enforcer", "entryRuleGroundTruth": "behaviour", "entryRuleRecoveryResult": "stored", "onlyGroundTruth": 0, "onlyRecoveryResult": 1, "interactionOverlap": 0.5, "orderAgreement": 1.0, "rankedInteractions": 1, "totalPairs": 0, "concordantPairs": 0, "discordantPairs": 0, "kendallTau": null, "prescribedActionSteps": 3, "prescribedStepsCovered": 2 }, "perRole": [ { "interaction": "Enforcer \u2192 Validator", "from": "Enforcer", "to": "Validator", "groundTruthRank": 1, "recoveryResultRank": 1, "outcome": "agreed", "discordantWith": 0, "evidence": { "groundTruth": { "file": "nos/server/http/_security.py", "line": 33, "via": [] }, "recoveryResult": { "file": "nos/server/http/_service.py", "line": 203, "via": [] } } }, { "interaction": "Validator \u2192 Token manager", "from": "Validator", "to": "Token manager", "groundTruthRank": null, "recoveryResultRank": 2, "outcome": "onlyRecoveryResult", "discordantWith": null, "evidence": { "groundTruth": null, "recoveryResult": { "file": "nos/server/http/_security.py", "line": 39, "via": [] } } } ], "notes": [ "Score = overlap \u00d7 order = 0.5 \u00d7 1. Overlap: 1 of 2 interactions were found by both sides (0 only in the ground truth, 1 only in the recovery result), |shared| / |union|.", "Order: 1 \u2014 a single shared interaction has no pair whose relative order could differ.", "Scored over the 3 roles both sides detected (Enforcer, Token manager, Validator). An interaction counts only when both of its endpoint roles were detected on both sides \u2014 whether a role was found at all is Level 1's measurement, and charging for it again here would say the same thing twice.", "The interaction order is derived statically from the code \u2014 no one annotated it. Each role's cited entity is located in the AST index and control flow is followed from it, through as many non-role helper functions as it takes to reach the next role. See secpat_scoring/sequence/JOERN.md.", "Left out \u2014 only one side detected it: Hasher. 1 role was excluded from both walks, so neither side's ordering mentions it. The walk still passes through the code behind such a role, attributing what it finds there to the nearest agreed role.", "Ground truth: the walk started at Enforcer \u2014 the earliest role the reference pattern has speaking that this side grounded in walkable code.", "Recovery result: the walk started at Enforcer \u2014 stored.", "An entry point the pattern did not name means its prescribed speakers were either ungrounded on that side or not where this code begins \u2014 a role something else reaches cannot open the flow \u2014 so the call graph had to stand in. Treat the ordering below as weaker evidence: it starts wherever the code happens to start." ] } ], "ground_truth_source": "recovery result", "vocabulary": [ "Enforcer", "Validator", "Hasher", "Token manager" ], "role_scope": "required", "excluded_roles": [ "Subject", "System" ], "dropped_ground_truth_rows": [ "Subject", "System" ], "dropped_recovery_rows": [ "Subject", "System" ], "gt_roles": 4, "rr_roles": 4, "struct_files": 149, "annotated_at": "2026-10-02 10:00:38.838000", "recovery_order": "stored" }, { "slug": "verifiable_token_based_authentication__Chainlit_chainlit", "id": "token-based/chainlit/ClaudeCodeopus4.7", "pattern": "verifiable_token_based_authentication", "repo": "Chainlit_chainlit", "error": null, "metrics": [ { "key": "level1", "level": "Level 1 \u2014 Role Detection", "formula": "agreement-rate (present/external/missing)", "available": true, "headlineLabel": "Roles agreed", "score": 1.0, "drift": 0.0, "detail": { "agreedRoles": 4, "totalRoles": 4 }, "perRole": [ { "patternRole": "Enforcer", "groundTruth": "present", "recoveryResult": "present", "groundTruthDetected": true, "recoveryResultDetected": true, "outcome": "agreed" }, { "patternRole": "Verifier", "groundTruth": "present", "recoveryResult": "present", "groundTruthDetected": true, "recoveryResultDetected": true, "outcome": "agreed" }, { "patternRole": "Key manager", "groundTruth": "present", "recoveryResult": "present", "groundTruthDetected": true, "recoveryResultDetected": true, "outcome": "agreed" }, { "patternRole": "Cryptographer", "groundTruth": "external", "recoveryResult": "external", "groundTruthDetected": true, "recoveryResultDetected": true, "outcome": "agreed" } ], "notes": [] }, { "key": "level2", "level": "Level 2 \u2014 Role Grounding", "formula": "path-length", "available": true, "headlineLabel": "Mean grounding score", "score": 0.8393, "drift": 0.1607, "detail": { "aggregate": 0.8393, "scoredRoles": 4, "skippedRoles": 2, "treeHeight": 7 }, "perRole": [ { "patternRole": "Cryptographer", "score": 1.0, "distance": 0, "groundTruth": { "entity": "decode", "loc": "backend/chainlit/auth/jwt.py:35" }, "recoveryResult": { "entity": "decode", "loc": "backend/chainlit/auth/jwt.py:37" }, "note": "path-length (d=0)" }, { "patternRole": "Enforcer", "score": 0.6429, "distance": 5, "groundTruth": { "entity": "_authenticate_connection", "loc": "backend/chainlit/socket.py:133\u2013141" }, "recoveryResult": { "entity": "get_current_user", "loc": "backend/chainlit/auth/__init__.py:90" }, "note": "path-length (d=5)" }, { "patternRole": "Key manager", "score": 1.0, "distance": 0, "groundTruth": { "entity": "get_jwt_secret", "loc": "backend/chainlit/auth/jwt.py:11\u201312" }, "recoveryResult": { "entity": "get_jwt_secret", "loc": "backend/chainlit/auth/jwt.py:17" }, "note": "path-length (d=0)" }, { "patternRole": "Verifier", "score": 0.7143, "distance": 4, "groundTruth": { "entity": "authenticate_user", "loc": "backend/chainlit/auth/__init__.py:58" }, "recoveryResult": { "entity": "decode_jwt", "loc": "backend/chainlit/auth/jwt.py:34" }, "note": "path-length (d=4)" } ], "notes": [ "Registrar: absent from the recovery result (Level 1 false negative)", "Token generator: absent from the recovery result (Level 1 false negative)" ] }, { "key": "level3", "level": "Level 3 \u2014 Interaction Order", "formula": "jaccard \u00d7 kendall-tau (shared interactions)", "available": true, "headlineLabel": "Interaction agreement", "score": 1.0, "drift": 0.0, "detail": { "agreedRoles": [ "Cryptographer", "Enforcer", "Key manager", "Verifier" ], "excludedRoles": [ "Registrar", "Token generator" ], "interactionsGroundTruth": 3, "interactionsRecoveryResult": 3, "sharedInteractions": 3, "unionSize": 3, "entryRoleGroundTruth": "Enforcer", "entryRoleRecoveryResult": "Enforcer", "entryRuleGroundTruth": "behaviour", "entryRuleRecoveryResult": "stored", "onlyGroundTruth": 0, "onlyRecoveryResult": 0, "interactionOverlap": 1.0, "orderAgreement": 1.0, "rankedInteractions": 3, "totalPairs": 3, "concordantPairs": 3, "discordantPairs": 0, "kendallTau": 1.0, "prescribedActionSteps": 3, "prescribedStepsCovered": 3 }, "perRole": [ { "interaction": "Enforcer \u2192 Verifier", "from": "Enforcer", "to": "Verifier", "groundTruthRank": 1, "recoveryResultRank": 1, "outcome": "agreed", "discordantWith": 0, "evidence": { "groundTruth": { "file": "backend/chainlit/auth/__init__.py", "line": 90, "via": [ "backend/chainlit/auth/__init__.py:<module>.get_current_user" ] }, "recoveryResult": { "file": "backend/chainlit/auth/__init__.py", "line": 41, "via": [] } } }, { "interaction": "Verifier \u2192 Key manager", "from": "Verifier", "to": "Key manager", "groundTruthRank": 2, "recoveryResultRank": 2, "outcome": "agreed", "discordantWith": 0, "evidence": { "groundTruth": { "file": "backend/chainlit/auth/jwt.py", "line": 32, "via": [ "backend/chainlit/auth/jwt.py:<module>.decode_jwt" ] }, "recoveryResult": { "file": "backend/chainlit/auth/jwt.py", "line": 35, "via": [] } } }, { "interaction": "Verifier \u2192 Cryptographer", "from": "Verifier", "to": "Cryptographer", "groundTruthRank": 3, "recoveryResultRank": 3, "outcome": "agreed", "discordantWith": 0, "evidence": { "groundTruth": { "file": "backend/chainlit/auth/jwt.py", "line": 35, "via": [ "backend/chainlit/auth/jwt.py:<module>.decode_jwt" ] }, "recoveryResult": { "file": "backend/chainlit/auth/jwt.py", "line": 37, "via": [] } } } ], "notes": [ "Score = overlap \u00d7 order = 1 \u00d7 1. Overlap: 3 of 3 interactions were found by both sides (0 only in the ground truth, 0 only in the recovery result), |shared| / |union|.", "Order: Kendall's Tau over the 3 shared interactions, in each side's own relative order \u2014 0 of 3 pairs run in opposite orders. Interactions only one side found take part in no pair (Fagin, Kumar & Sivakumar's p = 0, the induced sub-order); the overlap factor is what charges for them.", "Scored over the 4 roles both sides detected (Cryptographer, Enforcer, Key manager, Verifier). An interaction counts only when both of its endpoint roles were detected on both sides \u2014 whether a role was found at all is Level 1's measurement, and charging for it again here would say the same thing twice.", "The interaction order is derived statically from the code \u2014 no one annotated it. Each role's cited entity is located in the AST index and control flow is followed from it, through as many non-role helper functions as it takes to reach the next role. See secpat_scoring/sequence/JOERN.md.", "Left out \u2014 only one side detected them: Registrar, Token generator. 2 roles were excluded from both walks, so neither side's ordering mentions them. The walk still passes through the code behind such a role, attributing what it finds there to the nearest agreed role.", "Ground truth: the walk started at Enforcer \u2014 the earliest role the reference pattern has speaking that this side grounded in walkable code.", "Recovery result: the walk started at Enforcer \u2014 stored.", "An entry point the pattern did not name means its prescribed speakers were either ungrounded on that side or not where this code begins \u2014 a role something else reaches cannot open the flow \u2014 so the call graph had to stand in. Treat the ordering below as weaker evidence: it starts wherever the code happens to start.", "Ground truth: Cryptographer cites an entity that is not a symbol in this repository (a library or framework call). It can receive an interaction, but the walk cannot continue through it." ] } ], "ground_truth_source": "annotation", "vocabulary": [ "Enforcer", "Verifier", "Key manager", "Cryptographer" ], "role_scope": "required", "excluded_roles": [ "Subject", "System" ], "dropped_ground_truth_rows": [ "Registrar", "Subject", "System", "Token generator" ], "dropped_recovery_rows": [ "Subject", "System" ], "gt_roles": 4, "rr_roles": 4, "struct_files": 169, "annotated_at": "2026-08-06 09:00:07.327000", "recovery_order": "stored" }, { "slug": "verifiable_token_based_authentication__ddeutils_ddeutil-observe", "id": "token-based/ddeutil-observe/ClaudeCodeopus4.7", "pattern": "verifiable_token_based_authentication", "repo": "ddeutils_ddeutil-observe", "error": null, "metrics": [ { "key": "level1", "level": "Level 1 \u2014 Role Detection", "formula": "agreement-rate (present/external/missing)", "available": true, "headlineLabel": "Roles agreed", "score": 0.75, "drift": 0.25, "detail": { "agreedRoles": 3, "totalRoles": 4 }, "perRole": [ { "patternRole": "Enforcer", "groundTruth": "present", "recoveryResult": "present", "groundTruthDetected": true, "recoveryResultDetected": true, "outcome": "agreed" }, { "patternRole": "Verifier", "groundTruth": "present", "recoveryResult": "present", "groundTruthDetected": true, "recoveryResultDetected": true, "outcome": "agreed" }, { "patternRole": "Key manager", "groundTruth": "present", "recoveryResult": "present", "groundTruthDetected": true, "recoveryResultDetected": true, "outcome": "agreed" }, { "patternRole": "Cryptographer", "groundTruth": "present", "recoveryResult": "external", "groundTruthDetected": true, "recoveryResultDetected": true, "outcome": "mismatched" } ], "notes": [] }, { "key": "level2", "level": "Level 2 \u2014 Role Grounding", "formula": "path-length", "available": true, "headlineLabel": "Mean grounding score", "score": 0.75, "drift": 0.25, "detail": { "aggregate": 0.75, "scoredRoles": 4, "skippedRoles": 2, "treeHeight": 6 }, "perRole": [ { "patternRole": "Cryptographer", "score": 0.8333, "distance": 2, "groundTruth": { "entity": "decode", "loc": "src/ddeutil/observe/auth/securities.py:137" }, "recoveryResult": { "entity": "jwt", "loc": "src/ddeutil/observe/auth/securities.py:15" }, "note": "path-length (d=2)" }, { "patternRole": "Enforcer", "score": 0.8333, "distance": 2, "groundTruth": { "entity": "get_current_user", "loc": "src/ddeutil/observe/auth/deps.py:39\u201372" }, "recoveryResult": { "entity": "required_current_active_user", "loc": "src/ddeutil/observe/auth/deps.py:219" }, "note": "path-length (d=2)" }, { "patternRole": "Key manager", "score": 0.5, "distance": 6, "groundTruth": { "entity": "config", "loc": "src/ddeutil/observe/auth/securities.py:17" }, "recoveryResult": { "entity": "secret_key", "loc": "src/ddeutil/observe/conf.py:93" }, "note": "path-length (d=6)" }, { "patternRole": "Verifier", "score": 0.8333, "distance": 2, "groundTruth": { "entity": "verify_access_token", "loc": "src/ddeutil/observe/auth/crud.py:187" }, "recoveryResult": { "entity": "verify_refresh_token", "loc": "src/ddeutil/observe/auth/crud.py:145" }, "note": "path-length (d=2)" } ], "notes": [ "Registrar: absent from the recovery result (Level 1 false negative)", "Token generator: absent from the recovery result (Level 1 false negative)" ] }, { "key": "level3", "level": "Level 3 \u2014 Interaction Order", "formula": "jaccard \u00d7 kendall-tau (shared interactions)", "available": true, "headlineLabel": "Interaction agreement", "score": 0.6667, "drift": 0.3333, "detail": { "agreedRoles": [ "Cryptographer", "Enforcer", "Key manager", "Verifier" ], "excludedRoles": [ "Registrar", "Token generator" ], "interactionsGroundTruth": 2, "interactionsRecoveryResult": 3, "sharedInteractions": 2, "unionSize": 3, "entryRoleGroundTruth": "Enforcer", "entryRoleRecoveryResult": "Enforcer", "entryRuleGroundTruth": "behaviour", "entryRuleRecoveryResult": "stored", "onlyGroundTruth": 0, "onlyRecoveryResult": 1, "interactionOverlap": 0.6667, "orderAgreement": 1.0, "rankedInteractions": 2, "totalPairs": 1, "concordantPairs": 1, "discordantPairs": 0, "kendallTau": 1.0, "prescribedActionSteps": 3, "prescribedStepsCovered": 3 }, "perRole": [ { "interaction": "Enforcer \u2192 Verifier", "from": "Enforcer", "to": "Verifier", "groundTruthRank": 1, "recoveryResultRank": 1, "outcome": "agreed", "discordantWith": 0, "evidence": { "groundTruth": { "file": "src/ddeutil/observe/auth/deps.py", "line": 55, "via": [] }, "recoveryResult": { "file": "src/ddeutil/observe/backend.py", "line": 68, "via": [] } } }, { "interaction": "Verifier \u2192 Cryptographer", "from": "Verifier", "to": "Cryptographer", "groundTruthRank": 2, "recoveryResultRank": 3, "outcome": "agreed", "discordantWith": 0, "evidence": { "groundTruth": { "file": "src/ddeutil/observe/auth/securities.py", "line": 137, "via": [ "src/ddeutil/observe/auth/securities.py:<module>.decode_access_token" ] }, "recoveryResult": { "file": "src/ddeutil/observe/auth/securities.py", "line": 139, "via": [] } } }, { "interaction": "Verifier \u2192 Key manager", "from": "Verifier", "to": "Key manager", "groundTruthRank": null, "recoveryResultRank": 2, "outcome": "onlyRecoveryResult", "discordantWith": null, "evidence": { "groundTruth": null, "recoveryResult": { "file": "src/ddeutil/observe/auth/securities.py", "line": 139, "via": [] } } } ], "notes": [ "Score = overlap \u00d7 order = 0.6667 \u00d7 1. Overlap: 2 of 3 interactions were found by both sides (0 only in the ground truth, 1 only in the recovery result), |shared| / |union|.", "Order: Kendall's Tau over the 2 shared interactions, in each side's own relative order \u2014 0 of 1 pairs run in opposite orders. Interactions only one side found take part in no pair (Fagin, Kumar & Sivakumar's p = 0, the induced sub-order); the overlap factor is what charges for them.", "Only 1 ranked pair \u2014 the order factor can land on just a few values here and one disagreement swings it fully.", "Scored over the 4 roles both sides detected (Cryptographer, Enforcer, Key manager, Verifier). An interaction counts only when both of its endpoint roles were detected on both sides \u2014 whether a role was found at all is Level 1's measurement, and charging for it again here would say the same thing twice.", "The interaction order is derived statically from the code \u2014 no one annotated it. Each role's cited entity is located in the AST index and control flow is followed from it, through as many non-role helper functions as it takes to reach the next role. See secpat_scoring/sequence/JOERN.md.", "Left out \u2014 only one side detected them: Registrar, Token generator. 2 roles were excluded from both walks, so neither side's ordering mentions them. The walk still passes through the code behind such a role, attributing what it finds there to the nearest agreed role.", "Ground truth: the walk started at Enforcer \u2014 the earliest role the reference pattern has speaking that this side grounded in walkable code.", "Recovery result: the walk started at Enforcer \u2014 stored.", "An entry point the pattern did not name means its prescribed speakers were either ungrounded on that side or not where this code begins \u2014 a role something else reaches cannot open the flow \u2014 so the call graph had to stand in. Treat the ordering below as weaker evidence: it starts wherever the code happens to start.", "Ground truth: Cryptographer cites an entity that is not a symbol in this repository (a library or framework call). It can receive an interaction, but the walk cannot continue through it." ] } ], "ground_truth_source": "annotation", "vocabulary": [ "Enforcer", "Verifier", "Key manager", "Cryptographer" ], "role_scope": "required", "excluded_roles": [ "Subject", "System" ], "dropped_ground_truth_rows": [ "Registrar", "Subject", "System", "Token generator" ], "dropped_recovery_rows": [ "Subject", "System" ], "gt_roles": 4, "rr_roles": 4, "struct_files": 60, "annotated_at": "2026-08-11 09:34:44.397000", "recovery_order": "stored" }, { "slug": "verifiable_token_based_authentication__freqtrade_freqtrade", "id": "token-based/freqtrade/ClaudeCodeopus4.7", "pattern": "verifiable_token_based_authentication", "repo": "freqtrade_freqtrade", "error": null, "over_input_limit": true, "prompt_tokens": 1317633, "metrics": [ { "key": "level1", "level": "Level 1 \u2014 Role Detection", "available": true, "score": 0.0, "drift": 1.0, "notes": [ "Not run: the one-shot prompt is 1,317,633 tokens (o200k_base), over the baseline's 900,000 input limit (scripts/baseline_recover.py). Scored 0 by rule, not measured." ] }, { "key": "level2", "level": "Level 2 \u2014 Role Grounding", "available": true, "score": 0.0, "drift": 1.0, "notes": [ "Not run: the one-shot prompt is 1,317,633 tokens (o200k_base), over the baseline's 900,000 input limit (scripts/baseline_recover.py). Scored 0 by rule, not measured." ] }, { "key": "level3", "level": "Level 3 \u2014 Interaction Order", "available": true, "score": 0.0, "drift": 1.0, "notes": [ "Not run: the one-shot prompt is 1,317,633 tokens (o200k_base), over the baseline's 900,000 input limit (scripts/baseline_recover.py). Scored 0 by rule, not measured." ] } ] }, { "slug": "verifiable_token_based_authentication__Echo298327_auth-gateway-serverkit", "id": "token-based/auth-gateway-serverkit/ClaudeCodeopus4.7", "pattern": "verifiable_token_based_authentication", "repo": "Echo298327_auth-gateway-serverkit", "error": null, "metrics": [ { "key": "level1", "level": "Level 1 \u2014 Role Detection", "formula": "agreement-rate (present/external/missing)", "available": true, "headlineLabel": "Roles agreed", "score": 1.0, "drift": 0.0, "detail": { "agreedRoles": 4, "totalRoles": 4 }, "perRole": [ { "patternRole": "Enforcer", "groundTruth": "present", "recoveryResult": "present", "groundTruthDetected": true, "recoveryResultDetected": true, "outcome": "agreed" }, { "patternRole": "Verifier", "groundTruth": "present", "recoveryResult": "present", "groundTruthDetected": true, "recoveryResultDetected": true, "outcome": "agreed" }, { "patternRole": "Key manager", "groundTruth": "present", "recoveryResult": "present", "groundTruthDetected": true, "recoveryResultDetected": true, "outcome": "agreed" }, { "patternRole": "Cryptographer", "groundTruth": "external", "recoveryResult": "external", "groundTruthDetected": true, "recoveryResultDetected": true, "outcome": "agreed" } ], "notes": [] }, { "key": "level2", "level": "Level 2 \u2014 Role Grounding", "formula": "path-length", "available": true, "headlineLabel": "Mean grounding score", "score": 1.0, "drift": 0.0, "detail": { "aggregate": 1.0, "scoredRoles": 4, "skippedRoles": 2, "treeHeight": 5 }, "perRole": [ { "patternRole": "Cryptographer", "score": 1.0, "distance": 0, "groundTruth": { "entity": "decode", "loc": "src/auth_gateway_serverkit/middleware/auth.py:49" }, "recoveryResult": { "entity": "decode", "loc": "src/auth_gateway_serverkit/middleware/auth.py:48" }, "note": "path-length (d=0)" }, { "patternRole": "Enforcer", "score": 1.0, "distance": 0, "groundTruth": { "entity": "auth", "loc": "src/auth_gateway_serverkit/middleware/auth.py:147\u2013181" }, "recoveryResult": { "entity": "auth", "loc": "src/auth_gateway_serverkit/middleware/auth.py:146" }, "note": "path-length (d=0)" }, { "patternRole": "Key manager", "score": 1.0, "distance": 0, "groundTruth": { "entity": "get_idp_public_key", "loc": "src/auth_gateway_serverkit/middleware/auth.py:29" }, "recoveryResult": { "entity": "get_idp_public_key", "loc": "src/auth_gateway_serverkit/middleware/auth.py:28" }, "note": "path-length (d=0)" }, { "patternRole": "Verifier", "score": 1.0, "distance": 0, "groundTruth": { "entity": "get_payload", "loc": "src/auth_gateway_serverkit/middleware/auth.py:38" }, "recoveryResult": { "entity": "get_payload", "loc": "src/auth_gateway_serverkit/middleware/auth.py:37" }, "note": "path-length (d=0)" } ], "notes": [ "Registrar: absent from the ground truth (Level 1 negative)", "Token generator: absent from the ground truth (Level 1 negative)" ] }, { "key": "level3", "level": "Level 3 \u2014 Interaction Order", "formula": "jaccard \u00d7 kendall-tau (shared interactions)", "available": true, "headlineLabel": "Interaction agreement", "score": 1.0, "drift": 0.0, "detail": { "agreedRoles": [ "Cryptographer", "Enforcer", "Key manager", "Verifier" ], "excludedRoles": [ "Registrar", "Token generator" ], "interactionsGroundTruth": 3, "interactionsRecoveryResult": 3, "sharedInteractions": 3, "unionSize": 3, "entryRoleGroundTruth": "Enforcer", "entryRoleRecoveryResult": "Enforcer", "entryRuleGroundTruth": "behaviour", "entryRuleRecoveryResult": "stored", "onlyGroundTruth": 0, "onlyRecoveryResult": 0, "interactionOverlap": 1.0, "orderAgreement": 1.0, "rankedInteractions": 3, "totalPairs": 3, "concordantPairs": 3, "discordantPairs": 0, "kendallTau": 1.0, "prescribedActionSteps": 3, "prescribedStepsCovered": 3 }, "perRole": [ { "interaction": "Enforcer \u2192 Verifier", "from": "Enforcer", "to": "Verifier", "groundTruthRank": 1, "recoveryResultRank": 1, "outcome": "agreed", "discordantWith": 0, "evidence": { "groundTruth": { "file": "src/auth_gateway_serverkit/middleware/auth.py", "line": 92, "via": [ "src/auth_gateway_serverkit/middleware/auth.py:<module>.auth.decorator", "src/auth_gateway_serverkit/middleware/auth.py:<module>.auth.decorator.wrapper", "src/auth_gateway_serverkit/middleware/auth.py:<module>.get_user_info" ] }, "recoveryResult": { "file": "src/auth_gateway_serverkit/middleware/auth.py", "line": 157, "via": [] } } }, { "interaction": "Verifier \u2192 Key manager", "from": "Verifier", "to": "Key manager", "groundTruthRank": 2, "recoveryResultRank": 2, "outcome": "agreed", "discordantWith": 0, "evidence": { "groundTruth": { "file": "src/auth_gateway_serverkit/middleware/auth.py", "line": 46, "via": [] }, "recoveryResult": { "file": "src/auth_gateway_serverkit/middleware/auth.py", "line": 47, "via": [] } } }, { "interaction": "Verifier \u2192 Cryptographer", "from": "Verifier", "to": "Cryptographer", "groundTruthRank": 3, "recoveryResultRank": 3, "outcome": "agreed", "discordantWith": 0, "evidence": { "groundTruth": { "file": "src/auth_gateway_serverkit/middleware/auth.py", "line": 49, "via": [] }, "recoveryResult": { "file": "src/auth_gateway_serverkit/middleware/auth.py", "line": 48, "via": [] } } } ], "notes": [ "Score = overlap \u00d7 order = 1 \u00d7 1. Overlap: 3 of 3 interactions were found by both sides (0 only in the ground truth, 0 only in the recovery result), |shared| / |union|.", "Order: Kendall's Tau over the 3 shared interactions, in each side's own relative order \u2014 0 of 3 pairs run in opposite orders. Interactions only one side found take part in no pair (Fagin, Kumar & Sivakumar's p = 0, the induced sub-order); the overlap factor is what charges for them.", "Scored over the 4 roles both sides detected (Cryptographer, Enforcer, Key manager, Verifier). An interaction counts only when both of its endpoint roles were detected on both sides \u2014 whether a role was found at all is Level 1's measurement, and charging for it again here would say the same thing twice.", "The interaction order is derived statically from the code \u2014 no one annotated it. Each role's cited entity is located in the AST index and control flow is followed from it, through as many non-role helper functions as it takes to reach the next role. See secpat_scoring/sequence/JOERN.md.", "Left out \u2014 only one side detected them: Registrar, Token generator. 2 roles were excluded from both walks, so neither side's ordering mentions them. The walk still passes through the code behind such a role, attributing what it finds there to the nearest agreed role.", "Ground truth: the walk started at Enforcer \u2014 the earliest role the reference pattern has speaking that this side grounded in walkable code.", "Recovery result: the walk started at Enforcer \u2014 stored.", "An entry point the pattern did not name means its prescribed speakers were either ungrounded on that side or not where this code begins \u2014 a role something else reaches cannot open the flow \u2014 so the call graph had to stand in. Treat the ordering below as weaker evidence: it starts wherever the code happens to start.", "Ground truth: Cryptographer cites an entity that is not a symbol in this repository (a library or framework call). It can receive an interaction, but the walk cannot continue through it." ] } ], "ground_truth_source": "annotation", "vocabulary": [ "Enforcer", "Verifier", "Key manager", "Cryptographer" ], "role_scope": "required", "excluded_roles": [ "Subject", "System" ], "dropped_ground_truth_rows": [ "Registrar", "Subject", "System", "Token generator" ], "dropped_recovery_rows": [ "Subject", "System" ], "gt_roles": 4, "rr_roles": 4, "struct_files": 28, "annotated_at": "2026-08-13 13:21:53.091000", "recovery_order": "stored" }, { "slug": "verifiable_token_based_authentication__inteliver_inteliver", "id": "token-based/inteliver/ClaudeCodeopus4.7", "pattern": "verifiable_token_based_authentication", "repo": "inteliver_inteliver", "error": null, "metrics": [ { "key": "level1", "level": "Level 1 \u2014 Role Detection", "formula": "agreement-rate (present/external/missing)", "available": true, "headlineLabel": "Roles agreed", "score": 1.0, "drift": 0.0, "detail": { "agreedRoles": 4, "totalRoles": 4 }, "perRole": [ { "patternRole": "Enforcer", "groundTruth": "present", "recoveryResult": "present", "groundTruthDetected": true, "recoveryResultDetected": true, "outcome": "agreed" }, { "patternRole": "Verifier", "groundTruth": "present", "recoveryResult": "present", "groundTruthDetected": true, "recoveryResultDetected": true, "outcome": "agreed" }, { "patternRole": "Key manager", "groundTruth": "present", "recoveryResult": "present", "groundTruthDetected": true, "recoveryResultDetected": true, "outcome": "agreed" }, { "patternRole": "Cryptographer", "groundTruth": "external", "recoveryResult": "external", "groundTruthDetected": true, "recoveryResultDetected": true, "outcome": "agreed" } ], "notes": [] }, { "key": "level2", "level": "Level 2 \u2014 Role Grounding", "formula": "path-length", "available": true, "headlineLabel": "Mean grounding score", "score": 0.85, "drift": 0.15, "detail": { "aggregate": 0.85, "scoredRoles": 4, "skippedRoles": 2, "treeHeight": 5 }, "perRole": [ { "patternRole": "Cryptographer", "score": 0.8, "distance": 2, "groundTruth": { "entity": "decode", "loc": "src/inteliver/auth/service.py:323" }, "recoveryResult": { "entity": "jwt", "loc": "src/inteliver/auth/service.py:7" }, "note": "path-length (d=2)" }, { "patternRole": "Enforcer", "score": 1.0, "distance": 0, "groundTruth": { "entity": "get_current_user", "loc": "src/inteliver/auth/service.py:364\u2013366" }, "recoveryResult": { "entity": "get_current_user", "loc": "src/inteliver/auth/service.py:365" }, "note": "path-length (d=0)" }, { "patternRole": "Key manager", "score": 0.6, "distance": 4, "groundTruth": { "entity": "SECRET_KEY", "loc": "src/inteliver/auth/service.py:346" }, "recoveryResult": { "entity": "SECRET_KEY", "loc": "src/inteliver/auth/constants.py:3" }, "note": "path-length (d=4)" }, { "patternRole": "Verifier", "score": 1.0, "distance": 0, "groundTruth": { "entity": "decode_access_token", "loc": "src/inteliver/auth/service.py:331\u2013362" }, "recoveryResult": { "entity": "decode_access_token", "loc": "src/inteliver/auth/service.py:322" }, "note": "path-length (d=0)" } ], "notes": [ "Registrar: absent from the ground truth (Level 1 negative)", "Token generator: absent from the ground truth (Level 1 negative)" ] }, { "key": "level3", "level": "Level 3 \u2014 Interaction Order", "formula": "jaccard \u00d7 kendall-tau (shared interactions)", "available": true, "headlineLabel": "Interaction agreement", "score": 0.3333, "drift": 0.6667, "detail": { "agreedRoles": [ "Cryptographer", "Enforcer", "Key manager", "Verifier" ], "excludedRoles": [ "Registrar", "Token generator" ], "interactionsGroundTruth": 1, "interactionsRecoveryResult": 3, "sharedInteractions": 1, "unionSize": 3, "entryRoleGroundTruth": "Enforcer", "entryRoleRecoveryResult": "Enforcer", "entryRuleGroundTruth": "behaviour", "entryRuleRecoveryResult": "stored", "onlyGroundTruth": 0, "onlyRecoveryResult": 2, "interactionOverlap": 0.3333, "orderAgreement": 1.0, "rankedInteractions": 1, "totalPairs": 0, "concordantPairs": 0, "discordantPairs": 0, "kendallTau": null, "prescribedActionSteps": 3, "prescribedStepsCovered": 3 }, "perRole": [ { "interaction": "Enforcer \u2192 Verifier", "from": "Enforcer", "to": "Verifier", "groundTruthRank": 1, "recoveryResultRank": 1, "outcome": "agreed", "discordantWith": 0, "evidence": { "groundTruth": { "file": "src/inteliver/auth/service.py", "line": 366, "via": [] }, "recoveryResult": { "file": "src/inteliver/auth/service.py", "line": 366, "via": [] } } }, { "interaction": "Verifier \u2192 Key manager", "from": "Verifier", "to": "Key manager", "groundTruthRank": null, "recoveryResultRank": 2, "outcome": "onlyRecoveryResult", "discordantWith": null, "evidence": { "groundTruth": null, "recoveryResult": { "file": "src/inteliver/auth/constants.py", "line": 0, "via": [] } } }, { "interaction": "Verifier \u2192 Cryptographer", "from": "Verifier", "to": "Cryptographer", "groundTruthRank": null, "recoveryResultRank": 3, "outcome": "onlyRecoveryResult", "discordantWith": null, "evidence": { "groundTruth": null, "recoveryResult": { "file": "src/inteliver/auth/service.py", "line": 331, "via": [] } } } ], "notes": [ "Score = overlap \u00d7 order = 0.3333 \u00d7 1. Overlap: 1 of 3 interactions were found by both sides (0 only in the ground truth, 2 only in the recovery result), |shared| / |union|.", "Order: 1 \u2014 a single shared interaction has no pair whose relative order could differ.", "Scored over the 4 roles both sides detected (Cryptographer, Enforcer, Key manager, Verifier). An interaction counts only when both of its endpoint roles were detected on both sides \u2014 whether a role was found at all is Level 1's measurement, and charging for it again here would say the same thing twice.", "The interaction order is derived statically from the code \u2014 no one annotated it. Each role's cited entity is located in the AST index and control flow is followed from it, through as many non-role helper functions as it takes to reach the next role. See secpat_scoring/sequence/JOERN.md.", "Left out \u2014 only one side detected them: Registrar, Token generator. 2 roles were excluded from both walks, so neither side's ordering mentions them. The walk still passes through the code behind such a role, attributing what it finds there to the nearest agreed role.", "Ground truth: the walk started at Enforcer \u2014 the earliest role the reference pattern has speaking that this side grounded in walkable code.", "Recovery result: the walk started at Enforcer \u2014 stored.", "An entry point the pattern did not name means its prescribed speakers were either ungrounded on that side or not where this code begins \u2014 a role something else reaches cannot open the flow \u2014 so the call graph had to stand in. Treat the ordering below as weaker evidence: it starts wherever the code happens to start.", "Ground truth: Cryptographer cites an entity that is not a symbol in this repository (a library or framework call). It can receive an interaction, but the walk cannot continue through it." ] } ], "ground_truth_source": "annotation", "vocabulary": [ "Enforcer", "Verifier", "Key manager", "Cryptographer" ], "role_scope": "required", "excluded_roles": [ "Subject", "System" ], "dropped_ground_truth_rows": [ "Registrar", "Subject", "System", "Token generator" ], "dropped_recovery_rows": [ "Subject", "System" ], "gt_roles": 4, "rr_roles": 4, "struct_files": 71, "annotated_at": "2026-08-13 14:33:41.426000", "recovery_order": "stored" }, { "slug": "verifiable_token_based_authentication__amphetamarina_litellm", "id": "token-based/amphetamarina_litellm/ClaudeCodeOpus5.5", "pattern": "verifiable_token_based_authentication", "repo": "amphetamarina_litellm", "error": null, "over_input_limit": true, "prompt_tokens": 3250785, "metrics": [ { "key": "level1", "level": "Level 1 \u2014 Role Detection", "available": true, "score": 0.0, "drift": 1.0, "notes": [ "Not run: the one-shot prompt is 3,250,785 tokens (o200k_base), over the baseline's 900,000 input limit (scripts/baseline_recover.py). Scored 0 by rule, not measured." ] }, { "key": "level2", "level": "Level 2 \u2014 Role Grounding", "available": true, "score": 0.0, "drift": 1.0, "notes": [ "Not run: the one-shot prompt is 3,250,785 tokens (o200k_base), over the baseline's 900,000 input limit (scripts/baseline_recover.py). Scored 0 by rule, not measured." ] }, { "key": "level3", "level": "Level 3 \u2014 Interaction Order", "available": true, "score": 0.0, "drift": 1.0, "notes": [ "Not run: the one-shot prompt is 3,250,785 tokens (o200k_base), over the baseline's 900,000 input limit (scripts/baseline_recover.py). Scored 0 by rule, not measured." ] } ] }, { "slug": "verifiable_token_based_authentication__SINTEF-9012_SINDIT20", "id": "token-based/SINDIT20/ClaudeCodeOpus5.5", "pattern": "verifiable_token_based_authentication", "repo": "SINTEF-9012_SINDIT20", "error": null, "metrics": [ { "key": "level1", "level": "Level 1 \u2014 Role Detection", "formula": "agreement-rate (present/external/missing)", "available": true, "headlineLabel": "Roles agreed", "score": 1.0, "drift": 0.0, "detail": { "agreedRoles": 4, "totalRoles": 4 }, "perRole": [ { "patternRole": "Enforcer", "groundTruth": "present", "recoveryResult": "present", "groundTruthDetected": true, "recoveryResultDetected": true, "outcome": "agreed" }, { "patternRole": "Verifier", "groundTruth": "present", "recoveryResult": "present", "groundTruthDetected": true, "recoveryResultDetected": true, "outcome": "agreed" }, { "patternRole": "Key manager", "groundTruth": "present", "recoveryResult": "present", "groundTruthDetected": true, "recoveryResultDetected": true, "outcome": "agreed" }, { "patternRole": "Cryptographer", "groundTruth": "external", "recoveryResult": "external", "groundTruthDetected": true, "recoveryResultDetected": true, "outcome": "agreed" } ], "notes": [] }, { "key": "level2", "level": "Level 2 \u2014 Role Grounding", "formula": "path-length", "available": true, "headlineLabel": "Mean grounding score", "score": 0.9107, "drift": 0.0893, "detail": { "aggregate": 0.9107, "scoredRoles": 4, "skippedRoles": 0, "treeHeight": 7 }, "perRole": [ { "patternRole": "Cryptographer", "score": 0.8571, "distance": 2, "groundTruth": { "entity": "decode", "loc": "src/sindit/authentication/in_memory.py:125" }, "recoveryResult": { "entity": "jwt", "loc": "src/sindit/authentication/in_memory.py:12" }, "note": "path-length (d=2)" }, { "patternRole": "Enforcer", "score": 0.8571, "distance": 2, "groundTruth": { "entity": "get_current_user", "loc": "src/sindit/api/authentication_endpoints.py:21" }, "recoveryResult": { "entity": "get_current_active_user", "loc": "src/sindit/api/authentication_endpoints.py:29" }, "note": "path-length (d=2)" }, { "patternRole": "Key manager", "score": 0.9286, "distance": 1, "groundTruth": { "entity": "__init__", "loc": "src/sindit/authentication/in_memory.py:19" }, "recoveryResult": { "entity": "InMemoryAuthService", "loc": "src/sindit/authentication/in_memory.py:16" }, "note": "path-length (d=1)" }, { "patternRole": "Verifier", "score": 1.0, "distance": 0, "groundTruth": { "entity": "verify_token", "loc": "src/sindit/authentication/in_memory.py:118" }, "recoveryResult": { "entity": "verify_token", "loc": "src/sindit/authentication/in_memory.py:119" }, "note": "path-length (d=0)" } ], "notes": [] }, { "key": "level3", "level": "Level 3 \u2014 Interaction Order", "formula": "jaccard \u00d7 kendall-tau (shared interactions)", "available": true, "headlineLabel": "Interaction agreement", "score": 0.6667, "drift": 0.3333, "detail": { "agreedRoles": [ "Cryptographer", "Enforcer", "Key manager", "Verifier" ], "excludedRoles": [], "interactionsGroundTruth": 2, "interactionsRecoveryResult": 3, "sharedInteractions": 2, "unionSize": 3, "entryRoleGroundTruth": "Enforcer", "entryRoleRecoveryResult": "Enforcer", "entryRuleGroundTruth": "behaviour", "entryRuleRecoveryResult": "stored", "onlyGroundTruth": 0, "onlyRecoveryResult": 1, "interactionOverlap": 0.6667, "orderAgreement": 1.0, "rankedInteractions": 2, "totalPairs": 1, "concordantPairs": 1, "discordantPairs": 0, "kendallTau": 1.0, "prescribedActionSteps": 3, "prescribedStepsCovered": 3 }, "perRole": [ { "interaction": "Enforcer \u2192 Verifier", "from": "Enforcer", "to": "Verifier", "groundTruthRank": 1, "recoveryResultRank": 1, "outcome": "agreed", "discordantWith": 0, "evidence": { "groundTruth": { "file": "src/sindit/api/authentication_endpoints.py", "line": 22, "via": [] }, "recoveryResult": { "file": "src/sindit/api/authentication_endpoints.py", "line": 26, "via": [] } } }, { "interaction": "Verifier \u2192 Cryptographer", "from": "Verifier", "to": "Cryptographer", "groundTruthRank": 2, "recoveryResultRank": 3, "outcome": "agreed", "discordantWith": 0, "evidence": { "groundTruth": { "file": "src/sindit/authentication/in_memory.py", "line": 125, "via": [] }, "recoveryResult": { "file": "src/sindit/authentication/in_memory.py", "line": 122, "via": [] } } }, { "interaction": "Verifier \u2192 Key manager", "from": "Verifier", "to": "Key manager", "groundTruthRank": null, "recoveryResultRank": 2, "outcome": "onlyRecoveryResult", "discordantWith": null, "evidence": { "groundTruth": null, "recoveryResult": { "file": "src/sindit/authentication/in_memory.py", "line": 122, "via": [] } } } ], "notes": [ "Score = overlap \u00d7 order = 0.6667 \u00d7 1. Overlap: 2 of 3 interactions were found by both sides (0 only in the ground truth, 1 only in the recovery result), |shared| / |union|.", "Order: Kendall's Tau over the 2 shared interactions, in each side's own relative order \u2014 0 of 1 pairs run in opposite orders. Interactions only one side found take part in no pair (Fagin, Kumar & Sivakumar's p = 0, the induced sub-order); the overlap factor is what charges for them.", "Only 1 ranked pair \u2014 the order factor can land on just a few values here and one disagreement swings it fully.", "Scored over the 4 roles both sides detected (Cryptographer, Enforcer, Key manager, Verifier). An interaction counts only when both of its endpoint roles were detected on both sides \u2014 whether a role was found at all is Level 1's measurement, and charging for it again here would say the same thing twice.", "The interaction order is derived statically from the code \u2014 no one annotated it. Each role's cited entity is located in the AST index and control flow is followed from it, through as many non-role helper functions as it takes to reach the next role. See secpat_scoring/sequence/JOERN.md.", "Ground truth: the walk started at Enforcer \u2014 the earliest role the reference pattern has speaking that this side grounded in walkable code.", "Recovery result: the walk started at Enforcer \u2014 stored.", "An entry point the pattern did not name means its prescribed speakers were either ungrounded on that side or not where this code begins \u2014 a role something else reaches cannot open the flow \u2014 so the call graph had to stand in. Treat the ordering below as weaker evidence: it starts wherever the code happens to start.", "Ground truth: Cryptographer cites an entity that is not a symbol in this repository (a library or framework call). It can receive an interaction, but the walk cannot continue through it." ] } ], "ground_truth_source": "recovery result", "vocabulary": [ "Enforcer", "Verifier", "Key manager", "Cryptographer" ], "role_scope": "required", "excluded_roles": [ "Subject", "System" ], "dropped_ground_truth_rows": [ "Subject", "System" ], "dropped_recovery_rows": [ "Subject", "System" ], "gt_roles": 4, "rr_roles": 4, "struct_files": 65, "annotated_at": "2026-10-02 10:00:38.838000", "recovery_order": "stored" }, { "slug": "verifiable_token_based_authentication__sargsyanhayko_rsha_auth", "id": "token-based/rsha_auth/ClaudeCodeOpus5.5", "pattern": "verifiable_token_based_authentication", "repo": "sargsyanhayko_rsha_auth", "error": null, "metrics": [ { "key": "level1", "level": "Level 1 \u2014 Role Detection", "formula": "agreement-rate (present/external/missing)", "available": true, "headlineLabel": "Roles agreed", "score": 1.0, "drift": 0.0, "detail": { "agreedRoles": 4, "totalRoles": 4 }, "perRole": [ { "patternRole": "Enforcer", "groundTruth": "present", "recoveryResult": "present", "groundTruthDetected": true, "recoveryResultDetected": true, "outcome": "agreed" }, { "patternRole": "Verifier", "groundTruth": "present", "recoveryResult": "present", "groundTruthDetected": true, "recoveryResultDetected": true, "outcome": "agreed" }, { "patternRole": "Key manager", "groundTruth": "present", "recoveryResult": "present", "groundTruthDetected": true, "recoveryResultDetected": true, "outcome": "agreed" }, { "patternRole": "Cryptographer", "groundTruth": "external", "recoveryResult": "external", "groundTruthDetected": true, "recoveryResultDetected": true, "outcome": "agreed" } ], "notes": [] }, { "key": "level2", "level": "Level 2 \u2014 Role Grounding", "formula": "path-length", "available": true, "headlineLabel": "Mean grounding score", "score": 0.8334, "drift": 0.1666, "detail": { "aggregate": 0.8334, "scoredRoles": 4, "skippedRoles": 0, "treeHeight": 3 }, "perRole": [ { "patternRole": "Cryptographer", "score": 0.6667, "distance": 2, "groundTruth": { "entity": "decode", "loc": "src/auth_rsha/jwt.py:23" }, "recoveryResult": { "entity": "jwt", "loc": "src/auth_rsha/jwt.py:3" }, "note": "path-length (d=2)" }, { "patternRole": "Enforcer", "score": 1.0, "distance": 0, "groundTruth": { "entity": "current_user", "loc": "src/auth_rsha/deps.py:16" }, "recoveryResult": { "entity": "current_user", "loc": "src/auth_rsha/deps.py:19" }, "note": "path-length (d=0)" }, { "patternRole": "Key manager", "score": 0.6667, "distance": 2, "groundTruth": { "entity": "jwt_secret", "loc": "src/auth_rsha/config.py:6" }, "recoveryResult": { "entity": "AuthSettings", "loc": "src/auth_rsha/config.py:5" }, "note": "path-length (d=2)" }, { "patternRole": "Verifier", "score": 1.0, "distance": 0, "groundTruth": { "entity": "decode_token", "loc": "src/auth_rsha/jwt.py:21" }, "recoveryResult": { "entity": "decode_token", "loc": "src/auth_rsha/jwt.py:22" }, "note": "path-length (d=0)" } ], "notes": [] }, { "key": "level3", "level": "Level 3 \u2014 Interaction Order", "formula": "jaccard \u00d7 kendall-tau (shared interactions)", "available": true, "headlineLabel": "Interaction agreement", "score": 0.6667, "drift": 0.3333, "detail": { "agreedRoles": [ "Cryptographer", "Enforcer", "Key manager", "Verifier" ], "excludedRoles": [], "interactionsGroundTruth": 2, "interactionsRecoveryResult": 3, "sharedInteractions": 2, "unionSize": 3, "entryRoleGroundTruth": "Enforcer", "entryRoleRecoveryResult": "Enforcer", "entryRuleGroundTruth": "behaviour", "entryRuleRecoveryResult": "stored", "onlyGroundTruth": 0, "onlyRecoveryResult": 1, "interactionOverlap": 0.6667, "orderAgreement": 1.0, "rankedInteractions": 2, "totalPairs": 1, "concordantPairs": 1, "discordantPairs": 0, "kendallTau": 1.0, "prescribedActionSteps": 3, "prescribedStepsCovered": 3 }, "perRole": [ { "interaction": "Enforcer \u2192 Verifier", "from": "Enforcer", "to": "Verifier", "groundTruthRank": 1, "recoveryResultRank": 1, "outcome": "agreed", "discordantWith": 0, "evidence": { "groundTruth": { "file": "src/auth_rsha/deps.py", "line": 18, "via": [] }, "recoveryResult": { "file": "src/auth_rsha/deps.py", "line": 16, "via": [] } } }, { "interaction": "Verifier \u2192 Cryptographer", "from": "Verifier", "to": "Cryptographer", "groundTruthRank": 2, "recoveryResultRank": 3, "outcome": "agreed", "discordantWith": 0, "evidence": { "groundTruth": { "file": "src/auth_rsha/jwt.py", "line": 23, "via": [] }, "recoveryResult": { "file": "src/auth_rsha/jwt.py", "line": 23, "via": [] } } }, { "interaction": "Verifier \u2192 Key manager", "from": "Verifier", "to": "Key manager", "groundTruthRank": null, "recoveryResultRank": 2, "outcome": "onlyRecoveryResult", "discordantWith": null, "evidence": { "groundTruth": null, "recoveryResult": { "file": "src/auth_rsha/jwt.py", "line": 26, "via": [] } } } ], "notes": [ "Score = overlap \u00d7 order = 0.6667 \u00d7 1. Overlap: 2 of 3 interactions were found by both sides (0 only in the ground truth, 1 only in the recovery result), |shared| / |union|.", "Order: Kendall's Tau over the 2 shared interactions, in each side's own relative order \u2014 0 of 1 pairs run in opposite orders. Interactions only one side found take part in no pair (Fagin, Kumar & Sivakumar's p = 0, the induced sub-order); the overlap factor is what charges for them.", "Only 1 ranked pair \u2014 the order factor can land on just a few values here and one disagreement swings it fully.", "Scored over the 4 roles both sides detected (Cryptographer, Enforcer, Key manager, Verifier). An interaction counts only when both of its endpoint roles were detected on both sides \u2014 whether a role was found at all is Level 1's measurement, and charging for it again here would say the same thing twice.", "The interaction order is derived statically from the code \u2014 no one annotated it. Each role's cited entity is located in the AST index and control flow is followed from it, through as many non-role helper functions as it takes to reach the next role. See secpat_scoring/sequence/JOERN.md.", "Ground truth: the walk started at Enforcer \u2014 the earliest role the reference pattern has speaking that this side grounded in walkable code.", "Recovery result: the walk started at Enforcer \u2014 stored.", "An entry point the pattern did not name means its prescribed speakers were either ungrounded on that side or not where this code begins \u2014 a role something else reaches cannot open the flow \u2014 so the call graph had to stand in. Treat the ordering below as weaker evidence: it starts wherever the code happens to start.", "Ground truth: Cryptographer cites an entity that is not a symbol in this repository (a library or framework call). It can receive an interaction, but the walk cannot continue through it." ] } ], "ground_truth_source": "recovery result", "vocabulary": [ "Enforcer", "Verifier", "Key manager", "Cryptographer" ], "role_scope": "required", "excluded_roles": [ "Subject", "System" ], "dropped_ground_truth_rows": [ "Subject", "System" ], "dropped_recovery_rows": [ "Subject", "System" ], "gt_roles": 4, "rr_roles": 4, "struct_files": 9, "annotated_at": "2026-10-02 10:00:38.838000", "recovery_order": "stored" }, { "slug": "verifiable_token_based_authentication__beanone_apikey", "id": "token-based/apikey/ClaudeCodeOpus5.5", "pattern": "verifiable_token_based_authentication", "repo": "beanone_apikey", "error": null, "metrics": [ { "key": "level1", "level": "Level 1 \u2014 Role Detection", "formula": "agreement-rate (present/external/missing)", "available": true, "headlineLabel": "Roles agreed", "score": 1.0, "drift": 0.0, "detail": { "agreedRoles": 4, "totalRoles": 4 }, "perRole": [ { "patternRole": "Enforcer", "groundTruth": "present", "recoveryResult": "present", "groundTruthDetected": true, "recoveryResultDetected": true, "outcome": "agreed" }, { "patternRole": "Verifier", "groundTruth": "present", "recoveryResult": "present", "groundTruthDetected": true, "recoveryResultDetected": true, "outcome": "agreed" }, { "patternRole": "Key manager", "groundTruth": "present", "recoveryResult": "present", "groundTruthDetected": true, "recoveryResultDetected": true, "outcome": "agreed" }, { "patternRole": "Cryptographer", "groundTruth": "external", "recoveryResult": "external", "groundTruthDetected": true, "recoveryResultDetected": true, "outcome": "agreed" } ], "notes": [] }, { "key": "level2", "level": "Level 2 \u2014 Role Grounding", "formula": "path-length", "available": true, "headlineLabel": "Mean grounding score", "score": 0.9, "drift": 0.1, "detail": { "aggregate": 0.9, "scoredRoles": 4, "skippedRoles": 0, "treeHeight": 5 }, "perRole": [ { "patternRole": "Cryptographer", "score": 0.8, "distance": 2, "groundTruth": { "entity": "decode", "loc": "src/apikey/dependencies.py:74" }, "recoveryResult": { "entity": "jwt", "loc": "src/apikey/dependencies.py:13" }, "note": "path-length (d=2)" }, { "patternRole": "Enforcer", "score": 1.0, "distance": 0, "groundTruth": { "entity": "get_current_user", "loc": "src/apikey/dependencies.py:153" }, "recoveryResult": { "entity": "get_current_user", "loc": "src/apikey/dependencies.py:121" }, "note": "path-length (d=0)" }, { "patternRole": "Key manager", "score": 0.8, "distance": 2, "groundTruth": { "entity": "JWT_SECRET", "loc": "src/apikey/dependencies.py:55" }, "recoveryResult": { "entity": "Settings", "loc": "src/apikey/dependencies.py:25" }, "note": "path-length (d=2)" }, { "patternRole": "Verifier", "score": 1.0, "distance": 0, "groundTruth": { "entity": "validate_jwt", "loc": "src/apikey/dependencies.py:70" }, "recoveryResult": { "entity": "validate_jwt", "loc": "src/apikey/dependencies.py:62" }, "note": "path-length (d=0)" } ], "notes": [] }, { "key": "level3", "level": "Level 3 \u2014 Interaction Order", "formula": "jaccard \u00d7 kendall-tau (shared interactions)", "available": true, "headlineLabel": "Interaction agreement", "score": 0.6667, "drift": 0.3333, "detail": { "agreedRoles": [ "Cryptographer", "Enforcer", "Key manager", "Verifier" ], "excludedRoles": [], "interactionsGroundTruth": 2, "interactionsRecoveryResult": 3, "sharedInteractions": 2, "unionSize": 3, "entryRoleGroundTruth": "Enforcer", "entryRoleRecoveryResult": "Enforcer", "entryRuleGroundTruth": "behaviour", "entryRuleRecoveryResult": "stored", "onlyGroundTruth": 0, "onlyRecoveryResult": 1, "interactionOverlap": 0.6667, "orderAgreement": 1.0, "rankedInteractions": 2, "totalPairs": 1, "concordantPairs": 1, "discordantPairs": 0, "kendallTau": 1.0, "prescribedActionSteps": 3, "prescribedStepsCovered": 3 }, "perRole": [ { "interaction": "Enforcer \u2192 Verifier", "from": "Enforcer", "to": "Verifier", "groundTruthRank": 1, "recoveryResultRank": 1, "outcome": "agreed", "discordantWith": 0, "evidence": { "groundTruth": { "file": "src/apikey/dependencies.py", "line": 170, "via": [] }, "recoveryResult": { "file": "src/apikey/dependencies.py", "line": 135, "via": [] } } }, { "interaction": "Verifier \u2192 Cryptographer", "from": "Verifier", "to": "Cryptographer", "groundTruthRank": 2, "recoveryResultRank": 3, "outcome": "agreed", "discordantWith": 0, "evidence": { "groundTruth": { "file": "src/apikey/dependencies.py", "line": 74, "via": [] }, "recoveryResult": { "file": "src/apikey/dependencies.py", "line": 65, "via": [] } } }, { "interaction": "Verifier \u2192 Key manager", "from": "Verifier", "to": "Key manager", "groundTruthRank": null, "recoveryResultRank": 2, "outcome": "onlyRecoveryResult", "discordantWith": null, "evidence": { "groundTruth": null, "recoveryResult": { "file": "src/apikey/dependencies.py", "line": 65, "via": [] } } } ], "notes": [ "Score = overlap \u00d7 order = 0.6667 \u00d7 1. Overlap: 2 of 3 interactions were found by both sides (0 only in the ground truth, 1 only in the recovery result), |shared| / |union|.", "Order: Kendall's Tau over the 2 shared interactions, in each side's own relative order \u2014 0 of 1 pairs run in opposite orders. Interactions only one side found take part in no pair (Fagin, Kumar & Sivakumar's p = 0, the induced sub-order); the overlap factor is what charges for them.", "Only 1 ranked pair \u2014 the order factor can land on just a few values here and one disagreement swings it fully.", "Scored over the 4 roles both sides detected (Cryptographer, Enforcer, Key manager, Verifier). An interaction counts only when both of its endpoint roles were detected on both sides \u2014 whether a role was found at all is Level 1's measurement, and charging for it again here would say the same thing twice.", "The interaction order is derived statically from the code \u2014 no one annotated it. Each role's cited entity is located in the AST index and control flow is followed from it, through as many non-role helper functions as it takes to reach the next role. See secpat_scoring/sequence/JOERN.md.", "Ground truth: the walk started at Enforcer \u2014 the earliest role the reference pattern has speaking that this side grounded in walkable code.", "Recovery result: the walk started at Enforcer \u2014 stored.", "An entry point the pattern did not name means its prescribed speakers were either ungrounded on that side or not where this code begins \u2014 a role something else reaches cannot open the flow \u2014 so the call graph had to stand in. Treat the ordering below as weaker evidence: it starts wherever the code happens to start.", "Ground truth: Cryptographer cites an entity that is not a symbol in this repository (a library or framework call). It can receive an interaction, but the walk cannot continue through it." ] } ], "ground_truth_source": "recovery result", "vocabulary": [ "Enforcer", "Verifier", "Key manager", "Cryptographer" ], "role_scope": "required", "excluded_roles": [ "Subject", "System" ], "dropped_ground_truth_rows": [ "Subject", "System" ], "dropped_recovery_rows": [ "Subject", "System" ], "gt_roles": 4, "rr_roles": 4, "struct_files": 19, "annotated_at": "2026-10-02 10:00:38.838000", "recovery_order": "stored" }, { "slug": "verifiable_token_based_authentication__Lampx83_LakeFlow", "id": "token-based/Lampx83_LakeFlow/ClaudeCodeOpus5.5", "pattern": "verifiable_token_based_authentication", "repo": "Lampx83_LakeFlow", "error": null, "metrics": [ { "key": "level1", "level": "Level 1 \u2014 Role Detection", "formula": "agreement-rate (present/external/missing)", "available": true, "headlineLabel": "Roles agreed", "score": 1.0, "drift": 0.0, "detail": { "agreedRoles": 4, "totalRoles": 4 }, "perRole": [ { "patternRole": "Enforcer", "groundTruth": "present", "recoveryResult": "present", "groundTruthDetected": true, "recoveryResultDetected": true, "outcome": "agreed" }, { "patternRole": "Verifier", "groundTruth": "present", "recoveryResult": "present", "groundTruthDetected": true, "recoveryResultDetected": true, "outcome": "agreed" }, { "patternRole": "Key manager", "groundTruth": "present", "recoveryResult": "present", "groundTruthDetected": true, "recoveryResultDetected": true, "outcome": "agreed" }, { "patternRole": "Cryptographer", "groundTruth": "external", "recoveryResult": "external", "groundTruthDetected": true, "recoveryResultDetected": true, "outcome": "agreed" } ], "notes": [] }, { "key": "level2", "level": "Level 2 \u2014 Role Grounding", "formula": "path-length", "available": true, "headlineLabel": "Mean grounding score", "score": 0.875, "drift": 0.125, "detail": { "aggregate": 0.875, "scoredRoles": 4, "skippedRoles": 0, "treeHeight": 8 }, "perRole": [ { "patternRole": "Cryptographer", "score": 0.875, "distance": 2, "groundTruth": { "entity": "decode", "loc": "backend/src/lakeflow/core/auth.py:24" }, "recoveryResult": { "entity": "jwt", "loc": "backend/src/lakeflow/core/auth.py:6" }, "note": "path-length (d=2)" }, { "patternRole": "Enforcer", "score": 0.625, "distance": 6, "groundTruth": { "entity": "verify_token", "loc": "backend/src/lakeflow/core/auth.py:15" }, "recoveryResult": { "entity": "qa", "loc": "backend/src/lakeflow/api/search.py:211" }, "note": "path-length (d=6)" }, { "patternRole": "Key manager", "score": 1.0, "distance": 0, "groundTruth": { "entity": "JWT_SECRET_KEY", "loc": "backend/src/lakeflow/core/config.py:26" }, "recoveryResult": { "entity": "JWT_SECRET_KEY", "loc": "backend/src/lakeflow/core/config.py:30" }, "note": "path-length (d=0)" }, { "patternRole": "Verifier", "score": 1.0, "distance": 0, "groundTruth": { "entity": "verify_token", "loc": "backend/src/lakeflow/core/auth.py:15" }, "recoveryResult": { "entity": "verify_token", "loc": "backend/src/lakeflow/core/auth.py:13" }, "note": "path-length (d=0)" } ], "notes": [] }, { "key": "level3", "level": "Level 3 \u2014 Interaction Order", "formula": "jaccard \u00d7 kendall-tau (shared interactions)", "available": true, "headlineLabel": "Interaction agreement", "score": 0.5, "drift": 0.5, "detail": { "agreedRoles": [ "Cryptographer", "Enforcer", "Key manager", "Verifier" ], "excludedRoles": [], "interactionsGroundTruth": 3, "interactionsRecoveryResult": 3, "sharedInteractions": 2, "unionSize": 4, "entryRoleGroundTruth": "Enforcer", "entryRoleRecoveryResult": "Enforcer", "entryRuleGroundTruth": "behaviour", "entryRuleRecoveryResult": "stored", "onlyGroundTruth": 1, "onlyRecoveryResult": 1, "interactionOverlap": 0.5, "orderAgreement": 1.0, "rankedInteractions": 2, "totalPairs": 1, "concordantPairs": 1, "discordantPairs": 0, "kendallTau": 1.0, "prescribedActionSteps": 3, "prescribedStepsCovered": 3 }, "perRole": [ { "interaction": "Enforcer \u2192 Verifier", "from": "Enforcer", "to": "Verifier", "groundTruthRank": 1, "recoveryResultRank": 1, "outcome": "agreed", "discordantWith": 0, "evidence": { "groundTruth": { "file": "backend/src/lakeflow/core/auth.py", "line": 15, "via": [] }, "recoveryResult": { "file": "backend/src/lakeflow/api/search.py", "line": 212, "via": [] } } }, { "interaction": "Enforcer \u2192 Cryptographer", "from": "Enforcer", "to": "Cryptographer", "groundTruthRank": 2, "recoveryResultRank": null, "outcome": "onlyGroundTruth", "discordantWith": null, "evidence": { "groundTruth": { "file": "backend/src/lakeflow/core/auth.py", "line": 24, "via": [] }, "recoveryResult": null } }, { "interaction": "Verifier \u2192 Cryptographer", "from": "Verifier", "to": "Cryptographer", "groundTruthRank": 3, "recoveryResultRank": 3, "outcome": "agreed", "discordantWith": 0, "evidence": { "groundTruth": { "file": "backend/src/lakeflow/core/auth.py", "line": 24, "via": [] }, "recoveryResult": { "file": "backend/src/lakeflow/core/auth.py", "line": 22, "via": [] } } }, { "interaction": "Verifier \u2192 Key manager", "from": "Verifier", "to": "Key manager", "groundTruthRank": null, "recoveryResultRank": 2, "outcome": "onlyRecoveryResult", "discordantWith": null, "evidence": { "groundTruth": null, "recoveryResult": { "file": "backend/src/lakeflow/core/auth.py", "line": 23, "via": [] } } } ], "notes": [ "Score = overlap \u00d7 order = 0.5 \u00d7 1. Overlap: 2 of 4 interactions were found by both sides (1 only in the ground truth, 1 only in the recovery result), |shared| / |union|.", "Order: Kendall's Tau over the 2 shared interactions, in each side's own relative order \u2014 0 of 1 pairs run in opposite orders. Interactions only one side found take part in no pair (Fagin, Kumar & Sivakumar's p = 0, the induced sub-order); the overlap factor is what charges for them.", "Only 1 ranked pair \u2014 the order factor can land on just a few values here and one disagreement swings it fully.", "Scored over the 4 roles both sides detected (Cryptographer, Enforcer, Key manager, Verifier). An interaction counts only when both of its endpoint roles were detected on both sides \u2014 whether a role was found at all is Level 1's measurement, and charging for it again here would say the same thing twice.", "The interaction order is derived statically from the code \u2014 no one annotated it. Each role's cited entity is located in the AST index and control flow is followed from it, through as many non-role helper functions as it takes to reach the next role. See secpat_scoring/sequence/JOERN.md.", "Ground truth: the walk started at Enforcer \u2014 the earliest role the reference pattern has speaking that this side grounded in walkable code.", "Recovery result: the walk started at Enforcer \u2014 stored.", "An entry point the pattern did not name means its prescribed speakers were either ungrounded on that side or not where this code begins \u2014 a role something else reaches cannot open the flow \u2014 so the call graph had to stand in. Treat the ordering below as weaker evidence: it starts wherever the code happens to start.", "Ground truth: Cryptographer cites an entity that is not a symbol in this repository (a library or framework call). It can receive an interaction, but the walk cannot continue through it." ] } ], "ground_truth_source": "recovery result", "vocabulary": [ "Enforcer", "Verifier", "Key manager", "Cryptographer" ], "role_scope": "required", "excluded_roles": [ "Subject", "System" ], "dropped_ground_truth_rows": [ "Subject", "System" ], "dropped_recovery_rows": [ "Subject", "System" ], "gt_roles": 4, "rr_roles": 4, "struct_files": 113, "annotated_at": "2026-10-06 09:53:55.047000", "recovery_order": "stored" }, { "slug": "limit_request_rate__tumma72_mlx-manager", "id": "rate-limit/tumma72_mlx-manager/ClaudeCodeOpus5.5", "pattern": "limit_request_rate", "repo": "tumma72_mlx-manager", "error": null, "metrics": [ { "key": "level1", "level": "Level 1 \u2014 Role Detection", "formula": "agreement-rate", "available": true, "headlineLabel": "Roles agreed", "score": 1.0, "drift": 0.0, "detail": { "agreedRoles": 4, "totalRoles": 4 }, "perRole": [ { "patternRole": "Enforcer", "groundTruth": "present", "recoveryResult": "present", "groundTruthDetected": true, "recoveryResultDetected": true, "outcome": "agreed" }, { "patternRole": "Limiter", "groundTruth": "present", "recoveryResult": "present", "groundTruthDetected": true, "recoveryResultDetected": true, "outcome": "agreed" }, { "patternRole": "HistoryProvider", "groundTruth": "present", "recoveryResult": "present", "groundTruthDetected": true, "recoveryResultDetected": true, "outcome": "agreed" }, { "patternRole": "PolicyProvider", "groundTruth": "present", "recoveryResult": "present", "groundTruthDetected": true, "recoveryResultDetected": true, "outcome": "agreed" } ], "notes": [] }, { "key": "level2", "level": "Level 2 \u2014 Role Grounding", "formula": "path-length", "available": true, "headlineLabel": "Mean grounding score", "score": 0.9531, "drift": 0.0469, "detail": { "aggregate": 0.9531, "scoredRoles": 4, "skippedRoles": 0, "treeHeight": 8 }, "perRole": [ { "patternRole": "Enforcer", "score": 1.0, "distance": 0, "groundTruth": { "entity": "dispatch", "loc": "backend/mlx_manager/mlx_server/middleware/rate_limit.py:110" }, "recoveryResult": { "entity": "dispatch", "loc": "backend/mlx_manager/mlx_server/middleware/rate_limit.py:112" }, "note": "path-length (d=0)" }, { "patternRole": "HistoryProvider", "score": 0.8125, "distance": 3, "groundTruth": { "entity": "_get_bucket", "loc": "backend/mlx_manager/mlx_server/middleware/rate_limit.py:93" }, "recoveryResult": { "entity": "_buckets", "loc": "backend/mlx_manager/mlx_server/middleware/rate_limit.py:84" }, "note": "path-length (d=3)" }, { "patternRole": "Limiter", "score": 1.0, "distance": 0, "groundTruth": { "entity": "consume", "loc": "backend/mlx_manager/mlx_server/middleware/rate_limit.py:39" }, "recoveryResult": { "entity": "consume", "loc": "backend/mlx_manager/mlx_server/middleware/rate_limit.py:40" }, "note": "path-length (d=0)" }, { "patternRole": "PolicyProvider", "score": 1.0, "distance": 0, "groundTruth": { "entity": "rate_limit_rpm", "loc": "backend/mlx_manager/mlx_server/config.py:169" }, "recoveryResult": { "entity": "rate_limit_rpm", "loc": "backend/mlx_manager/mlx_server/config.py:139" }, "note": "path-length (d=0)" } ], "notes": [] }, { "key": "level3", "level": "Level 3 \u2014 Interaction Order", "formula": "jaccard \u00d7 kendall-tau (shared interactions)", "available": true, "headlineLabel": "Interaction agreement", "score": 0.5, "drift": 0.5, "detail": { "agreedRoles": [ "Enforcer", "HistoryProvider", "Limiter", "PolicyProvider" ], "excludedRoles": [], "interactionsGroundTruth": 2, "interactionsRecoveryResult": 4, "sharedInteractions": 2, "unionSize": 4, "entryRoleGroundTruth": "Enforcer", "entryRoleRecoveryResult": "PolicyProvider", "entryRuleGroundTruth": "behaviour", "entryRuleRecoveryResult": "stored", "onlyGroundTruth": 0, "onlyRecoveryResult": 2, "interactionOverlap": 0.5, "orderAgreement": 1.0, "rankedInteractions": 2, "totalPairs": 1, "concordantPairs": 1, "discordantPairs": 0, "kendallTau": 1.0, "prescribedActionSteps": 4, "prescribedStepsCovered": 3 }, "perRole": [ { "interaction": "Enforcer \u2192 HistoryProvider", "from": "Enforcer", "to": "HistoryProvider", "groundTruthRank": 1, "recoveryResultRank": 2, "outcome": "agreed", "discordantWith": 0, "evidence": { "groundTruth": { "file": "backend/mlx_manager/mlx_server/middleware/rate_limit.py", "line": 119, "via": [] }, "recoveryResult": { "file": "backend/mlx_manager/mlx_server/middleware/rate_limit.py", "line": 122, "via": [] } } }, { "interaction": "Enforcer \u2192 Limiter", "from": "Enforcer", "to": "Limiter", "groundTruthRank": 2, "recoveryResultRank": 3, "outcome": "agreed", "discordantWith": 0, "evidence": { "groundTruth": { "file": "backend/mlx_manager/mlx_server/middleware/rate_limit.py", "line": 121, "via": [] }, "recoveryResult": { "file": "backend/mlx_manager/mlx_server/middleware/rate_limit.py", "line": 124, "via": [] } } }, { "interaction": "PolicyProvider \u2192 Enforcer", "from": "PolicyProvider", "to": "Enforcer", "groundTruthRank": null, "recoveryResultRank": 1, "outcome": "onlyRecoveryResult", "discordantWith": null, "evidence": { "groundTruth": null, "recoveryResult": { "file": "backend/mlx_manager/mlx_server/main.py", "line": 169, "via": [] } } }, { "interaction": "Limiter \u2192 HistoryProvider", "from": "Limiter", "to": "HistoryProvider", "groundTruthRank": null, "recoveryResultRank": 4, "outcome": "onlyRecoveryResult", "discordantWith": null, "evidence": { "groundTruth": null, "recoveryResult": { "file": "backend/mlx_manager/mlx_server/middleware/rate_limit.py", "line": 42, "via": [] } } } ], "notes": [ "Score = overlap \u00d7 order = 0.5 \u00d7 1. Overlap: 2 of 4 interactions were found by both sides (0 only in the ground truth, 2 only in the recovery result), |shared| / |union|.", "Order: Kendall's Tau over the 2 shared interactions, in each side's own relative order \u2014 0 of 1 pairs run in opposite orders. Interactions only one side found take part in no pair (Fagin, Kumar & Sivakumar's p = 0, the induced sub-order); the overlap factor is what charges for them.", "Only 1 ranked pair \u2014 the order factor can land on just a few values here and one disagreement swings it fully.", "Scored over the 4 roles both sides detected (Enforcer, HistoryProvider, Limiter, PolicyProvider). An interaction counts only when both of its endpoint roles were detected on both sides \u2014 whether a role was found at all is Level 1's measurement, and charging for it again here would say the same thing twice.", "The interaction order is derived statically from the code \u2014 no one annotated it. Each role's cited entity is located in the AST index and control flow is followed from it, through as many non-role helper functions as it takes to reach the next role. See secpat_scoring/sequence/JOERN.md.", "Ground truth: the walk started at Enforcer \u2014 the earliest role the reference pattern has speaking that this side grounded in walkable code.", "Recovery result: the walk started at PolicyProvider \u2014 stored.", "An entry point the pattern did not name means its prescribed speakers were either ungrounded on that side or not where this code begins \u2014 a role something else reaches cannot open the flow \u2014 so the call graph had to stand in. Treat the ordering below as weaker evidence: it starts wherever the code happens to start." ] } ], "ground_truth_source": "recovery result", "vocabulary": [ "Enforcer", "Limiter", "HistoryProvider", "PolicyProvider" ], "role_scope": "required", "excluded_roles": [ "Entity", "System" ], "dropped_ground_truth_rows": [ "Entity", "System" ], "dropped_recovery_rows": [ "Entity", "System" ], "gt_roles": 4, "rr_roles": 4, "struct_files": 282, "annotated_at": "2026-10-02 10:00:38.838000", "recovery_order": "stored" }, { "slug": "limit_request_rate__luxman89_TradeArena", "id": "rate-limit/luxman89_TradeArena/ClaudeCodeOpus5.0", "pattern": "limit_request_rate", "repo": "luxman89_TradeArena", "error": null, "metrics": [ { "key": "level1", "level": "Level 1 \u2014 Role Detection", "formula": "agreement-rate (present/external/missing)", "available": true, "headlineLabel": "Roles agreed", "score": 1.0, "drift": 0.0, "detail": { "agreedRoles": 4, "totalRoles": 4 }, "perRole": [ { "patternRole": "Enforcer", "groundTruth": "present", "recoveryResult": "present", "groundTruthDetected": true, "recoveryResultDetected": true, "outcome": "agreed" }, { "patternRole": "Limiter", "groundTruth": "present", "recoveryResult": "present", "groundTruthDetected": true, "recoveryResultDetected": true, "outcome": "agreed" }, { "patternRole": "HistoryProvider", "groundTruth": "present", "recoveryResult": "present", "groundTruthDetected": true, "recoveryResultDetected": true, "outcome": "agreed" }, { "patternRole": "PolicyProvider", "groundTruth": "present", "recoveryResult": "present", "groundTruthDetected": true, "recoveryResultDetected": true, "outcome": "agreed" } ], "notes": [] }, { "key": "level2", "level": "Level 2 \u2014 Role Grounding", "formula": "path-length", "available": true, "headlineLabel": "Mean grounding score", "score": 0.9375, "drift": 0.0625, "detail": { "aggregate": 0.9375, "scoredRoles": 4, "skippedRoles": 0, "treeHeight": 6 }, "perRole": [ { "patternRole": "Enforcer", "score": 0.9167, "distance": 1, "groundTruth": { "entity": "dispatch", "loc": "src/tradearena/api/rate_limit.py:150" }, "recoveryResult": { "entity": "RateLimitMiddleware", "loc": "src/tradearena/api/rate_limit.py:126" }, "note": "path-length (d=1)" }, { "patternRole": "HistoryProvider", "score": 0.8333, "distance": 2, "groundTruth": { "entity": "_redis_check", "loc": "src/tradearena/api/rate_limit.py:65" }, "recoveryResult": { "entity": "_hits", "loc": "src/tradearena/api/rate_limit.py:141" }, "note": "path-length (d=2)" }, { "patternRole": "Limiter", "score": 1.0, "distance": 0, "groundTruth": { "entity": "_check", "loc": "src/tradearena/api/rate_limit.py:127" }, "recoveryResult": { "entity": "_check", "loc": "src/tradearena/api/rate_limit.py:145" }, "note": "path-length (d=0)" }, { "patternRole": "PolicyProvider", "score": 1.0, "distance": 0, "groundTruth": { "entity": "DEFAULT_RATE", "loc": "src/tradearena/api/rate_limit.py:19" }, "recoveryResult": { "entity": "DEFAULT_RATE", "loc": "src/tradearena/api/rate_limit.py:33" }, "note": "path-length (d=0)" } ], "notes": [] }, { "key": "level3", "level": "Level 3 \u2014 Interaction Order", "formula": "jaccard \u00d7 kendall-tau (shared interactions)", "available": true, "headlineLabel": "Interaction agreement", "score": 0.6667, "drift": 0.3333, "detail": { "agreedRoles": [ "Enforcer", "HistoryProvider", "Limiter", "PolicyProvider" ], "excludedRoles": [], "interactionsGroundTruth": 2, "interactionsRecoveryResult": 3, "sharedInteractions": 2, "unionSize": 3, "entryRoleGroundTruth": "Enforcer", "entryRoleRecoveryResult": "Enforcer", "entryRuleGroundTruth": "behaviour", "entryRuleRecoveryResult": "stored", "onlyGroundTruth": 0, "onlyRecoveryResult": 1, "interactionOverlap": 0.6667, "orderAgreement": 1.0, "rankedInteractions": 2, "totalPairs": 1, "concordantPairs": 1, "discordantPairs": 0, "kendallTau": 1.0, "prescribedActionSteps": 4, "prescribedStepsCovered": 4 }, "perRole": [ { "interaction": "Enforcer \u2192 Limiter", "from": "Enforcer", "to": "Limiter", "groundTruthRank": 1, "recoveryResultRank": 1, "outcome": "agreed", "discordantWith": 0, "evidence": { "groundTruth": { "file": "src/tradearena/api/rate_limit.py", "line": 159, "via": [] }, "recoveryResult": { "file": "src/tradearena/api/rate_limit.py", "line": 171, "via": [] } } }, { "interaction": "Limiter \u2192 HistoryProvider", "from": "Limiter", "to": "HistoryProvider", "groundTruthRank": 2, "recoveryResultRank": 3, "outcome": "agreed", "discordantWith": 0, "evidence": { "groundTruth": { "file": "src/tradearena/api/rate_limit.py", "line": 139, "via": [] }, "recoveryResult": { "file": "src/tradearena/api/rate_limit.py", "line": 155, "via": [] } } }, { "interaction": "Limiter \u2192 PolicyProvider", "from": "Limiter", "to": "PolicyProvider", "groundTruthRank": null, "recoveryResultRank": 2, "outcome": "onlyRecoveryResult", "discordantWith": null, "evidence": { "groundTruth": null, "recoveryResult": { "file": "src/tradearena/api/rate_limit.py", "line": 146, "via": [] } } } ], "notes": [ "Score = overlap \u00d7 order = 0.6667 \u00d7 1. Overlap: 2 of 3 interactions were found by both sides (0 only in the ground truth, 1 only in the recovery result), |shared| / |union|.", "Order: Kendall's Tau over the 2 shared interactions, in each side's own relative order \u2014 0 of 1 pairs run in opposite orders. Interactions only one side found take part in no pair (Fagin, Kumar & Sivakumar's p = 0, the induced sub-order); the overlap factor is what charges for them.", "Only 1 ranked pair \u2014 the order factor can land on just a few values here and one disagreement swings it fully.", "Scored over the 4 roles both sides detected (Enforcer, HistoryProvider, Limiter, PolicyProvider). An interaction counts only when both of its endpoint roles were detected on both sides \u2014 whether a role was found at all is Level 1's measurement, and charging for it again here would say the same thing twice.", "The interaction order is derived statically from the code \u2014 no one annotated it. Each role's cited entity is located in the AST index and control flow is followed from it, through as many non-role helper functions as it takes to reach the next role. See secpat_scoring/sequence/JOERN.md.", "Ground truth: the walk started at Enforcer \u2014 the earliest role the reference pattern has speaking that this side grounded in walkable code.", "Recovery result: the walk started at Enforcer \u2014 stored.", "An entry point the pattern did not name means its prescribed speakers were either ungrounded on that side or not where this code begins \u2014 a role something else reaches cannot open the flow \u2014 so the call graph had to stand in. Treat the ordering below as weaker evidence: it starts wherever the code happens to start." ] } ], "ground_truth_source": "recovery result", "vocabulary": [ "Enforcer", "Limiter", "HistoryProvider", "PolicyProvider" ], "role_scope": "required", "excluded_roles": [ "Entity", "System" ], "dropped_ground_truth_rows": [ "Entity", "System" ], "dropped_recovery_rows": [ "Entity", "System" ], "gt_roles": 4, "rr_roles": 4, "struct_files": 149, "annotated_at": "2026-08-22 09:09:45.666000", "recovery_order": "stored" }, { "slug": "limit_request_rate__stateset_stateset-agents", "id": "rate-limit/stateset_stateset-agents/ClaudeCodeOpus5.5", "pattern": "limit_request_rate", "repo": "stateset_stateset-agents", "error": null, "over_input_limit": true, "prompt_tokens": 1991977, "metrics": [ { "key": "level1", "level": "Level 1 \u2014 Role Detection", "available": true, "score": 0.0, "drift": 1.0, "notes": [ "Not run: the one-shot prompt is 1,991,977 tokens (o200k_base), over the baseline's 900,000 input limit (scripts/baseline_recover.py). Scored 0 by rule, not measured." ] }, { "key": "level2", "level": "Level 2 \u2014 Role Grounding", "available": true, "score": 0.0, "drift": 1.0, "notes": [ "Not run: the one-shot prompt is 1,991,977 tokens (o200k_base), over the baseline's 900,000 input limit (scripts/baseline_recover.py). Scored 0 by rule, not measured." ] }, { "key": "level3", "level": "Level 3 \u2014 Interaction Order", "available": true, "score": 0.0, "drift": 1.0, "notes": [ "Not run: the one-shot prompt is 1,991,977 tokens (o200k_base), over the baseline's 900,000 input limit (scripts/baseline_recover.py). Scored 0 by rule, not measured." ] } ] }, { "slug": "limit_request_rate__dchrnv_neurograph", "id": "rate-limit/dchrnv_neurograph/ClaudeCodeOpus5.5", "pattern": "limit_request_rate", "repo": "dchrnv_neurograph", "error": null, "metrics": [ { "key": "level1", "level": "Level 1 \u2014 Role Detection", "formula": "agreement-rate (present/external/missing)", "available": true, "headlineLabel": "Roles agreed", "score": 1.0, "drift": 0.0, "detail": { "agreedRoles": 4, "totalRoles": 4 }, "perRole": [ { "patternRole": "Enforcer", "groundTruth": "present", "recoveryResult": "present", "groundTruthDetected": true, "recoveryResultDetected": true, "outcome": "agreed" }, { "patternRole": "Limiter", "groundTruth": "present", "recoveryResult": "present", "groundTruthDetected": true, "recoveryResultDetected": true, "outcome": "agreed" }, { "patternRole": "HistoryProvider", "groundTruth": "present", "recoveryResult": "present", "groundTruthDetected": true, "recoveryResultDetected": true, "outcome": "agreed" }, { "patternRole": "PolicyProvider", "groundTruth": "present", "recoveryResult": "present", "groundTruthDetected": true, "recoveryResultDetected": true, "outcome": "agreed" } ], "notes": [] }, { "key": "level2", "level": "Level 2 \u2014 Role Grounding", "formula": "path-length", "available": true, "headlineLabel": "Mean grounding score", "score": 0.5208, "drift": 0.4792, "detail": { "aggregate": 0.5208, "scoredRoles": 4, "skippedRoles": 0, "treeHeight": 6 }, "perRole": [ { "patternRole": "Enforcer", "score": 0.9167, "distance": 1, "groundTruth": { "entity": "dispatch", "loc": "src/api/middlewares/rate_limit.py:178" }, "recoveryResult": { "entity": "RateLimitMiddleware", "loc": "src/api/middlewares/rate_limit.py:49" }, "note": "path-length (d=1)" }, { "patternRole": "HistoryProvider", "score": 0.4167, "distance": 7, "groundTruth": { "entity": "get_bucket", "loc": "src/api/middlewares/rate_limit.py:133" }, "recoveryResult": { "entity": "TokenBucket", "loc": "src/api/websocket/rate_limit.py:19" }, "note": "path-length (d=7)" }, { "patternRole": "Limiter", "score": 0.3333, "distance": 8, "groundTruth": { "entity": "consume", "loc": "src/api/middlewares/rate_limit.py:37" }, "recoveryResult": { "entity": "check_rate_limit", "loc": "src/api/websocket/rate_limit.py:150" }, "note": "path-length (d=8)" }, { "patternRole": "PolicyProvider", "score": 0.4167, "distance": 7, "groundTruth": { "entity": "get_bucket", "loc": "src/api/middlewares/rate_limit.py:133" }, "recoveryResult": { "entity": "WebSocketRateLimiter", "loc": "src/api/websocket/rate_limit.py:72" }, "note": "path-length (d=7)" } ], "notes": [] }, { "key": "level3", "level": "Level 3 \u2014 Interaction Order", "formula": "jaccard \u00d7 kendall-tau (shared interactions)", "available": true, "headlineLabel": "Interaction agreement", "score": 0.2, "drift": 0.8, "detail": { "agreedRoles": [ "Enforcer", "HistoryProvider", "Limiter", "PolicyProvider" ], "excludedRoles": [], "interactionsGroundTruth": 3, "interactionsRecoveryResult": 3, "sharedInteractions": 1, "unionSize": 5, "entryRoleGroundTruth": "Enforcer", "entryRoleRecoveryResult": "Enforcer", "entryRuleGroundTruth": "behaviour", "entryRuleRecoveryResult": "stored", "onlyGroundTruth": 2, "onlyRecoveryResult": 2, "interactionOverlap": 0.2, "orderAgreement": 1.0, "rankedInteractions": 1, "totalPairs": 0, "concordantPairs": 0, "discordantPairs": 0, "kendallTau": null, "prescribedActionSteps": 4, "prescribedStepsCovered": 4 }, "perRole": [ { "interaction": "Enforcer \u2192 PolicyProvider", "from": "Enforcer", "to": "PolicyProvider", "groundTruthRank": 1, "recoveryResultRank": null, "outcome": "onlyGroundTruth", "discordantWith": null, "evidence": { "groundTruth": { "file": "src/api/middlewares/rate_limit.py", "line": 188, "via": [] }, "recoveryResult": null } }, { "interaction": "Enforcer \u2192 HistoryProvider", "from": "Enforcer", "to": "HistoryProvider", "groundTruthRank": 2, "recoveryResultRank": null, "outcome": "onlyGroundTruth", "discordantWith": null, "evidence": { "groundTruth": { "file": "src/api/middlewares/rate_limit.py", "line": 188, "via": [] }, "recoveryResult": null } }, { "interaction": "Enforcer \u2192 Limiter", "from": "Enforcer", "to": "Limiter", "groundTruthRank": 3, "recoveryResultRank": 1, "outcome": "agreed", "discordantWith": 0, "evidence": { "groundTruth": { "file": "src/api/middlewares/rate_limit.py", "line": 191, "via": [] }, "recoveryResult": { "file": "src/api/websocket/connection.py", "line": 389, "via": [] } } }, { "interaction": "Limiter \u2192 PolicyProvider", "from": "Limiter", "to": "PolicyProvider", "groundTruthRank": null, "recoveryResultRank": 2, "outcome": "onlyRecoveryResult", "discordantWith": null, "evidence": { "groundTruth": null, "recoveryResult": { "file": "src/api/websocket/rate_limit.py", "line": 118, "via": [] } } }, { "interaction": "Limiter \u2192 HistoryProvider", "from": "Limiter", "to": "HistoryProvider", "groundTruthRank": null, "recoveryResultRank": 3, "outcome": "onlyRecoveryResult", "discordantWith": null, "evidence": { "groundTruth": null, "recoveryResult": { "file": "src/api/websocket/rate_limit.py", "line": 30, "via": [] } } } ], "notes": [ "Score = overlap \u00d7 order = 0.2 \u00d7 1. Overlap: 1 of 5 interactions were found by both sides (2 only in the ground truth, 2 only in the recovery result), |shared| / |union|.", "Order: 1 \u2014 a single shared interaction has no pair whose relative order could differ.", "Scored over the 4 roles both sides detected (Enforcer, HistoryProvider, Limiter, PolicyProvider). An interaction counts only when both of its endpoint roles were detected on both sides \u2014 whether a role was found at all is Level 1's measurement, and charging for it again here would say the same thing twice.", "The interaction order is derived statically from the code \u2014 no one annotated it. Each role's cited entity is located in the AST index and control flow is followed from it, through as many non-role helper functions as it takes to reach the next role. See secpat_scoring/sequence/JOERN.md.", "Ground truth: the walk started at Enforcer \u2014 the earliest role the reference pattern has speaking that this side grounded in walkable code.", "Recovery result: the walk started at Enforcer \u2014 stored.", "An entry point the pattern did not name means its prescribed speakers were either ungrounded on that side or not where this code begins \u2014 a role something else reaches cannot open the flow \u2014 so the call graph had to stand in. Treat the ordering below as weaker evidence: it starts wherever the code happens to start." ] } ], "ground_truth_source": "recovery result", "vocabulary": [ "Enforcer", "Limiter", "HistoryProvider", "PolicyProvider" ], "role_scope": "required", "excluded_roles": [ "Entity", "System" ], "dropped_ground_truth_rows": [ "Entity", "System" ], "dropped_recovery_rows": [ "Entity", "System" ], "gt_roles": 4, "rr_roles": 4, "struct_files": 202, "annotated_at": "2026-10-02 10:00:38.838000", "recovery_order": "stored" }, { "slug": "limit_request_rate__Daylily-Informatics_bloom", "id": "rate-limit/Daylily-Informatics_bloom/ClaudeCodeOpus5.5", "pattern": "limit_request_rate", "repo": "Daylily-Informatics_bloom", "error": null, "metrics": [ { "key": "level1", "level": "Level 1 \u2014 Role Detection", "formula": "agreement-rate", "available": true, "headlineLabel": "Roles agreed", "score": 1.0, "drift": 0.0, "detail": { "agreedRoles": 4, "totalRoles": 4 }, "perRole": [ { "patternRole": "Enforcer", "groundTruth": "present", "recoveryResult": "present", "groundTruthDetected": true, "recoveryResultDetected": true, "outcome": "agreed" }, { "patternRole": "Limiter", "groundTruth": "present", "recoveryResult": "present", "groundTruthDetected": true, "recoveryResultDetected": true, "outcome": "agreed" }, { "patternRole": "HistoryProvider", "groundTruth": "present", "recoveryResult": "present", "groundTruthDetected": true, "recoveryResultDetected": true, "outcome": "agreed" }, { "patternRole": "PolicyProvider", "groundTruth": "present", "recoveryResult": "present", "groundTruthDetected": true, "recoveryResultDetected": true, "outcome": "agreed" } ], "notes": [] }, { "key": "level2", "level": "Level 2 \u2014 Role Grounding", "formula": "path-length", "available": true, "headlineLabel": "Mean grounding score", "score": 0.875, "drift": 0.125, "detail": { "aggregate": 0.875, "scoredRoles": 4, "skippedRoles": 0, "treeHeight": 7 }, "perRole": [ { "patternRole": "Enforcer", "score": 0.9286, "distance": 1, "groundTruth": { "entity": "__call__", "loc": "bloom_lims/api/rate_limiting.py:322" }, "recoveryResult": { "entity": "RateLimitMiddleware", "loc": "bloom_lims/api/rate_limiting.py:309" }, "note": "path-length (d=1)" }, { "patternRole": "HistoryProvider", "score": 0.8571, "distance": 2, "groundTruth": { "entity": "_buckets", "loc": "bloom_lims/api/rate_limiting.py:104" }, "recoveryResult": { "entity": "ClientBucket", "loc": "bloom_lims/api/rate_limiting.py:56" }, "note": "path-length (d=2)" }, { "patternRole": "Limiter", "score": 0.9286, "distance": 1, "groundTruth": { "entity": "is_allowed", "loc": "bloom_lims/api/rate_limiting.py:119" }, "recoveryResult": { "entity": "RateLimiter", "loc": "bloom_lims/api/rate_limiting.py:67" }, "note": "path-length (d=1)" }, { "patternRole": "PolicyProvider", "score": 0.7857, "distance": 3, "groundTruth": { "entity": "is_allowed", "loc": "bloom_lims/api/rate_limiting.py:119" }, "recoveryResult": { "entity": "RateLimitConfig", "loc": "bloom_lims/api/rate_limiting.py:47" }, "note": "path-length (d=3)" } ], "notes": [] }, { "key": "level3", "level": "Level 3 \u2014 Interaction Order", "formula": "jaccard \u00d7 kendall-tau (shared interactions)", "available": true, "headlineLabel": "Interaction agreement", "score": 0.6, "drift": 0.4, "detail": { "agreedRoles": [ "Enforcer", "HistoryProvider", "Limiter", "PolicyProvider" ], "excludedRoles": [], "interactionsGroundTruth": 5, "interactionsRecoveryResult": 3, "sharedInteractions": 3, "unionSize": 5, "entryRoleGroundTruth": "Enforcer", "entryRoleRecoveryResult": "Enforcer", "entryRuleGroundTruth": "behaviour", "entryRuleRecoveryResult": "stored", "onlyGroundTruth": 2, "onlyRecoveryResult": 0, "interactionOverlap": 0.6, "orderAgreement": 1.0, "rankedInteractions": 3, "totalPairs": 3, "concordantPairs": 3, "discordantPairs": 0, "kendallTau": 1.0, "prescribedActionSteps": 4, "prescribedStepsCovered": 4 }, "perRole": [ { "interaction": "Enforcer \u2192 Limiter", "from": "Enforcer", "to": "Limiter", "groundTruthRank": 1, "recoveryResultRank": 1, "outcome": "agreed", "discordantWith": 0, "evidence": { "groundTruth": { "file": "bloom_lims/api/rate_limiting.py", "line": 343, "via": [] }, "recoveryResult": { "file": "bloom_lims/api/rate_limiting.py", "line": 347, "via": [] } } }, { "interaction": "Enforcer \u2192 PolicyProvider", "from": "Enforcer", "to": "PolicyProvider", "groundTruthRank": 2, "recoveryResultRank": null, "outcome": "onlyGroundTruth", "discordantWith": null, "evidence": { "groundTruth": { "file": "bloom_lims/api/rate_limiting.py", "line": 343, "via": [] }, "recoveryResult": null } }, { "interaction": "Limiter \u2192 PolicyProvider", "from": "Limiter", "to": "PolicyProvider", "groundTruthRank": 3, "recoveryResultRank": 2, "outcome": "agreed", "discordantWith": 0, "evidence": { "groundTruth": { "file": "bloom_lims/api/rate_limiting.py", "line": 343, "via": [] }, "recoveryResult": { "file": "bloom_lims/api/rate_limiting.py", "line": 113, "via": [] } } }, { "interaction": "Limiter \u2192 HistoryProvider", "from": "Limiter", "to": "HistoryProvider", "groundTruthRank": 4, "recoveryResultRank": 3, "outcome": "agreed", "discordantWith": 0, "evidence": { "groundTruth": { "file": "bloom_lims/api/rate_limiting.py", "line": 134, "via": [] }, "recoveryResult": { "file": "bloom_lims/api/rate_limiting.py", "line": 133, "via": [] } } }, { "interaction": "PolicyProvider \u2192 HistoryProvider", "from": "PolicyProvider", "to": "HistoryProvider", "groundTruthRank": 5, "recoveryResultRank": null, "outcome": "onlyGroundTruth", "discordantWith": null, "evidence": { "groundTruth": { "file": "bloom_lims/api/rate_limiting.py", "line": 134, "via": [] }, "recoveryResult": null } } ], "notes": [ "Score = overlap \u00d7 order = 0.6 \u00d7 1. Overlap: 3 of 5 interactions were found by both sides (2 only in the ground truth, 0 only in the recovery result), |shared| / |union|.", "Order: Kendall's Tau over the 3 shared interactions, in each side's own relative order \u2014 0 of 3 pairs run in opposite orders. Interactions only one side found take part in no pair (Fagin, Kumar & Sivakumar's p = 0, the induced sub-order); the overlap factor is what charges for them.", "Scored over the 4 roles both sides detected (Enforcer, HistoryProvider, Limiter, PolicyProvider). An interaction counts only when both of its endpoint roles were detected on both sides \u2014 whether a role was found at all is Level 1's measurement, and charging for it again here would say the same thing twice.", "The interaction order is derived statically from the code \u2014 no one annotated it. Each role's cited entity is located in the AST index and control flow is followed from it, through as many non-role helper functions as it takes to reach the next role. See secpat_scoring/sequence/JOERN.md.", "Ground truth: the walk started at Enforcer \u2014 the earliest role the reference pattern has speaking that this side grounded in walkable code.", "Recovery result: the walk started at Enforcer \u2014 stored.", "An entry point the pattern did not name means its prescribed speakers were either ungrounded on that side or not where this code begins \u2014 a role something else reaches cannot open the flow \u2014 so the call graph had to stand in. Treat the ordering below as weaker evidence: it starts wherever the code happens to start." ] } ], "ground_truth_source": "annotation", "vocabulary": [ "Enforcer", "Limiter", "HistoryProvider", "PolicyProvider" ], "role_scope": "required", "excluded_roles": [ "Entity", "System" ], "dropped_ground_truth_rows": [ "Entity", "System" ], "dropped_recovery_rows": [ "Entity", "System" ], "gt_roles": 4, "rr_roles": 4, "struct_files": 244, "annotated_at": "2026-10-16 11:32:26.011000", "recovery_order": "stored" }, { "slug": "limit_request_rate__ctrl-gaurav_effGen", "id": "rate-limit/ctrl-gaurav_effGen/ClaudeCodeOpus5.5", "pattern": "limit_request_rate", "repo": "ctrl-gaurav_effGen", "error": null, "over_input_limit": true, "prompt_tokens": 2806545, "metrics": [ { "key": "level1", "level": "Level 1 \u2014 Role Detection", "available": true, "score": 0.0, "drift": 1.0, "notes": [ "Not run: the one-shot prompt is 2,806,545 tokens (o200k_base), over the baseline's 900,000 input limit (scripts/baseline_recover.py). Scored 0 by rule, not measured." ] }, { "key": "level2", "level": "Level 2 \u2014 Role Grounding", "available": true, "score": 0.0, "drift": 1.0, "notes": [ "Not run: the one-shot prompt is 2,806,545 tokens (o200k_base), over the baseline's 900,000 input limit (scripts/baseline_recover.py). Scored 0 by rule, not measured." ] }, { "key": "level3", "level": "Level 3 \u2014 Interaction Order", "available": true, "score": 0.0, "drift": 1.0, "notes": [ "Not run: the one-shot prompt is 2,806,545 tokens (o200k_base), over the baseline's 900,000 input limit (scripts/baseline_recover.py). Scored 0 by rule, not measured." ] } ] }, { "slug": "limit_request_rate__shangyankeji_super-dev", "id": "rate-limit/shangyankeji_super-dev/ClaudeCodeOpus5.5", "pattern": "limit_request_rate", "repo": "shangyankeji_super-dev", "error": null, "over_input_limit": true, "prompt_tokens": 1443591, "metrics": [ { "key": "level1", "level": "Level 1 \u2014 Role Detection", "available": true, "score": 0.0, "drift": 1.0, "notes": [ "Not run: the one-shot prompt is 1,443,591 tokens (o200k_base), over the baseline's 900,000 input limit (scripts/baseline_recover.py). Scored 0 by rule, not measured." ] }, { "key": "level2", "level": "Level 2 \u2014 Role Grounding", "available": true, "score": 0.0, "drift": 1.0, "notes": [ "Not run: the one-shot prompt is 1,443,591 tokens (o200k_base), over the baseline's 900,000 input limit (scripts/baseline_recover.py). Scored 0 by rule, not measured." ] }, { "key": "level3", "level": "Level 3 \u2014 Interaction Order", "available": true, "score": 0.0, "drift": 1.0, "notes": [ "Not run: the one-shot prompt is 1,443,591 tokens (o200k_base), over the baseline's 900,000 input limit (scripts/baseline_recover.py). Scored 0 by rule, not measured." ] } ] }, { "slug": "limit_request_rate__cemde_WhatsNext", "id": "rate-limit/cemde_WhatsNext/ClaudeCodeOpus5.5", "pattern": "limit_request_rate", "repo": "cemde_WhatsNext", "error": null, "metrics": [ { "key": "level1", "level": "Level 1 \u2014 Role Detection", "formula": "agreement-rate (present/external/missing)", "available": true, "headlineLabel": "Roles agreed", "score": 1.0, "drift": 0.0, "detail": { "agreedRoles": 4, "totalRoles": 4 }, "perRole": [ { "patternRole": "Enforcer", "groundTruth": "present", "recoveryResult": "present", "groundTruthDetected": true, "recoveryResultDetected": true, "outcome": "agreed" }, { "patternRole": "Limiter", "groundTruth": "present", "recoveryResult": "present", "groundTruthDetected": true, "recoveryResultDetected": true, "outcome": "agreed" }, { "patternRole": "HistoryProvider", "groundTruth": "present", "recoveryResult": "present", "groundTruthDetected": true, "recoveryResultDetected": true, "outcome": "agreed" }, { "patternRole": "PolicyProvider", "groundTruth": "present", "recoveryResult": "present", "groundTruthDetected": true, "recoveryResultDetected": true, "outcome": "agreed" } ], "notes": [] }, { "key": "level2", "level": "Level 2 \u2014 Role Grounding", "formula": "path-length", "available": true, "headlineLabel": "Mean grounding score", "score": 0.8928, "drift": 0.1072, "detail": { "aggregate": 0.8928, "scoredRoles": 4, "skippedRoles": 0, "treeHeight": 7 }, "perRole": [ { "patternRole": "Enforcer", "score": 0.9286, "distance": 1, "groundTruth": { "entity": "dispatch", "loc": "whatsnext/api/server/middleware.py:85" }, "recoveryResult": { "entity": "RateLimitMiddleware", "loc": "whatsnext/api/server/middleware.py:63" }, "note": "path-length (d=1)" }, { "patternRole": "HistoryProvider", "score": 0.7857, "distance": 3, "groundTruth": { "entity": "_clean_old_requests", "loc": "whatsnext/api/server/middleware.py:80" }, "recoveryResult": { "entity": "request_times", "loc": "whatsnext/api/server/middleware.py:74" }, "note": "path-length (d=3)" }, { "patternRole": "Limiter", "score": 1.0, "distance": 0, "groundTruth": { "entity": "dispatch", "loc": "whatsnext/api/server/middleware.py:85" }, "recoveryResult": { "entity": "dispatch", "loc": "whatsnext/api/server/middleware.py:90" }, "note": "path-length (d=0)" }, { "patternRole": "PolicyProvider", "score": 0.8571, "distance": 2, "groundTruth": { "entity": "rate_limit_per_minute", "loc": "whatsnext/api/server/config.py:26" }, "recoveryResult": { "entity": "Settings", "loc": "whatsnext/api/server/config.py:11" }, "note": "path-length (d=2)" } ], "notes": [] }, { "key": "level3", "level": "Level 3 \u2014 Interaction Order", "formula": "jaccard \u00d7 kendall-tau (shared interactions)", "available": true, "headlineLabel": "Interaction agreement", "score": 0.5, "drift": 0.5, "detail": { "agreedRoles": [ "Enforcer", "HistoryProvider", "Limiter", "PolicyProvider" ], "excludedRoles": [], "interactionsGroundTruth": 3, "interactionsRecoveryResult": 3, "sharedInteractions": 2, "unionSize": 4, "entryRoleGroundTruth": "Enforcer", "entryRoleRecoveryResult": "Enforcer", "entryRuleGroundTruth": "behaviour", "entryRuleRecoveryResult": "stored", "onlyGroundTruth": 1, "onlyRecoveryResult": 1, "interactionOverlap": 0.5, "orderAgreement": 1.0, "rankedInteractions": 2, "totalPairs": 1, "concordantPairs": 1, "discordantPairs": 0, "kendallTau": 1.0, "prescribedActionSteps": 4, "prescribedStepsCovered": 4 }, "perRole": [ { "interaction": "Enforcer \u2192 Limiter", "from": "Enforcer", "to": "Limiter", "groundTruthRank": 1, "recoveryResultRank": 1, "outcome": "agreed", "discordantWith": 0, "evidence": { "groundTruth": { "file": "whatsnext/api/server/middleware.py", "line": 85, "via": [] }, "recoveryResult": { "file": "whatsnext/api/server/middleware.py", "line": 90, "via": [] } } }, { "interaction": "Enforcer \u2192 HistoryProvider", "from": "Enforcer", "to": "HistoryProvider", "groundTruthRank": 2, "recoveryResultRank": null, "outcome": "onlyGroundTruth", "discordantWith": null, "evidence": { "groundTruth": { "file": "whatsnext/api/server/middleware.py", "line": 94, "via": [] }, "recoveryResult": null } }, { "interaction": "Limiter \u2192 HistoryProvider", "from": "Limiter", "to": "HistoryProvider", "groundTruthRank": 3, "recoveryResultRank": 3, "outcome": "agreed", "discordantWith": 0, "evidence": { "groundTruth": { "file": "whatsnext/api/server/middleware.py", "line": 94, "via": [] }, "recoveryResult": { "file": "whatsnext/api/server/middleware.py", "line": 93, "via": [] } } }, { "interaction": "Limiter \u2192 PolicyProvider", "from": "Limiter", "to": "PolicyProvider", "groundTruthRank": null, "recoveryResultRank": 2, "outcome": "onlyRecoveryResult", "discordantWith": null, "evidence": { "groundTruth": null, "recoveryResult": { "file": "whatsnext/api/server/config.py", "line": 33, "via": [] } } } ], "notes": [ "Score = overlap \u00d7 order = 0.5 \u00d7 1. Overlap: 2 of 4 interactions were found by both sides (1 only in the ground truth, 1 only in the recovery result), |shared| / |union|.", "Order: Kendall's Tau over the 2 shared interactions, in each side's own relative order \u2014 0 of 1 pairs run in opposite orders. Interactions only one side found take part in no pair (Fagin, Kumar & Sivakumar's p = 0, the induced sub-order); the overlap factor is what charges for them.", "Only 1 ranked pair \u2014 the order factor can land on just a few values here and one disagreement swings it fully.", "Scored over the 4 roles both sides detected (Enforcer, HistoryProvider, Limiter, PolicyProvider). An interaction counts only when both of its endpoint roles were detected on both sides \u2014 whether a role was found at all is Level 1's measurement, and charging for it again here would say the same thing twice.", "The interaction order is derived statically from the code \u2014 no one annotated it. Each role's cited entity is located in the AST index and control flow is followed from it, through as many non-role helper functions as it takes to reach the next role. See secpat_scoring/sequence/JOERN.md.", "Ground truth: the walk started at Enforcer \u2014 the earliest role the reference pattern has speaking that this side grounded in walkable code.", "Recovery result: the walk started at Enforcer \u2014 stored.", "An entry point the pattern did not name means its prescribed speakers were either ungrounded on that side or not where this code begins \u2014 a role something else reaches cannot open the flow \u2014 so the call graph had to stand in. Treat the ordering below as weaker evidence: it starts wherever the code happens to start." ] } ], "ground_truth_source": "recovery result", "vocabulary": [ "Enforcer", "Limiter", "HistoryProvider", "PolicyProvider" ], "role_scope": "required", "excluded_roles": [ "Entity", "System" ], "dropped_ground_truth_rows": [ "Entity", "System" ], "dropped_recovery_rows": [ "Entity", "System" ], "gt_roles": 4, "rr_roles": 4, "struct_files": 65, "annotated_at": "2026-10-02 10:00:38.838000", "recovery_order": "stored" }, { "slug": "limit_request_rate__Helevon-Technologies-LTD_fabiplus", "id": "rate-limit/Helevon-Technologies-LTD_fabiplus/ClaudeCodeOpus5.5", "pattern": "limit_request_rate", "repo": "Helevon-Technologies-LTD_fabiplus", "error": null, "metrics": [ { "key": "level1", "level": "Level 1 \u2014 Role Detection", "formula": "agreement-rate (present/external/missing)", "available": true, "headlineLabel": "Roles agreed", "score": 1.0, "drift": 0.0, "detail": { "agreedRoles": 4, "totalRoles": 4 }, "perRole": [ { "patternRole": "Enforcer", "groundTruth": "present", "recoveryResult": "present", "groundTruthDetected": true, "recoveryResultDetected": true, "outcome": "agreed" }, { "patternRole": "Limiter", "groundTruth": "present", "recoveryResult": "present", "groundTruthDetected": true, "recoveryResultDetected": true, "outcome": "agreed" }, { "patternRole": "HistoryProvider", "groundTruth": "present", "recoveryResult": "present", "groundTruthDetected": true, "recoveryResultDetected": true, "outcome": "agreed" }, { "patternRole": "PolicyProvider", "groundTruth": "present", "recoveryResult": "present", "groundTruthDetected": true, "recoveryResultDetected": true, "outcome": "agreed" } ], "notes": [] }, { "key": "level2", "level": "Level 2 \u2014 Role Grounding", "formula": "path-length", "available": true, "headlineLabel": "Mean grounding score", "score": 0.825, "drift": 0.175, "detail": { "aggregate": 0.825, "scoredRoles": 4, "skippedRoles": 0, "treeHeight": 5 }, "perRole": [ { "patternRole": "Enforcer", "score": 0.9, "distance": 1, "groundTruth": { "entity": "dispatch", "loc": "fabiplus/middleware/security.py:272" }, "recoveryResult": { "entity": "RateLimitMiddleware", "loc": "fabiplus/middleware/security.py:281" }, "note": "path-length (d=1)" }, { "patternRole": "HistoryProvider", "score": 1.0, "distance": 0, "groundTruth": { "entity": "requests", "loc": "fabiplus/middleware/security.py:270" }, "recoveryResult": { "entity": "requests", "loc": "fabiplus/middleware/security.py:285" }, "note": "path-length (d=0)" }, { "patternRole": "Limiter", "score": 1.0, "distance": 0, "groundTruth": { "entity": "dispatch", "loc": "fabiplus/middleware/security.py:272" }, "recoveryResult": { "entity": "dispatch", "loc": "fabiplus/middleware/security.py:292" }, "note": "path-length (d=0)" }, { "patternRole": "PolicyProvider", "score": 0.4, "distance": 6, "groundTruth": { "entity": "requests_per_minute", "loc": "fabiplus/middleware/security.py:269" }, "recoveryResult": { "entity": "RATE_LIMIT_PER_MINUTE", "loc": "fabiplus/conf/settings.py:76" }, "note": "path-length (d=6)" } ], "notes": [] }, { "key": "level3", "level": "Level 3 \u2014 Interaction Order", "formula": "jaccard \u00d7 kendall-tau (shared interactions)", "available": true, "headlineLabel": "Interaction agreement", "score": 0.2, "drift": 0.8, "detail": { "agreedRoles": [ "Enforcer", "HistoryProvider", "Limiter", "PolicyProvider" ], "excludedRoles": [], "interactionsGroundTruth": 5, "interactionsRecoveryResult": 3, "sharedInteractions": 3, "unionSize": 5, "entryRoleGroundTruth": "Enforcer", "entryRoleRecoveryResult": "Enforcer", "entryRuleGroundTruth": "behaviour", "entryRuleRecoveryResult": "stored", "onlyGroundTruth": 2, "onlyRecoveryResult": 0, "interactionOverlap": 0.6, "orderAgreement": 0.3333, "rankedInteractions": 3, "totalPairs": 3, "concordantPairs": 1, "discordantPairs": 2, "kendallTau": -0.3333, "prescribedActionSteps": 4, "prescribedStepsCovered": 4 }, "perRole": [ { "interaction": "Enforcer \u2192 Limiter", "from": "Enforcer", "to": "Limiter", "groundTruthRank": 1, "recoveryResultRank": null, "outcome": "onlyGroundTruth", "discordantWith": null, "evidence": { "groundTruth": { "file": "fabiplus/middleware/security.py", "line": 272, "via": [] }, "recoveryResult": null } }, { "interaction": "Enforcer \u2192 HistoryProvider", "from": "Enforcer", "to": "HistoryProvider", "groundTruthRank": 2, "recoveryResultRank": 2, "outcome": "reordered", "discordantWith": 1, "evidence": { "groundTruth": { "file": "fabiplus/middleware/security.py", "line": 316, "via": [ "fabiplus/middleware/security.py:<module>.RateLimitMiddleware._cleanup_old_requests" ] }, "recoveryResult": { "file": "fabiplus/middleware/security.py", "line": 296, "via": [] } } }, { "interaction": "Limiter \u2192 HistoryProvider", "from": "Limiter", "to": "HistoryProvider", "groundTruthRank": 3, "recoveryResultRank": 3, "outcome": "reordered", "discordantWith": 1, "evidence": { "groundTruth": { "file": "fabiplus/middleware/security.py", "line": 316, "via": [ "fabiplus/middleware/security.py:<module>.RateLimitMiddleware._cleanup_old_requests" ] }, "recoveryResult": { "file": "fabiplus/middleware/security.py", "line": 306, "via": [] } } }, { "interaction": "Enforcer \u2192 PolicyProvider", "from": "Enforcer", "to": "PolicyProvider", "groundTruthRank": 4, "recoveryResultRank": 1, "outcome": "reordered", "discordantWith": 2, "evidence": { "groundTruth": { "file": "fabiplus/middleware/security.py", "line": 290, "via": [] }, "recoveryResult": { "file": "fabiplus/conf/settings.py", "line": 76, "via": [] } } }, { "interaction": "Limiter \u2192 PolicyProvider", "from": "Limiter", "to": "PolicyProvider", "groundTruthRank": 5, "recoveryResultRank": null, "outcome": "onlyGroundTruth", "discordantWith": null, "evidence": { "groundTruth": { "file": "fabiplus/middleware/security.py", "line": 290, "via": [] }, "recoveryResult": null } } ], "notes": [ "Score = overlap \u00d7 order = 0.6 \u00d7 0.3333. Overlap: 3 of 5 interactions were found by both sides (2 only in the ground truth, 0 only in the recovery result), |shared| / |union|.", "Order: Kendall's Tau over the 3 shared interactions, in each side's own relative order \u2014 2 of 3 pairs run in opposite orders. Interactions only one side found take part in no pair (Fagin, Kumar & Sivakumar's p = 0, the induced sub-order); the overlap factor is what charges for them.", "Scored over the 4 roles both sides detected (Enforcer, HistoryProvider, Limiter, PolicyProvider). An interaction counts only when both of its endpoint roles were detected on both sides \u2014 whether a role was found at all is Level 1's measurement, and charging for it again here would say the same thing twice.", "The interaction order is derived statically from the code \u2014 no one annotated it. Each role's cited entity is located in the AST index and control flow is followed from it, through as many non-role helper functions as it takes to reach the next role. See secpat_scoring/sequence/JOERN.md.", "Ground truth: the walk started at Enforcer \u2014 the earliest role the reference pattern has speaking that this side grounded in walkable code.", "Recovery result: the walk started at Enforcer \u2014 stored.", "An entry point the pattern did not name means its prescribed speakers were either ungrounded on that side or not where this code begins \u2014 a role something else reaches cannot open the flow \u2014 so the call graph had to stand in. Treat the ordering below as weaker evidence: it starts wherever the code happens to start." ] } ], "ground_truth_source": "recovery result", "vocabulary": [ "Enforcer", "Limiter", "HistoryProvider", "PolicyProvider" ], "role_scope": "required", "excluded_roles": [ "Entity", "System" ], "dropped_ground_truth_rows": [ "Entity", "System" ], "dropped_recovery_rows": [ "Entity", "System" ], "gt_roles": 4, "rr_roles": 4, "struct_files": 71, "annotated_at": "2026-10-06 10:57:21.305000", "recovery_order": "stored" }, { "slug": "limit_request_rate__engine-agi_engine-core", "id": "rate-limit/engine-agi_engine-core/ClaudeCodeOpus5.5", "pattern": "limit_request_rate", "repo": "engine-agi_engine-core", "error": null, "metrics": [ { "key": "level1", "level": "Level 1 \u2014 Role Detection", "formula": "agreement-rate (present/external/missing)", "available": true, "headlineLabel": "Roles agreed", "score": 1.0, "drift": 0.0, "detail": { "agreedRoles": 4, "totalRoles": 4 }, "perRole": [ { "patternRole": "Enforcer", "groundTruth": "present", "recoveryResult": "present", "groundTruthDetected": true, "recoveryResultDetected": true, "outcome": "agreed" }, { "patternRole": "Limiter", "groundTruth": "present", "recoveryResult": "present", "groundTruthDetected": true, "recoveryResultDetected": true, "outcome": "agreed" }, { "patternRole": "HistoryProvider", "groundTruth": "present", "recoveryResult": "present", "groundTruthDetected": true, "recoveryResultDetected": true, "outcome": "agreed" }, { "patternRole": "PolicyProvider", "groundTruth": "present", "recoveryResult": "present", "groundTruthDetected": true, "recoveryResultDetected": true, "outcome": "agreed" } ], "notes": [] }, { "key": "level2", "level": "Level 2 \u2014 Role Grounding", "formula": "path-length", "available": true, "headlineLabel": "Mean grounding score", "score": 0.9464, "drift": 0.0536, "detail": { "aggregate": 0.9464, "scoredRoles": 4, "skippedRoles": 0, "treeHeight": 7 }, "perRole": [ { "patternRole": "Enforcer", "score": 0.9286, "distance": 1, "groundTruth": { "entity": "dispatch", "loc": "src/engine_core/api/main.py:206" }, "recoveryResult": { "entity": "RateLimitMiddleware", "loc": "src/engine_core/api/main.py:210" }, "note": "path-length (d=1)" }, { "patternRole": "HistoryProvider", "score": 1.0, "distance": 0, "groundTruth": { "entity": "clients", "loc": "src/engine_core/api/main.py:204" }, "recoveryResult": { "entity": "clients", "loc": "src/engine_core/api/main.py:219" }, "note": "path-length (d=0)" }, { "patternRole": "Limiter", "score": 1.0, "distance": 0, "groundTruth": { "entity": "dispatch", "loc": "src/engine_core/api/main.py:206" }, "recoveryResult": { "entity": "dispatch", "loc": "src/engine_core/api/main.py:224" }, "note": "path-length (d=0)" }, { "patternRole": "PolicyProvider", "score": 0.8571, "distance": 2, "groundTruth": { "entity": "calls", "loc": "src/engine_core/api/main.py:202" }, "recoveryResult": { "entity": "configure_middleware", "loc": "src/engine_core/api/main.py:326" }, "note": "path-length (d=2)" } ], "notes": [] }, { "key": "level3", "level": "Level 3 \u2014 Interaction Order", "formula": "jaccard \u00d7 kendall-tau (shared interactions)", "available": true, "headlineLabel": "Interaction agreement", "score": 0.6, "drift": 0.4, "detail": { "agreedRoles": [ "Enforcer", "HistoryProvider", "Limiter", "PolicyProvider" ], "excludedRoles": [], "interactionsGroundTruth": 5, "interactionsRecoveryResult": 3, "sharedInteractions": 3, "unionSize": 5, "entryRoleGroundTruth": "Enforcer", "entryRoleRecoveryResult": "Enforcer", "entryRuleGroundTruth": "behaviour", "entryRuleRecoveryResult": "stored", "onlyGroundTruth": 2, "onlyRecoveryResult": 0, "interactionOverlap": 0.6, "orderAgreement": 1.0, "rankedInteractions": 3, "totalPairs": 3, "concordantPairs": 3, "discordantPairs": 0, "kendallTau": 1.0, "prescribedActionSteps": 4, "prescribedStepsCovered": 4 }, "perRole": [ { "interaction": "Enforcer \u2192 Limiter", "from": "Enforcer", "to": "Limiter", "groundTruthRank": 1, "recoveryResultRank": 1, "outcome": "agreed", "discordantWith": 0, "evidence": { "groundTruth": { "file": "src/engine_core/api/main.py", "line": 206, "via": [] }, "recoveryResult": { "file": "src/engine_core/api/main.py", "line": 227, "via": [] } } }, { "interaction": "Enforcer \u2192 HistoryProvider", "from": "Enforcer", "to": "HistoryProvider", "groundTruthRank": 2, "recoveryResultRank": null, "outcome": "onlyGroundTruth", "discordantWith": null, "evidence": { "groundTruth": { "file": "src/engine_core/api/main.py", "line": 216, "via": [] }, "recoveryResult": null } }, { "interaction": "Limiter \u2192 HistoryProvider", "from": "Limiter", "to": "HistoryProvider", "groundTruthRank": 3, "recoveryResultRank": 2, "outcome": "agreed", "discordantWith": 0, "evidence": { "groundTruth": { "file": "src/engine_core/api/main.py", "line": 216, "via": [] }, "recoveryResult": { "file": "src/engine_core/api/main.py", "line": 233, "via": [] } } }, { "interaction": "Enforcer \u2192 PolicyProvider", "from": "Enforcer", "to": "PolicyProvider", "groundTruthRank": 4, "recoveryResultRank": null, "outcome": "onlyGroundTruth", "discordantWith": null, "evidence": { "groundTruth": { "file": "src/engine_core/api/main.py", "line": 225, "via": [] }, "recoveryResult": null } }, { "interaction": "Limiter \u2192 PolicyProvider", "from": "Limiter", "to": "PolicyProvider", "groundTruthRank": 5, "recoveryResultRank": 3, "outcome": "agreed", "discordantWith": 0, "evidence": { "groundTruth": { "file": "src/engine_core/api/main.py", "line": 225, "via": [] }, "recoveryResult": { "file": "src/engine_core/api/main.py", "line": 240, "via": [] } } } ], "notes": [ "Score = overlap \u00d7 order = 0.6 \u00d7 1. Overlap: 3 of 5 interactions were found by both sides (2 only in the ground truth, 0 only in the recovery result), |shared| / |union|.", "Order: Kendall's Tau over the 3 shared interactions, in each side's own relative order \u2014 0 of 3 pairs run in opposite orders. Interactions only one side found take part in no pair (Fagin, Kumar & Sivakumar's p = 0, the induced sub-order); the overlap factor is what charges for them.", "Scored over the 4 roles both sides detected (Enforcer, HistoryProvider, Limiter, PolicyProvider). An interaction counts only when both of its endpoint roles were detected on both sides \u2014 whether a role was found at all is Level 1's measurement, and charging for it again here would say the same thing twice.", "The interaction order is derived statically from the code \u2014 no one annotated it. Each role's cited entity is located in the AST index and control flow is followed from it, through as many non-role helper functions as it takes to reach the next role. See secpat_scoring/sequence/JOERN.md.", "Ground truth: the walk started at Enforcer \u2014 the earliest role the reference pattern has speaking that this side grounded in walkable code.", "Recovery result: the walk started at Enforcer \u2014 stored.", "An entry point the pattern did not name means its prescribed speakers were either ungrounded on that side or not where this code begins \u2014 a role something else reaches cannot open the flow \u2014 so the call graph had to stand in. Treat the ordering below as weaker evidence: it starts wherever the code happens to start." ] } ], "ground_truth_source": "recovery result", "vocabulary": [ "Enforcer", "Limiter", "HistoryProvider", "PolicyProvider" ], "role_scope": "required", "excluded_roles": [ "Entity", "System" ], "dropped_ground_truth_rows": [ "Entity", "System" ], "dropped_recovery_rows": [ "Entity", "System" ], "gt_roles": 4, "rr_roles": 4, "struct_files": 67, "annotated_at": "2026-10-02 10:00:38.838000", "recovery_order": "stored" }, { "slug": "session_based_access_control__ParisNeo_lollms", "id": "session-based-ac/ParisNeo_lollms/ClaudeCodeOpus5.5", "pattern": "session_based_access_control", "repo": "ParisNeo_lollms", "error": null, "metrics": [ { "key": "level1", "level": "Level 1 \u2014 Role Detection", "formula": "agreement-rate (present/external/missing)", "available": true, "headlineLabel": "Roles agreed", "score": 0.5, "drift": 0.5, "detail": { "agreedRoles": 3, "totalRoles": 6 }, "perRole": [ { "patternRole": "Authentication enforcer", "groundTruth": "missing", "recoveryResult": "present", "groundTruthDetected": false, "recoveryResultDetected": true, "outcome": "mismatched" }, { "patternRole": "Verifier", "groundTruth": "missing", "recoveryResult": "present", "groundTruthDetected": false, "recoveryResultDetected": true, "outcome": "mismatched" }, { "patternRole": "Session manager", "groundTruth": "missing", "recoveryResult": "present", "groundTruthDetected": false, "recoveryResultDetected": true, "outcome": "mismatched" }, { "patternRole": "Authorisation enforcer", "groundTruth": "present", "recoveryResult": "present", "groundTruthDetected": true, "recoveryResultDetected": true, "outcome": "agreed" }, { "patternRole": "Decider", "groundTruth": "present", "recoveryResult": "present", "groundTruthDetected": true, "recoveryResultDetected": true, "outcome": "agreed" }, { "patternRole": "Policy provider", "groundTruth": "present", "recoveryResult": "present", "groundTruthDetected": true, "recoveryResultDetected": true, "outcome": "agreed" } ], "notes": [] }, { "key": "level2", "level": "Level 2 \u2014 Role Grounding", "formula": "path-length", "available": true, "headlineLabel": "Mean grounding score", "score": 0.7778, "drift": 0.2222, "detail": { "aggregate": 0.7778, "scoredRoles": 3, "skippedRoles": 3, "treeHeight": 6 }, "perRole": [ { "patternRole": "Authorisation enforcer", "score": 1.0, "distance": 0, "groundTruth": { "entity": "get_current_admin_user", "loc": "backend/session.py:402" }, "recoveryResult": { "entity": "get_current_admin_user", "loc": "backend/session.py:330" }, "note": "path-length (d=0)" }, { "patternRole": "Decider", "score": 0.5, "distance": 6, "groundTruth": { "entity": "get_current_admin_user", "loc": "backend/session.py:402" }, "recoveryResult": { "entity": "get_discussion_and_owner_for_request", "loc": "backend/routers/discussion/helpers.py:10" }, "note": "path-length (d=6)" }, { "patternRole": "Policy provider", "score": 0.8333, "distance": 2, "groundTruth": { "entity": "is_admin", "loc": "backend/db/models/user.py:27" }, "recoveryResult": { "entity": "User", "loc": "backend/db/models/user.py:14" }, "note": "path-length (d=2)" } ], "notes": [ "Authentication enforcer: absent from the ground truth (Level 1 negative)", "Session manager: absent from the ground truth (Level 1 negative)", "Verifier: absent from the ground truth (Level 1 negative)" ] }, { "key": "level3", "level": "Level 3 \u2014 Interaction Order", "formula": "jaccard \u00d7 kendall-tau (shared interactions)", "available": true, "headlineLabel": "Interaction agreement", "score": 0.5, "drift": 0.5, "detail": { "agreedRoles": [ "Authorisation enforcer", "Decider", "Policy provider" ], "excludedRoles": [ "Authentication enforcer", "Session manager", "Verifier" ], "interactionsGroundTruth": 1, "interactionsRecoveryResult": 2, "sharedInteractions": 1, "unionSize": 2, "entryRoleGroundTruth": "Authorisation enforcer", "entryRoleRecoveryResult": "Authorisation enforcer", "entryRuleGroundTruth": "behaviour", "entryRuleRecoveryResult": "stored", "onlyGroundTruth": 0, "onlyRecoveryResult": 1, "interactionOverlap": 0.5, "orderAgreement": 1.0, "rankedInteractions": 1, "totalPairs": 0, "concordantPairs": 0, "discordantPairs": 0, "kendallTau": null, "prescribedActionSteps": 5, "prescribedStepsCovered": 2 }, "perRole": [ { "interaction": "Authorisation enforcer \u2192 Decider", "from": "Authorisation enforcer", "to": "Decider", "groundTruthRank": 1, "recoveryResultRank": 1, "outcome": "agreed", "discordantWith": 0, "evidence": { "groundTruth": { "file": "backend/session.py", "line": 402, "via": [] }, "recoveryResult": { "file": "backend/routers/discussion/helpers.py", "line": 10, "via": [] } } }, { "interaction": "Decider \u2192 Policy provider", "from": "Decider", "to": "Policy provider", "groundTruthRank": null, "recoveryResultRank": 2, "outcome": "onlyRecoveryResult", "discordantWith": null, "evidence": { "groundTruth": null, "recoveryResult": { "file": "backend/routers/discussion/helpers.py", "line": 33, "via": [] } } } ], "notes": [ "Score = overlap \u00d7 order = 0.5 \u00d7 1. Overlap: 1 of 2 interactions were found by both sides (0 only in the ground truth, 1 only in the recovery result), |shared| / |union|.", "Order: 1 \u2014 a single shared interaction has no pair whose relative order could differ.", "Scored over the 3 roles both sides detected (Authorisation enforcer, Decider, Policy provider). An interaction counts only when both of its endpoint roles were detected on both sides \u2014 whether a role was found at all is Level 1's measurement, and charging for it again here would say the same thing twice.", "The interaction order is derived statically from the code \u2014 no one annotated it. Each role's cited entity is located in the AST index and control flow is followed from it, through as many non-role helper functions as it takes to reach the next role. See secpat_scoring/sequence/JOERN.md.", "Left out \u2014 only one side detected them: Authentication enforcer, Session manager, Verifier. 3 roles were excluded from both walks, so neither side's ordering mentions them. The walk still passes through the code behind such a role, attributing what it finds there to the nearest agreed role.", "Ground truth: the walk started at Authorisation enforcer \u2014 the earliest role the reference pattern has speaking that this side grounded in walkable code.", "Recovery result: the walk started at Authorisation enforcer \u2014 stored.", "An entry point the pattern did not name means its prescribed speakers were either ungrounded on that side or not where this code begins \u2014 a role something else reaches cannot open the flow \u2014 so the call graph had to stand in. Treat the ordering below as weaker evidence: it starts wherever the code happens to start." ] } ], "ground_truth_source": "recovery result", "vocabulary": [ "Authentication enforcer", "Verifier", "Session manager", "Authorisation enforcer", "Decider", "Policy provider" ], "role_scope": "required", "excluded_roles": [ "Subject", "System" ], "dropped_ground_truth_rows": [ "Subject", "System" ], "dropped_recovery_rows": [ "Subject", "System" ], "gt_roles": 6, "rr_roles": 6, "struct_files": 200, "annotated_at": "2026-10-02 10:00:38.838000", "recovery_order": "stored" }, { "slug": "session_based_access_control__namastexlabs_automagik-tools", "id": "session-based-ac/namastexlabs_automagik-tools/ClaudeCodeOpus5.5", "pattern": "session_based_access_control", "repo": "namastexlabs_automagik-tools", "error": null, "metrics": [ { "key": "level1", "level": "Level 1 \u2014 Role Detection", "formula": "agreement-rate", "available": true, "headlineLabel": "Roles agreed", "score": 0.5, "drift": 0.5, "detail": { "agreedRoles": 3, "totalRoles": 6 }, "perRole": [ { "patternRole": "Authentication enforcer", "groundTruth": "missing", "recoveryResult": "present", "groundTruthDetected": false, "recoveryResultDetected": true, "outcome": "mismatched" }, { "patternRole": "Verifier", "groundTruth": "missing", "recoveryResult": "present", "groundTruthDetected": false, "recoveryResultDetected": true, "outcome": "mismatched" }, { "patternRole": "Session manager", "groundTruth": "missing", "recoveryResult": "present", "groundTruthDetected": false, "recoveryResultDetected": true, "outcome": "mismatched" }, { "patternRole": "Authorisation enforcer", "groundTruth": "present", "recoveryResult": "present", "groundTruthDetected": true, "recoveryResultDetected": true, "outcome": "agreed" }, { "patternRole": "Decider", "groundTruth": "present", "recoveryResult": "present", "groundTruthDetected": true, "recoveryResultDetected": true, "outcome": "agreed" }, { "patternRole": "Policy provider", "groundTruth": "present", "recoveryResult": "present", "groundTruthDetected": true, "recoveryResultDetected": true, "outcome": "agreed" } ], "notes": [] }, { "key": "level2", "level": "Level 2 \u2014 Role Grounding", "formula": "path-length", "available": true, "headlineLabel": "Mean grounding score", "score": 0.7917, "drift": 0.2083, "detail": { "aggregate": 0.7917, "scoredRoles": 3, "skippedRoles": 3, "treeHeight": 8 }, "perRole": [ { "patternRole": "Authorisation enforcer", "score": 0.75, "distance": 4, "groundTruth": { "entity": "list_all_workspaces_api", "loc": "automagik_tools/hub/api_routes.py:462" }, "recoveryResult": { "entity": "require_permission", "loc": "automagik_tools/hub/authorization.py:231" }, "note": "path-length (d=4)" }, { "patternRole": "Decider", "score": 0.75, "distance": 4, "groundTruth": { "entity": "is_super_admin", "loc": "automagik_tools/hub/authorization.py:56" }, "recoveryResult": { "entity": "ThreeTierPermissionChecker", "loc": "automagik_tools/hub/permissions.py:31" }, "note": "path-length (d=4)" }, { "patternRole": "Policy provider", "score": 0.875, "distance": 2, "groundTruth": { "entity": "get_super_admin_emails", "loc": "automagik_tools/hub/authorization.py:34" }, "recoveryResult": { "entity": "require_permission", "loc": "automagik_tools/hub/authorization.py:231" }, "note": "path-length (d=2)" } ], "notes": [ "Authentication enforcer: absent from the ground truth (Level 1 negative)", "Session manager: absent from the ground truth (Level 1 negative)", "Verifier: absent from the ground truth (Level 1 negative)" ] }, { "key": "level3", "level": "Level 3 \u2014 Interaction Order", "formula": "jaccard \u00d7 kendall-tau (shared interactions)", "available": true, "headlineLabel": "Interaction agreement", "score": 1.0, "drift": 0.0, "detail": { "agreedRoles": [ "Authorisation enforcer", "Decider", "Policy provider" ], "excludedRoles": [ "Authentication enforcer", "Session manager", "Verifier" ], "interactionsGroundTruth": 2, "interactionsRecoveryResult": 2, "sharedInteractions": 2, "unionSize": 2, "entryRoleGroundTruth": "Authorisation enforcer", "entryRoleRecoveryResult": "Authorisation enforcer", "entryRuleGroundTruth": "behaviour", "entryRuleRecoveryResult": "stored", "onlyGroundTruth": 0, "onlyRecoveryResult": 0, "interactionOverlap": 1.0, "orderAgreement": 1.0, "rankedInteractions": 2, "totalPairs": 1, "concordantPairs": 1, "discordantPairs": 0, "kendallTau": 1.0, "prescribedActionSteps": 5, "prescribedStepsCovered": 2 }, "perRole": [ { "interaction": "Authorisation enforcer \u2192 Decider", "from": "Authorisation enforcer", "to": "Decider", "groundTruthRank": 1, "recoveryResultRank": 1, "outcome": "agreed", "discordantWith": 0, "evidence": { "groundTruth": { "file": "automagik_tools/hub/api_routes.py", "line": 485, "via": [] }, "recoveryResult": { "file": "automagik_tools/hub/permissions.py", "line": 74, "via": [] } } }, { "interaction": "Decider \u2192 Policy provider", "from": "Decider", "to": "Policy provider", "groundTruthRank": 2, "recoveryResultRank": 2, "outcome": "agreed", "discordantWith": 0, "evidence": { "groundTruth": { "file": "automagik_tools/hub/authorization.py", "line": 74, "via": [] }, "recoveryResult": { "file": "automagik_tools/hub/authorization.py", "line": 252, "via": [] } } } ], "notes": [ "Score = overlap \u00d7 order = 1 \u00d7 1. Overlap: 2 of 2 interactions were found by both sides (0 only in the ground truth, 0 only in the recovery result), |shared| / |union|.", "Order: Kendall's Tau over the 2 shared interactions, in each side's own relative order \u2014 0 of 1 pairs run in opposite orders. Interactions only one side found take part in no pair (Fagin, Kumar & Sivakumar's p = 0, the induced sub-order); the overlap factor is what charges for them.", "Only 1 ranked pair \u2014 the order factor can land on just a few values here and one disagreement swings it fully.", "Scored over the 3 roles both sides detected (Authorisation enforcer, Decider, Policy provider). An interaction counts only when both of its endpoint roles were detected on both sides \u2014 whether a role was found at all is Level 1's measurement, and charging for it again here would say the same thing twice.", "The interaction order is derived statically from the code \u2014 no one annotated it. Each role's cited entity is located in the AST index and control flow is followed from it, through as many non-role helper functions as it takes to reach the next role. See secpat_scoring/sequence/JOERN.md.", "Left out \u2014 only one side detected them: Authentication enforcer, Session manager, Verifier. 3 roles were excluded from both walks, so neither side's ordering mentions them. The walk still passes through the code behind such a role, attributing what it finds there to the nearest agreed role.", "Ground truth: the walk started at Authorisation enforcer \u2014 the earliest role the reference pattern has speaking that this side grounded in walkable code.", "Recovery result: the walk started at Authorisation enforcer \u2014 stored.", "An entry point the pattern did not name means its prescribed speakers were either ungrounded on that side or not where this code begins \u2014 a role something else reaches cannot open the flow \u2014 so the call graph had to stand in. Treat the ordering below as weaker evidence: it starts wherever the code happens to start." ] } ], "ground_truth_source": "recovery result", "vocabulary": [ "Authentication enforcer", "Verifier", "Session manager", "Authorisation enforcer", "Decider", "Policy provider" ], "role_scope": "required", "excluded_roles": [ "Subject", "System" ], "dropped_ground_truth_rows": [ "Subject", "System" ], "dropped_recovery_rows": [ "Subject", "System" ], "gt_roles": 6, "rr_roles": 6, "struct_files": 268, "annotated_at": "2026-10-02 10:00:38.838000", "recovery_order": "stored" }, { "slug": "session_based_access_control__nijaru_zenith", "id": "session-based-ac/nijaru_zenith/ClaudeCodeOpus5.0", "pattern": "session_based_access_control", "repo": "nijaru_zenith", "error": null, "metrics": [ { "key": "level1", "level": "Level 1 \u2014 Role Detection", "formula": "agreement-rate (present/external/missing)", "available": true, "headlineLabel": "Roles agreed", "score": 0.8333, "drift": 0.1667, "detail": { "agreedRoles": 5, "totalRoles": 6 }, "perRole": [ { "patternRole": "Authentication enforcer", "groundTruth": "present", "recoveryResult": "present", "groundTruthDetected": true, "recoveryResultDetected": true, "outcome": "agreed" }, { "patternRole": "Verifier", "groundTruth": "present", "recoveryResult": "present", "groundTruthDetected": true, "recoveryResultDetected": true, "outcome": "agreed" }, { "patternRole": "Session manager", "groundTruth": "present", "recoveryResult": "present", "groundTruthDetected": true, "recoveryResultDetected": true, "outcome": "agreed" }, { "patternRole": "Authorisation enforcer", "groundTruth": "present", "recoveryResult": "present", "groundTruthDetected": true, "recoveryResultDetected": true, "outcome": "agreed" }, { "patternRole": "Decider", "groundTruth": "present", "recoveryResult": "present", "groundTruthDetected": true, "recoveryResultDetected": true, "outcome": "agreed" }, { "patternRole": "Policy provider", "groundTruth": "present", "recoveryResult": "missing", "groundTruthDetected": true, "recoveryResultDetected": false, "outcome": "mismatched" } ], "notes": [] }, { "key": "level2", "level": "Level 2 \u2014 Role Grounding", "formula": "path-length", "available": true, "headlineLabel": "Mean grounding score", "score": 0.7857, "drift": 0.2143, "detail": { "aggregate": 0.7857, "scoredRoles": 5, "skippedRoles": 1, "treeHeight": 7 }, "perRole": [ { "patternRole": "Authentication enforcer", "score": 1.0, "distance": 0, "groundTruth": { "entity": "SessionMiddleware", "loc": "zenith/sessions/middleware.py:20" }, "recoveryResult": { "entity": "SessionMiddleware", "loc": "zenith/sessions/middleware.py:22" }, "note": "path-length (d=0)" }, { "patternRole": "Authorisation enforcer", "score": 0.6429, "distance": 5, "groundTruth": { "entity": "_resolve_auth", "loc": "zenith/core/routing/dependency_resolver.py:71" }, "recoveryResult": { "entity": "AuthDependency", "loc": "zenith/core/routing/dependencies.py:19" }, "note": "path-length (d=5)" }, { "patternRole": "Decider", "score": 0.5714, "distance": 6, "groundTruth": { "entity": "require_scopes", "loc": "zenith/middleware/auth.py:144" }, "recoveryResult": { "entity": "require_scopes", "loc": "zenith/auth/dependencies.py:62" }, "note": "path-length (d=6)" }, { "patternRole": "Session manager", "score": 0.7143, "distance": 4, "groundTruth": { "entity": "SessionStore", "loc": "zenith/sessions/store.py:16" }, "recoveryResult": { "entity": "SessionManager", "loc": "zenith/sessions/manager.py:115" }, "note": "path-length (d=4)" }, { "patternRole": "Verifier", "score": 1.0, "distance": 0, "groundTruth": { "entity": "get_session", "loc": "zenith/sessions/manager.py:223" }, "recoveryResult": { "entity": "get_session", "loc": "zenith/sessions/manager.py:207" }, "note": "path-length (d=0)" } ], "notes": [ "Policy provider: absent from the recovery result (Level 1 false negative)" ] }, { "key": "level3", "level": "Level 3 \u2014 Interaction Order", "formula": "jaccard \u00d7 kendall-tau (shared interactions)", "available": true, "headlineLabel": "Interaction agreement", "score": 0.4, "drift": 0.6, "detail": { "agreedRoles": [ "Authentication enforcer", "Authorisation enforcer", "Decider", "Session manager", "Verifier" ], "excludedRoles": [ "Policy provider" ], "interactionsGroundTruth": 3, "interactionsRecoveryResult": 4, "sharedInteractions": 2, "unionSize": 5, "entryRoleGroundTruth": "Authentication enforcer", "entryRoleRecoveryResult": "Authentication enforcer", "entryRuleGroundTruth": "behaviour", "entryRuleRecoveryResult": "stored", "onlyGroundTruth": 1, "onlyRecoveryResult": 2, "interactionOverlap": 0.4, "orderAgreement": 1.0, "rankedInteractions": 2, "totalPairs": 1, "concordantPairs": 1, "discordantPairs": 0, "kendallTau": 1.0, "prescribedActionSteps": 5, "prescribedStepsCovered": 4 }, "perRole": [ { "interaction": "Authentication enforcer \u2192 Session manager", "from": "Authentication enforcer", "to": "Session manager", "groundTruthRank": 1, "recoveryResultRank": null, "outcome": "onlyGroundTruth", "discordantWith": null, "evidence": { "groundTruth": { "file": "zenith/sessions/manager.py", "line": 240, "via": [ "zenith/sessions/manager.py:<module>.SessionManager.save_session" ] }, "recoveryResult": null } }, { "interaction": "Authentication enforcer \u2192 Verifier", "from": "Authentication enforcer", "to": "Verifier", "groundTruthRank": 2, "recoveryResultRank": 1, "outcome": "agreed", "discordantWith": 0, "evidence": { "groundTruth": { "file": "zenith/sessions/middleware.py", "line": 159, "via": [] }, "recoveryResult": { "file": "zenith/sessions/middleware.py", "line": 212, "via": [] } } }, { "interaction": "Verifier \u2192 Session manager", "from": "Verifier", "to": "Session manager", "groundTruthRank": 3, "recoveryResultRank": 2, "outcome": "agreed", "discordantWith": 0, "evidence": { "groundTruth": { "file": "zenith/sessions/manager.py", "line": 225, "via": [] }, "recoveryResult": { "file": "zenith/sessions/manager.py", "line": 208, "via": [] } } }, { "interaction": "Authentication enforcer \u2192 Authorisation enforcer", "from": "Authentication enforcer", "to": "Authorisation enforcer", "groundTruthRank": null, "recoveryResultRank": 3, "outcome": "onlyRecoveryResult", "discordantWith": null, "evidence": { "groundTruth": null, "recoveryResult": { "file": "zenith/core/routing/dependency_resolver.py", "line": 69, "via": [] } } }, { "interaction": "Authorisation enforcer \u2192 Decider", "from": "Authorisation enforcer", "to": "Decider", "groundTruthRank": null, "recoveryResultRank": 4, "outcome": "onlyRecoveryResult", "discordantWith": null, "evidence": { "groundTruth": null, "recoveryResult": { "file": "zenith/middleware/auth.py", "line": 150, "via": [] } } } ], "notes": [ "Score = overlap \u00d7 order = 0.4 \u00d7 1. Overlap: 2 of 5 interactions were found by both sides (1 only in the ground truth, 2 only in the recovery result), |shared| / |union|.", "Order: Kendall's Tau over the 2 shared interactions, in each side's own relative order \u2014 0 of 1 pairs run in opposite orders. Interactions only one side found take part in no pair (Fagin, Kumar & Sivakumar's p = 0, the induced sub-order); the overlap factor is what charges for them.", "Only 1 ranked pair \u2014 the order factor can land on just a few values here and one disagreement swings it fully.", "Scored over the 5 roles both sides detected (Authentication enforcer, Authorisation enforcer, Decider, Session manager, Verifier). An interaction counts only when both of its endpoint roles were detected on both sides \u2014 whether a role was found at all is Level 1's measurement, and charging for it again here would say the same thing twice.", "The interaction order is derived statically from the code \u2014 no one annotated it. Each role's cited entity is located in the AST index and control flow is followed from it, through as many non-role helper functions as it takes to reach the next role. See secpat_scoring/sequence/JOERN.md.", "Left out \u2014 only one side detected it: Policy provider. 1 role was excluded from both walks, so neither side's ordering mentions it. The walk still passes through the code behind such a role, attributing what it finds there to the nearest agreed role.", "Ground truth: the walk started at Authentication enforcer \u2014 the earliest role the reference pattern has speaking that this side grounded in walkable code.", "Recovery result: the walk started at Authentication enforcer \u2014 stored.", "An entry point the pattern did not name means its prescribed speakers were either ungrounded on that side or not where this code begins \u2014 a role something else reaches cannot open the flow \u2014 so the call graph had to stand in. Treat the ordering below as weaker evidence: it starts wherever the code happens to start." ] } ], "ground_truth_source": "recovery result", "vocabulary": [ "Authentication enforcer", "Verifier", "Session manager", "Authorisation enforcer", "Decider", "Policy provider" ], "role_scope": "required", "excluded_roles": [ "Subject", "System" ], "dropped_ground_truth_rows": [ "Subject", "System" ], "dropped_recovery_rows": [ "Subject", "System" ], "gt_roles": 6, "rr_roles": 6, "struct_files": 253, "annotated_at": "2026-08-22 09:09:19.498000", "recovery_order": "stored" }, { "slug": "session_based_access_control__DeepselSystems_deepsel", "id": "session-based-ac/DeepselSystems_deepsel/ClaudeCodeOpus5.5", "pattern": "session_based_access_control", "repo": "DeepselSystems_deepsel", "error": null, "metrics": [ { "key": "level1", "level": "Level 1 \u2014 Role Detection", "formula": "agreement-rate (present/external/missing)", "available": true, "headlineLabel": "Roles agreed", "score": 1.0, "drift": 0.0, "detail": { "agreedRoles": 6, "totalRoles": 6 }, "perRole": [ { "patternRole": "Authentication enforcer", "groundTruth": "present", "recoveryResult": "present", "groundTruthDetected": true, "recoveryResultDetected": true, "outcome": "agreed" }, { "patternRole": "Verifier", "groundTruth": "present", "recoveryResult": "present", "groundTruthDetected": true, "recoveryResultDetected": true, "outcome": "agreed" }, { "patternRole": "Session manager", "groundTruth": "present", "recoveryResult": "present", "groundTruthDetected": true, "recoveryResultDetected": true, "outcome": "agreed" }, { "patternRole": "Authorisation enforcer", "groundTruth": "present", "recoveryResult": "present", "groundTruthDetected": true, "recoveryResultDetected": true, "outcome": "agreed" }, { "patternRole": "Decider", "groundTruth": "present", "recoveryResult": "present", "groundTruthDetected": true, "recoveryResultDetected": true, "outcome": "agreed" }, { "patternRole": "Policy provider", "groundTruth": "present", "recoveryResult": "present", "groundTruthDetected": true, "recoveryResultDetected": true, "outcome": "agreed" } ], "notes": [] }, { "key": "level2", "level": "Level 2 \u2014 Role Grounding", "formula": "path-length", "available": true, "headlineLabel": "Mean grounding score", "score": 0.8333, "drift": 0.1667, "detail": { "aggregate": 0.8333, "scoredRoles": 6, "skippedRoles": 0, "treeHeight": 7 }, "perRole": [ { "patternRole": "Authentication enforcer", "score": 1.0, "distance": 0, "groundTruth": { "entity": "get_current_user", "loc": "deepsel/auth/get_current_user.py:56" }, "recoveryResult": { "entity": "get_current_user", "loc": "deepsel/auth/get_current_user.py:37" }, "note": "path-length (d=0)" }, { "patternRole": "Authorisation enforcer", "score": 0.9286, "distance": 1, "groundTruth": { "entity": "get_one", "loc": "deepsel/orm/mixin.py:835" }, "recoveryResult": { "entity": "ORMBaseMixin", "loc": "deepsel/orm/mixin.py:93" }, "note": "path-length (d=1)" }, { "patternRole": "Decider", "score": 1.0, "distance": 0, "groundTruth": { "entity": "_check_has_permission", "loc": "deepsel/orm/mixin.py:1115" }, "recoveryResult": { "entity": "_check_has_permission", "loc": "deepsel/orm/mixin.py:1690" }, "note": "path-length (d=0)" }, { "patternRole": "Policy provider", "score": 0.3571, "distance": 9, "groundTruth": { "entity": "get_user_permissions", "loc": "deepsel/orm/user_mixin.py:244" }, "recoveryResult": { "entity": "RoleModel", "loc": "deepsel/apps/core/models/role.py:27" }, "note": "path-length (d=9)" }, { "patternRole": "Session manager", "score": 0.7143, "distance": 4, "groundTruth": { "entity": "get", "loc": "deepsel/auth/get_current_user.py:40" }, "recoveryResult": { "entity": "SessionStore", "loc": "deepsel/auth/session.py:66" }, "note": "path-length (d=4)" }, { "patternRole": "Verifier", "score": 1.0, "distance": 0, "groundTruth": { "entity": "_resolve_user_from_session", "loc": "deepsel/auth/get_current_user.py:30" }, "recoveryResult": { "entity": "_resolve_user_from_session", "loc": "deepsel/auth/get_current_user.py:48" }, "note": "path-length (d=0)" } ], "notes": [] }, { "key": "level3", "level": "Level 3 \u2014 Interaction Order", "formula": "jaccard \u00d7 kendall-tau (shared interactions)", "available": true, "headlineLabel": "Interaction agreement", "score": 0.4, "drift": 0.6, "detail": { "agreedRoles": [ "Authentication enforcer", "Authorisation enforcer", "Decider", "Policy provider", "Session manager", "Verifier" ], "excludedRoles": [], "interactionsGroundTruth": 2, "interactionsRecoveryResult": 5, "sharedInteractions": 2, "unionSize": 5, "entryRoleGroundTruth": "Authentication enforcer", "entryRoleRecoveryResult": "Authentication enforcer", "entryRuleGroundTruth": "behaviour", "entryRuleRecoveryResult": "stored", "onlyGroundTruth": 0, "onlyRecoveryResult": 3, "interactionOverlap": 0.4, "orderAgreement": 1.0, "rankedInteractions": 2, "totalPairs": 1, "concordantPairs": 1, "discordantPairs": 0, "kendallTau": 1.0, "prescribedActionSteps": 5, "prescribedStepsCovered": 5 }, "perRole": [ { "interaction": "Authentication enforcer \u2192 Verifier", "from": "Authentication enforcer", "to": "Verifier", "groundTruthRank": 1, "recoveryResultRank": 1, "outcome": "agreed", "discordantWith": 0, "evidence": { "groundTruth": { "file": "deepsel/auth/get_current_user.py", "line": 85, "via": [] }, "recoveryResult": { "file": "deepsel/auth/get_current_user.py", "line": 95, "via": [] } } }, { "interaction": "Verifier \u2192 Session manager", "from": "Verifier", "to": "Session manager", "groundTruthRank": 2, "recoveryResultRank": 2, "outcome": "agreed", "discordantWith": 0, "evidence": { "groundTruth": { "file": "deepsel/auth/get_current_user.py", "line": 40, "via": [] }, "recoveryResult": { "file": "deepsel/auth/get_current_user.py", "line": 57, "via": [] } } }, { "interaction": "Authentication enforcer \u2192 Authorisation enforcer", "from": "Authentication enforcer", "to": "Authorisation enforcer", "groundTruthRank": null, "recoveryResultRank": 3, "outcome": "onlyRecoveryResult", "discordantWith": null, "evidence": { "groundTruth": null, "recoveryResult": { "file": "deepsel/utils/crud_router.py", "line": 243, "via": [] } } }, { "interaction": "Authorisation enforcer \u2192 Decider", "from": "Authorisation enforcer", "to": "Decider", "groundTruthRank": null, "recoveryResultRank": 4, "outcome": "onlyRecoveryResult", "discordantWith": null, "evidence": { "groundTruth": null, "recoveryResult": { "file": "deepsel/orm/mixin.py", "line": 105, "via": [] } } }, { "interaction": "Decider \u2192 Policy provider", "from": "Decider", "to": "Policy provider", "groundTruthRank": null, "recoveryResultRank": 5, "outcome": "onlyRecoveryResult", "discordantWith": null, "evidence": { "groundTruth": null, "recoveryResult": { "file": "deepsel/orm/mixin.py", "line": 1697, "via": [] } } } ], "notes": [ "Score = overlap \u00d7 order = 0.4 \u00d7 1. Overlap: 2 of 5 interactions were found by both sides (0 only in the ground truth, 3 only in the recovery result), |shared| / |union|.", "Order: Kendall's Tau over the 2 shared interactions, in each side's own relative order \u2014 0 of 1 pairs run in opposite orders. Interactions only one side found take part in no pair (Fagin, Kumar & Sivakumar's p = 0, the induced sub-order); the overlap factor is what charges for them.", "Only 1 ranked pair \u2014 the order factor can land on just a few values here and one disagreement swings it fully.", "Scored over the 6 roles both sides detected (Authentication enforcer, Authorisation enforcer, Decider, Policy provider, Session manager, Verifier). An interaction counts only when both of its endpoint roles were detected on both sides \u2014 whether a role was found at all is Level 1's measurement, and charging for it again here would say the same thing twice.", "The interaction order is derived statically from the code \u2014 no one annotated it. Each role's cited entity is located in the AST index and control flow is followed from it, through as many non-role helper functions as it takes to reach the next role. See secpat_scoring/sequence/JOERN.md.", "Ground truth: the walk started at Authentication enforcer \u2014 the earliest role the reference pattern has speaking that this side grounded in walkable code.", "Recovery result: the walk started at Authentication enforcer \u2014 stored.", "An entry point the pattern did not name means its prescribed speakers were either ungrounded on that side or not where this code begins \u2014 a role something else reaches cannot open the flow \u2014 so the call graph had to stand in. Treat the ordering below as weaker evidence: it starts wherever the code happens to start.", "Ground truth: Session manager cites an entity that is not a symbol in this repository (a library or framework call). It can receive an interaction, but the walk cannot continue through it." ] } ], "ground_truth_source": "annotation", "vocabulary": [ "Authentication enforcer", "Verifier", "Session manager", "Authorisation enforcer", "Decider", "Policy provider" ], "role_scope": "required", "excluded_roles": [ "Subject", "System" ], "dropped_ground_truth_rows": [ "Subject", "System" ], "dropped_recovery_rows": [ "Subject", "System" ], "gt_roles": 6, "rr_roles": 6, "struct_files": 262, "annotated_at": "2026-10-16 11:26:40.238000", "recovery_order": "stored" }, { "slug": "session_based_access_control__Quansight_ragna", "id": "session-based-ac/Quansight_ragna/ClaudeCodeOpus5.5", "pattern": "session_based_access_control", "repo": "Quansight_ragna", "error": null, "metrics": [ { "key": "level1", "level": "Level 1 \u2014 Role Detection", "formula": "agreement-rate (present/external/missing)", "available": true, "headlineLabel": "Roles agreed", "score": 0.8333, "drift": 0.1667, "detail": { "agreedRoles": 5, "totalRoles": 6 }, "perRole": [ { "patternRole": "Authentication enforcer", "groundTruth": "present", "recoveryResult": "present", "groundTruthDetected": true, "recoveryResultDetected": true, "outcome": "agreed" }, { "patternRole": "Verifier", "groundTruth": "present", "recoveryResult": "present", "groundTruthDetected": true, "recoveryResultDetected": true, "outcome": "agreed" }, { "patternRole": "Session manager", "groundTruth": "present", "recoveryResult": "present", "groundTruthDetected": true, "recoveryResultDetected": true, "outcome": "agreed" }, { "patternRole": "Authorisation enforcer", "groundTruth": "present", "recoveryResult": "present", "groundTruthDetected": true, "recoveryResultDetected": true, "outcome": "agreed" }, { "patternRole": "Decider", "groundTruth": "present", "recoveryResult": "present", "groundTruthDetected": true, "recoveryResultDetected": true, "outcome": "agreed" }, { "patternRole": "Policy provider", "groundTruth": "present", "recoveryResult": "missing", "groundTruthDetected": true, "recoveryResultDetected": false, "outcome": "mismatched" } ], "notes": [] }, { "key": "level2", "level": "Level 2 \u2014 Role Grounding", "formula": "path-length", "available": true, "headlineLabel": "Mean grounding score", "score": 0.8286, "drift": 0.1714, "detail": { "aggregate": 0.8286, "scoredRoles": 5, "skippedRoles": 1, "treeHeight": 7 }, "perRole": [ { "patternRole": "Authentication enforcer", "score": 0.9286, "distance": 1, "groundTruth": { "entity": "dispatch", "loc": "ragna/deploy/_auth.py:63" }, "recoveryResult": { "entity": "SessionMiddleware", "loc": "ragna/deploy/_auth.py:37" }, "note": "path-length (d=1)" }, { "patternRole": "Authorisation enforcer", "score": 0.6429, "distance": 5, "groundTruth": { "entity": "_get_orm_chat", "loc": "ragna/deploy/_database.py:231" }, "recoveryResult": { "entity": "_get_session", "loc": "ragna/deploy/_auth.py:281" }, "note": "path-length (d=5)" }, { "patternRole": "Decider", "score": 0.8571, "distance": 2, "groundTruth": { "entity": "_get_orm_chat", "loc": "ragna/deploy/_database.py:231" }, "recoveryResult": { "entity": "get_chats", "loc": "ragna/deploy/_database.py:293" }, "note": "path-length (d=2)" }, { "patternRole": "Session manager", "score": 0.7143, "distance": 4, "groundTruth": { "entity": "get", "loc": "ragna/deploy/_auth.py:113" }, "recoveryResult": { "entity": "KeyValueStore", "loc": "ragna/deploy/_key_value_store.py:32" }, "note": "path-length (d=4)" }, { "patternRole": "Verifier", "score": 1.0, "distance": 0, "groundTruth": { "entity": "_cookie_dispatch", "loc": "ragna/deploy/_auth.py:110" }, "recoveryResult": { "entity": "_cookie_dispatch", "loc": "ragna/deploy/_auth.py:166" }, "note": "path-length (d=0)" } ], "notes": [ "Policy provider: absent from the recovery result (Level 1 false negative)" ] }, { "key": "level3", "level": "Level 3 \u2014 Interaction Order", "formula": "jaccard \u00d7 kendall-tau (shared interactions)", "available": true, "headlineLabel": "Interaction agreement", "score": 0.5, "drift": 0.5, "detail": { "agreedRoles": [ "Authentication enforcer", "Authorisation enforcer", "Decider", "Session manager", "Verifier" ], "excludedRoles": [ "Policy provider" ], "interactionsGroundTruth": 2, "interactionsRecoveryResult": 4, "sharedInteractions": 2, "unionSize": 4, "entryRoleGroundTruth": "Authentication enforcer", "entryRoleRecoveryResult": "Authentication enforcer", "entryRuleGroundTruth": "behaviour", "entryRuleRecoveryResult": "stored", "onlyGroundTruth": 0, "onlyRecoveryResult": 2, "interactionOverlap": 0.5, "orderAgreement": 1.0, "rankedInteractions": 2, "totalPairs": 1, "concordantPairs": 1, "discordantPairs": 0, "kendallTau": 1.0, "prescribedActionSteps": 5, "prescribedStepsCovered": 4 }, "perRole": [ { "interaction": "Authentication enforcer \u2192 Verifier", "from": "Authentication enforcer", "to": "Verifier", "groundTruthRank": 1, "recoveryResultRank": 1, "outcome": "agreed", "discordantWith": 0, "evidence": { "groundTruth": { "file": "ragna/deploy/_auth.py", "line": 69, "via": [] }, "recoveryResult": { "file": "ragna/deploy/_auth.py", "line": 102, "via": [] } } }, { "interaction": "Verifier \u2192 Session manager", "from": "Verifier", "to": "Session manager", "groundTruthRank": 2, "recoveryResultRank": 2, "outcome": "agreed", "discordantWith": 0, "evidence": { "groundTruth": { "file": "ragna/deploy/_auth.py", "line": 113, "via": [] }, "recoveryResult": { "file": "ragna/deploy/_auth.py", "line": 171, "via": [] } } }, { "interaction": "Authentication enforcer \u2192 Authorisation enforcer", "from": "Authentication enforcer", "to": "Authorisation enforcer", "groundTruthRank": null, "recoveryResultRank": 3, "outcome": "onlyRecoveryResult", "discordantWith": null, "evidence": { "groundTruth": null, "recoveryResult": { "file": "ragna/deploy/_auth.py", "line": 289, "via": [] } } }, { "interaction": "Authorisation enforcer \u2192 Decider", "from": "Authorisation enforcer", "to": "Decider", "groundTruthRank": null, "recoveryResultRank": 4, "outcome": "onlyRecoveryResult", "discordantWith": null, "evidence": { "groundTruth": null, "recoveryResult": { "file": "ragna/deploy/_api.py", "line": 96, "via": [] } } } ], "notes": [ "Score = overlap \u00d7 order = 0.5 \u00d7 1. Overlap: 2 of 4 interactions were found by both sides (0 only in the ground truth, 2 only in the recovery result), |shared| / |union|.", "Order: Kendall's Tau over the 2 shared interactions, in each side's own relative order \u2014 0 of 1 pairs run in opposite orders. Interactions only one side found take part in no pair (Fagin, Kumar & Sivakumar's p = 0, the induced sub-order); the overlap factor is what charges for them.", "Only 1 ranked pair \u2014 the order factor can land on just a few values here and one disagreement swings it fully.", "Scored over the 5 roles both sides detected (Authentication enforcer, Authorisation enforcer, Decider, Session manager, Verifier). An interaction counts only when both of its endpoint roles were detected on both sides \u2014 whether a role was found at all is Level 1's measurement, and charging for it again here would say the same thing twice.", "The interaction order is derived statically from the code \u2014 no one annotated it. Each role's cited entity is located in the AST index and control flow is followed from it, through as many non-role helper functions as it takes to reach the next role. See secpat_scoring/sequence/JOERN.md.", "Left out \u2014 only one side detected it: Policy provider. 1 role was excluded from both walks, so neither side's ordering mentions it. The walk still passes through the code behind such a role, attributing what it finds there to the nearest agreed role.", "Ground truth: the walk started at Authentication enforcer \u2014 the earliest role the reference pattern has speaking that this side grounded in walkable code.", "Recovery result: the walk started at Authentication enforcer \u2014 stored.", "An entry point the pattern did not name means its prescribed speakers were either ungrounded on that side or not where this code begins \u2014 a role something else reaches cannot open the flow \u2014 so the call graph had to stand in. Treat the ordering below as weaker evidence: it starts wherever the code happens to start.", "Ground truth: Session manager cites an entity that is not a symbol in this repository (a library or framework call). It can receive an interaction, but the walk cannot continue through it." ] } ], "ground_truth_source": "annotation", "vocabulary": [ "Authentication enforcer", "Verifier", "Session manager", "Authorisation enforcer", "Decider", "Policy provider" ], "role_scope": "required", "excluded_roles": [ "Subject", "System" ], "dropped_ground_truth_rows": [ "Subject", "System" ], "dropped_recovery_rows": [ "Subject", "System" ], "gt_roles": 6, "rr_roles": 6, "struct_files": 89, "annotated_at": "2026-10-15 21:08:20.758000", "recovery_order": "stored" }, { "slug": "session_based_access_control__Percolation-Labs_percolate", "id": "session-based-ac/Percolation-Labs_percolate/ClaudeCodeOpus5.5", "pattern": "session_based_access_control", "repo": "Percolation-Labs_percolate", "error": null, "metrics": [ { "key": "level1", "level": "Level 1 \u2014 Role Detection", "formula": "agreement-rate", "available": true, "headlineLabel": "Roles agreed", "score": 0.5, "drift": 0.5, "detail": { "agreedRoles": 3, "totalRoles": 6 }, "perRole": [ { "patternRole": "Authentication enforcer", "groundTruth": "missing", "recoveryResult": "present", "groundTruthDetected": false, "recoveryResultDetected": true, "outcome": "mismatched" }, { "patternRole": "Verifier", "groundTruth": "missing", "recoveryResult": "present", "groundTruthDetected": false, "recoveryResultDetected": true, "outcome": "mismatched" }, { "patternRole": "Session manager", "groundTruth": "missing", "recoveryResult": "present", "groundTruthDetected": false, "recoveryResultDetected": true, "outcome": "mismatched" }, { "patternRole": "Authorisation enforcer", "groundTruth": "present", "recoveryResult": "present", "groundTruthDetected": true, "recoveryResultDetected": true, "outcome": "agreed" }, { "patternRole": "Decider", "groundTruth": "present", "recoveryResult": "present", "groundTruthDetected": true, "recoveryResultDetected": true, "outcome": "agreed" }, { "patternRole": "Policy provider", "groundTruth": "present", "recoveryResult": "present", "groundTruthDetected": true, "recoveryResultDetected": true, "outcome": "agreed" } ], "notes": [] }, { "key": "level2", "level": "Level 2 \u2014 Role Grounding", "formula": "path-length", "available": true, "headlineLabel": "Mean grounding score", "score": 0.6482, "drift": 0.3518, "detail": { "aggregate": 0.6482, "scoredRoles": 3, "skippedRoles": 3, "treeHeight": 9 }, "perRole": [ { "patternRole": "Authorisation enforcer", "score": 0.6667, "distance": 6, "groundTruth": { "entity": "prepare_agent_for_save", "loc": "clients/python/percolate/percolate/api/routes/entities/router.py:57" }, "recoveryResult": { "entity": "HybridAuth", "loc": "clients/python/percolate/percolate/api/routes/auth/__init__.py:249" }, "note": "path-length (d=6)" }, { "patternRole": "Decider", "score": 0.5, "distance": 9, "groundTruth": { "entity": "prepare_agent_for_save", "loc": "clients/python/percolate/percolate/api/routes/entities/router.py:57" }, "recoveryResult": { "entity": "get_functions_for_role_level", "loc": "clients/python/percolate/percolate/services/FunctionManager.py:130" }, "note": "path-length (d=9)" }, { "patternRole": "Policy provider", "score": 0.7778, "distance": 4, "groundTruth": { "entity": "role_level", "loc": "clients/python/percolate/percolate/models/p8/types.py:953" }, "recoveryResult": { "entity": "AccessLevel", "loc": "clients/python/percolate/percolate/models/p8/db_types.py:7" }, "note": "path-length (d=4)" } ], "notes": [ "Authentication enforcer: absent from the ground truth (Level 1 negative)", "Session manager: absent from the ground truth (Level 1 negative)", "Verifier: absent from the ground truth (Level 1 negative)" ] }, { "key": "level3", "level": "Level 3 \u2014 Interaction Order", "formula": "jaccard \u00d7 kendall-tau (shared interactions)", "available": true, "headlineLabel": "Interaction agreement", "score": 0.5, "drift": 0.5, "detail": { "agreedRoles": [ "Authorisation enforcer", "Decider", "Policy provider" ], "excludedRoles": [ "Authentication enforcer", "Session manager", "Verifier" ], "interactionsGroundTruth": 1, "interactionsRecoveryResult": 2, "sharedInteractions": 1, "unionSize": 2, "entryRoleGroundTruth": "Authorisation enforcer", "entryRoleRecoveryResult": "Authorisation enforcer", "entryRuleGroundTruth": "behaviour", "entryRuleRecoveryResult": "stored", "onlyGroundTruth": 0, "onlyRecoveryResult": 1, "interactionOverlap": 0.5, "orderAgreement": 1.0, "rankedInteractions": 1, "totalPairs": 0, "concordantPairs": 0, "discordantPairs": 0, "kendallTau": null, "prescribedActionSteps": 5, "prescribedStepsCovered": 2 }, "perRole": [ { "interaction": "Authorisation enforcer \u2192 Decider", "from": "Authorisation enforcer", "to": "Decider", "groundTruthRank": 1, "recoveryResultRank": 1, "outcome": "agreed", "discordantWith": 0, "evidence": { "groundTruth": { "file": "clients/python/percolate/percolate/api/routes/entities/router.py", "line": 57, "via": [] }, "recoveryResult": { "file": "clients/python/percolate/percolate/services/ModelRunner.py", "line": 213, "via": [] } } }, { "interaction": "Decider \u2192 Policy provider", "from": "Decider", "to": "Policy provider", "groundTruthRank": null, "recoveryResultRank": 2, "outcome": "onlyRecoveryResult", "discordantWith": null, "evidence": { "groundTruth": null, "recoveryResult": { "file": "clients/python/percolate/percolate/services/FunctionManager.py", "line": 139, "via": [] } } } ], "notes": [ "Score = overlap \u00d7 order = 0.5 \u00d7 1. Overlap: 1 of 2 interactions were found by both sides (0 only in the ground truth, 1 only in the recovery result), |shared| / |union|.", "Order: 1 \u2014 a single shared interaction has no pair whose relative order could differ.", "Scored over the 3 roles both sides detected (Authorisation enforcer, Decider, Policy provider). An interaction counts only when both of its endpoint roles were detected on both sides \u2014 whether a role was found at all is Level 1's measurement, and charging for it again here would say the same thing twice.", "The interaction order is derived statically from the code \u2014 no one annotated it. Each role's cited entity is located in the AST index and control flow is followed from it, through as many non-role helper functions as it takes to reach the next role. See secpat_scoring/sequence/JOERN.md.", "Left out \u2014 only one side detected them: Authentication enforcer, Session manager, Verifier. 3 roles were excluded from both walks, so neither side's ordering mentions them. The walk still passes through the code behind such a role, attributing what it finds there to the nearest agreed role.", "Ground truth: the walk started at Authorisation enforcer \u2014 the earliest role the reference pattern has speaking that this side grounded in walkable code.", "Recovery result: the walk started at Authorisation enforcer \u2014 stored.", "An entry point the pattern did not name means its prescribed speakers were either ungrounded on that side or not where this code begins \u2014 a role something else reaches cannot open the flow \u2014 so the call graph had to stand in. Treat the ordering below as weaker evidence: it starts wherever the code happens to start." ] } ], "ground_truth_source": "recovery result", "vocabulary": [ "Authentication enforcer", "Verifier", "Session manager", "Authorisation enforcer", "Decider", "Policy provider" ], "role_scope": "required", "excluded_roles": [ "Subject", "System" ], "dropped_ground_truth_rows": [ "Subject", "System" ], "dropped_recovery_rows": [ "Subject", "System" ], "gt_roles": 6, "rr_roles": 6, "struct_files": 289, "annotated_at": "2026-10-02 10:00:38.838000", "recovery_order": "stored" }, { "slug": "session_based_access_control__allen0099_FastAPI-CacheX", "id": "session-based-ac/allen0099_FastAPI-CacheX/ClaudeCodeOpus5.5", "pattern": "session_based_access_control", "repo": "allen0099_FastAPI-CacheX", "error": null, "metrics": [ { "key": "level1", "level": "Level 1 \u2014 Role Detection", "formula": "agreement-rate (present/external/missing)", "available": true, "headlineLabel": "Roles agreed", "score": 0.8333, "drift": 0.1667, "detail": { "agreedRoles": 5, "totalRoles": 6 }, "perRole": [ { "patternRole": "Authentication enforcer", "groundTruth": "present", "recoveryResult": "present", "groundTruthDetected": true, "recoveryResultDetected": true, "outcome": "agreed" }, { "patternRole": "Verifier", "groundTruth": "present", "recoveryResult": "present", "groundTruthDetected": true, "recoveryResultDetected": true, "outcome": "agreed" }, { "patternRole": "Session manager", "groundTruth": "present", "recoveryResult": "present", "groundTruthDetected": true, "recoveryResultDetected": true, "outcome": "agreed" }, { "patternRole": "Authorisation enforcer", "groundTruth": "missing", "recoveryResult": "present", "groundTruthDetected": false, "recoveryResultDetected": true, "outcome": "mismatched" }, { "patternRole": "Decider", "groundTruth": "missing", "recoveryResult": "missing", "groundTruthDetected": false, "recoveryResultDetected": false, "outcome": "agreed" }, { "patternRole": "Policy provider", "groundTruth": "missing", "recoveryResult": "missing", "groundTruthDetected": false, "recoveryResultDetected": false, "outcome": "agreed" } ], "notes": [] }, { "key": "level2", "level": "Level 2 \u2014 Role Grounding", "formula": "path-length", "available": true, "headlineLabel": "Mean grounding score", "score": 0.9333, "drift": 0.0667, "detail": { "aggregate": 0.9333, "scoredRoles": 3, "skippedRoles": 3, "treeHeight": 5 }, "perRole": [ { "patternRole": "Authentication enforcer", "score": 0.9, "distance": 1, "groundTruth": { "entity": "__call__", "loc": "fastapi_cachex/session/middleware.py:295" }, "recoveryResult": { "entity": "FastAPICacheXSessionMiddleware", "loc": "fastapi_cachex/session/middleware.py:176" }, "note": "path-length (d=1)" }, { "patternRole": "Session manager", "score": 0.9, "distance": 1, "groundTruth": { "entity": "_load_session_by_key", "loc": "fastapi_cachex/session/manager.py:469" }, "recoveryResult": { "entity": "SessionManager", "loc": "fastapi_cachex/session/manager.py:44" }, "note": "path-length (d=1)" }, { "patternRole": "Verifier", "score": 1.0, "distance": 0, "groundTruth": { "entity": "get_session", "loc": "fastapi_cachex/session/manager.py:168" }, "recoveryResult": { "entity": "get_session", "loc": "fastapi_cachex/session/manager.py:197" }, "note": "path-length (d=0)" } ], "notes": [ "Authorisation enforcer: absent from the ground truth (Level 1 negative)", "Decider: absent from the ground truth (Level 1 negative)", "Policy provider: absent from the ground truth (Level 1 negative)" ] }, { "key": "level3", "level": "Level 3 \u2014 Interaction Order", "formula": "jaccard \u00d7 kendall-tau (shared interactions)", "available": true, "headlineLabel": "Interaction agreement", "score": 1.0, "drift": 0.0, "detail": { "agreedRoles": [ "Authentication enforcer", "Session manager", "Verifier" ], "excludedRoles": [ "Authorisation enforcer", "Decider", "Policy provider" ], "interactionsGroundTruth": 2, "interactionsRecoveryResult": 2, "sharedInteractions": 2, "unionSize": 2, "entryRoleGroundTruth": "Authentication enforcer", "entryRoleRecoveryResult": "Authentication enforcer", "entryRuleGroundTruth": "behaviour", "entryRuleRecoveryResult": "stored", "onlyGroundTruth": 0, "onlyRecoveryResult": 0, "interactionOverlap": 1.0, "orderAgreement": 1.0, "rankedInteractions": 2, "totalPairs": 1, "concordantPairs": 1, "discordantPairs": 0, "kendallTau": 1.0, "prescribedActionSteps": 5, "prescribedStepsCovered": 2 }, "perRole": [ { "interaction": "Authentication enforcer \u2192 Verifier", "from": "Authentication enforcer", "to": "Verifier", "groundTruthRank": 1, "recoveryResultRank": 1, "outcome": "agreed", "discordantWith": 0, "evidence": { "groundTruth": { "file": "fastapi_cachex/session/middleware.py", "line": 328, "via": [] }, "recoveryResult": { "file": "fastapi_cachex/session/middleware.py", "line": 247, "via": [] } } }, { "interaction": "Verifier \u2192 Session manager", "from": "Verifier", "to": "Session manager", "groundTruthRank": 2, "recoveryResultRank": 2, "outcome": "agreed", "discordantWith": 0, "evidence": { "groundTruth": { "file": "fastapi_cachex/session/manager.py", "line": 467, "via": [ "fastapi_cachex/session/manager.py:<module>.SessionManager._load_session" ] }, "recoveryResult": { "file": "fastapi_cachex/session/manager.py", "line": 229, "via": [] } } } ], "notes": [ "Score = overlap \u00d7 order = 1 \u00d7 1. Overlap: 2 of 2 interactions were found by both sides (0 only in the ground truth, 0 only in the recovery result), |shared| / |union|.", "Order: Kendall's Tau over the 2 shared interactions, in each side's own relative order \u2014 0 of 1 pairs run in opposite orders. Interactions only one side found take part in no pair (Fagin, Kumar & Sivakumar's p = 0, the induced sub-order); the overlap factor is what charges for them.", "Only 1 ranked pair \u2014 the order factor can land on just a few values here and one disagreement swings it fully.", "Scored over the 3 roles both sides detected (Authentication enforcer, Session manager, Verifier). An interaction counts only when both of its endpoint roles were detected on both sides \u2014 whether a role was found at all is Level 1's measurement, and charging for it again here would say the same thing twice.", "The interaction order is derived statically from the code \u2014 no one annotated it. Each role's cited entity is located in the AST index and control flow is followed from it, through as many non-role helper functions as it takes to reach the next role. See secpat_scoring/sequence/JOERN.md.", "Left out \u2014 only one side detected them: Authorisation enforcer, Decider, Policy provider. 3 roles were excluded from both walks, so neither side's ordering mentions them. The walk still passes through the code behind such a role, attributing what it finds there to the nearest agreed role.", "Ground truth: the walk started at Authentication enforcer \u2014 the earliest role the reference pattern has speaking that this side grounded in walkable code.", "Recovery result: the walk started at Authentication enforcer \u2014 stored.", "An entry point the pattern did not name means its prescribed speakers were either ungrounded on that side or not where this code begins \u2014 a role something else reaches cannot open the flow \u2014 so the call graph had to stand in. Treat the ordering below as weaker evidence: it starts wherever the code happens to start." ] } ], "ground_truth_source": "annotation", "vocabulary": [ "Authentication enforcer", "Verifier", "Session manager", "Authorisation enforcer", "Decider", "Policy provider" ], "role_scope": "required", "excluded_roles": [ "Subject", "System" ], "dropped_ground_truth_rows": [ "Subject", "System" ], "dropped_recovery_rows": [ "Subject", "System" ], "gt_roles": 6, "rr_roles": 6, "struct_files": 64, "annotated_at": "2026-10-08 20:48:35.715000", "recovery_order": "stored" }, { "slug": "session_based_access_control__anelendata_zix", "id": "session-based-ac/anelendata_zix/ClaudeCodeOpus5.5", "pattern": "session_based_access_control", "repo": "anelendata_zix", "error": null, "metrics": [ { "key": "level1", "level": "Level 1 \u2014 Role Detection", "formula": "agreement-rate (present/external/missing)", "available": true, "headlineLabel": "Roles agreed", "score": 1.0, "drift": 0.0, "detail": { "agreedRoles": 6, "totalRoles": 6 }, "perRole": [ { "patternRole": "Authentication enforcer", "groundTruth": "present", "recoveryResult": "present", "groundTruthDetected": true, "recoveryResultDetected": true, "outcome": "agreed" }, { "patternRole": "Verifier", "groundTruth": "present", "recoveryResult": "present", "groundTruthDetected": true, "recoveryResultDetected": true, "outcome": "agreed" }, { "patternRole": "Session manager", "groundTruth": "present", "recoveryResult": "present", "groundTruthDetected": true, "recoveryResultDetected": true, "outcome": "agreed" }, { "patternRole": "Authorisation enforcer", "groundTruth": "present", "recoveryResult": "present", "groundTruthDetected": true, "recoveryResultDetected": true, "outcome": "agreed" }, { "patternRole": "Decider", "groundTruth": "present", "recoveryResult": "present", "groundTruthDetected": true, "recoveryResultDetected": true, "outcome": "agreed" }, { "patternRole": "Policy provider", "groundTruth": "present", "recoveryResult": "present", "groundTruthDetected": true, "recoveryResultDetected": true, "outcome": "agreed" } ], "notes": [] }, { "key": "level2", "level": "Level 2 \u2014 Role Grounding", "formula": "path-length", "available": true, "headlineLabel": "Mean grounding score", "score": 0.8229, "drift": 0.1771, "detail": { "aggregate": 0.8229, "scoredRoles": 6, "skippedRoles": 0, "treeHeight": 8 }, "perRole": [ { "patternRole": "Authentication enforcer", "score": 0.4375, "distance": 9, "groundTruth": { "entity": "get_current_user", "loc": "zix/default_project/app/plugins/users/crud.py:24" }, "recoveryResult": { "entity": "AuthenticationMiddleware", "loc": "zix/server/main.py:62" }, "note": "path-length (d=9)" }, { "patternRole": "Authorisation enforcer", "score": 0.875, "distance": 2, "groundTruth": { "entity": "get_current_active_admin_user", "loc": "zix/default_project/app/plugins/users/crud.py:76" }, "recoveryResult": { "entity": "get_current_active_user", "loc": "zix/default_project/app/plugins/users/crud.py:60" }, "note": "path-length (d=2)" }, { "patternRole": "Decider", "score": 1.0, "distance": 0, "groundTruth": { "entity": "get_current_active_admin_user", "loc": "zix/default_project/app/plugins/users/crud.py:76" }, "recoveryResult": { "entity": "get_current_active_admin_user", "loc": "zix/default_project/app/plugins/users/crud.py:70" }, "note": "path-length (d=0)" }, { "patternRole": "Policy provider", "score": 1.0, "distance": 0, "groundTruth": { "entity": "is_staff", "loc": "zix/default_project/app/plugins/users/models.py:65" }, "recoveryResult": { "entity": "is_staff", "loc": "zix/default_project/app/plugins/users/models.py:82" }, "note": "path-length (d=0)" }, { "patternRole": "Session manager", "score": 1.0, "distance": 0, "groundTruth": { "entity": "get_token", "loc": "zix/default_project/app/plugins/auth/crud.py:62" }, "recoveryResult": { "entity": "get_token", "loc": "zix/default_project/app/plugins/auth/crud.py:64" }, "note": "path-length (d=0)" }, { "patternRole": "Verifier", "score": 0.625, "distance": 6, "groundTruth": { "entity": "get_user_from_token", "loc": "zix/default_project/app/plugins/auth/crud.py:101" }, "recoveryResult": { "entity": "get_current_user", "loc": "zix/default_project/app/plugins/users/crud.py:22" }, "note": "path-length (d=6)" } ], "notes": [] }, { "key": "level3", "level": "Level 3 \u2014 Interaction Order", "formula": "jaccard \u00d7 kendall-tau (shared interactions)", "available": true, "headlineLabel": "Interaction agreement", "score": 0.2, "drift": 0.8, "detail": { "agreedRoles": [ "Authentication enforcer", "Authorisation enforcer", "Decider", "Policy provider", "Session manager", "Verifier" ], "excludedRoles": [], "interactionsGroundTruth": 1, "interactionsRecoveryResult": 5, "sharedInteractions": 1, "unionSize": 5, "entryRoleGroundTruth": "Verifier", "entryRoleRecoveryResult": "Authentication enforcer", "entryRuleGroundTruth": "code-root", "entryRuleRecoveryResult": "stored", "onlyGroundTruth": 0, "onlyRecoveryResult": 4, "interactionOverlap": 0.2, "orderAgreement": 1.0, "rankedInteractions": 1, "totalPairs": 0, "concordantPairs": 0, "discordantPairs": 0, "kendallTau": null, "prescribedActionSteps": 5, "prescribedStepsCovered": 5 }, "perRole": [ { "interaction": "Verifier \u2192 Session manager", "from": "Verifier", "to": "Session manager", "groundTruthRank": 1, "recoveryResultRank": 2, "outcome": "agreed", "discordantWith": 0, "evidence": { "groundTruth": { "file": "zix/default_project/app/plugins/auth/crud.py", "line": 103, "via": [] }, "recoveryResult": { "file": "zix/default_project/app/plugins/auth/crud.py", "line": 117, "via": [] } } }, { "interaction": "Authentication enforcer \u2192 Verifier", "from": "Authentication enforcer", "to": "Verifier", "groundTruthRank": null, "recoveryResultRank": 1, "outcome": "onlyRecoveryResult", "discordantWith": null, "evidence": { "groundTruth": null, "recoveryResult": { "file": "zix/default_project/app/plugins/users/crud.py", "line": 85, "via": [] } } }, { "interaction": "Authentication enforcer \u2192 Authorisation enforcer", "from": "Authentication enforcer", "to": "Authorisation enforcer", "groundTruthRank": null, "recoveryResultRank": 3, "outcome": "onlyRecoveryResult", "discordantWith": null, "evidence": { "groundTruth": null, "recoveryResult": { "file": "zix/default_project/app/plugins/users/crud.py", "line": 60, "via": [] } } }, { "interaction": "Authorisation enforcer \u2192 Decider", "from": "Authorisation enforcer", "to": "Decider", "groundTruthRank": null, "recoveryResultRank": 4, "outcome": "onlyRecoveryResult", "discordantWith": null, "evidence": { "groundTruth": null, "recoveryResult": { "file": "zix/default_project/app/plugins/users/crud.py", "line": 73, "via": [] } } }, { "interaction": "Decider \u2192 Policy provider", "from": "Decider", "to": "Policy provider", "groundTruthRank": null, "recoveryResultRank": 5, "outcome": "onlyRecoveryResult", "discordantWith": null, "evidence": { "groundTruth": null, "recoveryResult": { "file": "zix/default_project/app/plugins/users/models.py", "line": 82, "via": [] } } } ], "notes": [ "Score = overlap \u00d7 order = 0.2 \u00d7 1. Overlap: 1 of 5 interactions were found by both sides (0 only in the ground truth, 4 only in the recovery result), |shared| / |union|.", "Order: 1 \u2014 a single shared interaction has no pair whose relative order could differ.", "Scored over the 6 roles both sides detected (Authentication enforcer, Authorisation enforcer, Decider, Policy provider, Session manager, Verifier). An interaction counts only when both of its endpoint roles were detected on both sides \u2014 whether a role was found at all is Level 1's measurement, and charging for it again here would say the same thing twice.", "The interaction order is derived statically from the code \u2014 no one annotated it. Each role's cited entity is located in the AST index and control flow is followed from it, through as many non-role helper functions as it takes to reach the next role. See secpat_scoring/sequence/JOERN.md.", "Ground truth: the walk started at Verifier \u2014 the pattern's first speaker is not where this code begins \u2014 something else reaches it \u2014 so the code chose: no other role reaches this one, and it reaches more of the pattern than any other such role.", "Recovery result: the walk started at Authentication enforcer \u2014 stored.", "An entry point the pattern did not name means its prescribed speakers were either ungrounded on that side or not where this code begins \u2014 a role something else reaches cannot open the flow \u2014 so the call graph had to stand in. Treat the ordering below as weaker evidence: it starts wherever the code happens to start." ] } ], "ground_truth_source": "recovery result", "vocabulary": [ "Authentication enforcer", "Verifier", "Session manager", "Authorisation enforcer", "Decider", "Policy provider" ], "role_scope": "required", "excluded_roles": [ "Subject", "System" ], "dropped_ground_truth_rows": [ "Subject", "System" ], "dropped_recovery_rows": [ "Subject", "System" ], "gt_roles": 6, "rr_roles": 6, "struct_files": 49, "annotated_at": "2026-10-06 10:57:21.644000", "recovery_order": "stored" }, { "slug": "session_based_access_control__VForWaTer_metacatalog-api", "id": "session-based-ac/VForWaTer_metacatalog-api/ClaudeCodeOpus5.5", "pattern": "session_based_access_control", "repo": "VForWaTer_metacatalog-api", "error": null, "metrics": [ { "key": "level1", "level": "Level 1 \u2014 Role Detection", "formula": "agreement-rate (present/external/missing)", "available": true, "headlineLabel": "Roles agreed", "score": 0.8333, "drift": 0.1667, "detail": { "agreedRoles": 5, "totalRoles": 6 }, "perRole": [ { "patternRole": "Authentication enforcer", "groundTruth": "present", "recoveryResult": "present", "groundTruthDetected": true, "recoveryResultDetected": true, "outcome": "agreed" }, { "patternRole": "Verifier", "groundTruth": "present", "recoveryResult": "present", "groundTruthDetected": true, "recoveryResultDetected": true, "outcome": "agreed" }, { "patternRole": "Session manager", "groundTruth": "present", "recoveryResult": "present", "groundTruthDetected": true, "recoveryResultDetected": true, "outcome": "agreed" }, { "patternRole": "Authorisation enforcer", "groundTruth": "missing", "recoveryResult": "present", "groundTruthDetected": false, "recoveryResultDetected": true, "outcome": "mismatched" }, { "patternRole": "Decider", "groundTruth": "missing", "recoveryResult": "missing", "groundTruthDetected": false, "recoveryResultDetected": false, "outcome": "agreed" }, { "patternRole": "Policy provider", "groundTruth": "missing", "recoveryResult": "missing", "groundTruthDetected": false, "recoveryResultDetected": false, "outcome": "agreed" } ], "notes": [] }, { "key": "level2", "level": "Level 2 \u2014 Role Grounding", "formula": "path-length", "available": true, "headlineLabel": "Mean grounding score", "score": 0.9444, "drift": 0.0556, "detail": { "aggregate": 0.9444, "scoredRoles": 3, "skippedRoles": 3, "treeHeight": 6 }, "perRole": [ { "patternRole": "Authentication enforcer", "score": 1.0, "distance": 0, "groundTruth": { "entity": "validate_api_key", "loc": "metacatalog_api/router/api/security.py:9" }, "recoveryResult": { "entity": "validate_api_key", "loc": "metacatalog_api/router/api/security.py:10" }, "note": "path-length (d=0)" }, { "patternRole": "Session manager", "score": 0.8333, "distance": 2, "groundTruth": { "entity": "exec", "loc": "metacatalog_api/access_control.py:50" }, "recoveryResult": { "entity": "UserTokenTable", "loc": "metacatalog_api/access_control.py:12" }, "note": "path-length (d=2)" }, { "patternRole": "Verifier", "score": 1.0, "distance": 0, "groundTruth": { "entity": "validate_token", "loc": "metacatalog_api/access_control.py:46" }, "recoveryResult": { "entity": "validate_token", "loc": "metacatalog_api/access_control.py:48" }, "note": "path-length (d=0)" } ], "notes": [ "Authorisation enforcer: absent from the ground truth (Level 1 negative)", "Decider: absent from the ground truth (Level 1 negative)", "Policy provider: absent from the ground truth (Level 1 negative)" ] }, { "key": "level3", "level": "Level 3 \u2014 Interaction Order", "formula": "jaccard \u00d7 kendall-tau (shared interactions)", "available": true, "headlineLabel": "Interaction agreement", "score": 1.0, "drift": 0.0, "detail": { "agreedRoles": [ "Authentication enforcer", "Session manager", "Verifier" ], "excludedRoles": [ "Authorisation enforcer", "Decider", "Policy provider" ], "interactionsGroundTruth": 2, "interactionsRecoveryResult": 2, "sharedInteractions": 2, "unionSize": 2, "entryRoleGroundTruth": "Authentication enforcer", "entryRoleRecoveryResult": "Authentication enforcer", "entryRuleGroundTruth": "behaviour", "entryRuleRecoveryResult": "stored", "onlyGroundTruth": 0, "onlyRecoveryResult": 0, "interactionOverlap": 1.0, "orderAgreement": 1.0, "rankedInteractions": 2, "totalPairs": 1, "concordantPairs": 1, "discordantPairs": 0, "kendallTau": 1.0, "prescribedActionSteps": 5, "prescribedStepsCovered": 2 }, "perRole": [ { "interaction": "Authentication enforcer \u2192 Verifier", "from": "Authentication enforcer", "to": "Verifier", "groundTruthRank": 1, "recoveryResultRank": 1, "outcome": "agreed", "discordantWith": 0, "evidence": { "groundTruth": { "file": "metacatalog_api/router/api/security.py", "line": 11, "via": [] }, "recoveryResult": { "file": "metacatalog_api/router/api/security.py", "line": 12, "via": [] } } }, { "interaction": "Verifier \u2192 Session manager", "from": "Verifier", "to": "Session manager", "groundTruthRank": 2, "recoveryResultRank": 2, "outcome": "agreed", "discordantWith": 0, "evidence": { "groundTruth": { "file": "metacatalog_api/access_control.py", "line": 50, "via": [] }, "recoveryResult": { "file": "metacatalog_api/access_control.py", "line": 53, "via": [] } } } ], "notes": [ "Score = overlap \u00d7 order = 1 \u00d7 1. Overlap: 2 of 2 interactions were found by both sides (0 only in the ground truth, 0 only in the recovery result), |shared| / |union|.", "Order: Kendall's Tau over the 2 shared interactions, in each side's own relative order \u2014 0 of 1 pairs run in opposite orders. Interactions only one side found take part in no pair (Fagin, Kumar & Sivakumar's p = 0, the induced sub-order); the overlap factor is what charges for them.", "Only 1 ranked pair \u2014 the order factor can land on just a few values here and one disagreement swings it fully.", "Scored over the 3 roles both sides detected (Authentication enforcer, Session manager, Verifier). An interaction counts only when both of its endpoint roles were detected on both sides \u2014 whether a role was found at all is Level 1's measurement, and charging for it again here would say the same thing twice.", "The interaction order is derived statically from the code \u2014 no one annotated it. Each role's cited entity is located in the AST index and control flow is followed from it, through as many non-role helper functions as it takes to reach the next role. See secpat_scoring/sequence/JOERN.md.", "Left out \u2014 only one side detected them: Authorisation enforcer, Decider, Policy provider. 3 roles were excluded from both walks, so neither side's ordering mentions them. The walk still passes through the code behind such a role, attributing what it finds there to the nearest agreed role.", "Ground truth: the walk started at Authentication enforcer \u2014 the earliest role the reference pattern has speaking that this side grounded in walkable code.", "Recovery result: the walk started at Authentication enforcer \u2014 stored.", "An entry point the pattern did not name means its prescribed speakers were either ungrounded on that side or not where this code begins \u2014 a role something else reaches cannot open the flow \u2014 so the call graph had to stand in. Treat the ordering below as weaker evidence: it starts wherever the code happens to start.", "Ground truth: Session manager cites an entity that is not a symbol in this repository (a library or framework call). It can receive an interaction, but the walk cannot continue through it." ] } ], "ground_truth_source": "annotation", "vocabulary": [ "Authentication enforcer", "Verifier", "Session manager", "Authorisation enforcer", "Decider", "Policy provider" ], "role_scope": "required", "excluded_roles": [ "Subject", "System" ], "dropped_ground_truth_rows": [ "Subject", "System" ], "dropped_recovery_rows": [ "Subject", "System" ], "gt_roles": 6, "rr_roles": 6, "struct_files": 35, "annotated_at": "2026-10-15 20:59:40.361000", "recovery_order": "stored" }, { "slug": "session_based_access_control__kinde-oss_kinde-python-sdk", "id": "session-based-ac/kinde-oss_kinde-python-sdk/ClaudeCodeOpus5.5", "pattern": "session_based_access_control", "repo": "kinde-oss_kinde-python-sdk", "error": null, "over_input_limit": true, "prompt_tokens": 1001339, "metrics": [ { "key": "level1", "level": "Level 1 \u2014 Role Detection", "available": true, "score": 0.0, "drift": 1.0, "notes": [ "Not run: the one-shot prompt is 1,001,339 tokens (o200k_base), over the baseline's 900,000 input limit (scripts/baseline_recover.py). Scored 0 by rule, not measured." ] }, { "key": "level2", "level": "Level 2 \u2014 Role Grounding", "available": true, "score": 0.0, "drift": 1.0, "notes": [ "Not run: the one-shot prompt is 1,001,339 tokens (o200k_base), over the baseline's 900,000 input limit (scripts/baseline_recover.py). Scored 0 by rule, not measured." ] }, { "key": "level3", "level": "Level 3 \u2014 Interaction Order", "available": true, "score": 0.0, "drift": 1.0, "notes": [ "Not run: the one-shot prompt is 1,001,339 tokens (o200k_base), over the baseline's 900,000 input limit (scripts/baseline_recover.py). Scored 0 by rule, not measured." ] } ] } ]
[
{
"slug": "authentication_pwd__Clinical-Genomics_statina",
"id": "pwd-based/statina/ClaudeCodeopus4.7",
"pattern": "authentication_pwd",
"repo": "Clinical-Genomics_statina",
"error": null,
"metrics": [
{
"key": "level1",
"level": "Level 1 \u2014 Role Detection",
"formula": "agreement-rate (present/external/missing)",
"available": true,
"headlineLabel": "Roles agreed",
"score": 0.8571428571428571,
"drift": 0.1428571428571429,
"detail": {
"agreedRoles": 6,
"totalRoles": 7
},
"perRole": [
{
"patternRole": "Enforcer",
"groundTruth": "present",
"recoveryResult": "present",
"groundTruthDetected": true,
"recoveryResultDetected": true,
"outcome": "agreed"
},
{
"patternRole": "Verification manager",
"groundTruth": "present",
"recoveryResult": "present",
"groundTruthDetected": true,
"recoveryResultDetected": true,
"outcome": "agreed"
},
{
"patternRole": "Password store",
"groundTruth": "present",
"recoveryResult": "present",
"groundTruthDetected": true,
"recoveryResultDetected": true,
"outcome": "agreed"
},
{
"patternRole": "Hasher",
"groundTruth": "external",
"recoveryResult": "external",
"groundTruthDetected": true,
"recoveryResultDetected": true,
"outcome": "agreed"
},
{
"patternRole": "Pepper store",
"groundTruth": "missing",
"recoveryResult": "missing",
"groundTruthDetected": false,
"recoveryResultDetected": false,
"outcome": "agreed"
},
{
"patternRole": "Encrypter",
"groundTruth": "external",
"recoveryResult": "missing",
"groundTruthDetected": true,
"recoveryResultDetected": false,
"outcome": "mismatched"
},
{
"patternRole": "Comparator",
"groundTruth": "external",
"recoveryResult": "external",
"groundTruthDetected": true,
"recoveryResultDetected": true,
"outcome": "agreed"
}
],
"notes": []
},
{
"key": "level2",
"level": "Level 2 \u2014 Role Grounding",
"formula": "path-length",
"available": true,
"headlineLabel": "Mean grounding score",
"score": 0.9375,
"drift": 0.0625,
"detail": {
"aggregate": 0.9375,
"scoredRoles": 4,
"skippedRoles": 8,
"treeHeight": 8
},
"perRole": [
{
"patternRole": "Comparator",
"score": 1.0,
"distance": 0,
"groundTruth": {

Xet Storage Details

Size:
475 kB
·
Xet hash:
164af568c325b88b802e523174dbb65ede1611a90e53e7af74f2372a13f331c8

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.