deepseek-r1-14b-security / coverage.json
Volpidash's picture
Publish security benchmark protocol, measured coverage and available results
8a2e047 verified
Raw History Blame Contribute Delete
13 kB
{
"created_utc": "2026-11-09T13:40:21.918181+00:00",
"definitions": {
"ios_versions": "Release containing the fix, as named by Apple; not an affected-version range.",
"topics": "Non-exclusive keyword matches in prompt plus answer, not expert topic labels or measured model competence.",
"unique_cves": "Distinct CVE identifiers mentioned in retained records; not a fraction of all publicly known CVEs."
},
"train": {
"records": 3396,
"by_source": {
"apple_advisory": 1125,
"cisa_kev": 454,
"pentesting_explanation": 1817
},
"unique_cves": 1190,
"cve_years": {
"2026": 844,
"2025": 185,
"2020": 10,
"2009": 4,
"2008": 3,
"2023": 29,
"2014": 2,
"2016": 2,
"2021": 16,
"2013": 2,
"2017": 4,
"2024": 46,
"2022": 12,
"2018": 6,
"2011": 1,
"2012": 1,
"2019": 13,
"2015": 5,
"2010": 3,
"2007": 2
},
"ios_versions": {
"26.6": 97,
"26.7.2": 134,
"18.7.9": 49,
"18.7.5": 29,
"26.7": 95,
"26.4": 44,
"27": 163,
"18.7.10": 131,
"26.6.1": 35,
"27.2": 163,
"26.5.2": 38,
"26.5": 67,
"18.7.7": 26,
"26.3": 43,
"15.8.7": 4,
"16.7.15": 1,
"18.7.11": 3,
"18.7.3": 2,
"26.7.1": 1
},
"ios27_examples": 326,
"apple_components": {
"WebKit": 201,
"Kernel": 150,
"ImageIO": 62,
"Model I/O": 35,
"SceneKit": 30,
"CoreMedia": 22,
"AVEVideoEncoder": 21,
"CoreAudio": 18,
"Wi-Fi": 16,
"Security": 15,
"WebRTC": 15,
"Audio": 11,
"mDNSResponder": 11,
"Accounts": 10,
"FontParser": 10,
"Bluetooth": 10,
"Siri": 10,
"Spotlight": 9,
"Shortcuts": 9,
"Apple Neural Engine": 8,
"Contacts": 8,
"AppleAVD": 8,
"Accessibility": 8,
"Photos Storage": 8,
"App Store": 7,
"CoreGraphics": 7,
"CoreUI": 7,
"Messages": 7,
"Mail": 7,
"IOKit": 6,
"RealityKit": 6,
"libarchive": 6,
"Heimdal": 6,
"Baseband": 6,
"curl": 6,
"WebKit Canvas": 6,
"Sandbox Profiles": 6,
"Telephony": 6,
"Storage": 6,
"MediaRemote": 6,
"IOHIDFamily": 6,
"Accelerate Framework": 5,
"Game Center": 5,
"LaunchServices": 5,
"APFS": 5,
"Calendar": 5,
"Foundation": 5,
"CoreServices": 5,
"CoreText": 5,
"UIKit": 5,
"BackgroundAssets": 4,
"AppleDouble": 4,
"copyfile": 4,
"iCloud": 4,
"CloudKit": 4,
"FileProvider": 4,
"MobileBackup": 4,
"Power Management": 4,
"SpringBoard": 4,
"App Intents": 4,
"Apple Intelligence": 4,
"WebKit Process Model": 4,
"MobileAccessoryUpdater": 4,
"Screenshots": 3,
"TCC": 3,
"Quick Look": 3,
"AuthKit": 3,
"Core Bluetooth": 3,
"IOMobileFrameBuffer": 3,
"IOGPUFamily": 3,
"libxpc": 3,
"HFS": 3,
"Watch App": 3,
"AppleKeyStore": 3,
"NetworkExtension": 3,
"Managed Configuration": 3,
"WebKit Storage": 3,
"Maps": 3,
"Authentication Services": 3,
"Sandbox": 3,
"IOSurfaceAccelerator": 3,
"libexpat": 2,
"Vision": 2,
"Time Zone": 2,
"Focus": 2,
"Safari": 2,
"LinkPresentation": 2,
"Apple Account": 2,
"GeoServices": 2,
"Compression": 2,
"Music": 2,
"TextKit": 2,
"WebKit JavaScript Bindings": 2,
"Crash Reporter": 2,
"Live Captions": 2,
"libxslt": 2,
"IOSurface": 2,
"FrontBoard": 2,
"802.1X": 2,
"WorkoutKit": 2,
"USB": 2,
"SwiftUI": 2,
"CoreMotion": 2,
"CoreUtils": 2,
"zlib": 2,
"Safe Browsing": 2,
"Privacy": 2,
"CFNetwork": 2,
"exFAT": 2,
"AppleJPEG": 2,
"IOSkywalkFamily": 2,
"UIFoundation": 2,
"Pro Res": 2,
"Symptom Framework": 2,
"Disk Images": 2,
"MobileGestalt": 2,
"IOStorageFamily": 2,
"CoreML": 2,
"MediaAnalysis": 2,
"StoreKit": 2,
"file_cmds": 2,
"XPC": 2,
"CoreMedia Video Toolbox": 2,
"Books": 2,
"WebKit History": 2,
"Status Bar": 2,
"Clipboard": 2,
"Networking": 2,
"Multi-Touch": 2,
"Screen Time": 2,
"Photos": 2,
"AirDrop": 2,
"Graphics": 2,
"HomeKit": 2,
"DeviceCheck": 2,
"Libnotify": 2,
"DeviceLink": 2,
"AccessorySetupKit": 2,
"VoiceOver": 2,
"libc": 2,
"Siri Suggestions": 2,
"Calling Framework": 2,
"Reminders": 2,
"Accounts Framework": 1,
"NSColorPanel": 1,
"CoreVideo": 1,
"Single Sign-On": 1,
"MigrationKit": 1,
"Image Capture": 1,
"Wi-Fi Connectivity": 1,
"ppp": 1,
"Sync Services": 1,
"RawCamera": 1,
"Wallet": 1,
"Files": 1,
"DriverKit": 1,
"IOAcceleratorFamily": 1,
"Software Update": 1,
"CloudAttestation": 1,
"Call History": 1,
"ReplayKit": 1,
"File Bookmark": 1,
"Thread": 1,
"Mail Drafts": 1,
"Filters": 1,
"Web Extensions": 1,
"Safety Check": 1,
"App Protection": 1,
"System Preferences": 1,
"WidgetKit": 1,
"Voice Control": 1,
"WebKit Sandboxing": 1,
"libnetcore": 1,
"iWork": 1,
"CoreAnimation": 1,
"Camera": 1,
"Settings": 1,
"Accelerate": 1,
"Printing": 1,
"CoreSymbolication": 1,
"ColorSync": 1
},
"cisa_vendors": {
"Microsoft": 67,
"Cisco": 27,
"Fortinet": 16,
"Linux": 15,
"Google": 14,
"Ivanti": 13,
"Citrix": 11,
"Adobe": 10,
"Synacor": 9,
"SonicWall": 9,
"Oracle": 7,
"D-Link": 6,
"Apache": 6,
"Broadcom": 5,
"Palo Alto Networks": 5,
"Langflow": 5,
"Check Point": 4,
"Qualcomm": 4,
"SolarWinds": 4,
"Sangoma": 4,
"Gladinet": 4,
"Android": 4,
"Samsung": 4,
"Arista": 4,
"PaperCut": 4,
"GitLab": 4,
"SimpleHelp ": 4,
"Sitecore": 3,
"ConnectWise": 3,
"SmarterTools": 3,
"Kentico": 3,
"Roundcube": 3,
"Ubiquiti": 3,
"MikroTik": 3,
"VMware": 3,
"N-able": 3,
"BerriAI": 3,
"JFrog": 3,
"TP-Link": 3,
"Dassault Syst\u00e8mes": 3,
"WordPress": 3,
"Plex": 3,
"Craft CMS": 2,
"RARLAB": 2,
"F5": 2,
"Hitachi Vantara": 2,
"GNU": 2,
"Progress": 2,
"TrueConf": 2,
"JetBrains": 2,
"CrushFTP": 2,
"Advantive": 2,
"TeleMessage": 2,
"Ahsay": 2,
"GeoVision": 2,
"SysAid": 2,
"WatchGuard": 2,
"Juniper": 2,
"PHP Group": 2,
"LiteSpeed": 2,
"Red Hat": 2,
"OpenPLC": 2,
"Paessler": 2,
"Wing FTP Server": 2,
"Commvault": 2,
"BeyondTrust": 2,
"ownCloud": 1,
"Versa": 1,
"SAP": 1,
"OSGeo": 1,
"WSO2": 1,
"Edimax": 1,
"Hewlett Packard Enterprise (HPE)": 1,
"Qualitia": 1,
"ProFTPD": 1,
"KNX Association": 1,
"Soliton Systems K.K": 1,
"SKYSEA": 1,
"Hikvision": 1,
"Strapi": 1,
"MDaemon": 1,
"Lantronix": 1,
"Nx": 1,
"7-Zip": 1,
"Adminer": 1,
"Sudo": 1,
"Balbooa": 1,
"DrayTek": 1,
"DD-WRT": 1,
"CWP": 1,
"TeamT5": 1,
"Meta Platforms": 1,
"ISC": 1,
"AMI": 1,
"Trimble": 1,
"Wazuh": 1,
"Jenkins": 1,
"Prettier": 1,
"Collabora": 1,
"Motex": 1,
"Srimax": 1,
"Dell": 1,
"tj-actions": 1,
"Audinate": 1,
"Splunk": 1,
"Zyxel": 1,
"Kludex": 1,
"Looking Glass": 1,
"Grafana Labs": 1,
"Metabase": 1,
"Smartbedded": 1,
"Gitea": 1,
"Acronis": 1,
"Vite": 1,
"Atlassian": 1,
"Meta": 1,
"iCagenda": 1,
"PTC": 1,
"FreeType": 1,
"Array Networks ": 1,
"Mirasvit": 1,
"Quest": 1,
"Aquasecurity": 1,
"Daemon": 1,
"Mozilla": 1,
"Kestra": 1,
"Widget Factory": 1,
"Rockwell": 1,
"ASUS": 1,
"Laravel": 1,
"Drupal": 1,
"Fortra": 1,
"Sierra Wireless": 1,
"Sophos": 1,
"Digiever": 1,
"TanStack": 1,
"Gogs": 1,
"MLflow": 1,
"IGEL": 1,
"Libraesva": 1,
"Git": 1,
"XWiki": 1,
"ONLYOFFICE": 1,
"Marimo": 1,
"Ray-Project": 1,
"Trend Micro": 1,
"React Native Community": 1,
"reviewdog": 1,
"Erlang": 1,
"n8n": 1,
"NAKIVO": 1,
"IBM": 1,
"Mitel": 1,
"JQuery": 1,
"WebPros": 1,
"ZKTeco": 1,
"Ajax.NET Professional": 1,
"EG Secure Solutions": 1
},
"cisa_distinct_vendors": 158,
"cisa_distinct_products": 264,
"mcq_original_answer_labels": {
"A": 1167,
"B": 502,
"D": 25,
"C": 123
},
"keyword_topics": {
"iOS / Apple platform": 1178,
"Kernel / operating systems": 948,
"Memory safety / binary analysis": 664,
"Web / API security": 681,
"Network security": 502,
"Identity / authentication": 463,
"Cryptography": 203,
"Cloud / containers": 575,
"Malware / incident analysis": 276
}
},
"eval": {
"records": 264,
"by_source": {
"pentesting_explanation": 180,
"apple_advisory": 42,
"cisa_kev": 42
},
"unique_cves": 93,
"cve_years": {
"2025": 51,
"2023": 2,
"2021": 2,
"2026": 19,
"2024": 10,
"2014": 2,
"2010": 1,
"2016": 1,
"2017": 1,
"2019": 2,
"2020": 1,
"2012": 1
},
"ios_versions": {
"26.2": 41,
"16.7.16": 1
},
"ios27_examples": 0,
"apple_components": {
"WebKit": 11,
"Multi-Touch": 7,
"Foundation": 2,
"Screen Time": 2,
"curl": 2,
"Calling Framework": 1,
"Telephony": 1,
"BiometricKit": 1,
"Books": 1,
"Mail": 1,
"AppleJPEG": 1,
"Kernel": 1,
"Notification Services": 1,
"Security": 1,
"MediaExperience": 1,
"libarchive": 1,
"WebKit Web Inspector": 1,
"Photos": 1,
"Messages": 1,
"Icons": 1,
"App Store": 1,
"FaceTime": 1,
"iTunes Store": 1
},
"cisa_vendors": {
"Microsoft": 10,
"N-able": 2,
"SAP": 2,
"Zyxel": 2,
"ASUS": 2,
"Oracle": 1,
"Ivanti": 1,
"TeleMessage": 1,
"Omnissa": 1,
"WordPress": 1,
"Craft CMS": 1,
"MongoDB": 1,
"Linux": 1,
"JFrog": 1,
"Broadcom": 1,
"Sophos": 1,
"Aviatrix": 1,
"PHP": 1,
"Notepad++": 1,
"Siemens": 1,
"Rails": 1,
"Joomlack": 1,
"SonicWall": 1,
"Google": 1,
"TP-Link": 1,
"TrueConf": 1,
"Trend Micro": 1,
"Zammad GmbH": 1,
"JoomShaper": 1
},
"cisa_distinct_vendors": 29,
"cisa_distinct_products": 36,
"mcq_original_answer_labels": {
"A": 113,
"B": 47,
"C": 17,
"D": 3
},
"keyword_topics": {
"iOS / Apple platform": 47,
"Kernel / operating systems": 69,
"Memory safety / binary analysis": 25,
"Web / API security": 69,
"Network security": 48,
"Identity / authentication": 52,
"Cryptography": 11,
"Cloud / containers": 55,
"Malware / incident analysis": 26
}
},
"split_overlap": {
"groups": 0,
"cves": 0
},
"corpus_sha256": {
"train": "5f8b9b7e59e39dff1b759f9606394db33ec820e45a50dc9ff0efd0eab8e9e9e4",
"eval": "f2dc8e768048b0a38636839c9ce503dcd2045870a4998439f63c068a896ba39d"
},
"gaps": [
"No iOS 27 advisory examples are in the current held-out split; iOS 27 generalization remains unmeasured.",
"Apple examples primarily teach advisory interpretation, not kernel/WebKit source-code auditing.",
"Original MCQ answer positions are heavily skewed toward A; the separate benchmark reorders eligible choices to balance answer letters.",
"Exact group/CVE and lexical overlap checks cannot rule out semantic overlap or base-model pretraining contamination."
]
}