{ "dataset_manifest_sha256": "9af56087c7655f676c1cb64d214ed2d9d1a5cac78e09aaaee59abd16cd3a0f18", "split": "challenge", "documents": 14461, "models": { "valendra": { "metadata": { "model": "valendra", "dataset_manifest_sha256": "9af56087c7655f676c1cb64d214ed2d9d1a5cac78e09aaaee59abd16cd3a0f18", "split": "challenge", "document_order_sha256": "b7add247b64759824241aee97f400c291c878c1a3a16b8f8358d8fc9b733be84", "comparison_config_sha256": "808bf293786e733f809daa21722423c7c35719265e0ff977e809d6730f7873bb", "implementation_sha256": "4cf94a19586c67811e411a011bdfe680e82c1f2c844920dfe2be00ff8080e5f5", "documents": 14461, "limit": null, "sample_seed": null, "common_categories": [ "ADDRESS", "EMAIL", "PERSON", "PHONE", "SECRET", "URL" ], "device": "cuda", "hardware": "NVIDIA GeForce RTX 4090", "warmup_documents": 2, "batch_size": 1, "scoring": "exact_character_spans_v1", "model_details": { "requested_precision": "bf16", "quantization": "none", "torch_version": "2.8.0+cu128", "model_id": "Qwen/Qwen3-0.6B-Base", "weights_sha256": "130ea7b3a1de096aa5a1c32500d82a8157546c83607d5f71e361df0710fb9d7b", "parameter_dtypes": { "torch.bfloat16": 596091945 }, "attention_mode": "causal", "max_length": 512, "overlap": 64, "transformers_version": "4.55.4", "span_scores": true, "model_revision": "da87bfb608c14b7cf20ba1ce41287e8de496c0cd", "training_dataset_manifest_sha256": "9af56087c7655f676c1cb64d214ed2d9d1a5cac78e09aaaee59abd16cd3a0f18", "training_protocol_sha256": "3d55cc7f064fc68f3774d2078d8227b946308ffa4c6242412a83874ee898f073", "load_seconds": 5.278955338988453, "label_mapping": { "PERSON": "PERSON", "EMAIL": "EMAIL", "PHONE": "PHONE", "ADDRESS": "ADDRESS", "GOVERNMENT_ID": "GOVERNMENT_ID", "FINANCIAL_ID": "FINANCIAL_ID", "SECRET": "SECRET", "IP_ADDRESS": "IP_ADDRESS", "DATE_OF_BIRTH": "DATE_OF_BIRTH", "URL": "URL" } } }, "metrics": { "documents": 14461, "untyped": { "tp": 28169, "fp": 20817, "fn": 13721, "precision": 0.5750418486914629, "recall": 0.6724516591071855, "f1": 0.6199436594920551, "f2": 0.6504160778772179 }, "typed_common": { "tp": 18595, "fp": 17937, "fn": 10840, "precision": 0.5090058031315011, "recall": 0.6317309325632751, "f1": 0.5637667318507739, "f2": 0.6026693113461937 }, "common_categories": [ "ADDRESS", "EMAIL", "PERSON", "PHONE", "SECRET", "URL" ], "typed_by_category": { "ADDRESS": { "tp": 0, "fp": 8429, "fn": 1982, "precision": 0.0, "recall": 0.0, "f1": 0.0, "f2": 0.0 }, "EMAIL": { "tp": 2499, "fp": 377, "fn": 0, "precision": 0.8689151599443672, "recall": 1.0, "f1": 0.9298604651162791, "f2": 0.9707116221255438 }, "PERSON": { "tp": 6427, "fp": 1317, "fn": 1050, "precision": 0.8299328512396694, "recall": 0.8595693459943827, "f1": 0.8444911635240785, "f2": 0.853473919048125 }, "PHONE": { "tp": 2478, "fp": 6, "fn": 0, "precision": 0.9975845410628019, "recall": 1.0, "f1": 0.9987908101571946, "f2": 0.999515972894482 }, "SECRET": { "tp": 4691, "fp": 2808, "fn": 2808, "precision": 0.6255500733431124, "recall": 0.6255500733431124, "f1": 0.6255500733431124, "f2": 0.6255500733431124 }, "URL": { "tp": 2500, "fp": 5000, "fn": 5000, "precision": 0.3333333333333333, "recall": 0.3333333333333333, "f1": 0.3333333333333333, "f2": 0.3333333333333333 } }, "untyped_recall_by_gold_category": { "ADDRESS": { "matched": 0, "total": 1982, "recall": 0.0 }, "DATE_OF_BIRTH": { "matched": 2500, "total": 2500, "recall": 1.0 }, "EMAIL": { "matched": 2499, "total": 2499, "recall": 1.0 }, "FINANCIAL_ID": { "matched": 4978, "total": 4978, "recall": 1.0 }, "GOVERNMENT_ID": { "matched": 2096, "total": 2499, "recall": 0.8387354941976791 }, "IP_ADDRESS": { "matched": 0, "total": 2478, "recall": 0.0 }, "PERSON": { "matched": 6427, "total": 7477, "recall": 0.8595693459943827 }, "PHONE": { "matched": 2478, "total": 2478, "recall": 1.0 }, "SECRET": { "matched": 4691, "total": 7499, "recall": 0.6255500733431124 }, "URL": { "matched": 2500, "total": 7500, "recall": 0.3333333333333333 } }, "by_language": { "en": { "untyped": { "tp": 12413, "fp": 6659, "fn": 3569, "precision": 0.6508494127516778, "recall": 0.7766862720560631, "f1": 0.7082216009585212, "f2": 0.7477710843373494 }, "typed_common": { "tp": 7413, "fp": 6658, "fn": 3569, "precision": 0.5268282282709118, "recall": 0.6750136587142597, "f1": 0.5917854149203688, "f2": 0.6390627424610767 } }, "es": { "untyped": { "tp": 15756, "fp": 14158, "fn": 10152, "precision": 0.526709901718259, "recall": 0.6081519221861973, "f1": 0.5645086166744295, "f2": 0.5899090949934854 }, "typed_common": { "tp": 11182, "fp": 11279, "fn": 7271, "precision": 0.49784070166065625, "recall": 0.6059719286836829, "f1": 0.5466099623600724, "f2": 0.5807443416118745 } } }, "by_source": { "synthetic": { "untyped": { "tp": 28169, "fp": 20817, "fn": 13721, "precision": 0.5750418486914629, "recall": 0.6724516591071855, "f1": 0.6199436594920551, "f2": 0.6504160778772179 }, "typed_common": { "tp": 18595, "fp": 17937, "fn": 10840, "precision": 0.5090058031315011, "recall": 0.6317309325632751, "f1": 0.5637667318507739, "f2": 0.6026693113461937 } } }, "typed_excluded_gold": { "GOVERNMENT_ID": 2499, "DATE_OF_BIRTH": 2500, "IP_ADDRESS": 2478, "FINANCIAL_ID": 4978 }, "unmapped_predictions": {}, "ignored_predictions": 0, "false_positives_per_document": 1.43952700366503, "negative_documents": { "count": 2, "with_false_positive": 2, "false_positive_rate": 1.0 } }, "performance": { "latency_ms": { "p50": 32.25508285686374, "p95": 36.32426890544593, "p99": 38.85766505263746 }, "documents_per_second": 30.52227684407281, "characters_per_second": 3498.987195854014, "input_tokens_per_second": 1522.7726649514334, "input_tokens": 721467, "input_token_definition": "model tokenizer on original text; excludes prompt and repeated windows", "inference_seconds": 473.7851004325785, "end_to_end_seconds": 479.30525300791487, "sampled_peak_rss_bytes": 2073497600, "peak_vram_allocated_bytes": 1203220480, "peak_vram_reserved_bytes": 1218445312, "cpu_process_percent_one_core": 218.9168579762411 } }, "openai": { "metadata": { "model": "openai", "dataset_manifest_sha256": "9af56087c7655f676c1cb64d214ed2d9d1a5cac78e09aaaee59abd16cd3a0f18", "split": "challenge", "document_order_sha256": "b7add247b64759824241aee97f400c291c878c1a3a16b8f8358d8fc9b733be84", "comparison_config_sha256": "808bf293786e733f809daa21722423c7c35719265e0ff977e809d6730f7873bb", "implementation_sha256": "4cf94a19586c67811e411a011bdfe680e82c1f2c844920dfe2be00ff8080e5f5", "documents": 14461, "limit": null, "sample_seed": null, "common_categories": [ "ADDRESS", "EMAIL", "PERSON", "PHONE", "SECRET", "URL" ], "device": "cuda", "hardware": "NVIDIA GeForce RTX 4090", "warmup_documents": 2, "batch_size": 1, "scoring": "exact_character_spans_v1", "model_details": { "requested_precision": "native", "quantization": "none", "torch_version": "2.8.0+cu128", "model_id": "openai/privacy-filter", "model_revision": "7ffa9a043d54d1be65afb281eddf0ffbe629385b", "runtime_revision": "f7f00ca7fb869683eb732c010299d901457f19c3", "license": "Apache-2.0", "weights_sha256": "9c262cbe68a0c8a50590a648ef8341a2b7d3be1fa11dfb79893fe0b03ce57b5c", "parameter_dtypes": { "torch.bfloat16": 1399475840, "torch.float32": 10992 }, "runtime_context_window": 128000, "decode_mode": "viterbi", "calibration_sha256": "bbc8611ef08a55ed72d64856cbbbb9a91db8dfa881f0a92e2afbad6e4bbc775a", "span_scores": false, "opf_version": "0.1.0", "tiktoken_version": "0.11.0", "load_seconds": 4.347642426146194, "label_mapping": { "private_person": "PERSON", "private_email": "EMAIL", "private_phone": "PHONE", "private_address": "ADDRESS", "private_url": "URL", "secret": "SECRET", "private_date": null, "account_number": null } } }, "metrics": { "documents": 14461, "untyped": { "tp": 28402, "fp": 10624, "fn": 13488, "precision": 0.7277712294367857, "recall": 0.6780138457865839, "f1": 0.7020119630233822, "f2": 0.6874134742915783 }, "typed_common": { "tp": 16642, "fp": 10673, "fn": 12793, "precision": 0.6092623100860333, "recall": 0.5653813487344997, "f1": 0.5865022026431719, "f2": 0.5736444796801213 }, "common_categories": [ "ADDRESS", "EMAIL", "PERSON", "PHONE", "SECRET", "URL" ], "typed_by_category": { "ADDRESS": { "tp": 187, "fp": 5517, "fn": 1795, "precision": 0.032784011220196356, "recall": 0.09434914228052473, "f1": 0.04865990111891751, "f2": 0.06858861502347419 }, "EMAIL": { "tp": 2444, "fp": 33, "fn": 55, "precision": 0.9866774323778764, "recall": 0.9779911964785915, "f1": 0.9823151125401929, "f2": 0.9797161869638419 }, "PERSON": { "tp": 6602, "fp": 1081, "fn": 875, "precision": 0.859299752700768, "recall": 0.882974454995319, "f1": 0.8709762532981531, "f2": 0.8781357239764838 }, "PHONE": { "tp": 2419, "fp": 64, "fn": 59, "precision": 0.9742247281514297, "recall": 0.9761904761904762, "f1": 0.9752066115702479, "f2": 0.9757966922146027 }, "SECRET": { "tp": 11, "fp": 568, "fn": 7488, "precision": 0.018998272884283247, "recall": 0.0014668622482997733, "f1": 0.0027234463976231744, "f2": 0.0017988552739165984 }, "URL": { "tp": 4979, "fp": 3410, "fn": 2521, "precision": 0.5935153176779115, "recall": 0.6638666666666667, "f1": 0.6267228900497198, "f2": 0.648493057907213 } }, "untyped_recall_by_gold_category": { "ADDRESS": { "matched": 187, "total": 1982, "recall": 0.09434914228052473 }, "DATE_OF_BIRTH": { "matched": 2500, "total": 2500, "recall": 1.0 }, "EMAIL": { "matched": 2444, "total": 2499, "recall": 0.9779911964785915 }, "FINANCIAL_ID": { "matched": 4655, "total": 4978, "recall": 0.9351145038167938 }, "GOVERNMENT_ID": { "matched": 2421, "total": 2499, "recall": 0.9687875150060024 }, "IP_ADDRESS": { "matched": 326, "total": 2478, "recall": 0.13155770782889428 }, "PERSON": { "matched": 6634, "total": 7477, "recall": 0.8872542463554902 }, "PHONE": { "matched": 2419, "total": 2478, "recall": 0.9761904761904762 }, "SECRET": { "matched": 1837, "total": 7499, "recall": 0.24496599546606215 }, "URL": { "matched": 4979, "total": 7500, "recall": 0.6638666666666667 } }, "by_language": { "en": { "untyped": { "tp": 9841, "fp": 5819, "fn": 6141, "precision": 0.6284163473818646, "recall": 0.6157552246277062, "f1": 0.6220213640098603, "f2": 0.6182464693169825 }, "typed_common": { "tp": 5139, "fp": 5761, "fn": 5843, "precision": 0.4714678899082569, "recall": 0.46794755053724274, "f1": 0.4697011242116808, "f2": 0.4686474064346684 } }, "es": { "untyped": { "tp": 18561, "fp": 4805, "fn": 7347, "precision": 0.7943593255157065, "recall": 0.7164196387216304, "f1": 0.7533790640094168, "f2": 0.7307595395203074 }, "typed_common": { "tp": 11503, "fp": 4912, "fn": 6950, "precision": 0.7007614986293025, "recall": 0.6233674741234487, "f1": 0.659802684409774, "f2": 0.6374477706229841 } } }, "by_source": { "synthetic": { "untyped": { "tp": 28402, "fp": 10624, "fn": 13488, "precision": 0.7277712294367857, "recall": 0.6780138457865839, "f1": 0.7020119630233822, "f2": 0.6874134742915783 }, "typed_common": { "tp": 16642, "fp": 10673, "fn": 12793, "precision": 0.6092623100860333, "recall": 0.5653813487344997, "f1": 0.5865022026431719, "f2": 0.5736444796801213 } } }, "typed_excluded_gold": { "GOVERNMENT_ID": 2499, "DATE_OF_BIRTH": 2500, "IP_ADDRESS": 2478, "FINANCIAL_ID": 4978 }, "unmapped_predictions": { "account_number": 9206, "private_date": 2505 }, "ignored_predictions": 0, "false_positives_per_document": 0.7346656524445059, "negative_documents": { "count": 2, "with_false_positive": 2, "false_positive_rate": 1.0 } }, "performance": { "latency_ms": { "p50": 18.598005874082446, "p95": 20.177082857117057, "p99": 22.28398541919887 }, "documents_per_second": 53.1927473556665, "characters_per_second": 6097.86559700633, "input_tokens_per_second": 1904.7829560142147, "input_tokens": 517835, "input_token_definition": "model tokenizer on original text; excludes prompt and repeated windows", "inference_seconds": 271.8603704243433, "end_to_end_seconds": 275.67499247402884, "sampled_peak_rss_bytes": 1649799168, "peak_vram_allocated_bytes": 3080668160, "peak_vram_reserved_bytes": 3332374528, "cpu_process_percent_one_core": 274.424601669762 } }, "gliner": { "metadata": { "model": "gliner", "dataset_manifest_sha256": "9af56087c7655f676c1cb64d214ed2d9d1a5cac78e09aaaee59abd16cd3a0f18", "split": "challenge", "document_order_sha256": "b7add247b64759824241aee97f400c291c878c1a3a16b8f8358d8fc9b733be84", "comparison_config_sha256": "808bf293786e733f809daa21722423c7c35719265e0ff977e809d6730f7873bb", "implementation_sha256": "4cf94a19586c67811e411a011bdfe680e82c1f2c844920dfe2be00ff8080e5f5", "documents": 14461, "limit": null, "sample_seed": null, "common_categories": [ "ADDRESS", "EMAIL", "PERSON", "PHONE", "SECRET", "URL" ], "device": "cuda", "hardware": "NVIDIA GeForce RTX 4090", "warmup_documents": 2, "batch_size": 1, "scoring": "exact_character_spans_v1", "model_details": { "requested_precision": "bf16", "quantization": "none", "torch_version": "2.8.0+cu128", "model_id": "nvidia/gliner-PII", "model_revision": "bd23e8ef4425fd04e34c5204ab49ffaa706eae79", "license": "NVIDIA-Open-Model-License", "weights_sha256": "a4dfd0dcbd718acc86dca65fa99f1097d2796c8d7a681e1bc42f40f946c03802", "parameter_dtypes": { "torch.bfloat16": 445463040 }, "gliner_version": "0.2.24", "transformers_version": "4.57.3", "label_prompts": [ "first_name", "last_name", "name", "email", "phone_number", "street_address", "address", "url", "api_key", "password", "http_cookie", "cvv", "pin", "national_id", "ssn", "tax_id", "account_number", "bank_routing_number", "credit_debit_card", "swift_bic", "date_of_birth", "ipv4", "ipv6" ], "threshold": 0.3, "overlap_words": 32, "max_words": 384, "max_encoder_tokens": 512, "span_scores": true, "load_seconds": 11.4082525849808, "label_mapping": { "first_name": "PERSON", "last_name": "PERSON", "name": "PERSON", "email": "EMAIL", "phone_number": "PHONE", "street_address": "ADDRESS", "address": "ADDRESS", "url": "URL", "api_key": "SECRET", "password": "SECRET", "http_cookie": "SECRET", "cvv": "SECRET", "pin": "SECRET", "national_id": "GOVERNMENT_ID", "ssn": "GOVERNMENT_ID", "tax_id": "GOVERNMENT_ID", "account_number": "FINANCIAL_ID", "bank_routing_number": "FINANCIAL_ID", "credit_debit_card": "FINANCIAL_ID", "swift_bic": "FINANCIAL_ID", "date_of_birth": "DATE_OF_BIRTH", "ipv4": "IP_ADDRESS", "ipv6": "IP_ADDRESS" } } }, "metrics": { "documents": 14461, "untyped": { "tp": 15238, "fp": 24262, "fn": 26652, "precision": 0.38577215189873415, "recall": 0.3637622344234901, "f1": 0.3744440348937216, "f2": 0.3679609774944461 }, "typed_common": { "tp": 7790, "fp": 26030, "fn": 21645, "precision": 0.2303370786516854, "recall": 0.2646509257686428, "f1": 0.24630463994941112, "f2": 0.2569939297967802 }, "common_categories": [ "ADDRESS", "EMAIL", "PERSON", "PHONE", "SECRET", "URL" ], "typed_by_category": { "ADDRESS": { "tp": 286, "fp": 1621, "fn": 1696, "precision": 0.14997378080755114, "recall": 0.1442986881937437, "f1": 0.14708151195680125, "f2": 0.14539908490086428 }, "EMAIL": { "tp": 2499, "fp": 1, "fn": 0, "precision": 0.9996, "recall": 1.0, "f1": 0.9997999599919984, "f2": 0.9999199743918054 }, "PERSON": { "tp": 6, "fp": 17470, "fn": 7471, "precision": 0.0003433279926756695, "recall": 0.0008024608800320985, "f1": 0.00048090409970745, "f2": 0.0006331251055208511 }, "PHONE": { "tp": 0, "fp": 1, "fn": 2478, "precision": 0.0, "recall": 0.0, "f1": 0.0, "f2": 0.0 }, "SECRET": { "tp": 2499, "fp": 6937, "fn": 5000, "precision": 0.26483679525222553, "recall": 0.3332444325910121, "f1": 0.2951284322409211, "f2": 0.31687461959829577 }, "URL": { "tp": 2500, "fp": 0, "fn": 5000, "precision": 1.0, "recall": 0.3333333333333333, "f1": 0.5, "f2": 0.3846153846153846 } }, "untyped_recall_by_gold_category": { "ADDRESS": { "matched": 286, "total": 1982, "recall": 0.1442986881937437 }, "DATE_OF_BIRTH": { "matched": 2500, "total": 2500, "recall": 1.0 }, "EMAIL": { "matched": 2499, "total": 2499, "recall": 1.0 }, "FINANCIAL_ID": { "matched": 2498, "total": 4978, "recall": 0.5018079550020088 }, "GOVERNMENT_ID": { "matched": 2449, "total": 2499, "recall": 0.9799919967987195 }, "IP_ADDRESS": { "matched": 1, "total": 2478, "recall": 0.0004035512510088781 }, "PERSON": { "matched": 6, "total": 7477, "recall": 0.0008024608800320985 }, "PHONE": { "matched": 0, "total": 2478, "recall": 0.0 }, "SECRET": { "matched": 2499, "total": 7499, "recall": 0.3332444325910121 }, "URL": { "matched": 2500, "total": 7500, "recall": 0.3333333333333333 } }, "by_language": { "en": { "untyped": { "tp": 7784, "fp": 11083, "fn": 8198, "precision": 0.4125722160385859, "recall": 0.48704792892003507, "f1": 0.446727309248472, "f2": 0.47007669545262404 }, "typed_common": { "tp": 2786, "fp": 11077, "fn": 8196, "precision": 0.2009666017456539, "recall": 0.25368785285011836, "f1": 0.22427047695713423, "f2": 0.24104099254209135 } }, "es": { "untyped": { "tp": 7454, "fp": 13179, "fn": 18454, "precision": 0.36126593321378375, "recall": 0.287710359734445, "f1": 0.32031971809802107, "f2": 0.29992355047680364 }, "typed_common": { "tp": 5004, "fp": 14953, "fn": 13449, "precision": 0.2507390890414391, "recall": 0.2711754186311169, "f1": 0.26055714657641244, "f2": 0.2668259232795487 } } }, "by_source": { "synthetic": { "untyped": { "tp": 15238, "fp": 24262, "fn": 26652, "precision": 0.38577215189873415, "recall": 0.3637622344234901, "f1": 0.3744440348937216, "f2": 0.3679609774944461 }, "typed_common": { "tp": 7790, "fp": 26030, "fn": 21645, "precision": 0.2303370786516854, "recall": 0.2646509257686428, "f1": 0.24630463994941112, "f2": 0.2569939297967802 } } }, "typed_excluded_gold": { "GOVERNMENT_ID": 2499, "DATE_OF_BIRTH": 2500, "FINANCIAL_ID": 4978, "IP_ADDRESS": 2478 }, "unmapped_predictions": {}, "ignored_predictions": 0, "false_positives_per_document": 1.6777539589240025, "negative_documents": { "count": 2, "with_false_positive": 2, "false_positive_rate": 1.0 } }, "performance": { "latency_ms": { "p50": 40.188657119870186, "p95": 43.96042600274086, "p99": 46.425058972090476 }, "documents_per_second": 24.748330706898397, "characters_per_second": 2837.078397020506, "input_tokens_per_second": 1000.718373156295, "input_tokens": 584742, "input_token_definition": "model tokenizer on original text; excludes prompt and repeated windows", "inference_seconds": 584.3222385891713, "end_to_end_seconds": 590.5962648310233, "sampled_peak_rss_bytes": 2059366400, "peak_vram_allocated_bytes": 915900928, "peak_vram_reserved_bytes": 937426944, "cpu_process_percent_one_core": 100.08867905202999 } } } }