{ "dataset_manifest_sha256": "9af56087c7655f676c1cb64d214ed2d9d1a5cac78e09aaaee59abd16cd3a0f18", "split": "test", "documents": 22262, "models": { "valendra": { "metadata": { "model": "valendra", "dataset_manifest_sha256": "9af56087c7655f676c1cb64d214ed2d9d1a5cac78e09aaaee59abd16cd3a0f18", "split": "test", "document_order_sha256": "ca8c5376148ae463bfce0b324436e7f8e43091c3b9c6c3e1d54ca0bf52596f5a", "comparison_config_sha256": "808bf293786e733f809daa21722423c7c35719265e0ff977e809d6730f7873bb", "implementation_sha256": "4cf94a19586c67811e411a011bdfe680e82c1f2c844920dfe2be00ff8080e5f5", "documents": 22262, "limit": null, "sample_seed": null, "common_categories": [ "ADDRESS", "EMAIL", "PERSON", "PHONE", "SECRET", "URL" ], "device": "cuda", "hardware": "NVIDIA GeForce RTX 4090", "warmup_documents": 2, "batch_size": 1, "scoring": "exact_character_spans_v1", "model_details": { "requested_precision": "bf16", "quantization": "none", "torch_version": "2.8.0+cu128", "model_id": "Qwen/Qwen3-0.6B-Base", "weights_sha256": "130ea7b3a1de096aa5a1c32500d82a8157546c83607d5f71e361df0710fb9d7b", "parameter_dtypes": { "torch.bfloat16": 596091945 }, "attention_mode": "causal", "max_length": 512, "overlap": 64, "transformers_version": "4.55.4", "span_scores": true, "model_revision": "da87bfb608c14b7cf20ba1ce41287e8de496c0cd", "training_dataset_manifest_sha256": "9af56087c7655f676c1cb64d214ed2d9d1a5cac78e09aaaee59abd16cd3a0f18", "training_protocol_sha256": "3d55cc7f064fc68f3774d2078d8227b946308ffa4c6242412a83874ee898f073", "load_seconds": 5.230109900934622, "label_mapping": { "PERSON": "PERSON", "EMAIL": "EMAIL", "PHONE": "PHONE", "ADDRESS": "ADDRESS", "GOVERNMENT_ID": "GOVERNMENT_ID", "FINANCIAL_ID": "FINANCIAL_ID", "SECRET": "SECRET", "IP_ADDRESS": "IP_ADDRESS", "DATE_OF_BIRTH": "DATE_OF_BIRTH", "URL": "URL" } } }, "metrics": { "documents": 22262, "untyped": { "tp": 60977, "fp": 14750, "fn": 6384, "precision": 0.8052213873519353, "recall": 0.9052270601683466, "f1": 0.85230068209773, "f2": 0.8832868346413807 }, "typed_common": { "tp": 42487, "fp": 14719, "fn": 6437, "precision": 0.7427018144949831, "recall": 0.8684285831084948, "f1": 0.8006595684537832, "f2": 0.8399894030098616 }, "common_categories": [ "ADDRESS", "EMAIL", "PERSON", "PHONE", "SECRET", "URL" ], "typed_by_category": { "ADDRESS": { "tp": 11425, "fp": 10695, "fn": 2962, "precision": 0.516500904159132, "recall": 0.7941196913880587, "f1": 0.6259073602322841, "f2": 0.7170382085655471 }, "EMAIL": { "tp": 4669, "fp": 23, "fn": 15, "precision": 0.9950980392156863, "recall": 0.996797608881298, "f1": 0.9959470989761092, "f2": 0.9964572306641625 }, "PERSON": { "tp": 14703, "fp": 1321, "fn": 802, "precision": 0.9175611582626061, "recall": 0.9482747500806191, "f1": 0.9326651654032795, "f2": 0.941968633078776 }, "PHONE": { "tp": 1690, "fp": 178, "fn": 158, "precision": 0.9047109207708779, "recall": 0.9145021645021645, "f1": 0.9095801937567277, "f2": 0.9125269978401728 }, "SECRET": { "tp": 7500, "fp": 2499, "fn": 0, "precision": 0.7500750075007501, "recall": 1.0, "f1": 0.8571918395336876, "f2": 0.9375234380859522 }, "URL": { "tp": 2500, "fp": 3, "fn": 2500, "precision": 0.9988014382740711, "recall": 0.5, "f1": 0.666400106624017, "f2": 0.555481491356708 } }, "untyped_recall_by_gold_category": { "ADDRESS": { "matched": 11527, "total": 14387, "recall": 0.8012094251755056 }, "DATE_OF_BIRTH": { "matched": 2500, "total": 2500, "recall": 1.0 }, "EMAIL": { "matched": 4670, "total": 4684, "recall": 0.9970111016225448 }, "FINANCIAL_ID": { "matched": 4202, "total": 4229, "recall": 0.9936155119413573 }, "GOVERNMENT_ID": { "matched": 9165, "total": 9208, "recall": 0.9953301476976543 }, "IP_ADDRESS": { "matched": 2500, "total": 2500, "recall": 1.0 }, "PERSON": { "matched": 14723, "total": 15505, "recall": 0.9495646565623992 }, "PHONE": { "matched": 1690, "total": 1848, "recall": 0.9145021645021645 }, "SECRET": { "matched": 7500, "total": 7500, "recall": 1.0 }, "URL": { "matched": 2500, "total": 5000, "recall": 0.5 } }, "by_language": { "en": { "untyped": { "tp": 36451, "fp": 1163, "fn": 1266, "precision": 0.9690806614558409, "recall": 0.966434233899833, "f1": 0.9677556384489785, "f2": 0.966962362453709 }, "typed_common": { "tp": 22343, "fp": 1123, "fn": 1298, "precision": 0.9521435268047388, "recall": 0.9450953851359926, "f1": 0.9486063642346148, "f2": 0.9464966533932051 } }, "es": { "untyped": { "tp": 24526, "fp": 13587, "fn": 5118, "precision": 0.6435074646446095, "recall": 0.8273512346511942, "f1": 0.7239399619227534, "f2": 0.7826331140028975 }, "typed_common": { "tp": 20144, "fp": 13596, "fn": 5139, "precision": 0.597036158861885, "recall": 0.7967408930902187, "f1": 0.6825813665859072, "f2": 0.746782134171659 } } }, "by_source": { "openpii": { "untyped": { "tp": 31658, "fp": 585, "fn": 703, "precision": 0.9818565269981081, "recall": 0.9782763202620438, "f1": 0.9800631539842733, "f2": 0.9789902713266991 }, "typed_common": { "tp": 23178, "fp": 545, "fn": 746, "precision": 0.9770265143531593, "recall": 0.9688179234241766, "f1": 0.9729049048208702, "f2": 0.9704485885830564 } }, "synthetic": { "untyped": { "tp": 29319, "fp": 14165, "fn": 5681, "precision": 0.6742479992640972, "recall": 0.8376857142857143, "f1": 0.7471331736404873, "f2": 0.7989524972204661 }, "typed_common": { "tp": 19309, "fp": 14174, "fn": 5691, "precision": 0.5766807036406535, "recall": 0.77236, "f1": 0.6603286425114991, "f2": 0.7232756231130556 } } }, "typed_excluded_gold": { "DATE_OF_BIRTH": 2500, "GOVERNMENT_ID": 9208, "IP_ADDRESS": 2500, "FINANCIAL_ID": 4229 }, "unmapped_predictions": {}, "ignored_predictions": 3673, "false_positives_per_document": 0.6625640104213458, "negative_documents": { "count": 2, "with_false_positive": 2, "false_positive_rate": 1.0 } }, "performance": { "latency_ms": { "p50": 33.09264197014272, "p95": 37.26600594818592, "p99": 40.458257521968335 }, "documents_per_second": 29.708196863320453, "characters_per_second": 5214.450451661882, "input_tokens_per_second": 1761.4764185639576, "input_tokens": 1319972, "input_token_definition": "model tokenizer on original text; excludes prompt and repeated windows", "inference_seconds": 749.355475945631, "end_to_end_seconds": 759.7900253238622, "sampled_peak_rss_bytes": 2160828416, "peak_vram_allocated_bytes": 1218100736, "peak_vram_reserved_bytes": 1245708288, "cpu_process_percent_one_core": 232.7156110329722 } }, "openai": { "metadata": { "model": "openai", "dataset_manifest_sha256": "9af56087c7655f676c1cb64d214ed2d9d1a5cac78e09aaaee59abd16cd3a0f18", "split": "test", "document_order_sha256": "ca8c5376148ae463bfce0b324436e7f8e43091c3b9c6c3e1d54ca0bf52596f5a", "comparison_config_sha256": "808bf293786e733f809daa21722423c7c35719265e0ff977e809d6730f7873bb", "implementation_sha256": "4cf94a19586c67811e411a011bdfe680e82c1f2c844920dfe2be00ff8080e5f5", "documents": 22262, "limit": null, "sample_seed": null, "common_categories": [ "ADDRESS", "EMAIL", "PERSON", "PHONE", "SECRET", "URL" ], "device": "cuda", "hardware": "NVIDIA GeForce RTX 4090", "warmup_documents": 2, "batch_size": 1, "scoring": "exact_character_spans_v1", "model_details": { "requested_precision": "native", "quantization": "none", "torch_version": "2.8.0+cu128", "model_id": "openai/privacy-filter", "model_revision": "7ffa9a043d54d1be65afb281eddf0ffbe629385b", "runtime_revision": "f7f00ca7fb869683eb732c010299d901457f19c3", "license": "Apache-2.0", "weights_sha256": "9c262cbe68a0c8a50590a648ef8341a2b7d3be1fa11dfb79893fe0b03ce57b5c", "parameter_dtypes": { "torch.bfloat16": 1399475840, "torch.float32": 10992 }, "runtime_context_window": 128000, "decode_mode": "viterbi", "calibration_sha256": "bbc8611ef08a55ed72d64856cbbbb9a91db8dfa881f0a92e2afbad6e4bbc775a", "span_scores": false, "opf_version": "0.1.0", "tiktoken_version": "0.11.0", "load_seconds": 4.162106472067535, "label_mapping": { "private_person": "PERSON", "private_email": "EMAIL", "private_phone": "PHONE", "private_address": "ADDRESS", "private_url": "URL", "secret": "SECRET", "private_date": null, "account_number": null } } }, "metrics": { "documents": 22262, "untyped": { "tp": 44220, "fp": 10153, "fn": 23141, "precision": 0.8132712927372041, "recall": 0.6564629384955687, "f1": 0.7265020454433437, "f2": 0.6827930590426073 }, "typed_common": { "tp": 26775, "fp": 12780, "fn": 22149, "precision": 0.676905574516496, "recall": 0.5472774098601914, "f1": 0.6052283592716916, "f2": 0.5690730326332302 }, "common_categories": [ "ADDRESS", "EMAIL", "PERSON", "PHONE", "SECRET", "URL" ], "typed_by_category": { "ADDRESS": { "tp": 3545, "fp": 7427, "fn": 10842, "precision": 0.3230951512942034, "recall": 0.24640300271078056, "f1": 0.2795851571434205, "f2": 0.25868359603035607 }, "EMAIL": { "tp": 4593, "fp": 43, "fn": 91, "precision": 0.9907247627264884, "recall": 0.9805721605465414, "f1": 0.9856223175965665, "f2": 0.9825860003422898 }, "PERSON": { "tp": 9295, "fp": 2126, "fn": 6210, "precision": 0.8138516767358375, "recall": 0.599484037407288, "f1": 0.690410755403699, "f2": 0.6328209038547951 }, "PHONE": { "tp": 1561, "fp": 526, "fn": 287, "precision": 0.747963584091998, "recall": 0.8446969696969697, "f1": 0.7933926302414231, "f2": 0.8233990927313009 }, "SECRET": { "tp": 3198, "fp": 149, "fn": 4302, "precision": 0.9554825216611891, "recall": 0.4264, "f1": 0.5896561261178206, "f2": 0.4795034036045221 }, "URL": { "tp": 4583, "fp": 2509, "fn": 417, "precision": 0.6462210941906373, "recall": 0.9166, "f1": 0.7580218326166059, "f2": 0.8458216447659825 } }, "untyped_recall_by_gold_category": { "ADDRESS": { "matched": 3798, "total": 14387, "recall": 0.26398832279140894 }, "DATE_OF_BIRTH": { "matched": 2499, "total": 2500, "recall": 0.9996 }, "EMAIL": { "matched": 4593, "total": 4684, "recall": 0.9805721605465414 }, "FINANCIAL_ID": { "matched": 2131, "total": 4229, "recall": 0.5039016315913928 }, "GOVERNMENT_ID": { "matched": 8672, "total": 9208, "recall": 0.9417897480451781 }, "IP_ADDRESS": { "matched": 2500, "total": 2500, "recall": 1.0 }, "PERSON": { "matched": 9936, "total": 15505, "recall": 0.6408255401483393 }, "PHONE": { "matched": 1574, "total": 1848, "recall": 0.8517316017316018 }, "SECRET": { "matched": 3868, "total": 7500, "recall": 0.5157333333333334 }, "URL": { "matched": 4649, "total": 5000, "recall": 0.9298 } }, "by_language": { "en": { "untyped": { "tp": 21877, "fp": 3818, "fn": 15840, "precision": 0.8514107803074528, "recall": 0.5800302250974362, "f1": 0.6899955844319686, "f2": 0.6195239093128232 }, "typed_common": { "tp": 10196, "fp": 3402, "fn": 13445, "precision": 0.7498161494337403, "recall": 0.4312846326297534, "f1": 0.5475979483874432, "f2": 0.47133004197407596 } }, "es": { "untyped": { "tp": 22343, "fp": 6335, "fn": 7301, "precision": 0.7790989608759328, "recall": 0.7537107003103495, "f1": 0.7661945749459895, "f2": 0.7586551129341138 }, "typed_common": { "tp": 16579, "fp": 9378, "fn": 8704, "precision": 0.6387101745193975, "recall": 0.6557370565201914, "f1": 0.647111631537861, "f2": 0.6522594402347961 } } }, "by_source": { "openpii": { "untyped": { "tp": 18323, "fp": 3410, "fn": 14038, "precision": 0.8430957530023466, "recall": 0.5662062359012392, "f1": 0.6774503641808703, "f2": 0.6060114964577946 }, "typed_common": { "tp": 10379, "fp": 3932, "fn": 13545, "precision": 0.7252463140241772, "recall": 0.43383213509446583, "f1": 0.5429057146593436, "f2": 0.47174270728226386 } }, "synthetic": { "untyped": { "tp": 25897, "fp": 6743, "fn": 9103, "precision": 0.7934129901960785, "recall": 0.7399142857142857, "f1": 0.7657303370786517, "f2": 0.7500289620018536 }, "typed_common": { "tp": 16396, "fp": 8848, "fn": 8604, "precision": 0.6495008714942164, "recall": 0.65584, "f1": 0.6526550433882652, "f2": 0.6545622944013286 } } }, "typed_excluded_gold": { "DATE_OF_BIRTH": 2500, "GOVERNMENT_ID": 9208, "IP_ADDRESS": 2500, "FINANCIAL_ID": 4229 }, "unmapped_predictions": { "private_date": 2556, "account_number": 12262 }, "ignored_predictions": 9562, "false_positives_per_document": 0.4560686371395203, "negative_documents": { "count": 2, "with_false_positive": 1, "false_positive_rate": 0.5 } }, "performance": { "latency_ms": { "p50": 18.734499579295516, "p95": 23.633455566596243, "p99": 27.425626609474417 }, "documents_per_second": 51.418454390774706, "characters_per_second": 9025.084354842385, "input_tokens_per_second": 2411.1402536192577, "input_tokens": 1043921, "input_token_definition": "model tokenizer on original text; excludes prompt and repeated windows", "inference_seconds": 432.95739367837086, "end_to_end_seconds": 438.70551711996086, "sampled_peak_rss_bytes": 1698627584, "peak_vram_allocated_bytes": 3093583360, "peak_vram_reserved_bytes": 3334471680, "cpu_process_percent_one_core": 274.4118669633264 } }, "gliner": { "metadata": { "model": "gliner", "dataset_manifest_sha256": "9af56087c7655f676c1cb64d214ed2d9d1a5cac78e09aaaee59abd16cd3a0f18", "split": "test", "document_order_sha256": "ca8c5376148ae463bfce0b324436e7f8e43091c3b9c6c3e1d54ca0bf52596f5a", "comparison_config_sha256": "808bf293786e733f809daa21722423c7c35719265e0ff977e809d6730f7873bb", "implementation_sha256": "4cf94a19586c67811e411a011bdfe680e82c1f2c844920dfe2be00ff8080e5f5", "documents": 22262, "limit": null, "sample_seed": null, "common_categories": [ "ADDRESS", "EMAIL", "PERSON", "PHONE", "SECRET", "URL" ], "device": "cuda", "hardware": "NVIDIA GeForce RTX 4090", "warmup_documents": 2, "batch_size": 1, "scoring": "exact_character_spans_v1", "model_details": { "requested_precision": "bf16", "quantization": "none", "torch_version": "2.8.0+cu128", "model_id": "nvidia/gliner-PII", "model_revision": "bd23e8ef4425fd04e34c5204ab49ffaa706eae79", "license": "NVIDIA-Open-Model-License", "weights_sha256": "a4dfd0dcbd718acc86dca65fa99f1097d2796c8d7a681e1bc42f40f946c03802", "parameter_dtypes": { "torch.bfloat16": 445463040 }, "gliner_version": "0.2.24", "transformers_version": "4.57.3", "label_prompts": [ "first_name", "last_name", "name", "email", "phone_number", "street_address", "address", "url", "api_key", "password", "http_cookie", "cvv", "pin", "national_id", "ssn", "tax_id", "account_number", "bank_routing_number", "credit_debit_card", "swift_bic", "date_of_birth", "ipv4", "ipv6" ], "threshold": 0.3, "overlap_words": 32, "max_words": 384, "max_encoder_tokens": 512, "span_scores": true, "load_seconds": 11.514702324056998, "label_mapping": { "first_name": "PERSON", "last_name": "PERSON", "name": "PERSON", "email": "EMAIL", "phone_number": "PHONE", "street_address": "ADDRESS", "address": "ADDRESS", "url": "URL", "api_key": "SECRET", "password": "SECRET", "http_cookie": "SECRET", "cvv": "SECRET", "pin": "SECRET", "national_id": "GOVERNMENT_ID", "ssn": "GOVERNMENT_ID", "tax_id": "GOVERNMENT_ID", "account_number": "FINANCIAL_ID", "bank_routing_number": "FINANCIAL_ID", "credit_debit_card": "FINANCIAL_ID", "swift_bic": "FINANCIAL_ID", "date_of_birth": "DATE_OF_BIRTH", "ipv4": "IP_ADDRESS", "ipv6": "IP_ADDRESS" } } }, "metrics": { "documents": 22262, "untyped": { "tp": 38604, "fp": 38046, "fn": 28757, "precision": 0.5036399217221135, "recall": 0.5730912545835127, "f1": 0.536125712619175, "f2": 0.5577097551532243 }, "typed_common": { "tp": 20006, "fp": 39425, "fn": 28918, "precision": 0.33662566673958033, "recall": 0.4089199574850789, "f1": 0.3692676849245536, "f2": 0.39207923896725944 }, "common_categories": [ "ADDRESS", "EMAIL", "PERSON", "PHONE", "SECRET", "URL" ], "typed_by_category": { "ADDRESS": { "tp": 1893, "fp": 5266, "fn": 12494, "precision": 0.2644224053638776, "recall": 0.13157711823173698, "f1": 0.17571707045391255, "f2": 0.14627474616347538 }, "EMAIL": { "tp": 4673, "fp": 88, "fn": 11, "precision": 0.9815164881327452, "recall": 0.9976515798462853, "f1": 0.9895182636315512, "f2": 0.9943822615653062 }, "PERSON": { "tp": 1773, "fp": 32615, "fn": 13732, "precision": 0.051558683261602886, "recall": 0.11435020960980329, "f1": 0.07107209428176298, "f2": 0.09195294996265871 }, "PHONE": { "tp": 1667, "fp": 183, "fn": 181, "precision": 0.9010810810810811, "recall": 0.9020562770562771, "f1": 0.9015684153596538, "f2": 0.9018610690326769 }, "SECRET": { "tp": 7500, "fp": 1264, "fn": 0, "precision": 0.8557736193518941, "recall": 1.0, "f1": 0.9222823413674373, "f2": 0.9673924259622331 }, "URL": { "tp": 2500, "fp": 9, "fn": 2500, "precision": 0.9964129135113591, "recall": 0.5, "f1": 0.6658676255160474, "f2": 0.5553334221866809 } }, "untyped_recall_by_gold_category": { "ADDRESS": { "matched": 2801, "total": 14387, "recall": 0.1946896503788142 }, "DATE_OF_BIRTH": { "matched": 2500, "total": 2500, "recall": 1.0 }, "EMAIL": { "matched": 4673, "total": 4684, "recall": 0.9976515798462853 }, "FINANCIAL_ID": { "matched": 4091, "total": 4229, "recall": 0.967368172144715 }, "GOVERNMENT_ID": { "matched": 8516, "total": 9208, "recall": 0.924847958297133 }, "IP_ADDRESS": { "matched": 2500, "total": 2500, "recall": 1.0 }, "PERSON": { "matched": 1796, "total": 15505, "recall": 0.11583360206385038 }, "PHONE": { "matched": 1727, "total": 1848, "recall": 0.9345238095238095 }, "SECRET": { "matched": 7500, "total": 7500, "recall": 1.0 }, "URL": { "matched": 2500, "total": 5000, "recall": 0.5 } }, "by_language": { "en": { "untyped": { "tp": 22563, "fp": 19048, "fn": 15154, "precision": 0.5422364278676312, "recall": 0.5982183100458679, "f1": 0.5688533682936667, "f2": 0.586115887966999 }, "typed_common": { "tp": 8490, "fp": 20097, "fn": 15151, "precision": 0.2969881414629027, "recall": 0.35912186455733686, "f1": 0.32511296622501334, "f2": 0.34469878441912777 } }, "es": { "untyped": { "tp": 16041, "fp": 18998, "fn": 13603, "precision": 0.4578041610776563, "recall": 0.5411213061665092, "f1": 0.4959881267102639, "f2": 0.5221169807635974 }, "typed_common": { "tp": 11516, "fp": 19328, "fn": 13767, "precision": 0.3733627285695759, "recall": 0.45548392200292687, "f1": 0.4103550875692626, "f2": 0.4362914469297448 } } }, "by_source": { "openpii": { "untyped": { "tp": 16079, "fp": 18136, "fn": 16282, "precision": 0.469940084758147, "recall": 0.49686350854423533, "f1": 0.4830269166065849, "f2": 0.491234823627176 }, "typed_common": { "tp": 7485, "fp": 19502, "fn": 16439, "precision": 0.2773557638863156, "recall": 0.31286574151479685, "f1": 0.29404254483314024, "f2": 0.30505449002714313 } }, "synthetic": { "untyped": { "tp": 22525, "fp": 19910, "fn": 12475, "precision": 0.530811829857429, "recall": 0.6435714285714286, "f1": 0.5817782656421515, "f2": 0.6173431633184421 }, "typed_common": { "tp": 12521, "fp": 19923, "fn": 12479, "precision": 0.3859265195413636, "recall": 0.50084, "f1": 0.4359376088016155, "f2": 0.47269034459847187 } } }, "typed_excluded_gold": { "DATE_OF_BIRTH": 2500, "GOVERNMENT_ID": 9208, "IP_ADDRESS": 2500, "FINANCIAL_ID": 4229 }, "unmapped_predictions": {}, "ignored_predictions": 7486, "false_positives_per_document": 1.7090108705417304, "negative_documents": { "count": 2, "with_false_positive": 2, "false_positive_rate": 1.0 } }, "performance": { "latency_ms": { "p50": 38.92561804968864, "p95": 50.08259153692052, "p99": 59.749936242587864 }, "documents_per_second": 24.541077877489485, "characters_per_second": 4307.505945624785, "input_tokens_per_second": 1203.8577138053818, "input_tokens": 1092058, "input_token_definition": "model tokenizer on original text; excludes prompt and repeated windows", "inference_seconds": 907.1321199147496, "end_to_end_seconds": 918.7948913769796, "sampled_peak_rss_bytes": 2114744320, "peak_vram_allocated_bytes": 981291520, "peak_vram_reserved_bytes": 1019215872, "cpu_process_percent_one_core": 100.08653820678396 } } } }