| { |
| "dataset_manifest_sha256": "9af56087c7655f676c1cb64d214ed2d9d1a5cac78e09aaaee59abd16cd3a0f18", |
| "split": "test", |
| "documents": 22262, |
| "models": { |
| "valendra": { |
| "metadata": { |
| "model": "valendra", |
| "dataset_manifest_sha256": "9af56087c7655f676c1cb64d214ed2d9d1a5cac78e09aaaee59abd16cd3a0f18", |
| "split": "test", |
| "document_order_sha256": "ca8c5376148ae463bfce0b324436e7f8e43091c3b9c6c3e1d54ca0bf52596f5a", |
| "comparison_config_sha256": "808bf293786e733f809daa21722423c7c35719265e0ff977e809d6730f7873bb", |
| "implementation_sha256": "4cf94a19586c67811e411a011bdfe680e82c1f2c844920dfe2be00ff8080e5f5", |
| "documents": 22262, |
| "limit": null, |
| "sample_seed": null, |
| "common_categories": [ |
| "ADDRESS", |
| "EMAIL", |
| "PERSON", |
| "PHONE", |
| "SECRET", |
| "URL" |
| ], |
| "device": "cuda", |
| "hardware": "NVIDIA GeForce RTX 4090", |
| "warmup_documents": 2, |
| "batch_size": 1, |
| "scoring": "exact_character_spans_v1", |
| "model_details": { |
| "requested_precision": "bf16", |
| "quantization": "none", |
| "torch_version": "2.8.0+cu128", |
| "model_id": "Qwen/Qwen3-0.6B-Base", |
| "weights_sha256": "130ea7b3a1de096aa5a1c32500d82a8157546c83607d5f71e361df0710fb9d7b", |
| "parameter_dtypes": { |
| "torch.bfloat16": 596091945 |
| }, |
| "attention_mode": "causal", |
| "max_length": 512, |
| "overlap": 64, |
| "transformers_version": "4.55.4", |
| "span_scores": true, |
| "model_revision": "da87bfb608c14b7cf20ba1ce41287e8de496c0cd", |
| "training_dataset_manifest_sha256": "9af56087c7655f676c1cb64d214ed2d9d1a5cac78e09aaaee59abd16cd3a0f18", |
| "training_protocol_sha256": "3d55cc7f064fc68f3774d2078d8227b946308ffa4c6242412a83874ee898f073", |
| "load_seconds": 5.230109900934622, |
| "label_mapping": { |
| "PERSON": "PERSON", |
| "EMAIL": "EMAIL", |
| "PHONE": "PHONE", |
| "ADDRESS": "ADDRESS", |
| "GOVERNMENT_ID": "GOVERNMENT_ID", |
| "FINANCIAL_ID": "FINANCIAL_ID", |
| "SECRET": "SECRET", |
| "IP_ADDRESS": "IP_ADDRESS", |
| "DATE_OF_BIRTH": "DATE_OF_BIRTH", |
| "URL": "URL" |
| } |
| } |
| }, |
| "metrics": { |
| "documents": 22262, |
| "untyped": { |
| "tp": 60977, |
| "fp": 14750, |
| "fn": 6384, |
| "precision": 0.8052213873519353, |
| "recall": 0.9052270601683466, |
| "f1": 0.85230068209773, |
| "f2": 0.8832868346413807 |
| }, |
| "typed_common": { |
| "tp": 42487, |
| "fp": 14719, |
| "fn": 6437, |
| "precision": 0.7427018144949831, |
| "recall": 0.8684285831084948, |
| "f1": 0.8006595684537832, |
| "f2": 0.8399894030098616 |
| }, |
| "common_categories": [ |
| "ADDRESS", |
| "EMAIL", |
| "PERSON", |
| "PHONE", |
| "SECRET", |
| "URL" |
| ], |
| "typed_by_category": { |
| "ADDRESS": { |
| "tp": 11425, |
| "fp": 10695, |
| "fn": 2962, |
| "precision": 0.516500904159132, |
| "recall": 0.7941196913880587, |
| "f1": 0.6259073602322841, |
| "f2": 0.7170382085655471 |
| }, |
| "EMAIL": { |
| "tp": 4669, |
| "fp": 23, |
| "fn": 15, |
| "precision": 0.9950980392156863, |
| "recall": 0.996797608881298, |
| "f1": 0.9959470989761092, |
| "f2": 0.9964572306641625 |
| }, |
| "PERSON": { |
| "tp": 14703, |
| "fp": 1321, |
| "fn": 802, |
| "precision": 0.9175611582626061, |
| "recall": 0.9482747500806191, |
| "f1": 0.9326651654032795, |
| "f2": 0.941968633078776 |
| }, |
| "PHONE": { |
| "tp": 1690, |
| "fp": 178, |
| "fn": 158, |
| "precision": 0.9047109207708779, |
| "recall": 0.9145021645021645, |
| "f1": 0.9095801937567277, |
| "f2": 0.9125269978401728 |
| }, |
| "SECRET": { |
| "tp": 7500, |
| "fp": 2499, |
| "fn": 0, |
| "precision": 0.7500750075007501, |
| "recall": 1.0, |
| "f1": 0.8571918395336876, |
| "f2": 0.9375234380859522 |
| }, |
| "URL": { |
| "tp": 2500, |
| "fp": 3, |
| "fn": 2500, |
| "precision": 0.9988014382740711, |
| "recall": 0.5, |
| "f1": 0.666400106624017, |
| "f2": 0.555481491356708 |
| } |
| }, |
| "untyped_recall_by_gold_category": { |
| "ADDRESS": { |
| "matched": 11527, |
| "total": 14387, |
| "recall": 0.8012094251755056 |
| }, |
| "DATE_OF_BIRTH": { |
| "matched": 2500, |
| "total": 2500, |
| "recall": 1.0 |
| }, |
| "EMAIL": { |
| "matched": 4670, |
| "total": 4684, |
| "recall": 0.9970111016225448 |
| }, |
| "FINANCIAL_ID": { |
| "matched": 4202, |
| "total": 4229, |
| "recall": 0.9936155119413573 |
| }, |
| "GOVERNMENT_ID": { |
| "matched": 9165, |
| "total": 9208, |
| "recall": 0.9953301476976543 |
| }, |
| "IP_ADDRESS": { |
| "matched": 2500, |
| "total": 2500, |
| "recall": 1.0 |
| }, |
| "PERSON": { |
| "matched": 14723, |
| "total": 15505, |
| "recall": 0.9495646565623992 |
| }, |
| "PHONE": { |
| "matched": 1690, |
| "total": 1848, |
| "recall": 0.9145021645021645 |
| }, |
| "SECRET": { |
| "matched": 7500, |
| "total": 7500, |
| "recall": 1.0 |
| }, |
| "URL": { |
| "matched": 2500, |
| "total": 5000, |
| "recall": 0.5 |
| } |
| }, |
| "by_language": { |
| "en": { |
| "untyped": { |
| "tp": 36451, |
| "fp": 1163, |
| "fn": 1266, |
| "precision": 0.9690806614558409, |
| "recall": 0.966434233899833, |
| "f1": 0.9677556384489785, |
| "f2": 0.966962362453709 |
| }, |
| "typed_common": { |
| "tp": 22343, |
| "fp": 1123, |
| "fn": 1298, |
| "precision": 0.9521435268047388, |
| "recall": 0.9450953851359926, |
| "f1": 0.9486063642346148, |
| "f2": 0.9464966533932051 |
| } |
| }, |
| "es": { |
| "untyped": { |
| "tp": 24526, |
| "fp": 13587, |
| "fn": 5118, |
| "precision": 0.6435074646446095, |
| "recall": 0.8273512346511942, |
| "f1": 0.7239399619227534, |
| "f2": 0.7826331140028975 |
| }, |
| "typed_common": { |
| "tp": 20144, |
| "fp": 13596, |
| "fn": 5139, |
| "precision": 0.597036158861885, |
| "recall": 0.7967408930902187, |
| "f1": 0.6825813665859072, |
| "f2": 0.746782134171659 |
| } |
| } |
| }, |
| "by_source": { |
| "openpii": { |
| "untyped": { |
| "tp": 31658, |
| "fp": 585, |
| "fn": 703, |
| "precision": 0.9818565269981081, |
| "recall": 0.9782763202620438, |
| "f1": 0.9800631539842733, |
| "f2": 0.9789902713266991 |
| }, |
| "typed_common": { |
| "tp": 23178, |
| "fp": 545, |
| "fn": 746, |
| "precision": 0.9770265143531593, |
| "recall": 0.9688179234241766, |
| "f1": 0.9729049048208702, |
| "f2": 0.9704485885830564 |
| } |
| }, |
| "synthetic": { |
| "untyped": { |
| "tp": 29319, |
| "fp": 14165, |
| "fn": 5681, |
| "precision": 0.6742479992640972, |
| "recall": 0.8376857142857143, |
| "f1": 0.7471331736404873, |
| "f2": 0.7989524972204661 |
| }, |
| "typed_common": { |
| "tp": 19309, |
| "fp": 14174, |
| "fn": 5691, |
| "precision": 0.5766807036406535, |
| "recall": 0.77236, |
| "f1": 0.6603286425114991, |
| "f2": 0.7232756231130556 |
| } |
| } |
| }, |
| "typed_excluded_gold": { |
| "DATE_OF_BIRTH": 2500, |
| "GOVERNMENT_ID": 9208, |
| "IP_ADDRESS": 2500, |
| "FINANCIAL_ID": 4229 |
| }, |
| "unmapped_predictions": {}, |
| "ignored_predictions": 3673, |
| "false_positives_per_document": 0.6625640104213458, |
| "negative_documents": { |
| "count": 2, |
| "with_false_positive": 2, |
| "false_positive_rate": 1.0 |
| } |
| }, |
| "performance": { |
| "latency_ms": { |
| "p50": 33.09264197014272, |
| "p95": 37.26600594818592, |
| "p99": 40.458257521968335 |
| }, |
| "documents_per_second": 29.708196863320453, |
| "characters_per_second": 5214.450451661882, |
| "input_tokens_per_second": 1761.4764185639576, |
| "input_tokens": 1319972, |
| "input_token_definition": "model tokenizer on original text; excludes prompt and repeated windows", |
| "inference_seconds": 749.355475945631, |
| "end_to_end_seconds": 759.7900253238622, |
| "sampled_peak_rss_bytes": 2160828416, |
| "peak_vram_allocated_bytes": 1218100736, |
| "peak_vram_reserved_bytes": 1245708288, |
| "cpu_process_percent_one_core": 232.7156110329722 |
| } |
| }, |
| "openai": { |
| "metadata": { |
| "model": "openai", |
| "dataset_manifest_sha256": "9af56087c7655f676c1cb64d214ed2d9d1a5cac78e09aaaee59abd16cd3a0f18", |
| "split": "test", |
| "document_order_sha256": "ca8c5376148ae463bfce0b324436e7f8e43091c3b9c6c3e1d54ca0bf52596f5a", |
| "comparison_config_sha256": "808bf293786e733f809daa21722423c7c35719265e0ff977e809d6730f7873bb", |
| "implementation_sha256": "4cf94a19586c67811e411a011bdfe680e82c1f2c844920dfe2be00ff8080e5f5", |
| "documents": 22262, |
| "limit": null, |
| "sample_seed": null, |
| "common_categories": [ |
| "ADDRESS", |
| "EMAIL", |
| "PERSON", |
| "PHONE", |
| "SECRET", |
| "URL" |
| ], |
| "device": "cuda", |
| "hardware": "NVIDIA GeForce RTX 4090", |
| "warmup_documents": 2, |
| "batch_size": 1, |
| "scoring": "exact_character_spans_v1", |
| "model_details": { |
| "requested_precision": "native", |
| "quantization": "none", |
| "torch_version": "2.8.0+cu128", |
| "model_id": "openai/privacy-filter", |
| "model_revision": "7ffa9a043d54d1be65afb281eddf0ffbe629385b", |
| "runtime_revision": "f7f00ca7fb869683eb732c010299d901457f19c3", |
| "license": "Apache-2.0", |
| "weights_sha256": "9c262cbe68a0c8a50590a648ef8341a2b7d3be1fa11dfb79893fe0b03ce57b5c", |
| "parameter_dtypes": { |
| "torch.bfloat16": 1399475840, |
| "torch.float32": 10992 |
| }, |
| "runtime_context_window": 128000, |
| "decode_mode": "viterbi", |
| "calibration_sha256": "bbc8611ef08a55ed72d64856cbbbb9a91db8dfa881f0a92e2afbad6e4bbc775a", |
| "span_scores": false, |
| "opf_version": "0.1.0", |
| "tiktoken_version": "0.11.0", |
| "load_seconds": 4.162106472067535, |
| "label_mapping": { |
| "private_person": "PERSON", |
| "private_email": "EMAIL", |
| "private_phone": "PHONE", |
| "private_address": "ADDRESS", |
| "private_url": "URL", |
| "secret": "SECRET", |
| "private_date": null, |
| "account_number": null |
| } |
| } |
| }, |
| "metrics": { |
| "documents": 22262, |
| "untyped": { |
| "tp": 44220, |
| "fp": 10153, |
| "fn": 23141, |
| "precision": 0.8132712927372041, |
| "recall": 0.6564629384955687, |
| "f1": 0.7265020454433437, |
| "f2": 0.6827930590426073 |
| }, |
| "typed_common": { |
| "tp": 26775, |
| "fp": 12780, |
| "fn": 22149, |
| "precision": 0.676905574516496, |
| "recall": 0.5472774098601914, |
| "f1": 0.6052283592716916, |
| "f2": 0.5690730326332302 |
| }, |
| "common_categories": [ |
| "ADDRESS", |
| "EMAIL", |
| "PERSON", |
| "PHONE", |
| "SECRET", |
| "URL" |
| ], |
| "typed_by_category": { |
| "ADDRESS": { |
| "tp": 3545, |
| "fp": 7427, |
| "fn": 10842, |
| "precision": 0.3230951512942034, |
| "recall": 0.24640300271078056, |
| "f1": 0.2795851571434205, |
| "f2": 0.25868359603035607 |
| }, |
| "EMAIL": { |
| "tp": 4593, |
| "fp": 43, |
| "fn": 91, |
| "precision": 0.9907247627264884, |
| "recall": 0.9805721605465414, |
| "f1": 0.9856223175965665, |
| "f2": 0.9825860003422898 |
| }, |
| "PERSON": { |
| "tp": 9295, |
| "fp": 2126, |
| "fn": 6210, |
| "precision": 0.8138516767358375, |
| "recall": 0.599484037407288, |
| "f1": 0.690410755403699, |
| "f2": 0.6328209038547951 |
| }, |
| "PHONE": { |
| "tp": 1561, |
| "fp": 526, |
| "fn": 287, |
| "precision": 0.747963584091998, |
| "recall": 0.8446969696969697, |
| "f1": 0.7933926302414231, |
| "f2": 0.8233990927313009 |
| }, |
| "SECRET": { |
| "tp": 3198, |
| "fp": 149, |
| "fn": 4302, |
| "precision": 0.9554825216611891, |
| "recall": 0.4264, |
| "f1": 0.5896561261178206, |
| "f2": 0.4795034036045221 |
| }, |
| "URL": { |
| "tp": 4583, |
| "fp": 2509, |
| "fn": 417, |
| "precision": 0.6462210941906373, |
| "recall": 0.9166, |
| "f1": 0.7580218326166059, |
| "f2": 0.8458216447659825 |
| } |
| }, |
| "untyped_recall_by_gold_category": { |
| "ADDRESS": { |
| "matched": 3798, |
| "total": 14387, |
| "recall": 0.26398832279140894 |
| }, |
| "DATE_OF_BIRTH": { |
| "matched": 2499, |
| "total": 2500, |
| "recall": 0.9996 |
| }, |
| "EMAIL": { |
| "matched": 4593, |
| "total": 4684, |
| "recall": 0.9805721605465414 |
| }, |
| "FINANCIAL_ID": { |
| "matched": 2131, |
| "total": 4229, |
| "recall": 0.5039016315913928 |
| }, |
| "GOVERNMENT_ID": { |
| "matched": 8672, |
| "total": 9208, |
| "recall": 0.9417897480451781 |
| }, |
| "IP_ADDRESS": { |
| "matched": 2500, |
| "total": 2500, |
| "recall": 1.0 |
| }, |
| "PERSON": { |
| "matched": 9936, |
| "total": 15505, |
| "recall": 0.6408255401483393 |
| }, |
| "PHONE": { |
| "matched": 1574, |
| "total": 1848, |
| "recall": 0.8517316017316018 |
| }, |
| "SECRET": { |
| "matched": 3868, |
| "total": 7500, |
| "recall": 0.5157333333333334 |
| }, |
| "URL": { |
| "matched": 4649, |
| "total": 5000, |
| "recall": 0.9298 |
| } |
| }, |
| "by_language": { |
| "en": { |
| "untyped": { |
| "tp": 21877, |
| "fp": 3818, |
| "fn": 15840, |
| "precision": 0.8514107803074528, |
| "recall": 0.5800302250974362, |
| "f1": 0.6899955844319686, |
| "f2": 0.6195239093128232 |
| }, |
| "typed_common": { |
| "tp": 10196, |
| "fp": 3402, |
| "fn": 13445, |
| "precision": 0.7498161494337403, |
| "recall": 0.4312846326297534, |
| "f1": 0.5475979483874432, |
| "f2": 0.47133004197407596 |
| } |
| }, |
| "es": { |
| "untyped": { |
| "tp": 22343, |
| "fp": 6335, |
| "fn": 7301, |
| "precision": 0.7790989608759328, |
| "recall": 0.7537107003103495, |
| "f1": 0.7661945749459895, |
| "f2": 0.7586551129341138 |
| }, |
| "typed_common": { |
| "tp": 16579, |
| "fp": 9378, |
| "fn": 8704, |
| "precision": 0.6387101745193975, |
| "recall": 0.6557370565201914, |
| "f1": 0.647111631537861, |
| "f2": 0.6522594402347961 |
| } |
| } |
| }, |
| "by_source": { |
| "openpii": { |
| "untyped": { |
| "tp": 18323, |
| "fp": 3410, |
| "fn": 14038, |
| "precision": 0.8430957530023466, |
| "recall": 0.5662062359012392, |
| "f1": 0.6774503641808703, |
| "f2": 0.6060114964577946 |
| }, |
| "typed_common": { |
| "tp": 10379, |
| "fp": 3932, |
| "fn": 13545, |
| "precision": 0.7252463140241772, |
| "recall": 0.43383213509446583, |
| "f1": 0.5429057146593436, |
| "f2": 0.47174270728226386 |
| } |
| }, |
| "synthetic": { |
| "untyped": { |
| "tp": 25897, |
| "fp": 6743, |
| "fn": 9103, |
| "precision": 0.7934129901960785, |
| "recall": 0.7399142857142857, |
| "f1": 0.7657303370786517, |
| "f2": 0.7500289620018536 |
| }, |
| "typed_common": { |
| "tp": 16396, |
| "fp": 8848, |
| "fn": 8604, |
| "precision": 0.6495008714942164, |
| "recall": 0.65584, |
| "f1": 0.6526550433882652, |
| "f2": 0.6545622944013286 |
| } |
| } |
| }, |
| "typed_excluded_gold": { |
| "DATE_OF_BIRTH": 2500, |
| "GOVERNMENT_ID": 9208, |
| "IP_ADDRESS": 2500, |
| "FINANCIAL_ID": 4229 |
| }, |
| "unmapped_predictions": { |
| "private_date": 2556, |
| "account_number": 12262 |
| }, |
| "ignored_predictions": 9562, |
| "false_positives_per_document": 0.4560686371395203, |
| "negative_documents": { |
| "count": 2, |
| "with_false_positive": 1, |
| "false_positive_rate": 0.5 |
| } |
| }, |
| "performance": { |
| "latency_ms": { |
| "p50": 18.734499579295516, |
| "p95": 23.633455566596243, |
| "p99": 27.425626609474417 |
| }, |
| "documents_per_second": 51.418454390774706, |
| "characters_per_second": 9025.084354842385, |
| "input_tokens_per_second": 2411.1402536192577, |
| "input_tokens": 1043921, |
| "input_token_definition": "model tokenizer on original text; excludes prompt and repeated windows", |
| "inference_seconds": 432.95739367837086, |
| "end_to_end_seconds": 438.70551711996086, |
| "sampled_peak_rss_bytes": 1698627584, |
| "peak_vram_allocated_bytes": 3093583360, |
| "peak_vram_reserved_bytes": 3334471680, |
| "cpu_process_percent_one_core": 274.4118669633264 |
| } |
| }, |
| "gliner": { |
| "metadata": { |
| "model": "gliner", |
| "dataset_manifest_sha256": "9af56087c7655f676c1cb64d214ed2d9d1a5cac78e09aaaee59abd16cd3a0f18", |
| "split": "test", |
| "document_order_sha256": "ca8c5376148ae463bfce0b324436e7f8e43091c3b9c6c3e1d54ca0bf52596f5a", |
| "comparison_config_sha256": "808bf293786e733f809daa21722423c7c35719265e0ff977e809d6730f7873bb", |
| "implementation_sha256": "4cf94a19586c67811e411a011bdfe680e82c1f2c844920dfe2be00ff8080e5f5", |
| "documents": 22262, |
| "limit": null, |
| "sample_seed": null, |
| "common_categories": [ |
| "ADDRESS", |
| "EMAIL", |
| "PERSON", |
| "PHONE", |
| "SECRET", |
| "URL" |
| ], |
| "device": "cuda", |
| "hardware": "NVIDIA GeForce RTX 4090", |
| "warmup_documents": 2, |
| "batch_size": 1, |
| "scoring": "exact_character_spans_v1", |
| "model_details": { |
| "requested_precision": "bf16", |
| "quantization": "none", |
| "torch_version": "2.8.0+cu128", |
| "model_id": "nvidia/gliner-PII", |
| "model_revision": "bd23e8ef4425fd04e34c5204ab49ffaa706eae79", |
| "license": "NVIDIA-Open-Model-License", |
| "weights_sha256": "a4dfd0dcbd718acc86dca65fa99f1097d2796c8d7a681e1bc42f40f946c03802", |
| "parameter_dtypes": { |
| "torch.bfloat16": 445463040 |
| }, |
| "gliner_version": "0.2.24", |
| "transformers_version": "4.57.3", |
| "label_prompts": [ |
| "first_name", |
| "last_name", |
| "name", |
| "email", |
| "phone_number", |
| "street_address", |
| "address", |
| "url", |
| "api_key", |
| "password", |
| "http_cookie", |
| "cvv", |
| "pin", |
| "national_id", |
| "ssn", |
| "tax_id", |
| "account_number", |
| "bank_routing_number", |
| "credit_debit_card", |
| "swift_bic", |
| "date_of_birth", |
| "ipv4", |
| "ipv6" |
| ], |
| "threshold": 0.3, |
| "overlap_words": 32, |
| "max_words": 384, |
| "max_encoder_tokens": 512, |
| "span_scores": true, |
| "load_seconds": 11.514702324056998, |
| "label_mapping": { |
| "first_name": "PERSON", |
| "last_name": "PERSON", |
| "name": "PERSON", |
| "email": "EMAIL", |
| "phone_number": "PHONE", |
| "street_address": "ADDRESS", |
| "address": "ADDRESS", |
| "url": "URL", |
| "api_key": "SECRET", |
| "password": "SECRET", |
| "http_cookie": "SECRET", |
| "cvv": "SECRET", |
| "pin": "SECRET", |
| "national_id": "GOVERNMENT_ID", |
| "ssn": "GOVERNMENT_ID", |
| "tax_id": "GOVERNMENT_ID", |
| "account_number": "FINANCIAL_ID", |
| "bank_routing_number": "FINANCIAL_ID", |
| "credit_debit_card": "FINANCIAL_ID", |
| "swift_bic": "FINANCIAL_ID", |
| "date_of_birth": "DATE_OF_BIRTH", |
| "ipv4": "IP_ADDRESS", |
| "ipv6": "IP_ADDRESS" |
| } |
| } |
| }, |
| "metrics": { |
| "documents": 22262, |
| "untyped": { |
| "tp": 38604, |
| "fp": 38046, |
| "fn": 28757, |
| "precision": 0.5036399217221135, |
| "recall": 0.5730912545835127, |
| "f1": 0.536125712619175, |
| "f2": 0.5577097551532243 |
| }, |
| "typed_common": { |
| "tp": 20006, |
| "fp": 39425, |
| "fn": 28918, |
| "precision": 0.33662566673958033, |
| "recall": 0.4089199574850789, |
| "f1": 0.3692676849245536, |
| "f2": 0.39207923896725944 |
| }, |
| "common_categories": [ |
| "ADDRESS", |
| "EMAIL", |
| "PERSON", |
| "PHONE", |
| "SECRET", |
| "URL" |
| ], |
| "typed_by_category": { |
| "ADDRESS": { |
| "tp": 1893, |
| "fp": 5266, |
| "fn": 12494, |
| "precision": 0.2644224053638776, |
| "recall": 0.13157711823173698, |
| "f1": 0.17571707045391255, |
| "f2": 0.14627474616347538 |
| }, |
| "EMAIL": { |
| "tp": 4673, |
| "fp": 88, |
| "fn": 11, |
| "precision": 0.9815164881327452, |
| "recall": 0.9976515798462853, |
| "f1": 0.9895182636315512, |
| "f2": 0.9943822615653062 |
| }, |
| "PERSON": { |
| "tp": 1773, |
| "fp": 32615, |
| "fn": 13732, |
| "precision": 0.051558683261602886, |
| "recall": 0.11435020960980329, |
| "f1": 0.07107209428176298, |
| "f2": 0.09195294996265871 |
| }, |
| "PHONE": { |
| "tp": 1667, |
| "fp": 183, |
| "fn": 181, |
| "precision": 0.9010810810810811, |
| "recall": 0.9020562770562771, |
| "f1": 0.9015684153596538, |
| "f2": 0.9018610690326769 |
| }, |
| "SECRET": { |
| "tp": 7500, |
| "fp": 1264, |
| "fn": 0, |
| "precision": 0.8557736193518941, |
| "recall": 1.0, |
| "f1": 0.9222823413674373, |
| "f2": 0.9673924259622331 |
| }, |
| "URL": { |
| "tp": 2500, |
| "fp": 9, |
| "fn": 2500, |
| "precision": 0.9964129135113591, |
| "recall": 0.5, |
| "f1": 0.6658676255160474, |
| "f2": 0.5553334221866809 |
| } |
| }, |
| "untyped_recall_by_gold_category": { |
| "ADDRESS": { |
| "matched": 2801, |
| "total": 14387, |
| "recall": 0.1946896503788142 |
| }, |
| "DATE_OF_BIRTH": { |
| "matched": 2500, |
| "total": 2500, |
| "recall": 1.0 |
| }, |
| "EMAIL": { |
| "matched": 4673, |
| "total": 4684, |
| "recall": 0.9976515798462853 |
| }, |
| "FINANCIAL_ID": { |
| "matched": 4091, |
| "total": 4229, |
| "recall": 0.967368172144715 |
| }, |
| "GOVERNMENT_ID": { |
| "matched": 8516, |
| "total": 9208, |
| "recall": 0.924847958297133 |
| }, |
| "IP_ADDRESS": { |
| "matched": 2500, |
| "total": 2500, |
| "recall": 1.0 |
| }, |
| "PERSON": { |
| "matched": 1796, |
| "total": 15505, |
| "recall": 0.11583360206385038 |
| }, |
| "PHONE": { |
| "matched": 1727, |
| "total": 1848, |
| "recall": 0.9345238095238095 |
| }, |
| "SECRET": { |
| "matched": 7500, |
| "total": 7500, |
| "recall": 1.0 |
| }, |
| "URL": { |
| "matched": 2500, |
| "total": 5000, |
| "recall": 0.5 |
| } |
| }, |
| "by_language": { |
| "en": { |
| "untyped": { |
| "tp": 22563, |
| "fp": 19048, |
| "fn": 15154, |
| "precision": 0.5422364278676312, |
| "recall": 0.5982183100458679, |
| "f1": 0.5688533682936667, |
| "f2": 0.586115887966999 |
| }, |
| "typed_common": { |
| "tp": 8490, |
| "fp": 20097, |
| "fn": 15151, |
| "precision": 0.2969881414629027, |
| "recall": 0.35912186455733686, |
| "f1": 0.32511296622501334, |
| "f2": 0.34469878441912777 |
| } |
| }, |
| "es": { |
| "untyped": { |
| "tp": 16041, |
| "fp": 18998, |
| "fn": 13603, |
| "precision": 0.4578041610776563, |
| "recall": 0.5411213061665092, |
| "f1": 0.4959881267102639, |
| "f2": 0.5221169807635974 |
| }, |
| "typed_common": { |
| "tp": 11516, |
| "fp": 19328, |
| "fn": 13767, |
| "precision": 0.3733627285695759, |
| "recall": 0.45548392200292687, |
| "f1": 0.4103550875692626, |
| "f2": 0.4362914469297448 |
| } |
| } |
| }, |
| "by_source": { |
| "openpii": { |
| "untyped": { |
| "tp": 16079, |
| "fp": 18136, |
| "fn": 16282, |
| "precision": 0.469940084758147, |
| "recall": 0.49686350854423533, |
| "f1": 0.4830269166065849, |
| "f2": 0.491234823627176 |
| }, |
| "typed_common": { |
| "tp": 7485, |
| "fp": 19502, |
| "fn": 16439, |
| "precision": 0.2773557638863156, |
| "recall": 0.31286574151479685, |
| "f1": 0.29404254483314024, |
| "f2": 0.30505449002714313 |
| } |
| }, |
| "synthetic": { |
| "untyped": { |
| "tp": 22525, |
| "fp": 19910, |
| "fn": 12475, |
| "precision": 0.530811829857429, |
| "recall": 0.6435714285714286, |
| "f1": 0.5817782656421515, |
| "f2": 0.6173431633184421 |
| }, |
| "typed_common": { |
| "tp": 12521, |
| "fp": 19923, |
| "fn": 12479, |
| "precision": 0.3859265195413636, |
| "recall": 0.50084, |
| "f1": 0.4359376088016155, |
| "f2": 0.47269034459847187 |
| } |
| } |
| }, |
| "typed_excluded_gold": { |
| "DATE_OF_BIRTH": 2500, |
| "GOVERNMENT_ID": 9208, |
| "IP_ADDRESS": 2500, |
| "FINANCIAL_ID": 4229 |
| }, |
| "unmapped_predictions": {}, |
| "ignored_predictions": 7486, |
| "false_positives_per_document": 1.7090108705417304, |
| "negative_documents": { |
| "count": 2, |
| "with_false_positive": 2, |
| "false_positive_rate": 1.0 |
| } |
| }, |
| "performance": { |
| "latency_ms": { |
| "p50": 38.92561804968864, |
| "p95": 50.08259153692052, |
| "p99": 59.749936242587864 |
| }, |
| "documents_per_second": 24.541077877489485, |
| "characters_per_second": 4307.505945624785, |
| "input_tokens_per_second": 1203.8577138053818, |
| "input_tokens": 1092058, |
| "input_token_definition": "model tokenizer on original text; excludes prompt and repeated windows", |
| "inference_seconds": 907.1321199147496, |
| "end_to_end_seconds": 918.7948913769796, |
| "sampled_peak_rss_bytes": 2114744320, |
| "peak_vram_allocated_bytes": 981291520, |
| "peak_vram_reserved_bytes": 1019215872, |
| "cpu_process_percent_one_core": 100.08653820678396 |
| } |
| } |
| } |
| } |
