| { |
| "dataset_manifest_sha256": "9af56087c7655f676c1cb64d214ed2d9d1a5cac78e09aaaee59abd16cd3a0f18", |
| "split": "challenge", |
| "documents": 14461, |
| "models": { |
| "valendra": { |
| "metadata": { |
| "model": "valendra", |
| "dataset_manifest_sha256": "9af56087c7655f676c1cb64d214ed2d9d1a5cac78e09aaaee59abd16cd3a0f18", |
| "split": "challenge", |
| "document_order_sha256": "b7add247b64759824241aee97f400c291c878c1a3a16b8f8358d8fc9b733be84", |
| "comparison_config_sha256": "808bf293786e733f809daa21722423c7c35719265e0ff977e809d6730f7873bb", |
| "implementation_sha256": "4cf94a19586c67811e411a011bdfe680e82c1f2c844920dfe2be00ff8080e5f5", |
| "documents": 14461, |
| "limit": null, |
| "sample_seed": null, |
| "common_categories": [ |
| "ADDRESS", |
| "EMAIL", |
| "PERSON", |
| "PHONE", |
| "SECRET", |
| "URL" |
| ], |
| "device": "cuda", |
| "hardware": "NVIDIA GeForce RTX 4090", |
| "warmup_documents": 2, |
| "batch_size": 1, |
| "scoring": "exact_character_spans_v1", |
| "model_details": { |
| "requested_precision": "bf16", |
| "quantization": "none", |
| "torch_version": "2.8.0+cu128", |
| "model_id": "Qwen/Qwen3-0.6B-Base", |
| "weights_sha256": "130ea7b3a1de096aa5a1c32500d82a8157546c83607d5f71e361df0710fb9d7b", |
| "parameter_dtypes": { |
| "torch.bfloat16": 596091945 |
| }, |
| "attention_mode": "causal", |
| "max_length": 512, |
| "overlap": 64, |
| "transformers_version": "4.55.4", |
| "span_scores": true, |
| "model_revision": "da87bfb608c14b7cf20ba1ce41287e8de496c0cd", |
| "training_dataset_manifest_sha256": "9af56087c7655f676c1cb64d214ed2d9d1a5cac78e09aaaee59abd16cd3a0f18", |
| "training_protocol_sha256": "3d55cc7f064fc68f3774d2078d8227b946308ffa4c6242412a83874ee898f073", |
| "load_seconds": 5.278955338988453, |
| "label_mapping": { |
| "PERSON": "PERSON", |
| "EMAIL": "EMAIL", |
| "PHONE": "PHONE", |
| "ADDRESS": "ADDRESS", |
| "GOVERNMENT_ID": "GOVERNMENT_ID", |
| "FINANCIAL_ID": "FINANCIAL_ID", |
| "SECRET": "SECRET", |
| "IP_ADDRESS": "IP_ADDRESS", |
| "DATE_OF_BIRTH": "DATE_OF_BIRTH", |
| "URL": "URL" |
| } |
| } |
| }, |
| "metrics": { |
| "documents": 14461, |
| "untyped": { |
| "tp": 28169, |
| "fp": 20817, |
| "fn": 13721, |
| "precision": 0.5750418486914629, |
| "recall": 0.6724516591071855, |
| "f1": 0.6199436594920551, |
| "f2": 0.6504160778772179 |
| }, |
| "typed_common": { |
| "tp": 18595, |
| "fp": 17937, |
| "fn": 10840, |
| "precision": 0.5090058031315011, |
| "recall": 0.6317309325632751, |
| "f1": 0.5637667318507739, |
| "f2": 0.6026693113461937 |
| }, |
| "common_categories": [ |
| "ADDRESS", |
| "EMAIL", |
| "PERSON", |
| "PHONE", |
| "SECRET", |
| "URL" |
| ], |
| "typed_by_category": { |
| "ADDRESS": { |
| "tp": 0, |
| "fp": 8429, |
| "fn": 1982, |
| "precision": 0.0, |
| "recall": 0.0, |
| "f1": 0.0, |
| "f2": 0.0 |
| }, |
| "EMAIL": { |
| "tp": 2499, |
| "fp": 377, |
| "fn": 0, |
| "precision": 0.8689151599443672, |
| "recall": 1.0, |
| "f1": 0.9298604651162791, |
| "f2": 0.9707116221255438 |
| }, |
| "PERSON": { |
| "tp": 6427, |
| "fp": 1317, |
| "fn": 1050, |
| "precision": 0.8299328512396694, |
| "recall": 0.8595693459943827, |
| "f1": 0.8444911635240785, |
| "f2": 0.853473919048125 |
| }, |
| "PHONE": { |
| "tp": 2478, |
| "fp": 6, |
| "fn": 0, |
| "precision": 0.9975845410628019, |
| "recall": 1.0, |
| "f1": 0.9987908101571946, |
| "f2": 0.999515972894482 |
| }, |
| "SECRET": { |
| "tp": 4691, |
| "fp": 2808, |
| "fn": 2808, |
| "precision": 0.6255500733431124, |
| "recall": 0.6255500733431124, |
| "f1": 0.6255500733431124, |
| "f2": 0.6255500733431124 |
| }, |
| "URL": { |
| "tp": 2500, |
| "fp": 5000, |
| "fn": 5000, |
| "precision": 0.3333333333333333, |
| "recall": 0.3333333333333333, |
| "f1": 0.3333333333333333, |
| "f2": 0.3333333333333333 |
| } |
| }, |
| "untyped_recall_by_gold_category": { |
| "ADDRESS": { |
| "matched": 0, |
| "total": 1982, |
| "recall": 0.0 |
| }, |
| "DATE_OF_BIRTH": { |
| "matched": 2500, |
| "total": 2500, |
| "recall": 1.0 |
| }, |
| "EMAIL": { |
| "matched": 2499, |
| "total": 2499, |
| "recall": 1.0 |
| }, |
| "FINANCIAL_ID": { |
| "matched": 4978, |
| "total": 4978, |
| "recall": 1.0 |
| }, |
| "GOVERNMENT_ID": { |
| "matched": 2096, |
| "total": 2499, |
| "recall": 0.8387354941976791 |
| }, |
| "IP_ADDRESS": { |
| "matched": 0, |
| "total": 2478, |
| "recall": 0.0 |
| }, |
| "PERSON": { |
| "matched": 6427, |
| "total": 7477, |
| "recall": 0.8595693459943827 |
| }, |
| "PHONE": { |
| "matched": 2478, |
| "total": 2478, |
| "recall": 1.0 |
| }, |
| "SECRET": { |
| "matched": 4691, |
| "total": 7499, |
| "recall": 0.6255500733431124 |
| }, |
| "URL": { |
| "matched": 2500, |
| "total": 7500, |
| "recall": 0.3333333333333333 |
| } |
| }, |
| "by_language": { |
| "en": { |
| "untyped": { |
| "tp": 12413, |
| "fp": 6659, |
| "fn": 3569, |
| "precision": 0.6508494127516778, |
| "recall": 0.7766862720560631, |
| "f1": 0.7082216009585212, |
| "f2": 0.7477710843373494 |
| }, |
| "typed_common": { |
| "tp": 7413, |
| "fp": 6658, |
| "fn": 3569, |
| "precision": 0.5268282282709118, |
| "recall": 0.6750136587142597, |
| "f1": 0.5917854149203688, |
| "f2": 0.6390627424610767 |
| } |
| }, |
| "es": { |
| "untyped": { |
| "tp": 15756, |
| "fp": 14158, |
| "fn": 10152, |
| "precision": 0.526709901718259, |
| "recall": 0.6081519221861973, |
| "f1": 0.5645086166744295, |
| "f2": 0.5899090949934854 |
| }, |
| "typed_common": { |
| "tp": 11182, |
| "fp": 11279, |
| "fn": 7271, |
| "precision": 0.49784070166065625, |
| "recall": 0.6059719286836829, |
| "f1": 0.5466099623600724, |
| "f2": 0.5807443416118745 |
| } |
| } |
| }, |
| "by_source": { |
| "synthetic": { |
| "untyped": { |
| "tp": 28169, |
| "fp": 20817, |
| "fn": 13721, |
| "precision": 0.5750418486914629, |
| "recall": 0.6724516591071855, |
| "f1": 0.6199436594920551, |
| "f2": 0.6504160778772179 |
| }, |
| "typed_common": { |
| "tp": 18595, |
| "fp": 17937, |
| "fn": 10840, |
| "precision": 0.5090058031315011, |
| "recall": 0.6317309325632751, |
| "f1": 0.5637667318507739, |
| "f2": 0.6026693113461937 |
| } |
| } |
| }, |
| "typed_excluded_gold": { |
| "GOVERNMENT_ID": 2499, |
| "DATE_OF_BIRTH": 2500, |
| "IP_ADDRESS": 2478, |
| "FINANCIAL_ID": 4978 |
| }, |
| "unmapped_predictions": {}, |
| "ignored_predictions": 0, |
| "false_positives_per_document": 1.43952700366503, |
| "negative_documents": { |
| "count": 2, |
| "with_false_positive": 2, |
| "false_positive_rate": 1.0 |
| } |
| }, |
| "performance": { |
| "latency_ms": { |
| "p50": 32.25508285686374, |
| "p95": 36.32426890544593, |
| "p99": 38.85766505263746 |
| }, |
| "documents_per_second": 30.52227684407281, |
| "characters_per_second": 3498.987195854014, |
| "input_tokens_per_second": 1522.7726649514334, |
| "input_tokens": 721467, |
| "input_token_definition": "model tokenizer on original text; excludes prompt and repeated windows", |
| "inference_seconds": 473.7851004325785, |
| "end_to_end_seconds": 479.30525300791487, |
| "sampled_peak_rss_bytes": 2073497600, |
| "peak_vram_allocated_bytes": 1203220480, |
| "peak_vram_reserved_bytes": 1218445312, |
| "cpu_process_percent_one_core": 218.9168579762411 |
| } |
| }, |
| "openai": { |
| "metadata": { |
| "model": "openai", |
| "dataset_manifest_sha256": "9af56087c7655f676c1cb64d214ed2d9d1a5cac78e09aaaee59abd16cd3a0f18", |
| "split": "challenge", |
| "document_order_sha256": "b7add247b64759824241aee97f400c291c878c1a3a16b8f8358d8fc9b733be84", |
| "comparison_config_sha256": "808bf293786e733f809daa21722423c7c35719265e0ff977e809d6730f7873bb", |
| "implementation_sha256": "4cf94a19586c67811e411a011bdfe680e82c1f2c844920dfe2be00ff8080e5f5", |
| "documents": 14461, |
| "limit": null, |
| "sample_seed": null, |
| "common_categories": [ |
| "ADDRESS", |
| "EMAIL", |
| "PERSON", |
| "PHONE", |
| "SECRET", |
| "URL" |
| ], |
| "device": "cuda", |
| "hardware": "NVIDIA GeForce RTX 4090", |
| "warmup_documents": 2, |
| "batch_size": 1, |
| "scoring": "exact_character_spans_v1", |
| "model_details": { |
| "requested_precision": "native", |
| "quantization": "none", |
| "torch_version": "2.8.0+cu128", |
| "model_id": "openai/privacy-filter", |
| "model_revision": "7ffa9a043d54d1be65afb281eddf0ffbe629385b", |
| "runtime_revision": "f7f00ca7fb869683eb732c010299d901457f19c3", |
| "license": "Apache-2.0", |
| "weights_sha256": "9c262cbe68a0c8a50590a648ef8341a2b7d3be1fa11dfb79893fe0b03ce57b5c", |
| "parameter_dtypes": { |
| "torch.bfloat16": 1399475840, |
| "torch.float32": 10992 |
| }, |
| "runtime_context_window": 128000, |
| "decode_mode": "viterbi", |
| "calibration_sha256": "bbc8611ef08a55ed72d64856cbbbb9a91db8dfa881f0a92e2afbad6e4bbc775a", |
| "span_scores": false, |
| "opf_version": "0.1.0", |
| "tiktoken_version": "0.11.0", |
| "load_seconds": 4.347642426146194, |
| "label_mapping": { |
| "private_person": "PERSON", |
| "private_email": "EMAIL", |
| "private_phone": "PHONE", |
| "private_address": "ADDRESS", |
| "private_url": "URL", |
| "secret": "SECRET", |
| "private_date": null, |
| "account_number": null |
| } |
| } |
| }, |
| "metrics": { |
| "documents": 14461, |
| "untyped": { |
| "tp": 28402, |
| "fp": 10624, |
| "fn": 13488, |
| "precision": 0.7277712294367857, |
| "recall": 0.6780138457865839, |
| "f1": 0.7020119630233822, |
| "f2": 0.6874134742915783 |
| }, |
| "typed_common": { |
| "tp": 16642, |
| "fp": 10673, |
| "fn": 12793, |
| "precision": 0.6092623100860333, |
| "recall": 0.5653813487344997, |
| "f1": 0.5865022026431719, |
| "f2": 0.5736444796801213 |
| }, |
| "common_categories": [ |
| "ADDRESS", |
| "EMAIL", |
| "PERSON", |
| "PHONE", |
| "SECRET", |
| "URL" |
| ], |
| "typed_by_category": { |
| "ADDRESS": { |
| "tp": 187, |
| "fp": 5517, |
| "fn": 1795, |
| "precision": 0.032784011220196356, |
| "recall": 0.09434914228052473, |
| "f1": 0.04865990111891751, |
| "f2": 0.06858861502347419 |
| }, |
| "EMAIL": { |
| "tp": 2444, |
| "fp": 33, |
| "fn": 55, |
| "precision": 0.9866774323778764, |
| "recall": 0.9779911964785915, |
| "f1": 0.9823151125401929, |
| "f2": 0.9797161869638419 |
| }, |
| "PERSON": { |
| "tp": 6602, |
| "fp": 1081, |
| "fn": 875, |
| "precision": 0.859299752700768, |
| "recall": 0.882974454995319, |
| "f1": 0.8709762532981531, |
| "f2": 0.8781357239764838 |
| }, |
| "PHONE": { |
| "tp": 2419, |
| "fp": 64, |
| "fn": 59, |
| "precision": 0.9742247281514297, |
| "recall": 0.9761904761904762, |
| "f1": 0.9752066115702479, |
| "f2": 0.9757966922146027 |
| }, |
| "SECRET": { |
| "tp": 11, |
| "fp": 568, |
| "fn": 7488, |
| "precision": 0.018998272884283247, |
| "recall": 0.0014668622482997733, |
| "f1": 0.0027234463976231744, |
| "f2": 0.0017988552739165984 |
| }, |
| "URL": { |
| "tp": 4979, |
| "fp": 3410, |
| "fn": 2521, |
| "precision": 0.5935153176779115, |
| "recall": 0.6638666666666667, |
| "f1": 0.6267228900497198, |
| "f2": 0.648493057907213 |
| } |
| }, |
| "untyped_recall_by_gold_category": { |
| "ADDRESS": { |
| "matched": 187, |
| "total": 1982, |
| "recall": 0.09434914228052473 |
| }, |
| "DATE_OF_BIRTH": { |
| "matched": 2500, |
| "total": 2500, |
| "recall": 1.0 |
| }, |
| "EMAIL": { |
| "matched": 2444, |
| "total": 2499, |
| "recall": 0.9779911964785915 |
| }, |
| "FINANCIAL_ID": { |
| "matched": 4655, |
| "total": 4978, |
| "recall": 0.9351145038167938 |
| }, |
| "GOVERNMENT_ID": { |
| "matched": 2421, |
| "total": 2499, |
| "recall": 0.9687875150060024 |
| }, |
| "IP_ADDRESS": { |
| "matched": 326, |
| "total": 2478, |
| "recall": 0.13155770782889428 |
| }, |
| "PERSON": { |
| "matched": 6634, |
| "total": 7477, |
| "recall": 0.8872542463554902 |
| }, |
| "PHONE": { |
| "matched": 2419, |
| "total": 2478, |
| "recall": 0.9761904761904762 |
| }, |
| "SECRET": { |
| "matched": 1837, |
| "total": 7499, |
| "recall": 0.24496599546606215 |
| }, |
| "URL": { |
| "matched": 4979, |
| "total": 7500, |
| "recall": 0.6638666666666667 |
| } |
| }, |
| "by_language": { |
| "en": { |
| "untyped": { |
| "tp": 9841, |
| "fp": 5819, |
| "fn": 6141, |
| "precision": 0.6284163473818646, |
| "recall": 0.6157552246277062, |
| "f1": 0.6220213640098603, |
| "f2": 0.6182464693169825 |
| }, |
| "typed_common": { |
| "tp": 5139, |
| "fp": 5761, |
| "fn": 5843, |
| "precision": 0.4714678899082569, |
| "recall": 0.46794755053724274, |
| "f1": 0.4697011242116808, |
| "f2": 0.4686474064346684 |
| } |
| }, |
| "es": { |
| "untyped": { |
| "tp": 18561, |
| "fp": 4805, |
| "fn": 7347, |
| "precision": 0.7943593255157065, |
| "recall": 0.7164196387216304, |
| "f1": 0.7533790640094168, |
| "f2": 0.7307595395203074 |
| }, |
| "typed_common": { |
| "tp": 11503, |
| "fp": 4912, |
| "fn": 6950, |
| "precision": 0.7007614986293025, |
| "recall": 0.6233674741234487, |
| "f1": 0.659802684409774, |
| "f2": 0.6374477706229841 |
| } |
| } |
| }, |
| "by_source": { |
| "synthetic": { |
| "untyped": { |
| "tp": 28402, |
| "fp": 10624, |
| "fn": 13488, |
| "precision": 0.7277712294367857, |
| "recall": 0.6780138457865839, |
| "f1": 0.7020119630233822, |
| "f2": 0.6874134742915783 |
| }, |
| "typed_common": { |
| "tp": 16642, |
| "fp": 10673, |
| "fn": 12793, |
| "precision": 0.6092623100860333, |
| "recall": 0.5653813487344997, |
| "f1": 0.5865022026431719, |
| "f2": 0.5736444796801213 |
| } |
| } |
| }, |
| "typed_excluded_gold": { |
| "GOVERNMENT_ID": 2499, |
| "DATE_OF_BIRTH": 2500, |
| "IP_ADDRESS": 2478, |
| "FINANCIAL_ID": 4978 |
| }, |
| "unmapped_predictions": { |
| "account_number": 9206, |
| "private_date": 2505 |
| }, |
| "ignored_predictions": 0, |
| "false_positives_per_document": 0.7346656524445059, |
| "negative_documents": { |
| "count": 2, |
| "with_false_positive": 2, |
| "false_positive_rate": 1.0 |
| } |
| }, |
| "performance": { |
| "latency_ms": { |
| "p50": 18.598005874082446, |
| "p95": 20.177082857117057, |
| "p99": 22.28398541919887 |
| }, |
| "documents_per_second": 53.1927473556665, |
| "characters_per_second": 6097.86559700633, |
| "input_tokens_per_second": 1904.7829560142147, |
| "input_tokens": 517835, |
| "input_token_definition": "model tokenizer on original text; excludes prompt and repeated windows", |
| "inference_seconds": 271.8603704243433, |
| "end_to_end_seconds": 275.67499247402884, |
| "sampled_peak_rss_bytes": 1649799168, |
| "peak_vram_allocated_bytes": 3080668160, |
| "peak_vram_reserved_bytes": 3332374528, |
| "cpu_process_percent_one_core": 274.424601669762 |
| } |
| }, |
| "gliner": { |
| "metadata": { |
| "model": "gliner", |
| "dataset_manifest_sha256": "9af56087c7655f676c1cb64d214ed2d9d1a5cac78e09aaaee59abd16cd3a0f18", |
| "split": "challenge", |
| "document_order_sha256": "b7add247b64759824241aee97f400c291c878c1a3a16b8f8358d8fc9b733be84", |
| "comparison_config_sha256": "808bf293786e733f809daa21722423c7c35719265e0ff977e809d6730f7873bb", |
| "implementation_sha256": "4cf94a19586c67811e411a011bdfe680e82c1f2c844920dfe2be00ff8080e5f5", |
| "documents": 14461, |
| "limit": null, |
| "sample_seed": null, |
| "common_categories": [ |
| "ADDRESS", |
| "EMAIL", |
| "PERSON", |
| "PHONE", |
| "SECRET", |
| "URL" |
| ], |
| "device": "cuda", |
| "hardware": "NVIDIA GeForce RTX 4090", |
| "warmup_documents": 2, |
| "batch_size": 1, |
| "scoring": "exact_character_spans_v1", |
| "model_details": { |
| "requested_precision": "bf16", |
| "quantization": "none", |
| "torch_version": "2.8.0+cu128", |
| "model_id": "nvidia/gliner-PII", |
| "model_revision": "bd23e8ef4425fd04e34c5204ab49ffaa706eae79", |
| "license": "NVIDIA-Open-Model-License", |
| "weights_sha256": "a4dfd0dcbd718acc86dca65fa99f1097d2796c8d7a681e1bc42f40f946c03802", |
| "parameter_dtypes": { |
| "torch.bfloat16": 445463040 |
| }, |
| "gliner_version": "0.2.24", |
| "transformers_version": "4.57.3", |
| "label_prompts": [ |
| "first_name", |
| "last_name", |
| "name", |
| "email", |
| "phone_number", |
| "street_address", |
| "address", |
| "url", |
| "api_key", |
| "password", |
| "http_cookie", |
| "cvv", |
| "pin", |
| "national_id", |
| "ssn", |
| "tax_id", |
| "account_number", |
| "bank_routing_number", |
| "credit_debit_card", |
| "swift_bic", |
| "date_of_birth", |
| "ipv4", |
| "ipv6" |
| ], |
| "threshold": 0.3, |
| "overlap_words": 32, |
| "max_words": 384, |
| "max_encoder_tokens": 512, |
| "span_scores": true, |
| "load_seconds": 11.4082525849808, |
| "label_mapping": { |
| "first_name": "PERSON", |
| "last_name": "PERSON", |
| "name": "PERSON", |
| "email": "EMAIL", |
| "phone_number": "PHONE", |
| "street_address": "ADDRESS", |
| "address": "ADDRESS", |
| "url": "URL", |
| "api_key": "SECRET", |
| "password": "SECRET", |
| "http_cookie": "SECRET", |
| "cvv": "SECRET", |
| "pin": "SECRET", |
| "national_id": "GOVERNMENT_ID", |
| "ssn": "GOVERNMENT_ID", |
| "tax_id": "GOVERNMENT_ID", |
| "account_number": "FINANCIAL_ID", |
| "bank_routing_number": "FINANCIAL_ID", |
| "credit_debit_card": "FINANCIAL_ID", |
| "swift_bic": "FINANCIAL_ID", |
| "date_of_birth": "DATE_OF_BIRTH", |
| "ipv4": "IP_ADDRESS", |
| "ipv6": "IP_ADDRESS" |
| } |
| } |
| }, |
| "metrics": { |
| "documents": 14461, |
| "untyped": { |
| "tp": 15238, |
| "fp": 24262, |
| "fn": 26652, |
| "precision": 0.38577215189873415, |
| "recall": 0.3637622344234901, |
| "f1": 0.3744440348937216, |
| "f2": 0.3679609774944461 |
| }, |
| "typed_common": { |
| "tp": 7790, |
| "fp": 26030, |
| "fn": 21645, |
| "precision": 0.2303370786516854, |
| "recall": 0.2646509257686428, |
| "f1": 0.24630463994941112, |
| "f2": 0.2569939297967802 |
| }, |
| "common_categories": [ |
| "ADDRESS", |
| "EMAIL", |
| "PERSON", |
| "PHONE", |
| "SECRET", |
| "URL" |
| ], |
| "typed_by_category": { |
| "ADDRESS": { |
| "tp": 286, |
| "fp": 1621, |
| "fn": 1696, |
| "precision": 0.14997378080755114, |
| "recall": 0.1442986881937437, |
| "f1": 0.14708151195680125, |
| "f2": 0.14539908490086428 |
| }, |
| "EMAIL": { |
| "tp": 2499, |
| "fp": 1, |
| "fn": 0, |
| "precision": 0.9996, |
| "recall": 1.0, |
| "f1": 0.9997999599919984, |
| "f2": 0.9999199743918054 |
| }, |
| "PERSON": { |
| "tp": 6, |
| "fp": 17470, |
| "fn": 7471, |
| "precision": 0.0003433279926756695, |
| "recall": 0.0008024608800320985, |
| "f1": 0.00048090409970745, |
| "f2": 0.0006331251055208511 |
| }, |
| "PHONE": { |
| "tp": 0, |
| "fp": 1, |
| "fn": 2478, |
| "precision": 0.0, |
| "recall": 0.0, |
| "f1": 0.0, |
| "f2": 0.0 |
| }, |
| "SECRET": { |
| "tp": 2499, |
| "fp": 6937, |
| "fn": 5000, |
| "precision": 0.26483679525222553, |
| "recall": 0.3332444325910121, |
| "f1": 0.2951284322409211, |
| "f2": 0.31687461959829577 |
| }, |
| "URL": { |
| "tp": 2500, |
| "fp": 0, |
| "fn": 5000, |
| "precision": 1.0, |
| "recall": 0.3333333333333333, |
| "f1": 0.5, |
| "f2": 0.3846153846153846 |
| } |
| }, |
| "untyped_recall_by_gold_category": { |
| "ADDRESS": { |
| "matched": 286, |
| "total": 1982, |
| "recall": 0.1442986881937437 |
| }, |
| "DATE_OF_BIRTH": { |
| "matched": 2500, |
| "total": 2500, |
| "recall": 1.0 |
| }, |
| "EMAIL": { |
| "matched": 2499, |
| "total": 2499, |
| "recall": 1.0 |
| }, |
| "FINANCIAL_ID": { |
| "matched": 2498, |
| "total": 4978, |
| "recall": 0.5018079550020088 |
| }, |
| "GOVERNMENT_ID": { |
| "matched": 2449, |
| "total": 2499, |
| "recall": 0.9799919967987195 |
| }, |
| "IP_ADDRESS": { |
| "matched": 1, |
| "total": 2478, |
| "recall": 0.0004035512510088781 |
| }, |
| "PERSON": { |
| "matched": 6, |
| "total": 7477, |
| "recall": 0.0008024608800320985 |
| }, |
| "PHONE": { |
| "matched": 0, |
| "total": 2478, |
| "recall": 0.0 |
| }, |
| "SECRET": { |
| "matched": 2499, |
| "total": 7499, |
| "recall": 0.3332444325910121 |
| }, |
| "URL": { |
| "matched": 2500, |
| "total": 7500, |
| "recall": 0.3333333333333333 |
| } |
| }, |
| "by_language": { |
| "en": { |
| "untyped": { |
| "tp": 7784, |
| "fp": 11083, |
| "fn": 8198, |
| "precision": 0.4125722160385859, |
| "recall": 0.48704792892003507, |
| "f1": 0.446727309248472, |
| "f2": 0.47007669545262404 |
| }, |
| "typed_common": { |
| "tp": 2786, |
| "fp": 11077, |
| "fn": 8196, |
| "precision": 0.2009666017456539, |
| "recall": 0.25368785285011836, |
| "f1": 0.22427047695713423, |
| "f2": 0.24104099254209135 |
| } |
| }, |
| "es": { |
| "untyped": { |
| "tp": 7454, |
| "fp": 13179, |
| "fn": 18454, |
| "precision": 0.36126593321378375, |
| "recall": 0.287710359734445, |
| "f1": 0.32031971809802107, |
| "f2": 0.29992355047680364 |
| }, |
| "typed_common": { |
| "tp": 5004, |
| "fp": 14953, |
| "fn": 13449, |
| "precision": 0.2507390890414391, |
| "recall": 0.2711754186311169, |
| "f1": 0.26055714657641244, |
| "f2": 0.2668259232795487 |
| } |
| } |
| }, |
| "by_source": { |
| "synthetic": { |
| "untyped": { |
| "tp": 15238, |
| "fp": 24262, |
| "fn": 26652, |
| "precision": 0.38577215189873415, |
| "recall": 0.3637622344234901, |
| "f1": 0.3744440348937216, |
| "f2": 0.3679609774944461 |
| }, |
| "typed_common": { |
| "tp": 7790, |
| "fp": 26030, |
| "fn": 21645, |
| "precision": 0.2303370786516854, |
| "recall": 0.2646509257686428, |
| "f1": 0.24630463994941112, |
| "f2": 0.2569939297967802 |
| } |
| } |
| }, |
| "typed_excluded_gold": { |
| "GOVERNMENT_ID": 2499, |
| "DATE_OF_BIRTH": 2500, |
| "FINANCIAL_ID": 4978, |
| "IP_ADDRESS": 2478 |
| }, |
| "unmapped_predictions": {}, |
| "ignored_predictions": 0, |
| "false_positives_per_document": 1.6777539589240025, |
| "negative_documents": { |
| "count": 2, |
| "with_false_positive": 2, |
| "false_positive_rate": 1.0 |
| } |
| }, |
| "performance": { |
| "latency_ms": { |
| "p50": 40.188657119870186, |
| "p95": 43.96042600274086, |
| "p99": 46.425058972090476 |
| }, |
| "documents_per_second": 24.748330706898397, |
| "characters_per_second": 2837.078397020506, |
| "input_tokens_per_second": 1000.718373156295, |
| "input_tokens": 584742, |
| "input_token_definition": "model tokenizer on original text; excludes prompt and repeated windows", |
| "inference_seconds": 584.3222385891713, |
| "end_to_end_seconds": 590.5962648310233, |
| "sampled_peak_rss_bytes": 2059366400, |
| "peak_vram_allocated_bytes": 915900928, |
| "peak_vram_reserved_bytes": 937426944, |
| "cpu_process_percent_one_core": 100.08867905202999 |
| } |
| } |
| } |
| } |
