{ "schema_version": "gate03e-development-comparison/v1", "source_sha": "094ad6052cde66e5a7204fb614eff10c5572f313", "corpus_sha256": { "qualification": "6ea032a54babf57a5d90daa4ede63f32f13c750513bdc00f2d8992deea9321b0", "validation": "e65b71349101d25768f7e330e5d3a73e327919e2fa251c4835eb12d1af5c56e4", "development03c": "0174fb41d34d0a4db18877c68fb70149eaa67c05da86608d5da4ff8765a52c15", "development03d": "7180e176f6d8bd259da56ad729ede6fd2c798aca15a786ba0ff0adcd862f21a1" }, "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "prompts": { "PERSON": "person", "CITY": "location", "RELIGION": "religion", "EMAIL": "email address", "PHONE": "phone number", "ORGANIZATION": "organization" }, "threshold": 0.5, "loading_ms": 15253.841200996249, "actual_inference_calls": 330, "experiments": { "gate03d_literal_control": { "qualification": { "schema_version": "development-candidate-outcomes/v1", "accuracy": { "documents": 40, "ner_entity_metrics": { "PERSON": { "tp": 18, "fp": 3, "fn": 2, "precision": 0.8571428571428571, "recall": 0.9, "f1": 0.8780487804878048 }, "CITY": { "tp": 16, "fp": 2, "fn": 1, "precision": 0.8888888888888888, "recall": 0.9411764705882353, "f1": 0.9142857142857143 }, "RELIGION": { "tp": 6, "fp": 0, "fn": 0, "precision": 1.0, "recall": 1.0, "f1": 1.0 }, "EMAIL": { "tp": 1, "fp": 0, "fn": 1, "precision": 1.0, "recall": 0.5, "f1": 0.6666666666666666 }, "PHONE": { "tp": 1, "fp": 0, "fn": 0, "precision": 1.0, "recall": 1.0, "f1": 1.0 }, "ORGANIZATION": { "tp": 1, "fp": 2, "fn": 1, "precision": 0.3333333333333333, "recall": 0.5, "f1": 0.4 } }, "combined_entity_metrics": { "PERSON": { "tp": 18, "fp": 3, "fn": 2, "precision": 0.8571428571428571, "recall": 0.9, "f1": 0.8780487804878048 }, "CITY": { "tp": 16, "fp": 1, "fn": 1, "precision": 0.9411764705882353, "recall": 0.9411764705882353, "f1": 0.9411764705882353 }, "RELIGION": { "tp": 6, "fp": 0, "fn": 0, "precision": 1.0, "recall": 1.0, "f1": 1.0 }, "EMAIL": { "tp": 2, "fp": 0, "fn": 0, "precision": 1.0, "recall": 1.0, "f1": 1.0 }, "PHONE": { "tp": 1, "fp": 0, "fn": 0, "precision": 1.0, "recall": 1.0, "f1": 1.0 }, "ORGANIZATION": { "tp": 1, "fp": 2, "fn": 1, "precision": 0.3333333333333333, "recall": 0.5, "f1": 0.4 } }, "document_exact_match_rate": 0.875, "groups": { "de": { "documents": 10, "exact_matches": 8, "false_negatives": 1 }, "en": { "documents": 29, "exact_matches": 26, "false_negatives": 3 }, "mixed": { "documents": 1, "exact_matches": 1, "false_negatives": 0 }, "ocr_noise": { "documents": 4, "exact_matches": 2, "false_negatives": 2 } }, "blocked_documents": 0, "failure_codes": {}, "automated_release_eligible": false, "uncovered_source_canaries_including_incomplete": { "PERSON": 0, "CITY": 0, "RELIGION": 0, "EMAIL": 0, "PHONE": 0, "ORGANIZATION": 0 }, "literal_presence_or_incomplete": { "PERSON": 0, "CITY": 0, "RELIGION": 0, "EMAIL": 0, "PHONE": 0, "ORGANIZATION": 0 }, "partially_covered_source_canaries_including_incomplete": { "PERSON": 0, "CITY": 0, "RELIGION": 0, "EMAIL": 0, "PHONE": 0, "ORGANIZATION": 0 }, "fully_uncovered_source_canaries_including_incomplete": { "PERSON": 0, "CITY": 0, "RELIGION": 0, "EMAIL": 0, "PHONE": 0, "ORGANIZATION": 0 }, "all_document_source_coverage": { "PERSON": { "covered": 208, "total": 208 }, "CITY": { "covered": 118, "total": 118 }, "RELIGION": { "covered": 50, "total": 50 }, "EMAIL": { "covered": 27, "total": 27 }, "PHONE": { "covered": 7, "total": 7 }, "ORGANIZATION": { "covered": 11, "total": 11 } }, "conservative_accuracy_gate_passed": true, "scope": "all source spans; incomplete processing counts as missing, not leaked output", "deterministic_replay_latency_ms": { "median": 0.0705559978086967, "max": 2.214542000729125 } }, "candidate_outcomes": { "complete_no_measured_leakage": 40, "candidate_partial_mask": 0, "candidate_fully_unmasked": 0, "blocked_conflicting_spans": 0, "blocked_model_runtime": 0, "other_incomplete": 0 }, "candidate_return_rate": 1.0, "safe_abstention_rate": 0.0, "incomplete_rate": 0.0, "no_measured_leakage_rate_among_returned": 1.0, "candidate_partial_canaries": { "PERSON": 0, "CITY": 0, "RELIGION": 0, "EMAIL": 0, "PHONE": 0, "ORGANIZATION": 0 }, "candidate_fully_unmasked_canaries": { "PERSON": 0, "CITY": 0, "RELIGION": 0, "EMAIL": 0, "PHONE": 0, "ORGANIZATION": 0 }, "returned_candidate_entity_metrics": { "PERSON": { "tp": 18, "fp": 3, "fn": 2, "precision": 0.8571428571428571, "recall": 0.9, "f1": 0.8780487804878048 }, "CITY": { "tp": 16, "fp": 1, "fn": 1, "precision": 0.9411764705882353, "recall": 0.9411764705882353, "f1": 0.9411764705882353 }, "RELIGION": { "tp": 6, "fp": 0, "fn": 0, "precision": 1.0, "recall": 1.0, "f1": 1.0 }, "EMAIL": { "tp": 2, "fp": 0, "fn": 0, "precision": 1.0, "recall": 1.0, "f1": 1.0 }, "PHONE": { "tp": 1, "fp": 0, "fn": 0, "precision": 1.0, "recall": 1.0, "f1": 1.0 }, "ORGANIZATION": { "tp": 1, "fp": 2, "fn": 1, "precision": 0.3333333333333333, "recall": 0.5, "f1": 0.4 } }, "per_case": [ { "case_id": "baseline-00", "scenario": null, "predictions": [ { "start": 0, "end": 16, "label": "PERSON", "text": "Lina Wolkenstein", "score": 0.9969016313552856, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 26, "end": 32, "label": "CITY", "text": "Berlin", "score": 0.9999364614486694, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 0, "end": 16, "label": "PERSON", "text": "Lina Wolkenstein", "score": 0.9969016313552856, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 26, "end": 32, "label": "CITY", "text": "Berlin", "score": 0.9999364614486694, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 0, "end": 16, "label": "PERSON", "text": "Lina Wolkenstein", "uncovered_positions": [] }, { "start": 26, "end": 32, "label": "CITY", "text": "Berlin", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "baseline-01", "scenario": null, "predictions": [ { "start": 0, "end": 11, "label": "PERSON", "text": "Hans Müller", "score": 0.9912292957305908, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 0, "end": 11, "label": "PERSON", "text": "Hans Müller", "score": 0.9912292957305908, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 0, "end": 11, "label": "PERSON", "text": "Hans Müller", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "baseline-02", "scenario": null, "predictions": [ { "start": 13, "end": 30, "label": "PERSON", "text": "Dr. Klaus Schmidt", "score": 0.9196021556854248, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 13, "end": 30, "label": "PERSON", "text": "Dr. Klaus Schmidt", "score": 0.9196021556854248, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [ [ 13, 30, "PERSON" ] ], "false_negatives": [ [ 17, 30, "PERSON" ] ], "exact_match": false, "error_code": null, "canaries": [ { "start": 17, "end": 30, "label": "PERSON", "text": "Klaus Schmidt", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "baseline-03", "scenario": null, "predictions": [ { "start": 0, "end": 17, "label": "PERSON", "text": "Friedrich Wilhelm", "score": 0.9821144938468933, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 0, "end": 17, "label": "PERSON", "text": "Friedrich Wilhelm", "score": 0.9821144938468933, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 0, "end": 17, "label": "PERSON", "text": "Friedrich Wilhelm", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "baseline-04", "scenario": null, "predictions": [ { "start": 15, "end": 35, "label": "PERSON", "text": "Anne-Marie Schneider", "score": 0.981816291809082, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 15, "end": 35, "label": "PERSON", "text": "Anne-Marie Schneider", "score": 0.981816291809082, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 15, "end": 35, "label": "PERSON", "text": "Anne-Marie Schneider", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "baseline-05", "scenario": null, "predictions": [ { "start": 0, "end": 8, "label": "PERSON", "text": "John Doe", "score": 0.9945249557495117, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 0, "end": 8, "label": "PERSON", "text": "John Doe", "score": 0.9945249557495117, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 0, "end": 8, "label": "PERSON", "text": "John Doe", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "baseline-06", "scenario": null, "predictions": [ { "start": 0, "end": 5, "label": "PERSON", "text": "Alice", "score": 0.9985244870185852, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 10, "end": 13, "label": "PERSON", "text": "Bob", "score": 0.9991118311882019, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 26, "end": 31, "label": "CITY", "text": "store", "score": 0.5467662811279297, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 0, "end": 5, "label": "PERSON", "text": "Alice", "score": 0.9985244870185852, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 10, "end": 13, "label": "PERSON", "text": "Bob", "score": 0.9991118311882019, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 26, "end": 31, "label": "CITY", "text": "store", "score": 0.5467662811279297, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [ [ 26, 31, "CITY" ] ], "false_negatives": [], "exact_match": false, "error_code": null, "canaries": [ { "start": 0, "end": 5, "label": "PERSON", "text": "Alice", "uncovered_positions": [] }, { "start": 10, "end": 13, "label": "PERSON", "text": "Bob", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "baseline-07", "scenario": null, "predictions": [ { "start": 3, "end": 15, "label": "PERSON", "text": "Mary Johnson", "score": 0.9960722923278809, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 3, "end": 15, "label": "PERSON", "text": "Mary Johnson", "score": 0.9960722923278809, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 3, "end": 15, "label": "PERSON", "text": "Mary Johnson", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "baseline-08", "scenario": null, "predictions": [ { "start": 17, "end": 35, "label": "PERSON", "text": "Charles Montgomery", "score": 0.9641899466514587, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 17, "end": 35, "label": "PERSON", "text": "Charles Montgomery", "score": 0.9641899466514587, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 17, "end": 35, "label": "PERSON", "text": "Charles Montgomery", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "baseline-09", "scenario": null, "predictions": [ { "start": 0, "end": 16, "label": "PERSON", "text": "Elizabeth Taylor", "score": 0.9948871731758118, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 0, "end": 16, "label": "PERSON", "text": "Elizabeth Taylor", "score": 0.9948871731758118, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 0, "end": 16, "label": "PERSON", "text": "Elizabeth Taylor", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "baseline-10", "scenario": null, "predictions": [ { "start": 10, "end": 18, "label": "CITY", "text": "New York", "score": 0.9997181296348572, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 10, "end": 18, "label": "CITY", "text": "New York", "score": 0.9997181296348572, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 10, "end": 18, "label": "CITY", "text": "New York", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "baseline-11", "scenario": null, "predictions": [ { "start": 13, "end": 20, "label": "CITY", "text": "München", "score": 0.9988282322883606, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 13, "end": 20, "label": "CITY", "text": "München", "score": 0.9988282322883606, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 13, "end": 20, "label": "CITY", "text": "München", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "baseline-12", "scenario": null, "predictions": [ { "start": 0, "end": 5, "label": "CITY", "text": "Paris", "score": 0.9994774460792542, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 0, "end": 5, "label": "CITY", "text": "Paris", "score": 0.9994774460792542, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 0, "end": 5, "label": "CITY", "text": "Paris", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "baseline-13", "scenario": null, "predictions": [ { "start": 9, "end": 18, "label": "CITY", "text": "Frankfurt", "score": 0.9987022876739502, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 24, "end": 29, "label": "CITY", "text": "Tokyo", "score": 0.9982670545578003, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 9, "end": 18, "label": "CITY", "text": "Frankfurt", "score": 0.9987022876739502, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 24, "end": 29, "label": "CITY", "text": "Tokyo", "score": 0.9982670545578003, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 9, "end": 18, "label": "CITY", "text": "Frankfurt", "uncovered_positions": [] }, { "start": 24, "end": 29, "label": "CITY", "text": "Tokyo", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "baseline-14", "scenario": null, "predictions": [ { "start": 11, "end": 22, "label": "CITY", "text": "Los Angeles", "score": 0.9923924207687378, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 24, "end": 34, "label": "CITY", "text": "California", "score": 0.9569051861763, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 11, "end": 22, "label": "CITY", "text": "Los Angeles", "score": 0.9923924207687378, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 24, "end": 34, "label": "CITY", "text": "California", "score": 0.9569051861763, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 11, "end": 22, "label": "CITY", "text": "Los Angeles", "uncovered_positions": [] }, { "start": 24, "end": 34, "label": "CITY", "text": "California", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "baseline-15", "scenario": null, "predictions": [ { "start": 32, "end": 43, "label": "RELIGION", "text": "evangelisch", "score": 0.8318354487419128, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 32, "end": 43, "label": "RELIGION", "text": "evangelisch", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 32, "end": 43, "label": "RELIGION", "text": "evangelisch", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "baseline-16", "scenario": null, "predictions": [ { "start": 0, "end": 2, "label": "PERSON", "text": "Er", "score": 0.6436470746994019, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 7, "end": 17, "label": "RELIGION", "text": "katholisch", "score": 0.7684664726257324, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 0, "end": 2, "label": "PERSON", "text": "Er", "score": 0.6436470746994019, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 7, "end": 17, "label": "RELIGION", "text": "katholisch", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [ [ 0, 2, "PERSON" ] ], "false_negatives": [], "exact_match": false, "error_code": null, "canaries": [ { "start": 7, "end": 17, "label": "RELIGION", "text": "katholisch", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "baseline-17", "scenario": null, "predictions": [ { "start": 14, "end": 21, "label": "RELIGION", "text": "Judaism", "score": 0.9957650899887085, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 14, "end": 21, "label": "RELIGION", "text": "Judaism", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 14, "end": 21, "label": "RELIGION", "text": "Judaism", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "baseline-18", "scenario": null, "predictions": [ { "start": 15, "end": 21, "label": "RELIGION", "text": "Muslim", "score": 0.966416597366333, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 15, "end": 21, "label": "RELIGION", "text": "Muslim", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 15, "end": 21, "label": "RELIGION", "text": "Muslim", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "baseline-19", "scenario": null, "predictions": [ { "start": 0, "end": 8, "label": "RELIGION", "text": "Buddhist", "score": 0.7523731589317322, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 0, "end": 8, "label": "RELIGION", "text": "Buddhist", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 0, "end": 8, "label": "RELIGION", "text": "Buddhist", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "baseline-20", "scenario": null, "predictions": [ { "start": 0, "end": 8, "label": "PERSON", "text": "John Doe", "score": 0.9968609809875488, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 14, "end": 22, "label": "CITY", "text": "New York", "score": 0.9980940222740173, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 27, "end": 43, "label": "PERSON", "text": "Lina Wolkenstein", "score": 0.9969224333763123, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 47, "end": 53, "label": "CITY", "text": "Berlin", "score": 0.9998062252998352, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 0, "end": 8, "label": "PERSON", "text": "John Doe", "score": 0.9968609809875488, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 14, "end": 22, "label": "CITY", "text": "New York", "score": 0.9980940222740173, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 27, "end": 43, "label": "PERSON", "text": "Lina Wolkenstein", "score": 0.9969224333763123, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 47, "end": 53, "label": "CITY", "text": "Berlin", "score": 0.9998062252998352, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 0, "end": 8, "label": "PERSON", "text": "John Doe", "uncovered_positions": [] }, { "start": 14, "end": 22, "label": "CITY", "text": "New York", "uncovered_positions": [] }, { "start": 27, "end": 43, "label": "PERSON", "text": "Lina Wolkenstein", "uncovered_positions": [] }, { "start": 47, "end": 53, "label": "CITY", "text": "Berlin", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "baseline-21", "scenario": null, "predictions": [ { "start": 0, "end": 5, "label": "PERSON", "text": "David", "score": 0.9995999932289124, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 9, "end": 17, "label": "RELIGION", "text": "Catholic", "score": 0.9957641363143921, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 31, "end": 35, "label": "CITY", "text": "Rome", "score": 0.9998972415924072, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 0, "end": 5, "label": "PERSON", "text": "David", "score": 0.9995999932289124, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 9, "end": 17, "label": "RELIGION", "text": "Catholic", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" }, { "start": 31, "end": 35, "label": "CITY", "text": "Rome", "score": 0.9998972415924072, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 0, "end": 5, "label": "PERSON", "text": "David", "uncovered_positions": [] }, { "start": 9, "end": 17, "label": "RELIGION", "text": "Catholic", "uncovered_positions": [] }, { "start": 31, "end": 35, "label": "CITY", "text": "Rome", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "baseline-22", "scenario": null, "predictions": [ { "start": 0, "end": 4, "label": "PERSON", "text": "John", "score": 0.9999161958694458, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 11, "end": 15, "label": "PERSON", "text": "John", "score": 0.9968172311782837, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 0, "end": 4, "label": "PERSON", "text": "John", "score": 0.9999161958694458, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 11, "end": 15, "label": "PERSON", "text": "John", "score": 0.9968172311782837, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 0, "end": 4, "label": "PERSON", "text": "John", "uncovered_positions": [] }, { "start": 11, "end": 15, "label": "PERSON", "text": "John", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "baseline-23", "scenario": null, "predictions": [ { "start": 0, "end": 6, "label": "CITY", "text": "Berlin", "score": 0.9994149208068848, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 11, "end": 17, "label": "CITY", "text": "Berlin", "score": 0.9993422627449036, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 0, "end": 6, "label": "CITY", "text": "Berlin", "score": 0.9994149208068848, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 11, "end": 17, "label": "CITY", "text": "Berlin", "score": 0.9993422627449036, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 0, "end": 6, "label": "CITY", "text": "Berlin", "uncovered_positions": [] }, { "start": 11, "end": 17, "label": "CITY", "text": "Berlin", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "baseline-24", "scenario": null, "predictions": [ { "start": 0, "end": 10, "label": "CITY", "text": "Washington", "score": 0.9995170831680298, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 0, "end": 10, "label": "CITY", "text": "Washington", "score": 0.9995170831680298, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 0, "end": 10, "label": "CITY", "text": "Washington", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "baseline-25", "scenario": null, "predictions": [ { "start": 0, "end": 17, "label": "PERSON", "text": "George Washington", "score": 0.9903205037117004, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 0, "end": 17, "label": "PERSON", "text": "George Washington", "score": 0.9903205037117004, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 0, "end": 17, "label": "PERSON", "text": "George Washington", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "baseline-26", "scenario": null, "predictions": [ { "start": 14, "end": 20, "label": "CITY", "text": "Jordan", "score": 0.9931551814079285, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 14, "end": 20, "label": "CITY", "text": "Jordan", "score": 0.9931551814079285, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 14, "end": 20, "label": "CITY", "text": "Jordan", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "baseline-27", "scenario": null, "predictions": [ { "start": 0, "end": 14, "label": "PERSON", "text": "Michael Jordan", "score": 0.9887990355491638, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 0, "end": 14, "label": "PERSON", "text": "Michael Jordan", "score": 0.9887990355491638, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 0, "end": 14, "label": "PERSON", "text": "Michael Jordan", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "baseline-28", "scenario": null, "predictions": [ { "start": 0, "end": 14, "label": "PERSON", "text": "Mr. Smith-\nson", "score": 0.9715586304664612, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "dehyphenation-v1" } ], "combined_predictions": [ { "start": 0, "end": 14, "label": "PERSON", "text": "Mr. Smith-\nson", "score": 0.9715586304664612, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "dehyphenation-v1" } ], "false_positives": [ [ 0, 14, "PERSON" ] ], "false_negatives": [ [ 4, 14, "PERSON" ] ], "exact_match": false, "error_code": null, "canaries": [ { "start": 4, "end": 14, "label": "PERSON", "text": "Smith-\nson", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "baseline-29", "scenario": null, "predictions": [ { "start": 0, "end": 8, "label": "CITY", "text": "Ber-\nlin", "score": 0.9711049795150757, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "dehyphenation-v1" } ], "combined_predictions": [ { "start": 0, "end": 8, "label": "CITY", "text": "Ber-\nlin", "score": 0.9711049795150757, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "dehyphenation-v1" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 0, "end": 8, "label": "CITY", "text": "Ber-\nlin", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "baseline-30", "scenario": null, "predictions": [ { "start": 0, "end": 8, "label": "PERSON", "text": "Jchn D0e", "score": 0.6583331823348999, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 0, "end": 8, "label": "PERSON", "text": "Jchn D0e", "score": 0.6583331823348999, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 0, "end": 8, "label": "PERSON", "text": "Jchn D0e", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "baseline-31", "scenario": null, "predictions": [ { "start": 11, "end": 20, "label": "ORGANIZATION", "text": "Nevv Y0rk", "score": 0.8232883810997009, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 11, "end": 20, "label": "ORGANIZATION", "text": "Nevv Y0rk", "score": 0.8232883810997009, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [ [ 11, 20, "ORGANIZATION" ] ], "false_negatives": [ [ 11, 20, "CITY" ] ], "exact_match": false, "error_code": null, "canaries": [ { "start": 11, "end": 20, "label": "CITY", "text": "Nevv Y0rk", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "baseline-32", "scenario": null, "predictions": [], "combined_predictions": [], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "baseline-33", "scenario": null, "predictions": [], "combined_predictions": [], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "baseline-34", "scenario": null, "predictions": [], "combined_predictions": [], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "baseline-35", "scenario": null, "predictions": [ { "start": 12, "end": 28, "label": "EMAIL", "text": "test@example.com", "score": 0.8803917765617371, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 12, "end": 28, "label": "EMAIL", "text": "test@example.com", "score": null, "source": "deterministic", "model_revision": "contact-patterns-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 12, "end": 28, "label": "EMAIL", "text": "test@example.com", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "baseline-36", "scenario": null, "predictions": [ { "start": 11, "end": 19, "label": "PHONE", "text": "555-0199", "score": 0.9762608408927917, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 11, "end": 19, "label": "PHONE", "text": "555-0199", "score": null, "source": "deterministic", "model_revision": "contact-patterns-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 11, "end": 19, "label": "PHONE", "text": "555-0199", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "baseline-37", "scenario": null, "predictions": [ { "start": 11, "end": 21, "label": "ORGANIZATION", "text": "Google Inc", "score": 0.7403514981269836, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 11, "end": 21, "label": "ORGANIZATION", "text": "Google Inc", "score": 0.7403514981269836, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [ [ 11, 21, "ORGANIZATION" ] ], "false_negatives": [ [ 11, 22, "ORGANIZATION" ] ], "exact_match": true, "error_code": null, "canaries": [ { "start": 11, "end": 22, "label": "ORGANIZATION", "text": "Google Inc.", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "baseline-38", "scenario": null, "predictions": [ { "start": 13, "end": 23, "label": "CITY", "text": "company.de", "score": 0.8622952103614807, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 8, "end": 23, "label": "EMAIL", "text": "info@company.de", "score": null, "source": "deterministic", "model_revision": "contact-patterns-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 8, "end": 23, "label": "EMAIL", "text": "info@company.de", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "baseline-39", "scenario": null, "predictions": [ { "start": 4, "end": 6, "label": "ORGANIZATION", "text": "UN", "score": 0.9981075525283813, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 4, "end": 6, "label": "ORGANIZATION", "text": "UN", "score": 0.9981075525283813, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 4, "end": 6, "label": "ORGANIZATION", "text": "UN", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" } ], "automated_release_eligible": false, "recognition_qualified": false }, "validation": { "schema_version": "development-candidate-outcomes/v1", "accuracy": { "documents": 10, "ner_entity_metrics": { "PERSON": { "tp": 7, "fp": 0, "fn": 0, "precision": 1.0, "recall": 1.0, "f1": 1.0 }, "CITY": { "tp": 5, "fp": 1, "fn": 0, "precision": 0.8333333333333334, "recall": 1.0, "f1": 0.9090909090909091 }, "RELIGION": { "tp": 2, "fp": 1, "fn": 0, "precision": 0.6666666666666666, "recall": 1.0, "f1": 0.8 }, "EMAIL": { "tp": 2, "fp": 0, "fn": 0, "precision": 1.0, "recall": 1.0, "f1": 1.0 }, "PHONE": { "tp": 2, "fp": 0, "fn": 0, "precision": 1.0, "recall": 1.0, "f1": 1.0 }, "ORGANIZATION": { "tp": 2, "fp": 0, "fn": 0, "precision": 1.0, "recall": 1.0, "f1": 1.0 } }, "combined_entity_metrics": { "PERSON": { "tp": 7, "fp": 0, "fn": 0, "precision": 1.0, "recall": 1.0, "f1": 1.0 }, "CITY": { "tp": 5, "fp": 1, "fn": 0, "precision": 0.8333333333333334, "recall": 1.0, "f1": 0.9090909090909091 }, "RELIGION": { "tp": 2, "fp": 1, "fn": 0, "precision": 0.6666666666666666, "recall": 1.0, "f1": 0.8 }, "EMAIL": { "tp": 2, "fp": 0, "fn": 0, "precision": 1.0, "recall": 1.0, "f1": 1.0 }, "PHONE": { "tp": 2, "fp": 0, "fn": 0, "precision": 1.0, "recall": 1.0, "f1": 1.0 }, "ORGANIZATION": { "tp": 2, "fp": 0, "fn": 0, "precision": 1.0, "recall": 1.0, "f1": 1.0 } }, "document_exact_match_rate": 0.8, "groups": { "de": { "documents": 5, "exact_matches": 4, "false_negatives": 0 }, "en": { "documents": 5, "exact_matches": 4, "false_negatives": 0 }, "ocr_noise": { "documents": 2, "exact_matches": 2, "false_negatives": 0 } }, "blocked_documents": 0, "failure_codes": {}, "automated_release_eligible": false, "uncovered_source_canaries_including_incomplete": { "PERSON": 0, "CITY": 0, "RELIGION": 0, "EMAIL": 0, "PHONE": 0, "ORGANIZATION": 0 }, "literal_presence_or_incomplete": { "PERSON": 0, "CITY": 0, "RELIGION": 0, "EMAIL": 0, "PHONE": 0, "ORGANIZATION": 0 }, "partially_covered_source_canaries_including_incomplete": { "PERSON": 0, "CITY": 0, "RELIGION": 0, "EMAIL": 0, "PHONE": 0, "ORGANIZATION": 0 }, "fully_uncovered_source_canaries_including_incomplete": { "PERSON": 0, "CITY": 0, "RELIGION": 0, "EMAIL": 0, "PHONE": 0, "ORGANIZATION": 0 }, "all_document_source_coverage": { "PERSON": { "covered": 75, "total": 75 }, "CITY": { "covered": 30, "total": 30 }, "RELIGION": { "covered": 18, "total": 18 }, "EMAIL": { "covered": 33, "total": 33 }, "PHONE": { "covered": 23, "total": 23 }, "ORGANIZATION": { "covered": 22, "total": 22 } }, "conservative_accuracy_gate_passed": true, "scope": "all source spans; incomplete processing counts as missing, not leaked output", "deterministic_replay_latency_ms": { "median": 0.10431899863760918, "max": 1.984617003472522 } }, "candidate_outcomes": { "complete_no_measured_leakage": 10, "candidate_partial_mask": 0, "candidate_fully_unmasked": 0, "blocked_conflicting_spans": 0, "blocked_model_runtime": 0, "other_incomplete": 0 }, "candidate_return_rate": 1.0, "safe_abstention_rate": 0.0, "incomplete_rate": 0.0, "no_measured_leakage_rate_among_returned": 1.0, "candidate_partial_canaries": { "PERSON": 0, "CITY": 0, "RELIGION": 0, "EMAIL": 0, "PHONE": 0, "ORGANIZATION": 0 }, "candidate_fully_unmasked_canaries": { "PERSON": 0, "CITY": 0, "RELIGION": 0, "EMAIL": 0, "PHONE": 0, "ORGANIZATION": 0 }, "returned_candidate_entity_metrics": { "PERSON": { "tp": 7, "fp": 0, "fn": 0, "precision": 1.0, "recall": 1.0, "f1": 1.0 }, "CITY": { "tp": 5, "fp": 1, "fn": 0, "precision": 0.8333333333333334, "recall": 1.0, "f1": 0.9090909090909091 }, "RELIGION": { "tp": 2, "fp": 1, "fn": 0, "precision": 0.6666666666666666, "recall": 1.0, "f1": 0.8 }, "EMAIL": { "tp": 2, "fp": 0, "fn": 0, "precision": 1.0, "recall": 1.0, "f1": 1.0 }, "PHONE": { "tp": 2, "fp": 0, "fn": 0, "precision": 1.0, "recall": 1.0, "f1": 1.0 }, "ORGANIZATION": { "tp": 2, "fp": 0, "fn": 0, "precision": 1.0, "recall": 1.0, "f1": 1.0 } }, "per_case": [ { "case_id": "validation-00", "scenario": null, "predictions": [ { "start": 0, "end": 9, "label": "PERSON", "text": "Mara Kühn", "score": 0.9990392923355103, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 23, "end": 30, "label": "ORGANIZATION", "text": "Siemens", "score": 0.9998852014541626, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 34, "end": 41, "label": "CITY", "text": "Dresden", "score": 0.9993748068809509, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 0, "end": 9, "label": "PERSON", "text": "Mara Kühn", "score": 0.9990392923355103, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 23, "end": 30, "label": "ORGANIZATION", "text": "Siemens", "score": 0.9998852014541626, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 34, "end": 41, "label": "CITY", "text": "Dresden", "score": 0.9993748068809509, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 0, "end": 9, "label": "PERSON", "text": "Mara Kühn", "uncovered_positions": [] }, { "start": 23, "end": 30, "label": "ORGANIZATION", "text": "Siemens", "uncovered_positions": [] }, { "start": 34, "end": 41, "label": "CITY", "text": "Dresden", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "validation-01", "scenario": null, "predictions": [ { "start": 0, "end": 11, "label": "PERSON", "text": "Emma Wilson", "score": 0.9975014328956604, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 22, "end": 38, "label": "ORGANIZATION", "text": "Acme Corporation", "score": 0.9995630383491516, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 42, "end": 48, "label": "CITY", "text": "London", "score": 0.9999483823776245, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 0, "end": 11, "label": "PERSON", "text": "Emma Wilson", "score": 0.9975014328956604, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 22, "end": 38, "label": "ORGANIZATION", "text": "Acme Corporation", "score": 0.9995630383491516, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 42, "end": 48, "label": "CITY", "text": "London", "score": 0.9999483823776245, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 0, "end": 11, "label": "PERSON", "text": "Emma Wilson", "uncovered_positions": [] }, { "start": 22, "end": 38, "label": "ORGANIZATION", "text": "Acme Corporation", "uncovered_positions": [] }, { "start": 42, "end": 48, "label": "CITY", "text": "London", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "validation-02", "scenario": null, "predictions": [ { "start": 8, "end": 18, "label": "RELIGION", "text": "muslimisch", "score": 0.9497504234313965, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 29, "end": 51, "label": "EMAIL", "text": "mara.kuehn@example.org", "score": 0.9375261664390564, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 53, "end": 68, "label": "PHONE", "text": "+49 351 7654321", "score": 0.9705594182014465, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 8, "end": 18, "label": "RELIGION", "text": "muslimisch", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" }, { "start": 29, "end": 51, "label": "EMAIL", "text": "mara.kuehn@example.org", "score": null, "source": "deterministic", "model_revision": "contact-patterns-v1", "normalization": "identity" }, { "start": 53, "end": 68, "label": "PHONE", "text": "+49 351 7654321", "score": null, "source": "deterministic", "model_revision": "contact-patterns-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 8, "end": 18, "label": "RELIGION", "text": "muslimisch", "uncovered_positions": [] }, { "start": 29, "end": 51, "label": "EMAIL", "text": "mara.kuehn@example.org", "uncovered_positions": [] }, { "start": 53, "end": 68, "label": "PHONE", "text": "+49 351 7654321", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "validation-03", "scenario": null, "predictions": [ { "start": 16, "end": 24, "label": "RELIGION", "text": "Hinduism", "score": 0.9999716281890869, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 34, "end": 50, "label": "EMAIL", "text": "emma@example.net", "score": 0.903052568435669, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 54, "end": 69, "label": "PHONE", "text": "+1 212 555 0188", "score": 0.951524555683136, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 16, "end": 24, "label": "RELIGION", "text": "Hinduism", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" }, { "start": 34, "end": 50, "label": "EMAIL", "text": "emma@example.net", "score": null, "source": "deterministic", "model_revision": "contact-patterns-v1", "normalization": "identity" }, { "start": 54, "end": 69, "label": "PHONE", "text": "+1 212 555 0188", "score": null, "source": "deterministic", "model_revision": "contact-patterns-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 16, "end": 24, "label": "RELIGION", "text": "Hinduism", "uncovered_positions": [] }, { "start": 34, "end": 50, "label": "EMAIL", "text": "emma@example.net", "uncovered_positions": [] }, { "start": 54, "end": 69, "label": "PHONE", "text": "+1 212 555 0188", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "validation-04", "scenario": null, "predictions": [ { "start": 0, "end": 11, "label": "PERSON", "text": "Nora Winter", "score": 0.9975367784500122, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 20, "end": 24, "label": "CITY", "text": "Köln", "score": 0.9991357922554016, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 26, "end": 37, "label": "PERSON", "text": "Nora Winter", "score": 0.99813312292099, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 0, "end": 11, "label": "PERSON", "text": "Nora Winter", "score": 0.9975367784500122, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 20, "end": 24, "label": "CITY", "text": "Köln", "score": 0.9991357922554016, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 26, "end": 37, "label": "PERSON", "text": "Nora Winter", "score": 0.99813312292099, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 0, "end": 11, "label": "PERSON", "text": "Nora Winter", "uncovered_positions": [] }, { "start": 20, "end": 24, "label": "CITY", "text": "Köln", "uncovered_positions": [] }, { "start": 26, "end": 37, "label": "PERSON", "text": "Nora Winter", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "validation-05", "scenario": null, "predictions": [ { "start": 0, "end": 6, "label": "CITY", "text": "Jordan", "score": 0.9668874144554138, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 23, "end": 36, "label": "PERSON", "text": "Jordan Miller", "score": 0.982417643070221, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 0, "end": 6, "label": "CITY", "text": "Jordan", "score": 0.9668874144554138, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 23, "end": 36, "label": "PERSON", "text": "Jordan Miller", "score": 0.982417643070221, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [ [ 0, 6, "CITY" ] ], "false_negatives": [], "exact_match": false, "error_code": null, "canaries": [ { "start": 23, "end": 36, "label": "PERSON", "text": "Jordan Miller", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "validation-06", "scenario": null, "predictions": [ { "start": 0, "end": 8, "label": "RELIGION", "text": "Religion", "score": 0.6538923978805542, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 0, "end": 8, "label": "RELIGION", "text": "Religion", "score": 0.6538923978805542, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [ [ 0, 8, "RELIGION" ] ], "false_negatives": [], "exact_match": false, "error_code": null, "canaries": [], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "validation-07", "scenario": null, "predictions": [], "combined_predictions": [], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "validation-08", "scenario": null, "predictions": [ { "start": 5, "end": 23, "label": "PERSON", "text": "Eva-Luise Schröder", "score": 0.986770510673523, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "dehyphenation-v1" }, { "start": 33, "end": 42, "label": "CITY", "text": "Ham-\nburg", "score": 0.9998313188552856, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "dehyphenation-v1" } ], "combined_predictions": [ { "start": 5, "end": 23, "label": "PERSON", "text": "Eva-Luise Schröder", "score": 0.986770510673523, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "dehyphenation-v1" }, { "start": 33, "end": 42, "label": "CITY", "text": "Ham-\nburg", "score": 0.9998313188552856, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "dehyphenation-v1" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 5, "end": 23, "label": "PERSON", "text": "Eva-Luise Schröder", "uncovered_positions": [] }, { "start": 33, "end": 42, "label": "CITY", "text": "Ham-\nburg", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "validation-09", "scenario": null, "predictions": [ { "start": 0, "end": 10, "label": "PERSON", "text": "M4ry Br0wn", "score": 0.9766657948493958, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 20, "end": 26, "label": "CITY", "text": "L0ndon", "score": 0.996916651725769, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 0, "end": 10, "label": "PERSON", "text": "M4ry Br0wn", "score": 0.9766657948493958, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 20, "end": 26, "label": "CITY", "text": "L0ndon", "score": 0.996916651725769, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 0, "end": 10, "label": "PERSON", "text": "M4ry Br0wn", "uncovered_positions": [] }, { "start": 20, "end": 26, "label": "CITY", "text": "L0ndon", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" } ], "automated_release_eligible": false, "recognition_qualified": false }, "development03c": { "schema_version": "development-candidate-outcomes/v1", "accuracy": { "documents": 120, "ner_entity_metrics": { "PERSON": { "tp": 132, "fp": 12, "fn": 0, "precision": 0.9166666666666666, "recall": 1.0, "f1": 0.9565217391304348 }, "CITY": { "tp": 84, "fp": 13, "fn": 0, "precision": 0.865979381443299, "recall": 1.0, "f1": 0.9281767955801106 }, "RELIGION": { "tp": 34, "fp": 0, "fn": 2, "precision": 1.0, "recall": 0.9444444444444444, "f1": 0.9714285714285714 }, "EMAIL": { "tp": 23, "fp": 0, "fn": 1, "precision": 1.0, "recall": 0.9583333333333334, "f1": 0.9787234042553191 }, "PHONE": { "tp": 24, "fp": 0, "fn": 0, "precision": 1.0, "recall": 1.0, "f1": 1.0 }, "ORGANIZATION": { "tp": 84, "fp": 12, "fn": 0, "precision": 0.875, "recall": 1.0, "f1": 0.9333333333333333 } }, "combined_entity_metrics": { "PERSON": { "tp": 132, "fp": 12, "fn": 0, "precision": 0.9166666666666666, "recall": 1.0, "f1": 0.9565217391304348 }, "CITY": { "tp": 84, "fp": 12, "fn": 0, "precision": 0.875, "recall": 1.0, "f1": 0.9333333333333333 }, "RELIGION": { "tp": 36, "fp": 12, "fn": 0, "precision": 0.75, "recall": 1.0, "f1": 0.8571428571428571 }, "EMAIL": { "tp": 24, "fp": 0, "fn": 0, "precision": 1.0, "recall": 1.0, "f1": 1.0 }, "PHONE": { "tp": 24, "fp": 0, "fn": 0, "precision": 1.0, "recall": 1.0, "f1": 1.0 }, "ORGANIZATION": { "tp": 84, "fp": 12, "fn": 0, "precision": 0.875, "recall": 1.0, "f1": 0.9333333333333333 } }, "document_exact_match_rate": 0.9, "groups": { "de": { "documents": 60, "exact_matches": 60, "false_negatives": 0 }, "en": { "documents": 60, "exact_matches": 48, "false_negatives": 0 }, "ocr_noise": { "documents": 24, "exact_matches": 24, "false_negatives": 0 } }, "blocked_documents": 0, "failure_codes": {}, "automated_release_eligible": false, "uncovered_source_canaries_including_incomplete": { "PERSON": 0, "CITY": 0, "RELIGION": 0, "EMAIL": 0, "PHONE": 0, "ORGANIZATION": 0 }, "literal_presence_or_incomplete": { "PERSON": 0, "CITY": 0, "RELIGION": 0, "EMAIL": 0, "PHONE": 0, "ORGANIZATION": 0 }, "partially_covered_source_canaries_including_incomplete": { "PERSON": 0, "CITY": 0, "RELIGION": 0, "EMAIL": 0, "PHONE": 0, "ORGANIZATION": 0 }, "fully_uncovered_source_canaries_including_incomplete": { "PERSON": 0, "CITY": 0, "RELIGION": 0, "EMAIL": 0, "PHONE": 0, "ORGANIZATION": 0 }, "all_document_source_coverage": { "PERSON": { "covered": 1727, "total": 1727 }, "CITY": { "covered": 588, "total": 588 }, "RELIGION": { "covered": 358, "total": 358 }, "EMAIL": { "covered": 648, "total": 648 }, "PHONE": { "covered": 264, "total": 264 }, "ORGANIZATION": { "covered": 1631, "total": 1631 } }, "conservative_accuracy_gate_passed": true, "scope": "all source spans; incomplete processing counts as missing, not leaked output", "deterministic_replay_latency_ms": { "median": 0.16394650083384477, "max": 3.3133899996755645 } }, "candidate_outcomes": { "complete_no_measured_leakage": 120, "candidate_partial_mask": 0, "candidate_fully_unmasked": 0, "blocked_conflicting_spans": 0, "blocked_model_runtime": 0, "other_incomplete": 0 }, "candidate_return_rate": 1.0, "safe_abstention_rate": 0.0, "incomplete_rate": 0.0, "no_measured_leakage_rate_among_returned": 1.0, "candidate_partial_canaries": { "PERSON": 0, "CITY": 0, "RELIGION": 0, "EMAIL": 0, "PHONE": 0, "ORGANIZATION": 0 }, "candidate_fully_unmasked_canaries": { "PERSON": 0, "CITY": 0, "RELIGION": 0, "EMAIL": 0, "PHONE": 0, "ORGANIZATION": 0 }, "returned_candidate_entity_metrics": { "PERSON": { "tp": 132, "fp": 12, "fn": 0, "precision": 0.9166666666666666, "recall": 1.0, "f1": 0.9565217391304348 }, "CITY": { "tp": 84, "fp": 12, "fn": 0, "precision": 0.875, "recall": 1.0, "f1": 0.9333333333333333 }, "RELIGION": { "tp": 36, "fp": 12, "fn": 0, "precision": 0.75, "recall": 1.0, "f1": 0.8571428571428571 }, "EMAIL": { "tp": 24, "fp": 0, "fn": 0, "precision": 1.0, "recall": 1.0, "f1": 1.0 }, "PHONE": { "tp": 24, "fp": 0, "fn": 0, "precision": 1.0, "recall": 1.0, "f1": 1.0 }, "ORGANIZATION": { "tp": 84, "fp": 12, "fn": 0, "precision": 0.875, "recall": 1.0, "f1": 0.9333333333333333 } }, "per_case": [ { "case_id": "holdout03c-000", "scenario": null, "predictions": [ { "start": 10, "end": 25, "label": "PERSON", "text": "Saskia Dornfeld", "score": 0.9995438456535339, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 39, "end": 63, "label": "ORGANIZATION", "text": "Aster Research Institute", "score": 0.9974908828735352, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 67, "end": 76, "label": "CITY", "text": "Osnabrück", "score": 0.999496579170227, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 10, "end": 25, "label": "PERSON", "text": "Saskia Dornfeld", "score": 0.9995438456535339, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 39, "end": 63, "label": "ORGANIZATION", "text": "Aster Research Institute", "score": 0.9974908828735352, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 67, "end": 76, "label": "CITY", "text": "Osnabrück", "score": 0.999496579170227, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 10, "end": 25, "label": "PERSON", "text": "Saskia Dornfeld", "uncovered_positions": [] }, { "start": 39, "end": 63, "label": "ORGANIZATION", "text": "Aster Research Institute", "uncovered_positions": [] }, { "start": 67, "end": 76, "label": "CITY", "text": "Osnabrück", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-001", "scenario": null, "predictions": [ { "start": 12, "end": 27, "label": "PERSON", "text": "Saskia Dornfeld", "score": 0.9990620017051697, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 39, "end": 63, "label": "ORGANIZATION", "text": "Aster Research Institute", "score": 0.997700035572052, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 69, "end": 78, "label": "CITY", "text": "Osnabrück", "score": 0.9995871186256409, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 12, "end": 27, "label": "PERSON", "text": "Saskia Dornfeld", "score": 0.9990620017051697, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 39, "end": 63, "label": "ORGANIZATION", "text": "Aster Research Institute", "score": 0.997700035572052, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 69, "end": 78, "label": "CITY", "text": "Osnabrück", "score": 0.9995871186256409, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 12, "end": 27, "label": "PERSON", "text": "Saskia Dornfeld", "uncovered_positions": [] }, { "start": 39, "end": 63, "label": "ORGANIZATION", "text": "Aster Research Institute", "uncovered_positions": [] }, { "start": 69, "end": 78, "label": "CITY", "text": "Osnabrück", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-002", "scenario": null, "predictions": [ { "start": 0, "end": 15, "label": "PERSON", "text": "Saskia Dornfeld", "score": 0.9997074007987976, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 22, "end": 33, "label": "RELIGION", "text": "sikhistisch", "score": 0.9321887493133545, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 61, "end": 91, "label": "EMAIL", "text": "case101.review@example.invalid", "score": 0.9007220268249512, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 102, "end": 116, "label": "PHONE", "text": "+49 541 730101", "score": 0.9946468472480774, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 0, "end": 15, "label": "PERSON", "text": "Saskia Dornfeld", "score": 0.9997074007987976, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 22, "end": 33, "label": "RELIGION", "text": "sikhistisch", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" }, { "start": 61, "end": 91, "label": "EMAIL", "text": "case101.review@example.invalid", "score": null, "source": "deterministic", "model_revision": "contact-patterns-v1", "normalization": "identity" }, { "start": 102, "end": 116, "label": "PHONE", "text": "+49 541 730101", "score": null, "source": "deterministic", "model_revision": "contact-patterns-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 0, "end": 15, "label": "PERSON", "text": "Saskia Dornfeld", "uncovered_positions": [] }, { "start": 22, "end": 33, "label": "RELIGION", "text": "sikhistisch", "uncovered_positions": [] }, { "start": 61, "end": 91, "label": "EMAIL", "text": "case101.review@example.invalid", "uncovered_positions": [] }, { "start": 102, "end": 116, "label": "PHONE", "text": "+49 541 730101", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-003", "scenario": null, "predictions": [ { "start": 0, "end": 15, "label": "PERSON", "text": "Saskia Dornfeld", "score": 0.9992882609367371, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 33, "end": 40, "label": "RELIGION", "text": "Sikhism", "score": 0.9966249465942383, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 66, "end": 96, "label": "EMAIL", "text": "case101.review@example.invalid", "score": 0.908982515335083, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 105, "end": 119, "label": "PHONE", "text": "+49 541 730101", "score": 0.9701294898986816, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 0, "end": 15, "label": "PERSON", "text": "Saskia Dornfeld", "score": 0.9992882609367371, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 33, "end": 40, "label": "RELIGION", "text": "Sikhism", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" }, { "start": 66, "end": 96, "label": "EMAIL", "text": "case101.review@example.invalid", "score": null, "source": "deterministic", "model_revision": "contact-patterns-v1", "normalization": "identity" }, { "start": 105, "end": 119, "label": "PHONE", "text": "+49 541 730101", "score": null, "source": "deterministic", "model_revision": "contact-patterns-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 0, "end": 15, "label": "PERSON", "text": "Saskia Dornfeld", "uncovered_positions": [] }, { "start": 33, "end": 40, "label": "RELIGION", "text": "Sikhism", "uncovered_positions": [] }, { "start": 66, "end": 96, "label": "EMAIL", "text": "case101.review@example.invalid", "uncovered_positions": [] }, { "start": 105, "end": 119, "label": "PHONE", "text": "+49 541 730101", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-004", "scenario": null, "predictions": [ { "start": 0, "end": 15, "label": "PERSON", "text": "Saskia Dornfeld", "score": 0.9996077418327332, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 24, "end": 33, "label": "CITY", "text": "Osnabrück", "score": 0.998096764087677, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 36, "end": 51, "label": "PERSON", "text": "Saskia Dornfeld", "score": 0.9998266100883484, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 77, "end": 101, "label": "ORGANIZATION", "text": "Aster Research Institute", "score": 0.9952711462974548, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 0, "end": 15, "label": "PERSON", "text": "Saskia Dornfeld", "score": 0.9996077418327332, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 24, "end": 33, "label": "CITY", "text": "Osnabrück", "score": 0.998096764087677, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 36, "end": 51, "label": "PERSON", "text": "Saskia Dornfeld", "score": 0.9998266100883484, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 77, "end": 101, "label": "ORGANIZATION", "text": "Aster Research Institute", "score": 0.9952711462974548, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 0, "end": 15, "label": "PERSON", "text": "Saskia Dornfeld", "uncovered_positions": [] }, { "start": 24, "end": 33, "label": "CITY", "text": "Osnabrück", "uncovered_positions": [] }, { "start": 36, "end": 51, "label": "PERSON", "text": "Saskia Dornfeld", "uncovered_positions": [] }, { "start": 77, "end": 101, "label": "ORGANIZATION", "text": "Aster Research Institute", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-005", "scenario": null, "predictions": [ { "start": 3, "end": 12, "label": "CITY", "text": "Osnabrück", "score": 0.9999159574508667, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 14, "end": 29, "label": "PERSON", "text": "Saskia Dornfeld", "score": 0.9998455047607422, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 41, "end": 65, "label": "ORGANIZATION", "text": "Aster Research Institute", "score": 0.9969690442085266, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 86, "end": 101, "label": "PERSON", "text": "Saskia Dornfeld", "score": 0.9999145269393921, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 114, "end": 123, "label": "CITY", "text": "Osnabrück", "score": 0.999828577041626, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 3, "end": 12, "label": "CITY", "text": "Osnabrück", "score": 0.9999159574508667, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 14, "end": 29, "label": "PERSON", "text": "Saskia Dornfeld", "score": 0.9998455047607422, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 41, "end": 65, "label": "ORGANIZATION", "text": "Aster Research Institute", "score": 0.9969690442085266, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 86, "end": 101, "label": "PERSON", "text": "Saskia Dornfeld", "score": 0.9999145269393921, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 114, "end": 123, "label": "CITY", "text": "Osnabrück", "score": 0.999828577041626, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 3, "end": 12, "label": "CITY", "text": "Osnabrück", "uncovered_positions": [] }, { "start": 14, "end": 29, "label": "PERSON", "text": "Saskia Dornfeld", "uncovered_positions": [] }, { "start": 41, "end": 65, "label": "ORGANIZATION", "text": "Aster Research Institute", "uncovered_positions": [] }, { "start": 86, "end": 101, "label": "PERSON", "text": "Saskia Dornfeld", "uncovered_positions": [] }, { "start": 114, "end": 123, "label": "CITY", "text": "Osnabrück", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-006", "scenario": null, "predictions": [ { "start": 6, "end": 22, "label": "PERSON", "text": "Saskia-\nDornfeld", "score": 0.9996308088302612, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "dehyphenation-v1" }, { "start": 28, "end": 39, "label": "CITY", "text": "Osna-\nbrück", "score": 0.9999524354934692, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "dehyphenation-v1" }, { "start": 53, "end": 77, "label": "ORGANIZATION", "text": "Aster Research Institute", "score": 0.9986594915390015, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "dehyphenation-v1" } ], "combined_predictions": [ { "start": 6, "end": 22, "label": "PERSON", "text": "Saskia-\nDornfeld", "score": 0.9996308088302612, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "dehyphenation-v1" }, { "start": 28, "end": 39, "label": "CITY", "text": "Osna-\nbrück", "score": 0.9999524354934692, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "dehyphenation-v1" }, { "start": 53, "end": 77, "label": "ORGANIZATION", "text": "Aster Research Institute", "score": 0.9986594915390015, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "dehyphenation-v1" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 6, "end": 22, "label": "PERSON", "text": "Saskia-\nDornfeld", "uncovered_positions": [] }, { "start": 28, "end": 39, "label": "CITY", "text": "Osna-\nbrück", "uncovered_positions": [] }, { "start": 53, "end": 77, "label": "ORGANIZATION", "text": "Aster Research Institute", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-007", "scenario": null, "predictions": [ { "start": 16, "end": 31, "label": "PERSON", "text": "Sask1a D0rnfeld", "score": 0.9964213371276855, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 43, "end": 52, "label": "CITY", "text": "Osn4brück", "score": 0.999607503414154, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 66, "end": 90, "label": "ORGANIZATION", "text": "Aster Research Institute", "score": 0.9983013868331909, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 16, "end": 31, "label": "PERSON", "text": "Sask1a D0rnfeld", "score": 0.9964213371276855, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 43, "end": 52, "label": "CITY", "text": "Osn4brück", "score": 0.999607503414154, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 66, "end": 90, "label": "ORGANIZATION", "text": "Aster Research Institute", "score": 0.9983013868331909, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 16, "end": 31, "label": "PERSON", "text": "Sask1a D0rnfeld", "uncovered_positions": [] }, { "start": 43, "end": 52, "label": "CITY", "text": "Osn4brück", "uncovered_positions": [] }, { "start": 66, "end": 90, "label": "ORGANIZATION", "text": "Aster Research Institute", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-008", "scenario": null, "predictions": [ { "start": 10, "end": 21, "label": "RELIGION", "text": "sikhistisch", "score": 0.820220947265625, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 48, "end": 72, "label": "ORGANIZATION", "text": "Aster Research Institute", "score": 0.983711302280426, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 86, "end": 101, "label": "PERSON", "text": "Saskia Dornfeld", "score": 0.9986269474029541, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 10, "end": 21, "label": "RELIGION", "text": "sikhistisch", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" }, { "start": 48, "end": 72, "label": "ORGANIZATION", "text": "Aster Research Institute", "score": 0.983711302280426, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 86, "end": 101, "label": "PERSON", "text": "Saskia Dornfeld", "score": 0.9986269474029541, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 10, "end": 21, "label": "RELIGION", "text": "sikhistisch", "uncovered_positions": [] }, { "start": 48, "end": 72, "label": "ORGANIZATION", "text": "Aster Research Institute", "uncovered_positions": [] }, { "start": 86, "end": 101, "label": "PERSON", "text": "Saskia Dornfeld", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-009", "scenario": null, "predictions": [ { "start": 101, "end": 107, "label": "PERSON", "text": "person", "score": 0.7040723562240601, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 109, "end": 114, "label": "CITY", "text": "place", "score": 0.7776683568954468, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 116, "end": 128, "label": "ORGANIZATION", "text": "organization", "score": 0.8319209218025208, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 40, "end": 64, "label": "RELIGION", "text": "no religious affiliation", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" }, { "start": 101, "end": 107, "label": "PERSON", "text": "person", "score": 0.7040723562240601, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 109, "end": 114, "label": "CITY", "text": "place", "score": 0.7776683568954468, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 116, "end": 128, "label": "ORGANIZATION", "text": "organization", "score": 0.8319209218025208, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [ [ 40, 64, "RELIGION" ], [ 101, 107, "PERSON" ], [ 109, 114, "CITY" ], [ 116, 128, "ORGANIZATION" ] ], "false_negatives": [], "exact_match": false, "error_code": null, "canaries": [], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-010", "scenario": null, "predictions": [ { "start": 10, "end": 21, "label": "PERSON", "text": "Emil Farber", "score": 0.9987824559211731, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 35, "end": 56, "label": "ORGANIZATION", "text": "Northbridge Analytics", "score": 0.9969447255134583, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 60, "end": 67, "label": "CITY", "text": "Potsdam", "score": 0.9994106292724609, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 10, "end": 21, "label": "PERSON", "text": "Emil Farber", "score": 0.9987824559211731, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 35, "end": 56, "label": "ORGANIZATION", "text": "Northbridge Analytics", "score": 0.9969447255134583, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 60, "end": 67, "label": "CITY", "text": "Potsdam", "score": 0.9994106292724609, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 10, "end": 21, "label": "PERSON", "text": "Emil Farber", "uncovered_positions": [] }, { "start": 35, "end": 56, "label": "ORGANIZATION", "text": "Northbridge Analytics", "uncovered_positions": [] }, { "start": 60, "end": 67, "label": "CITY", "text": "Potsdam", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-011", "scenario": null, "predictions": [ { "start": 12, "end": 23, "label": "PERSON", "text": "Emil Farber", "score": 0.9989939332008362, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 35, "end": 56, "label": "ORGANIZATION", "text": "Northbridge Analytics", "score": 0.9984185695648193, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 62, "end": 69, "label": "CITY", "text": "Potsdam", "score": 0.9994024038314819, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 12, "end": 23, "label": "PERSON", "text": "Emil Farber", "score": 0.9989939332008362, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 35, "end": 56, "label": "ORGANIZATION", "text": "Northbridge Analytics", "score": 0.9984185695648193, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 62, "end": 69, "label": "CITY", "text": "Potsdam", "score": 0.9994024038314819, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 12, "end": 23, "label": "PERSON", "text": "Emil Farber", "uncovered_positions": [] }, { "start": 35, "end": 56, "label": "ORGANIZATION", "text": "Northbridge Analytics", "uncovered_positions": [] }, { "start": 62, "end": 69, "label": "CITY", "text": "Potsdam", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-012", "scenario": null, "predictions": [ { "start": 0, "end": 11, "label": "PERSON", "text": "Emil Farber", "score": 0.9991351962089539, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 18, "end": 26, "label": "RELIGION", "text": "orthodox", "score": 0.9795414805412292, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 54, "end": 84, "label": "EMAIL", "text": "case102.review@example.invalid", "score": 0.9129301309585571, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 95, "end": 109, "label": "PHONE", "text": "+49 541 730102", "score": 0.9935047626495361, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 0, "end": 11, "label": "PERSON", "text": "Emil Farber", "score": 0.9991351962089539, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 18, "end": 26, "label": "RELIGION", "text": "orthodox", "score": 0.9795414805412292, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 54, "end": 84, "label": "EMAIL", "text": "case102.review@example.invalid", "score": null, "source": "deterministic", "model_revision": "contact-patterns-v1", "normalization": "identity" }, { "start": 95, "end": 109, "label": "PHONE", "text": "+49 541 730102", "score": null, "source": "deterministic", "model_revision": "contact-patterns-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 0, "end": 11, "label": "PERSON", "text": "Emil Farber", "uncovered_positions": [] }, { "start": 18, "end": 26, "label": "RELIGION", "text": "orthodox", "uncovered_positions": [] }, { "start": 54, "end": 84, "label": "EMAIL", "text": "case102.review@example.invalid", "uncovered_positions": [] }, { "start": 95, "end": 109, "label": "PHONE", "text": "+49 541 730102", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-013", "scenario": null, "predictions": [ { "start": 0, "end": 11, "label": "PERSON", "text": "Emil Farber", "score": 0.9992079138755798, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 29, "end": 50, "label": "RELIGION", "text": "Orthodox Christianity", "score": 0.9964632391929626, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 76, "end": 106, "label": "EMAIL", "text": "case102.review@example.invalid", "score": 0.9135122895240784, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 115, "end": 129, "label": "PHONE", "text": "+49 541 730102", "score": 0.9687923192977905, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 0, "end": 11, "label": "PERSON", "text": "Emil Farber", "score": 0.9992079138755798, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 29, "end": 50, "label": "RELIGION", "text": "Orthodox Christianity", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" }, { "start": 76, "end": 106, "label": "EMAIL", "text": "case102.review@example.invalid", "score": null, "source": "deterministic", "model_revision": "contact-patterns-v1", "normalization": "identity" }, { "start": 115, "end": 129, "label": "PHONE", "text": "+49 541 730102", "score": null, "source": "deterministic", "model_revision": "contact-patterns-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 0, "end": 11, "label": "PERSON", "text": "Emil Farber", "uncovered_positions": [] }, { "start": 29, "end": 50, "label": "RELIGION", "text": "Orthodox Christianity", "uncovered_positions": [] }, { "start": 76, "end": 106, "label": "EMAIL", "text": "case102.review@example.invalid", "uncovered_positions": [] }, { "start": 115, "end": 129, "label": "PHONE", "text": "+49 541 730102", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-014", "scenario": null, "predictions": [ { "start": 0, "end": 11, "label": "PERSON", "text": "Emil Farber", "score": 0.9997215867042542, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 20, "end": 27, "label": "CITY", "text": "Potsdam", "score": 0.999298095703125, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 30, "end": 41, "label": "PERSON", "text": "Emil Farber", "score": 0.9996708631515503, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 67, "end": 88, "label": "ORGANIZATION", "text": "Northbridge Analytics", "score": 0.9961026906967163, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 0, "end": 11, "label": "PERSON", "text": "Emil Farber", "score": 0.9997215867042542, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 20, "end": 27, "label": "CITY", "text": "Potsdam", "score": 0.999298095703125, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 30, "end": 41, "label": "PERSON", "text": "Emil Farber", "score": 0.9996708631515503, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 67, "end": 88, "label": "ORGANIZATION", "text": "Northbridge Analytics", "score": 0.9961026906967163, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 0, "end": 11, "label": "PERSON", "text": "Emil Farber", "uncovered_positions": [] }, { "start": 20, "end": 27, "label": "CITY", "text": "Potsdam", "uncovered_positions": [] }, { "start": 30, "end": 41, "label": "PERSON", "text": "Emil Farber", "uncovered_positions": [] }, { "start": 67, "end": 88, "label": "ORGANIZATION", "text": "Northbridge Analytics", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-015", "scenario": null, "predictions": [ { "start": 3, "end": 10, "label": "CITY", "text": "Potsdam", "score": 0.9998461008071899, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 12, "end": 23, "label": "PERSON", "text": "Emil Farber", "score": 0.9998921155929565, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 35, "end": 56, "label": "ORGANIZATION", "text": "Northbridge Analytics", "score": 0.9982470273971558, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 77, "end": 88, "label": "PERSON", "text": "Emil Farber", "score": 0.9998248219490051, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 101, "end": 108, "label": "CITY", "text": "Potsdam", "score": 0.9998860359191895, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 3, "end": 10, "label": "CITY", "text": "Potsdam", "score": 0.9998461008071899, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 12, "end": 23, "label": "PERSON", "text": "Emil Farber", "score": 0.9998921155929565, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 35, "end": 56, "label": "ORGANIZATION", "text": "Northbridge Analytics", "score": 0.9982470273971558, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 77, "end": 88, "label": "PERSON", "text": "Emil Farber", "score": 0.9998248219490051, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 101, "end": 108, "label": "CITY", "text": "Potsdam", "score": 0.9998860359191895, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 3, "end": 10, "label": "CITY", "text": "Potsdam", "uncovered_positions": [] }, { "start": 12, "end": 23, "label": "PERSON", "text": "Emil Farber", "uncovered_positions": [] }, { "start": 35, "end": 56, "label": "ORGANIZATION", "text": "Northbridge Analytics", "uncovered_positions": [] }, { "start": 77, "end": 88, "label": "PERSON", "text": "Emil Farber", "uncovered_positions": [] }, { "start": 101, "end": 108, "label": "CITY", "text": "Potsdam", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-016", "scenario": null, "predictions": [ { "start": 6, "end": 18, "label": "PERSON", "text": "Emil-\nFarber", "score": 0.9998793601989746, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "dehyphenation-v1" }, { "start": 24, "end": 33, "label": "CITY", "text": "Pot-\nsdam", "score": 0.9999624490737915, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "dehyphenation-v1" }, { "start": 47, "end": 68, "label": "ORGANIZATION", "text": "Northbridge Analytics", "score": 0.997786283493042, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "dehyphenation-v1" } ], "combined_predictions": [ { "start": 6, "end": 18, "label": "PERSON", "text": "Emil-\nFarber", "score": 0.9998793601989746, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "dehyphenation-v1" }, { "start": 24, "end": 33, "label": "CITY", "text": "Pot-\nsdam", "score": 0.9999624490737915, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "dehyphenation-v1" }, { "start": 47, "end": 68, "label": "ORGANIZATION", "text": "Northbridge Analytics", "score": 0.997786283493042, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "dehyphenation-v1" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 6, "end": 18, "label": "PERSON", "text": "Emil-\nFarber", "uncovered_positions": [] }, { "start": 24, "end": 33, "label": "CITY", "text": "Pot-\nsdam", "uncovered_positions": [] }, { "start": 47, "end": 68, "label": "ORGANIZATION", "text": "Northbridge Analytics", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-017", "scenario": null, "predictions": [ { "start": 16, "end": 27, "label": "PERSON", "text": "Em1l Farber", "score": 0.9948354959487915, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 39, "end": 46, "label": "CITY", "text": "P0tsd4m", "score": 0.9718403220176697, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 60, "end": 81, "label": "ORGANIZATION", "text": "Northbridge Analytics", "score": 0.9991218447685242, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 16, "end": 27, "label": "PERSON", "text": "Em1l Farber", "score": 0.9948354959487915, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 39, "end": 46, "label": "CITY", "text": "P0tsd4m", "score": 0.9718403220176697, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 60, "end": 81, "label": "ORGANIZATION", "text": "Northbridge Analytics", "score": 0.9991218447685242, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 16, "end": 27, "label": "PERSON", "text": "Em1l Farber", "uncovered_positions": [] }, { "start": 39, "end": 46, "label": "CITY", "text": "P0tsd4m", "uncovered_positions": [] }, { "start": 60, "end": 81, "label": "ORGANIZATION", "text": "Northbridge Analytics", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-018", "scenario": null, "predictions": [ { "start": 10, "end": 18, "label": "RELIGION", "text": "orthodox", "score": 0.8374165296554565, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 45, "end": 66, "label": "ORGANIZATION", "text": "Northbridge Analytics", "score": 0.9889349341392517, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 80, "end": 91, "label": "PERSON", "text": "Emil Farber", "score": 0.9984270334243774, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 10, "end": 18, "label": "RELIGION", "text": "orthodox", "score": 0.8374165296554565, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 45, "end": 66, "label": "ORGANIZATION", "text": "Northbridge Analytics", "score": 0.9889349341392517, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 80, "end": 91, "label": "PERSON", "text": "Emil Farber", "score": 0.9984270334243774, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 10, "end": 18, "label": "RELIGION", "text": "orthodox", "uncovered_positions": [] }, { "start": 45, "end": 66, "label": "ORGANIZATION", "text": "Northbridge Analytics", "uncovered_positions": [] }, { "start": 80, "end": 91, "label": "PERSON", "text": "Emil Farber", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-019", "scenario": null, "predictions": [ { "start": 101, "end": 107, "label": "PERSON", "text": "person", "score": 0.7027997374534607, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 109, "end": 114, "label": "CITY", "text": "place", "score": 0.7789004445075989, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 116, "end": 128, "label": "ORGANIZATION", "text": "organization", "score": 0.8365603685379028, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 40, "end": 64, "label": "RELIGION", "text": "no religious affiliation", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" }, { "start": 101, "end": 107, "label": "PERSON", "text": "person", "score": 0.7027997374534607, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 109, "end": 114, "label": "CITY", "text": "place", "score": 0.7789004445075989, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 116, "end": 128, "label": "ORGANIZATION", "text": "organization", "score": 0.8365603685379028, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [ [ 40, 64, "RELIGION" ], [ 101, 107, "PERSON" ], [ 109, 114, "CITY" ], [ 116, 128, "ORGANIZATION" ] ], "false_negatives": [], "exact_match": false, "error_code": null, "canaries": [], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-020", "scenario": null, "predictions": [ { "start": 10, "end": 25, "label": "PERSON", "text": "Cora Lindenfels", "score": 0.9973998069763184, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 39, "end": 62, "label": "ORGANIZATION", "text": "Amber Health Foundation", "score": 0.9973830580711365, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 66, "end": 72, "label": "CITY", "text": "Dundee", "score": 0.9992653727531433, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 10, "end": 25, "label": "PERSON", "text": "Cora Lindenfels", "score": 0.9973998069763184, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 39, "end": 62, "label": "ORGANIZATION", "text": "Amber Health Foundation", "score": 0.9973830580711365, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 66, "end": 72, "label": "CITY", "text": "Dundee", "score": 0.9992653727531433, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 10, "end": 25, "label": "PERSON", "text": "Cora Lindenfels", "uncovered_positions": [] }, { "start": 39, "end": 62, "label": "ORGANIZATION", "text": "Amber Health Foundation", "uncovered_positions": [] }, { "start": 66, "end": 72, "label": "CITY", "text": "Dundee", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-021", "scenario": null, "predictions": [ { "start": 12, "end": 27, "label": "PERSON", "text": "Cora Lindenfels", "score": 0.997750461101532, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 39, "end": 62, "label": "ORGANIZATION", "text": "Amber Health Foundation", "score": 0.9969223141670227, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 68, "end": 74, "label": "CITY", "text": "Dundee", "score": 0.9994328618049622, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 12, "end": 27, "label": "PERSON", "text": "Cora Lindenfels", "score": 0.997750461101532, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 39, "end": 62, "label": "ORGANIZATION", "text": "Amber Health Foundation", "score": 0.9969223141670227, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 68, "end": 74, "label": "CITY", "text": "Dundee", "score": 0.9994328618049622, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 12, "end": 27, "label": "PERSON", "text": "Cora Lindenfels", "uncovered_positions": [] }, { "start": 39, "end": 62, "label": "ORGANIZATION", "text": "Amber Health Foundation", "uncovered_positions": [] }, { "start": 68, "end": 74, "label": "CITY", "text": "Dundee", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-022", "scenario": null, "predictions": [ { "start": 0, "end": 15, "label": "PERSON", "text": "Cora Lindenfels", "score": 0.9937719702720642, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 22, "end": 33, "label": "RELIGION", "text": "jainistisch", "score": 0.9889543056488037, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 61, "end": 91, "label": "EMAIL", "text": "case103.review@example.invalid", "score": 0.9086615443229675, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 102, "end": 116, "label": "PHONE", "text": "+49 541 730103", "score": 0.9945864677429199, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 0, "end": 15, "label": "PERSON", "text": "Cora Lindenfels", "score": 0.9937719702720642, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 22, "end": 33, "label": "RELIGION", "text": "jainistisch", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" }, { "start": 61, "end": 91, "label": "EMAIL", "text": "case103.review@example.invalid", "score": null, "source": "deterministic", "model_revision": "contact-patterns-v1", "normalization": "identity" }, { "start": 102, "end": 116, "label": "PHONE", "text": "+49 541 730103", "score": null, "source": "deterministic", "model_revision": "contact-patterns-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 0, "end": 15, "label": "PERSON", "text": "Cora Lindenfels", "uncovered_positions": [] }, { "start": 22, "end": 33, "label": "RELIGION", "text": "jainistisch", "uncovered_positions": [] }, { "start": 61, "end": 91, "label": "EMAIL", "text": "case103.review@example.invalid", "uncovered_positions": [] }, { "start": 102, "end": 116, "label": "PHONE", "text": "+49 541 730103", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-023", "scenario": null, "predictions": [ { "start": 0, "end": 15, "label": "PERSON", "text": "Cora Lindenfels", "score": 0.9916738867759705, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 33, "end": 40, "label": "RELIGION", "text": "Jainism", "score": 0.9980966448783875, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 66, "end": 96, "label": "EMAIL", "text": "case103.review@example.invalid", "score": 0.9247252345085144, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 105, "end": 119, "label": "PHONE", "text": "+49 541 730103", "score": 0.9735237956047058, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 0, "end": 15, "label": "PERSON", "text": "Cora Lindenfels", "score": 0.9916738867759705, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 33, "end": 40, "label": "RELIGION", "text": "Jainism", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" }, { "start": 66, "end": 96, "label": "EMAIL", "text": "case103.review@example.invalid", "score": null, "source": "deterministic", "model_revision": "contact-patterns-v1", "normalization": "identity" }, { "start": 105, "end": 119, "label": "PHONE", "text": "+49 541 730103", "score": null, "source": "deterministic", "model_revision": "contact-patterns-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 0, "end": 15, "label": "PERSON", "text": "Cora Lindenfels", "uncovered_positions": [] }, { "start": 33, "end": 40, "label": "RELIGION", "text": "Jainism", "uncovered_positions": [] }, { "start": 66, "end": 96, "label": "EMAIL", "text": "case103.review@example.invalid", "uncovered_positions": [] }, { "start": 105, "end": 119, "label": "PHONE", "text": "+49 541 730103", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-024", "scenario": null, "predictions": [ { "start": 0, "end": 15, "label": "PERSON", "text": "Cora Lindenfels", "score": 0.9980031847953796, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 24, "end": 30, "label": "CITY", "text": "Dundee", "score": 0.9969285130500793, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 33, "end": 48, "label": "PERSON", "text": "Cora Lindenfels", "score": 0.9981155395507812, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 74, "end": 97, "label": "ORGANIZATION", "text": "Amber Health Foundation", "score": 0.9936736822128296, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 0, "end": 15, "label": "PERSON", "text": "Cora Lindenfels", "score": 0.9980031847953796, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 24, "end": 30, "label": "CITY", "text": "Dundee", "score": 0.9969285130500793, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 33, "end": 48, "label": "PERSON", "text": "Cora Lindenfels", "score": 0.9981155395507812, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 74, "end": 97, "label": "ORGANIZATION", "text": "Amber Health Foundation", "score": 0.9936736822128296, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 0, "end": 15, "label": "PERSON", "text": "Cora Lindenfels", "uncovered_positions": [] }, { "start": 24, "end": 30, "label": "CITY", "text": "Dundee", "uncovered_positions": [] }, { "start": 33, "end": 48, "label": "PERSON", "text": "Cora Lindenfels", "uncovered_positions": [] }, { "start": 74, "end": 97, "label": "ORGANIZATION", "text": "Amber Health Foundation", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-025", "scenario": null, "predictions": [ { "start": 3, "end": 9, "label": "CITY", "text": "Dundee", "score": 0.9998049139976501, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 11, "end": 26, "label": "PERSON", "text": "Cora Lindenfels", "score": 0.9993300437927246, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 38, "end": 61, "label": "ORGANIZATION", "text": "Amber Health Foundation", "score": 0.9960338473320007, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 82, "end": 97, "label": "PERSON", "text": "Cora Lindenfels", "score": 0.9993378520011902, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 110, "end": 116, "label": "CITY", "text": "Dundee", "score": 0.9997878670692444, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 3, "end": 9, "label": "CITY", "text": "Dundee", "score": 0.9998049139976501, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 11, "end": 26, "label": "PERSON", "text": "Cora Lindenfels", "score": 0.9993300437927246, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 38, "end": 61, "label": "ORGANIZATION", "text": "Amber Health Foundation", "score": 0.9960338473320007, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 82, "end": 97, "label": "PERSON", "text": "Cora Lindenfels", "score": 0.9993378520011902, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 110, "end": 116, "label": "CITY", "text": "Dundee", "score": 0.9997878670692444, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 3, "end": 9, "label": "CITY", "text": "Dundee", "uncovered_positions": [] }, { "start": 11, "end": 26, "label": "PERSON", "text": "Cora Lindenfels", "uncovered_positions": [] }, { "start": 38, "end": 61, "label": "ORGANIZATION", "text": "Amber Health Foundation", "uncovered_positions": [] }, { "start": 82, "end": 97, "label": "PERSON", "text": "Cora Lindenfels", "uncovered_positions": [] }, { "start": 110, "end": 116, "label": "CITY", "text": "Dundee", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-026", "scenario": null, "predictions": [ { "start": 6, "end": 22, "label": "PERSON", "text": "Cora-\nLindenfels", "score": 0.997488260269165, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "dehyphenation-v1" }, { "start": 28, "end": 36, "label": "CITY", "text": "Dun-\ndee", "score": 0.999882698059082, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "dehyphenation-v1" }, { "start": 50, "end": 73, "label": "ORGANIZATION", "text": "Amber Health Foundation", "score": 0.9973509311676025, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "dehyphenation-v1" } ], "combined_predictions": [ { "start": 6, "end": 22, "label": "PERSON", "text": "Cora-\nLindenfels", "score": 0.997488260269165, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "dehyphenation-v1" }, { "start": 28, "end": 36, "label": "CITY", "text": "Dun-\ndee", "score": 0.999882698059082, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "dehyphenation-v1" }, { "start": 50, "end": 73, "label": "ORGANIZATION", "text": "Amber Health Foundation", "score": 0.9973509311676025, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "dehyphenation-v1" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 6, "end": 22, "label": "PERSON", "text": "Cora-\nLindenfels", "uncovered_positions": [] }, { "start": 28, "end": 36, "label": "CITY", "text": "Dun-\ndee", "uncovered_positions": [] }, { "start": 50, "end": 73, "label": "ORGANIZATION", "text": "Amber Health Foundation", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-027", "scenario": null, "predictions": [ { "start": 16, "end": 31, "label": "PERSON", "text": "C0ra L1ndenfels", "score": 0.9803281426429749, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 43, "end": 49, "label": "CITY", "text": "Dundee", "score": 0.9996110796928406, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 63, "end": 86, "label": "ORGANIZATION", "text": "Amber Health Foundation", "score": 0.9959501028060913, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 16, "end": 31, "label": "PERSON", "text": "C0ra L1ndenfels", "score": 0.9803281426429749, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 43, "end": 49, "label": "CITY", "text": "Dundee", "score": 0.9996110796928406, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 63, "end": 86, "label": "ORGANIZATION", "text": "Amber Health Foundation", "score": 0.9959501028060913, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 16, "end": 31, "label": "PERSON", "text": "C0ra L1ndenfels", "uncovered_positions": [] }, { "start": 43, "end": 49, "label": "CITY", "text": "Dundee", "uncovered_positions": [] }, { "start": 63, "end": 86, "label": "ORGANIZATION", "text": "Amber Health Foundation", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-028", "scenario": null, "predictions": [ { "start": 10, "end": 21, "label": "RELIGION", "text": "jainistisch", "score": 0.7595014572143555, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 48, "end": 71, "label": "ORGANIZATION", "text": "Amber Health Foundation", "score": 0.9929428100585938, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 85, "end": 100, "label": "PERSON", "text": "Cora Lindenfels", "score": 0.9964152574539185, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 10, "end": 21, "label": "RELIGION", "text": "jainistisch", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" }, { "start": 48, "end": 71, "label": "ORGANIZATION", "text": "Amber Health Foundation", "score": 0.9929428100585938, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 85, "end": 100, "label": "PERSON", "text": "Cora Lindenfels", "score": 0.9964152574539185, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 10, "end": 21, "label": "RELIGION", "text": "jainistisch", "uncovered_positions": [] }, { "start": 48, "end": 71, "label": "ORGANIZATION", "text": "Amber Health Foundation", "uncovered_positions": [] }, { "start": 85, "end": 100, "label": "PERSON", "text": "Cora Lindenfels", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-029", "scenario": null, "predictions": [ { "start": 101, "end": 107, "label": "PERSON", "text": "person", "score": 0.7094536423683167, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 109, "end": 114, "label": "CITY", "text": "place", "score": 0.7874830365180969, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 116, "end": 128, "label": "ORGANIZATION", "text": "organization", "score": 0.8446553945541382, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 40, "end": 64, "label": "RELIGION", "text": "no religious affiliation", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" }, { "start": 101, "end": 107, "label": "PERSON", "text": "person", "score": 0.7094536423683167, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 109, "end": 114, "label": "CITY", "text": "place", "score": 0.7874830365180969, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 116, "end": 128, "label": "ORGANIZATION", "text": "organization", "score": 0.8446553945541382, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [ [ 40, 64, "RELIGION" ], [ 101, 107, "PERSON" ], [ 109, 114, "CITY" ], [ 116, 128, "ORGANIZATION" ] ], "false_negatives": [], "exact_match": false, "error_code": null, "canaries": [], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-030", "scenario": null, "predictions": [ { "start": 10, "end": 22, "label": "PERSON", "text": "Jasper Keene", "score": 0.997222900390625, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 36, "end": 57, "label": "ORGANIZATION", "text": "Silver Finch Robotics", "score": 0.9982925057411194, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 61, "end": 69, "label": "CITY", "text": "Aberdeen", "score": 0.9969948530197144, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 10, "end": 22, "label": "PERSON", "text": "Jasper Keene", "score": 0.997222900390625, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 36, "end": 57, "label": "ORGANIZATION", "text": "Silver Finch Robotics", "score": 0.9982925057411194, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 61, "end": 69, "label": "CITY", "text": "Aberdeen", "score": 0.9969948530197144, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 10, "end": 22, "label": "PERSON", "text": "Jasper Keene", "uncovered_positions": [] }, { "start": 36, "end": 57, "label": "ORGANIZATION", "text": "Silver Finch Robotics", "uncovered_positions": [] }, { "start": 61, "end": 69, "label": "CITY", "text": "Aberdeen", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-031", "scenario": null, "predictions": [ { "start": 12, "end": 24, "label": "PERSON", "text": "Jasper Keene", "score": 0.9982971549034119, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 36, "end": 57, "label": "ORGANIZATION", "text": "Silver Finch Robotics", "score": 0.9989097118377686, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 63, "end": 71, "label": "CITY", "text": "Aberdeen", "score": 0.9978512525558472, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 12, "end": 24, "label": "PERSON", "text": "Jasper Keene", "score": 0.9982971549034119, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 36, "end": 57, "label": "ORGANIZATION", "text": "Silver Finch Robotics", "score": 0.9989097118377686, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 63, "end": 71, "label": "CITY", "text": "Aberdeen", "score": 0.9978512525558472, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 12, "end": 24, "label": "PERSON", "text": "Jasper Keene", "uncovered_positions": [] }, { "start": 36, "end": 57, "label": "ORGANIZATION", "text": "Silver Finch Robotics", "uncovered_positions": [] }, { "start": 63, "end": 71, "label": "CITY", "text": "Aberdeen", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-032", "scenario": null, "predictions": [ { "start": 0, "end": 12, "label": "PERSON", "text": "Jasper Keene", "score": 0.9964504241943359, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 19, "end": 31, "label": "RELIGION", "text": "hinduistisch", "score": 0.9917290806770325, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 59, "end": 89, "label": "EMAIL", "text": "case104.review@example.invalid", "score": 0.9179330468177795, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 100, "end": 114, "label": "PHONE", "text": "+49 541 730104", "score": 0.9929746389389038, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 0, "end": 12, "label": "PERSON", "text": "Jasper Keene", "score": 0.9964504241943359, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 19, "end": 31, "label": "RELIGION", "text": "hinduistisch", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" }, { "start": 59, "end": 89, "label": "EMAIL", "text": "case104.review@example.invalid", "score": null, "source": "deterministic", "model_revision": "contact-patterns-v1", "normalization": "identity" }, { "start": 100, "end": 114, "label": "PHONE", "text": "+49 541 730104", "score": null, "source": "deterministic", "model_revision": "contact-patterns-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 0, "end": 12, "label": "PERSON", "text": "Jasper Keene", "uncovered_positions": [] }, { "start": 19, "end": 31, "label": "RELIGION", "text": "hinduistisch", "uncovered_positions": [] }, { "start": 59, "end": 89, "label": "EMAIL", "text": "case104.review@example.invalid", "uncovered_positions": [] }, { "start": 100, "end": 114, "label": "PHONE", "text": "+49 541 730104", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-033", "scenario": null, "predictions": [ { "start": 0, "end": 12, "label": "PERSON", "text": "Jasper Keene", "score": 0.9973819851875305, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 30, "end": 38, "label": "RELIGION", "text": "Hinduism", "score": 0.9998874664306641, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 64, "end": 94, "label": "EMAIL", "text": "case104.review@example.invalid", "score": 0.9190782308578491, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 103, "end": 117, "label": "PHONE", "text": "+49 541 730104", "score": 0.9675639271736145, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 0, "end": 12, "label": "PERSON", "text": "Jasper Keene", "score": 0.9973819851875305, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 30, "end": 38, "label": "RELIGION", "text": "Hinduism", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" }, { "start": 64, "end": 94, "label": "EMAIL", "text": "case104.review@example.invalid", "score": null, "source": "deterministic", "model_revision": "contact-patterns-v1", "normalization": "identity" }, { "start": 103, "end": 117, "label": "PHONE", "text": "+49 541 730104", "score": null, "source": "deterministic", "model_revision": "contact-patterns-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 0, "end": 12, "label": "PERSON", "text": "Jasper Keene", "uncovered_positions": [] }, { "start": 30, "end": 38, "label": "RELIGION", "text": "Hinduism", "uncovered_positions": [] }, { "start": 64, "end": 94, "label": "EMAIL", "text": "case104.review@example.invalid", "uncovered_positions": [] }, { "start": 103, "end": 117, "label": "PHONE", "text": "+49 541 730104", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-034", "scenario": null, "predictions": [ { "start": 0, "end": 12, "label": "PERSON", "text": "Jasper Keene", "score": 0.9992870688438416, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 21, "end": 29, "label": "CITY", "text": "Aberdeen", "score": 0.9939911961555481, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 32, "end": 44, "label": "PERSON", "text": "Jasper Keene", "score": 0.9992408752441406, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 70, "end": 91, "label": "ORGANIZATION", "text": "Silver Finch Robotics", "score": 0.9981598258018494, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 0, "end": 12, "label": "PERSON", "text": "Jasper Keene", "score": 0.9992870688438416, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 21, "end": 29, "label": "CITY", "text": "Aberdeen", "score": 0.9939911961555481, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 32, "end": 44, "label": "PERSON", "text": "Jasper Keene", "score": 0.9992408752441406, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 70, "end": 91, "label": "ORGANIZATION", "text": "Silver Finch Robotics", "score": 0.9981598258018494, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 0, "end": 12, "label": "PERSON", "text": "Jasper Keene", "uncovered_positions": [] }, { "start": 21, "end": 29, "label": "CITY", "text": "Aberdeen", "uncovered_positions": [] }, { "start": 32, "end": 44, "label": "PERSON", "text": "Jasper Keene", "uncovered_positions": [] }, { "start": 70, "end": 91, "label": "ORGANIZATION", "text": "Silver Finch Robotics", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-035", "scenario": null, "predictions": [ { "start": 3, "end": 11, "label": "CITY", "text": "Aberdeen", "score": 0.9991244673728943, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 13, "end": 25, "label": "PERSON", "text": "Jasper Keene", "score": 0.9997237324714661, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 37, "end": 58, "label": "ORGANIZATION", "text": "Silver Finch Robotics", "score": 0.9979164004325867, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 79, "end": 91, "label": "PERSON", "text": "Jasper Keene", "score": 0.9997982382774353, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 104, "end": 112, "label": "CITY", "text": "Aberdeen", "score": 0.9984569549560547, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 3, "end": 11, "label": "CITY", "text": "Aberdeen", "score": 0.9991244673728943, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 13, "end": 25, "label": "PERSON", "text": "Jasper Keene", "score": 0.9997237324714661, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 37, "end": 58, "label": "ORGANIZATION", "text": "Silver Finch Robotics", "score": 0.9979164004325867, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 79, "end": 91, "label": "PERSON", "text": "Jasper Keene", "score": 0.9997982382774353, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 104, "end": 112, "label": "CITY", "text": "Aberdeen", "score": 0.9984569549560547, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 3, "end": 11, "label": "CITY", "text": "Aberdeen", "uncovered_positions": [] }, { "start": 13, "end": 25, "label": "PERSON", "text": "Jasper Keene", "uncovered_positions": [] }, { "start": 37, "end": 58, "label": "ORGANIZATION", "text": "Silver Finch Robotics", "uncovered_positions": [] }, { "start": 79, "end": 91, "label": "PERSON", "text": "Jasper Keene", "uncovered_positions": [] }, { "start": 104, "end": 112, "label": "CITY", "text": "Aberdeen", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-036", "scenario": null, "predictions": [ { "start": 6, "end": 19, "label": "PERSON", "text": "Jasper-\nKeene", "score": 0.9998306035995483, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "dehyphenation-v1" }, { "start": 25, "end": 35, "label": "CITY", "text": "Aber-\ndeen", "score": 0.9998304843902588, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "dehyphenation-v1" }, { "start": 49, "end": 70, "label": "ORGANIZATION", "text": "Silver Finch Robotics", "score": 0.9984242916107178, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "dehyphenation-v1" } ], "combined_predictions": [ { "start": 6, "end": 19, "label": "PERSON", "text": "Jasper-\nKeene", "score": 0.9998306035995483, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "dehyphenation-v1" }, { "start": 25, "end": 35, "label": "CITY", "text": "Aber-\ndeen", "score": 0.9998304843902588, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "dehyphenation-v1" }, { "start": 49, "end": 70, "label": "ORGANIZATION", "text": "Silver Finch Robotics", "score": 0.9984242916107178, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "dehyphenation-v1" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 6, "end": 19, "label": "PERSON", "text": "Jasper-\nKeene", "uncovered_positions": [] }, { "start": 25, "end": 35, "label": "CITY", "text": "Aber-\ndeen", "uncovered_positions": [] }, { "start": 49, "end": 70, "label": "ORGANIZATION", "text": "Silver Finch Robotics", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-037", "scenario": null, "predictions": [ { "start": 16, "end": 28, "label": "PERSON", "text": "Jasper Keene", "score": 0.9981817007064819, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 40, "end": 48, "label": "CITY", "text": "Aberdeen", "score": 0.999523401260376, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 62, "end": 83, "label": "ORGANIZATION", "text": "Silver Finch Robotics", "score": 0.9983817338943481, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 16, "end": 28, "label": "PERSON", "text": "Jasper Keene", "score": 0.9981817007064819, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 40, "end": 48, "label": "CITY", "text": "Aberdeen", "score": 0.999523401260376, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 62, "end": 83, "label": "ORGANIZATION", "text": "Silver Finch Robotics", "score": 0.9983817338943481, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 16, "end": 28, "label": "PERSON", "text": "Jasper Keene", "uncovered_positions": [] }, { "start": 40, "end": 48, "label": "CITY", "text": "Aberdeen", "uncovered_positions": [] }, { "start": 62, "end": 83, "label": "ORGANIZATION", "text": "Silver Finch Robotics", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-038", "scenario": null, "predictions": [ { "start": 10, "end": 22, "label": "RELIGION", "text": "hinduistisch", "score": 0.8530606627464294, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 49, "end": 70, "label": "ORGANIZATION", "text": "Silver Finch Robotics", "score": 0.991775631904602, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 84, "end": 96, "label": "PERSON", "text": "Jasper Keene", "score": 0.9976971745491028, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 10, "end": 22, "label": "RELIGION", "text": "hinduistisch", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" }, { "start": 49, "end": 70, "label": "ORGANIZATION", "text": "Silver Finch Robotics", "score": 0.991775631904602, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 84, "end": 96, "label": "PERSON", "text": "Jasper Keene", "score": 0.9976971745491028, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 10, "end": 22, "label": "RELIGION", "text": "hinduistisch", "uncovered_positions": [] }, { "start": 49, "end": 70, "label": "ORGANIZATION", "text": "Silver Finch Robotics", "uncovered_positions": [] }, { "start": 84, "end": 96, "label": "PERSON", "text": "Jasper Keene", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-039", "scenario": null, "predictions": [ { "start": 101, "end": 107, "label": "PERSON", "text": "person", "score": 0.7161614894866943, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 109, "end": 114, "label": "CITY", "text": "place", "score": 0.7881067395210266, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 116, "end": 128, "label": "ORGANIZATION", "text": "organization", "score": 0.847603976726532, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 40, "end": 64, "label": "RELIGION", "text": "no religious affiliation", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" }, { "start": 101, "end": 107, "label": "PERSON", "text": "person", "score": 0.7161614894866943, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 109, "end": 114, "label": "CITY", "text": "place", "score": 0.7881067395210266, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 116, "end": 128, "label": "ORGANIZATION", "text": "organization", "score": 0.847603976726532, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [ [ 40, 64, "RELIGION" ], [ 101, 107, "PERSON" ], [ 109, 114, "CITY" ], [ 116, 128, "ORGANIZATION" ] ], "false_negatives": [], "exact_match": false, "error_code": null, "canaries": [], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-040", "scenario": null, "predictions": [ { "start": 10, "end": 23, "label": "PERSON", "text": "Frida Östberg", "score": 0.9995619654655457, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 37, "end": 59, "label": "ORGANIZATION", "text": "Blue Cedar Cooperative", "score": 0.9986785054206848, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 63, "end": 70, "label": "CITY", "text": "Uppsala", "score": 0.9995537400245667, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 10, "end": 23, "label": "PERSON", "text": "Frida Östberg", "score": 0.9995619654655457, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 37, "end": 59, "label": "ORGANIZATION", "text": "Blue Cedar Cooperative", "score": 0.9986785054206848, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 63, "end": 70, "label": "CITY", "text": "Uppsala", "score": 0.9995537400245667, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 10, "end": 23, "label": "PERSON", "text": "Frida Östberg", "uncovered_positions": [] }, { "start": 37, "end": 59, "label": "ORGANIZATION", "text": "Blue Cedar Cooperative", "uncovered_positions": [] }, { "start": 63, "end": 70, "label": "CITY", "text": "Uppsala", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-041", "scenario": null, "predictions": [ { "start": 12, "end": 25, "label": "PERSON", "text": "Frida Östberg", "score": 0.9993315935134888, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 37, "end": 59, "label": "ORGANIZATION", "text": "Blue Cedar Cooperative", "score": 0.9984630346298218, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 65, "end": 72, "label": "CITY", "text": "Uppsala", "score": 0.9996377229690552, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 12, "end": 25, "label": "PERSON", "text": "Frida Östberg", "score": 0.9993315935134888, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 37, "end": 59, "label": "ORGANIZATION", "text": "Blue Cedar Cooperative", "score": 0.9984630346298218, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 65, "end": 72, "label": "CITY", "text": "Uppsala", "score": 0.9996377229690552, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 12, "end": 25, "label": "PERSON", "text": "Frida Östberg", "uncovered_positions": [] }, { "start": 37, "end": 59, "label": "ORGANIZATION", "text": "Blue Cedar Cooperative", "uncovered_positions": [] }, { "start": 65, "end": 72, "label": "CITY", "text": "Uppsala", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-042", "scenario": null, "predictions": [ { "start": 0, "end": 13, "label": "PERSON", "text": "Frida Östberg", "score": 0.9994117021560669, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 20, "end": 25, "label": "RELIGION", "text": "bahai", "score": 0.9852348566055298, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 53, "end": 83, "label": "EMAIL", "text": "case105.review@example.invalid", "score": 0.8997676968574524, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 94, "end": 108, "label": "PHONE", "text": "+49 541 730105", "score": 0.9945896863937378, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 0, "end": 13, "label": "PERSON", "text": "Frida Östberg", "score": 0.9994117021560669, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 20, "end": 25, "label": "RELIGION", "text": "bahai", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" }, { "start": 53, "end": 83, "label": "EMAIL", "text": "case105.review@example.invalid", "score": null, "source": "deterministic", "model_revision": "contact-patterns-v1", "normalization": "identity" }, { "start": 94, "end": 108, "label": "PHONE", "text": "+49 541 730105", "score": null, "source": "deterministic", "model_revision": "contact-patterns-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 0, "end": 13, "label": "PERSON", "text": "Frida Östberg", "uncovered_positions": [] }, { "start": 20, "end": 25, "label": "RELIGION", "text": "bahai", "uncovered_positions": [] }, { "start": 53, "end": 83, "label": "EMAIL", "text": "case105.review@example.invalid", "uncovered_positions": [] }, { "start": 94, "end": 108, "label": "PHONE", "text": "+49 541 730105", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-043", "scenario": null, "predictions": [ { "start": 0, "end": 13, "label": "PERSON", "text": "Frida Östberg", "score": 0.9995088577270508, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 31, "end": 43, "label": "RELIGION", "text": "Baháʼí Faith", "score": 0.9641181826591492, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 69, "end": 99, "label": "EMAIL", "text": "case105.review@example.invalid", "score": 0.9057281017303467, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 108, "end": 122, "label": "PHONE", "text": "+49 541 730105", "score": 0.9755640625953674, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 0, "end": 13, "label": "PERSON", "text": "Frida Östberg", "score": 0.9995088577270508, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 31, "end": 43, "label": "RELIGION", "text": "Baháʼí Faith", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" }, { "start": 69, "end": 99, "label": "EMAIL", "text": "case105.review@example.invalid", "score": null, "source": "deterministic", "model_revision": "contact-patterns-v1", "normalization": "identity" }, { "start": 108, "end": 122, "label": "PHONE", "text": "+49 541 730105", "score": null, "source": "deterministic", "model_revision": "contact-patterns-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 0, "end": 13, "label": "PERSON", "text": "Frida Östberg", "uncovered_positions": [] }, { "start": 31, "end": 43, "label": "RELIGION", "text": "Baháʼí Faith", "uncovered_positions": [] }, { "start": 69, "end": 99, "label": "EMAIL", "text": "case105.review@example.invalid", "uncovered_positions": [] }, { "start": 108, "end": 122, "label": "PHONE", "text": "+49 541 730105", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-044", "scenario": null, "predictions": [ { "start": 0, "end": 13, "label": "PERSON", "text": "Frida Östberg", "score": 0.9998176693916321, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 22, "end": 29, "label": "CITY", "text": "Uppsala", "score": 0.9994066953659058, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 32, "end": 45, "label": "PERSON", "text": "Frida Östberg", "score": 0.999852180480957, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 71, "end": 93, "label": "ORGANIZATION", "text": "Blue Cedar Cooperative", "score": 0.9967365860939026, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 0, "end": 13, "label": "PERSON", "text": "Frida Östberg", "score": 0.9998176693916321, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 22, "end": 29, "label": "CITY", "text": "Uppsala", "score": 0.9994066953659058, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 32, "end": 45, "label": "PERSON", "text": "Frida Östberg", "score": 0.999852180480957, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 71, "end": 93, "label": "ORGANIZATION", "text": "Blue Cedar Cooperative", "score": 0.9967365860939026, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 0, "end": 13, "label": "PERSON", "text": "Frida Östberg", "uncovered_positions": [] }, { "start": 22, "end": 29, "label": "CITY", "text": "Uppsala", "uncovered_positions": [] }, { "start": 32, "end": 45, "label": "PERSON", "text": "Frida Östberg", "uncovered_positions": [] }, { "start": 71, "end": 93, "label": "ORGANIZATION", "text": "Blue Cedar Cooperative", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-045", "scenario": null, "predictions": [ { "start": 3, "end": 10, "label": "CITY", "text": "Uppsala", "score": 0.9998781681060791, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 12, "end": 25, "label": "PERSON", "text": "Frida Östberg", "score": 0.9998877048492432, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 37, "end": 59, "label": "ORGANIZATION", "text": "Blue Cedar Cooperative", "score": 0.9978963136672974, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 80, "end": 93, "label": "PERSON", "text": "Frida Östberg", "score": 0.9998670816421509, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 106, "end": 113, "label": "CITY", "text": "Uppsala", "score": 0.9998447895050049, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 3, "end": 10, "label": "CITY", "text": "Uppsala", "score": 0.9998781681060791, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 12, "end": 25, "label": "PERSON", "text": "Frida Östberg", "score": 0.9998877048492432, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 37, "end": 59, "label": "ORGANIZATION", "text": "Blue Cedar Cooperative", "score": 0.9978963136672974, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 80, "end": 93, "label": "PERSON", "text": "Frida Östberg", "score": 0.9998670816421509, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 106, "end": 113, "label": "CITY", "text": "Uppsala", "score": 0.9998447895050049, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 3, "end": 10, "label": "CITY", "text": "Uppsala", "uncovered_positions": [] }, { "start": 12, "end": 25, "label": "PERSON", "text": "Frida Östberg", "uncovered_positions": [] }, { "start": 37, "end": 59, "label": "ORGANIZATION", "text": "Blue Cedar Cooperative", "uncovered_positions": [] }, { "start": 80, "end": 93, "label": "PERSON", "text": "Frida Östberg", "uncovered_positions": [] }, { "start": 106, "end": 113, "label": "CITY", "text": "Uppsala", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-046", "scenario": null, "predictions": [ { "start": 6, "end": 20, "label": "PERSON", "text": "Frida-\nÖstberg", "score": 0.9996815919876099, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "dehyphenation-v1" }, { "start": 26, "end": 35, "label": "CITY", "text": "Upp-\nsala", "score": 0.9999419450759888, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "dehyphenation-v1" }, { "start": 49, "end": 71, "label": "ORGANIZATION", "text": "Blue Cedar Cooperative", "score": 0.9979609251022339, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "dehyphenation-v1" } ], "combined_predictions": [ { "start": 6, "end": 20, "label": "PERSON", "text": "Frida-\nÖstberg", "score": 0.9996815919876099, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "dehyphenation-v1" }, { "start": 26, "end": 35, "label": "CITY", "text": "Upp-\nsala", "score": 0.9999419450759888, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "dehyphenation-v1" }, { "start": 49, "end": 71, "label": "ORGANIZATION", "text": "Blue Cedar Cooperative", "score": 0.9979609251022339, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "dehyphenation-v1" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 6, "end": 20, "label": "PERSON", "text": "Frida-\nÖstberg", "uncovered_positions": [] }, { "start": 26, "end": 35, "label": "CITY", "text": "Upp-\nsala", "uncovered_positions": [] }, { "start": 49, "end": 71, "label": "ORGANIZATION", "text": "Blue Cedar Cooperative", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-047", "scenario": null, "predictions": [ { "start": 16, "end": 29, "label": "PERSON", "text": "Fr1da Östberg", "score": 0.9966031312942505, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 41, "end": 48, "label": "CITY", "text": "Upps4l4", "score": 0.990062415599823, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 62, "end": 84, "label": "ORGANIZATION", "text": "Blue Cedar Cooperative", "score": 0.9982581734657288, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 16, "end": 29, "label": "PERSON", "text": "Fr1da Östberg", "score": 0.9966031312942505, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 41, "end": 48, "label": "CITY", "text": "Upps4l4", "score": 0.990062415599823, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 62, "end": 84, "label": "ORGANIZATION", "text": "Blue Cedar Cooperative", "score": 0.9982581734657288, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 16, "end": 29, "label": "PERSON", "text": "Fr1da Östberg", "uncovered_positions": [] }, { "start": 41, "end": 48, "label": "CITY", "text": "Upps4l4", "uncovered_positions": [] }, { "start": 62, "end": 84, "label": "ORGANIZATION", "text": "Blue Cedar Cooperative", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-048", "scenario": null, "predictions": [ { "start": 10, "end": 15, "label": "RELIGION", "text": "bahai", "score": 0.8015578985214233, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 42, "end": 64, "label": "ORGANIZATION", "text": "Blue Cedar Cooperative", "score": 0.9946749210357666, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 78, "end": 91, "label": "PERSON", "text": "Frida Östberg", "score": 0.9991986155509949, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 10, "end": 15, "label": "RELIGION", "text": "bahai", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" }, { "start": 42, "end": 64, "label": "ORGANIZATION", "text": "Blue Cedar Cooperative", "score": 0.9946749210357666, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 78, "end": 91, "label": "PERSON", "text": "Frida Östberg", "score": 0.9991986155509949, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 10, "end": 15, "label": "RELIGION", "text": "bahai", "uncovered_positions": [] }, { "start": 42, "end": 64, "label": "ORGANIZATION", "text": "Blue Cedar Cooperative", "uncovered_positions": [] }, { "start": 78, "end": 91, "label": "PERSON", "text": "Frida Östberg", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-049", "scenario": null, "predictions": [ { "start": 101, "end": 107, "label": "PERSON", "text": "person", "score": 0.7069818377494812, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 109, "end": 114, "label": "CITY", "text": "place", "score": 0.785081684589386, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 116, "end": 128, "label": "ORGANIZATION", "text": "organization", "score": 0.8447502255439758, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 40, "end": 64, "label": "RELIGION", "text": "no religious affiliation", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" }, { "start": 101, "end": 107, "label": "PERSON", "text": "person", "score": 0.7069818377494812, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 109, "end": 114, "label": "CITY", "text": "place", "score": 0.785081684589386, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 116, "end": 128, "label": "ORGANIZATION", "text": "organization", "score": 0.8447502255439758, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [ [ 40, 64, "RELIGION" ], [ 101, 107, "PERSON" ], [ 109, 114, "CITY" ], [ 116, 128, "ORGANIZATION" ] ], "false_negatives": [], "exact_match": false, "error_code": null, "canaries": [], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-050", "scenario": null, "predictions": [ { "start": 10, "end": 24, "label": "PERSON", "text": "Owen Fairchild", "score": 0.9981813430786133, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 38, "end": 60, "label": "ORGANIZATION", "text": "Oakwell Public Library", "score": 0.9969713687896729, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 64, "end": 73, "label": "CITY", "text": "Leicester", "score": 0.999505877494812, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 10, "end": 24, "label": "PERSON", "text": "Owen Fairchild", "score": 0.9981813430786133, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 38, "end": 60, "label": "ORGANIZATION", "text": "Oakwell Public Library", "score": 0.9969713687896729, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 64, "end": 73, "label": "CITY", "text": "Leicester", "score": 0.999505877494812, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 10, "end": 24, "label": "PERSON", "text": "Owen Fairchild", "uncovered_positions": [] }, { "start": 38, "end": 60, "label": "ORGANIZATION", "text": "Oakwell Public Library", "uncovered_positions": [] }, { "start": 64, "end": 73, "label": "CITY", "text": "Leicester", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-051", "scenario": null, "predictions": [ { "start": 12, "end": 26, "label": "PERSON", "text": "Owen Fairchild", "score": 0.9984040856361389, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 38, "end": 60, "label": "ORGANIZATION", "text": "Oakwell Public Library", "score": 0.9967267513275146, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 66, "end": 75, "label": "CITY", "text": "Leicester", "score": 0.9997547268867493, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 12, "end": 26, "label": "PERSON", "text": "Owen Fairchild", "score": 0.9984040856361389, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 38, "end": 60, "label": "ORGANIZATION", "text": "Oakwell Public Library", "score": 0.9967267513275146, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 66, "end": 75, "label": "CITY", "text": "Leicester", "score": 0.9997547268867493, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 12, "end": 26, "label": "PERSON", "text": "Owen Fairchild", "uncovered_positions": [] }, { "start": 38, "end": 60, "label": "ORGANIZATION", "text": "Oakwell Public Library", "uncovered_positions": [] }, { "start": 66, "end": 75, "label": "CITY", "text": "Leicester", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-052", "scenario": null, "predictions": [ { "start": 0, "end": 14, "label": "PERSON", "text": "Owen Fairchild", "score": 0.9966481328010559, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 21, "end": 33, "label": "RELIGION", "text": "buddhistisch", "score": 0.9927167296409607, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 61, "end": 91, "label": "EMAIL", "text": "case106.review@example.invalid", "score": 0.9082765579223633, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 102, "end": 116, "label": "PHONE", "text": "+49 541 730106", "score": 0.9930872321128845, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 0, "end": 14, "label": "PERSON", "text": "Owen Fairchild", "score": 0.9966481328010559, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 21, "end": 33, "label": "RELIGION", "text": "buddhistisch", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" }, { "start": 61, "end": 91, "label": "EMAIL", "text": "case106.review@example.invalid", "score": null, "source": "deterministic", "model_revision": "contact-patterns-v1", "normalization": "identity" }, { "start": 102, "end": 116, "label": "PHONE", "text": "+49 541 730106", "score": null, "source": "deterministic", "model_revision": "contact-patterns-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 0, "end": 14, "label": "PERSON", "text": "Owen Fairchild", "uncovered_positions": [] }, { "start": 21, "end": 33, "label": "RELIGION", "text": "buddhistisch", "uncovered_positions": [] }, { "start": 61, "end": 91, "label": "EMAIL", "text": "case106.review@example.invalid", "uncovered_positions": [] }, { "start": 102, "end": 116, "label": "PHONE", "text": "+49 541 730106", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-053", "scenario": null, "predictions": [ { "start": 0, "end": 14, "label": "PERSON", "text": "Owen Fairchild", "score": 0.9962939620018005, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 32, "end": 40, "label": "RELIGION", "text": "Buddhism", "score": 0.9998692274093628, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 66, "end": 96, "label": "EMAIL", "text": "case106.review@example.invalid", "score": 0.9211816191673279, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 105, "end": 119, "label": "PHONE", "text": "+49 541 730106", "score": 0.9646264910697937, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 0, "end": 14, "label": "PERSON", "text": "Owen Fairchild", "score": 0.9962939620018005, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 32, "end": 40, "label": "RELIGION", "text": "Buddhism", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" }, { "start": 66, "end": 96, "label": "EMAIL", "text": "case106.review@example.invalid", "score": null, "source": "deterministic", "model_revision": "contact-patterns-v1", "normalization": "identity" }, { "start": 105, "end": 119, "label": "PHONE", "text": "+49 541 730106", "score": null, "source": "deterministic", "model_revision": "contact-patterns-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 0, "end": 14, "label": "PERSON", "text": "Owen Fairchild", "uncovered_positions": [] }, { "start": 32, "end": 40, "label": "RELIGION", "text": "Buddhism", "uncovered_positions": [] }, { "start": 66, "end": 96, "label": "EMAIL", "text": "case106.review@example.invalid", "uncovered_positions": [] }, { "start": 105, "end": 119, "label": "PHONE", "text": "+49 541 730106", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-054", "scenario": null, "predictions": [ { "start": 0, "end": 14, "label": "PERSON", "text": "Owen Fairchild", "score": 0.9986757636070251, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 23, "end": 32, "label": "CITY", "text": "Leicester", "score": 0.9993718266487122, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 35, "end": 49, "label": "PERSON", "text": "Owen Fairchild", "score": 0.9993817806243896, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 75, "end": 97, "label": "ORGANIZATION", "text": "Oakwell Public Library", "score": 0.9926812052726746, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 0, "end": 14, "label": "PERSON", "text": "Owen Fairchild", "score": 0.9986757636070251, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 23, "end": 32, "label": "CITY", "text": "Leicester", "score": 0.9993718266487122, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 35, "end": 49, "label": "PERSON", "text": "Owen Fairchild", "score": 0.9993817806243896, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 75, "end": 97, "label": "ORGANIZATION", "text": "Oakwell Public Library", "score": 0.9926812052726746, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 0, "end": 14, "label": "PERSON", "text": "Owen Fairchild", "uncovered_positions": [] }, { "start": 23, "end": 32, "label": "CITY", "text": "Leicester", "uncovered_positions": [] }, { "start": 35, "end": 49, "label": "PERSON", "text": "Owen Fairchild", "uncovered_positions": [] }, { "start": 75, "end": 97, "label": "ORGANIZATION", "text": "Oakwell Public Library", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-055", "scenario": null, "predictions": [ { "start": 3, "end": 12, "label": "CITY", "text": "Leicester", "score": 0.9998503923416138, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 14, "end": 28, "label": "PERSON", "text": "Owen Fairchild", "score": 0.999387264251709, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 40, "end": 62, "label": "ORGANIZATION", "text": "Oakwell Public Library", "score": 0.9959281086921692, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 83, "end": 97, "label": "PERSON", "text": "Owen Fairchild", "score": 0.9997162222862244, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 110, "end": 119, "label": "CITY", "text": "Leicester", "score": 0.9999486207962036, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 3, "end": 12, "label": "CITY", "text": "Leicester", "score": 0.9998503923416138, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 14, "end": 28, "label": "PERSON", "text": "Owen Fairchild", "score": 0.999387264251709, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 40, "end": 62, "label": "ORGANIZATION", "text": "Oakwell Public Library", "score": 0.9959281086921692, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 83, "end": 97, "label": "PERSON", "text": "Owen Fairchild", "score": 0.9997162222862244, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 110, "end": 119, "label": "CITY", "text": "Leicester", "score": 0.9999486207962036, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 3, "end": 12, "label": "CITY", "text": "Leicester", "uncovered_positions": [] }, { "start": 14, "end": 28, "label": "PERSON", "text": "Owen Fairchild", "uncovered_positions": [] }, { "start": 40, "end": 62, "label": "ORGANIZATION", "text": "Oakwell Public Library", "uncovered_positions": [] }, { "start": 83, "end": 97, "label": "PERSON", "text": "Owen Fairchild", "uncovered_positions": [] }, { "start": 110, "end": 119, "label": "CITY", "text": "Leicester", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-056", "scenario": null, "predictions": [ { "start": 6, "end": 21, "label": "PERSON", "text": "Owen-\nFairchild", "score": 0.9996999502182007, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "dehyphenation-v1" }, { "start": 27, "end": 38, "label": "CITY", "text": "Leic-\nester", "score": 0.9999775886535645, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "dehyphenation-v1" }, { "start": 52, "end": 74, "label": "ORGANIZATION", "text": "Oakwell Public Library", "score": 0.9970940351486206, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "dehyphenation-v1" } ], "combined_predictions": [ { "start": 6, "end": 21, "label": "PERSON", "text": "Owen-\nFairchild", "score": 0.9996999502182007, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "dehyphenation-v1" }, { "start": 27, "end": 38, "label": "CITY", "text": "Leic-\nester", "score": 0.9999775886535645, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "dehyphenation-v1" }, { "start": 52, "end": 74, "label": "ORGANIZATION", "text": "Oakwell Public Library", "score": 0.9970940351486206, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "dehyphenation-v1" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 6, "end": 21, "label": "PERSON", "text": "Owen-\nFairchild", "uncovered_positions": [] }, { "start": 27, "end": 38, "label": "CITY", "text": "Leic-\nester", "uncovered_positions": [] }, { "start": 52, "end": 74, "label": "ORGANIZATION", "text": "Oakwell Public Library", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-057", "scenario": null, "predictions": [ { "start": 16, "end": 30, "label": "PERSON", "text": "Owen Fa1rch1ld", "score": 0.9971356391906738, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 42, "end": 51, "label": "CITY", "text": "Leicester", "score": 0.9998725652694702, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 65, "end": 87, "label": "ORGANIZATION", "text": "Oakwell Public Library", "score": 0.9962441921234131, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 16, "end": 30, "label": "PERSON", "text": "Owen Fa1rch1ld", "score": 0.9971356391906738, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 42, "end": 51, "label": "CITY", "text": "Leicester", "score": 0.9998725652694702, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 65, "end": 87, "label": "ORGANIZATION", "text": "Oakwell Public Library", "score": 0.9962441921234131, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 16, "end": 30, "label": "PERSON", "text": "Owen Fa1rch1ld", "uncovered_positions": [] }, { "start": 42, "end": 51, "label": "CITY", "text": "Leicester", "uncovered_positions": [] }, { "start": 65, "end": 87, "label": "ORGANIZATION", "text": "Oakwell Public Library", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-058", "scenario": null, "predictions": [ { "start": 10, "end": 22, "label": "RELIGION", "text": "buddhistisch", "score": 0.8650351166725159, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 49, "end": 71, "label": "ORGANIZATION", "text": "Oakwell Public Library", "score": 0.9895272254943848, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 85, "end": 99, "label": "PERSON", "text": "Owen Fairchild", "score": 0.9970881342887878, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 10, "end": 22, "label": "RELIGION", "text": "buddhistisch", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" }, { "start": 49, "end": 71, "label": "ORGANIZATION", "text": "Oakwell Public Library", "score": 0.9895272254943848, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 85, "end": 99, "label": "PERSON", "text": "Owen Fairchild", "score": 0.9970881342887878, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 10, "end": 22, "label": "RELIGION", "text": "buddhistisch", "uncovered_positions": [] }, { "start": 49, "end": 71, "label": "ORGANIZATION", "text": "Oakwell Public Library", "uncovered_positions": [] }, { "start": 85, "end": 99, "label": "PERSON", "text": "Owen Fairchild", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-059", "scenario": null, "predictions": [ { "start": 101, "end": 107, "label": "PERSON", "text": "person", "score": 0.7122743725776672, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 109, "end": 114, "label": "CITY", "text": "place", "score": 0.7866309881210327, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 116, "end": 128, "label": "ORGANIZATION", "text": "organization", "score": 0.8471381068229675, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 40, "end": 64, "label": "RELIGION", "text": "no religious affiliation", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" }, { "start": 101, "end": 107, "label": "PERSON", "text": "person", "score": 0.7122743725776672, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 109, "end": 114, "label": "CITY", "text": "place", "score": 0.7866309881210327, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 116, "end": 128, "label": "ORGANIZATION", "text": "organization", "score": 0.8471381068229675, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [ [ 40, 64, "RELIGION" ], [ 101, 107, "PERSON" ], [ 109, 114, "CITY" ], [ 116, 128, "ORGANIZATION" ] ], "false_negatives": [], "exact_match": false, "error_code": null, "canaries": [], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-060", "scenario": null, "predictions": [ { "start": 10, "end": 23, "label": "PERSON", "text": "Nele Brückner", "score": 0.9995220899581909, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 37, "end": 62, "label": "ORGANIZATION", "text": "Westhaven Water Authority", "score": 0.9957793951034546, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 66, "end": 74, "label": "CITY", "text": "Würzburg", "score": 0.9995125532150269, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 10, "end": 23, "label": "PERSON", "text": "Nele Brückner", "score": 0.9995220899581909, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 37, "end": 62, "label": "ORGANIZATION", "text": "Westhaven Water Authority", "score": 0.9957793951034546, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 66, "end": 74, "label": "CITY", "text": "Würzburg", "score": 0.9995125532150269, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 10, "end": 23, "label": "PERSON", "text": "Nele Brückner", "uncovered_positions": [] }, { "start": 37, "end": 62, "label": "ORGANIZATION", "text": "Westhaven Water Authority", "uncovered_positions": [] }, { "start": 66, "end": 74, "label": "CITY", "text": "Würzburg", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-061", "scenario": null, "predictions": [ { "start": 12, "end": 25, "label": "PERSON", "text": "Nele Brückner", "score": 0.9993897676467896, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 37, "end": 62, "label": "ORGANIZATION", "text": "Westhaven Water Authority", "score": 0.9940574169158936, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 68, "end": 76, "label": "CITY", "text": "Würzburg", "score": 0.9994066953659058, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 12, "end": 25, "label": "PERSON", "text": "Nele Brückner", "score": 0.9993897676467896, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 37, "end": 62, "label": "ORGANIZATION", "text": "Westhaven Water Authority", "score": 0.9940574169158936, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 68, "end": 76, "label": "CITY", "text": "Würzburg", "score": 0.9994066953659058, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 12, "end": 25, "label": "PERSON", "text": "Nele Brückner", "uncovered_positions": [] }, { "start": 37, "end": 62, "label": "ORGANIZATION", "text": "Westhaven Water Authority", "uncovered_positions": [] }, { "start": 68, "end": 76, "label": "CITY", "text": "Würzburg", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-062", "scenario": null, "predictions": [ { "start": 0, "end": 13, "label": "PERSON", "text": "Nele Brückner", "score": 0.9983347058296204, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 20, "end": 30, "label": "RELIGION", "text": "katholisch", "score": 0.9748141169548035, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 73, "end": 88, "label": "CITY", "text": "example.invalid", "score": 0.9023637175559998, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 99, "end": 113, "label": "PHONE", "text": "+49 541 730107", "score": 0.9952542781829834, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 0, "end": 13, "label": "PERSON", "text": "Nele Brückner", "score": 0.9983347058296204, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 20, "end": 30, "label": "RELIGION", "text": "katholisch", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" }, { "start": 58, "end": 88, "label": "EMAIL", "text": "case107.review@example.invalid", "score": null, "source": "deterministic", "model_revision": "contact-patterns-v1", "normalization": "identity" }, { "start": 99, "end": 113, "label": "PHONE", "text": "+49 541 730107", "score": null, "source": "deterministic", "model_revision": "contact-patterns-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 0, "end": 13, "label": "PERSON", "text": "Nele Brückner", "uncovered_positions": [] }, { "start": 20, "end": 30, "label": "RELIGION", "text": "katholisch", "uncovered_positions": [] }, { "start": 58, "end": 88, "label": "EMAIL", "text": "case107.review@example.invalid", "uncovered_positions": [] }, { "start": 99, "end": 113, "label": "PHONE", "text": "+49 541 730107", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-063", "scenario": null, "predictions": [ { "start": 0, "end": 13, "label": "PERSON", "text": "Nele Brückner", "score": 0.9967458248138428, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 31, "end": 42, "label": "RELIGION", "text": "Catholicism", "score": 0.9980359673500061, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 68, "end": 98, "label": "EMAIL", "text": "case107.review@example.invalid", "score": 0.9019798040390015, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 107, "end": 121, "label": "PHONE", "text": "+49 541 730107", "score": 0.9717420339584351, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 0, "end": 13, "label": "PERSON", "text": "Nele Brückner", "score": 0.9967458248138428, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 31, "end": 42, "label": "RELIGION", "text": "Catholicism", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" }, { "start": 68, "end": 98, "label": "EMAIL", "text": "case107.review@example.invalid", "score": null, "source": "deterministic", "model_revision": "contact-patterns-v1", "normalization": "identity" }, { "start": 107, "end": 121, "label": "PHONE", "text": "+49 541 730107", "score": null, "source": "deterministic", "model_revision": "contact-patterns-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 0, "end": 13, "label": "PERSON", "text": "Nele Brückner", "uncovered_positions": [] }, { "start": 31, "end": 42, "label": "RELIGION", "text": "Catholicism", "uncovered_positions": [] }, { "start": 68, "end": 98, "label": "EMAIL", "text": "case107.review@example.invalid", "uncovered_positions": [] }, { "start": 107, "end": 121, "label": "PHONE", "text": "+49 541 730107", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-064", "scenario": null, "predictions": [ { "start": 0, "end": 13, "label": "PERSON", "text": "Nele Brückner", "score": 0.9998051524162292, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 22, "end": 30, "label": "CITY", "text": "Würzburg", "score": 0.9990678429603577, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 33, "end": 46, "label": "PERSON", "text": "Nele Brückner", "score": 0.9998308420181274, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 72, "end": 97, "label": "ORGANIZATION", "text": "Westhaven Water Authority", "score": 0.9937987923622131, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 0, "end": 13, "label": "PERSON", "text": "Nele Brückner", "score": 0.9998051524162292, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 22, "end": 30, "label": "CITY", "text": "Würzburg", "score": 0.9990678429603577, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 33, "end": 46, "label": "PERSON", "text": "Nele Brückner", "score": 0.9998308420181274, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 72, "end": 97, "label": "ORGANIZATION", "text": "Westhaven Water Authority", "score": 0.9937987923622131, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 0, "end": 13, "label": "PERSON", "text": "Nele Brückner", "uncovered_positions": [] }, { "start": 22, "end": 30, "label": "CITY", "text": "Würzburg", "uncovered_positions": [] }, { "start": 33, "end": 46, "label": "PERSON", "text": "Nele Brückner", "uncovered_positions": [] }, { "start": 72, "end": 97, "label": "ORGANIZATION", "text": "Westhaven Water Authority", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-065", "scenario": null, "predictions": [ { "start": 3, "end": 11, "label": "CITY", "text": "Würzburg", "score": 0.9997674822807312, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 13, "end": 26, "label": "PERSON", "text": "Nele Brückner", "score": 0.9999117851257324, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 38, "end": 63, "label": "ORGANIZATION", "text": "Westhaven Water Authority", "score": 0.9946510195732117, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 84, "end": 97, "label": "PERSON", "text": "Nele Brückner", "score": 0.9998905658721924, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 110, "end": 118, "label": "CITY", "text": "Würzburg", "score": 0.9998799562454224, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 3, "end": 11, "label": "CITY", "text": "Würzburg", "score": 0.9997674822807312, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 13, "end": 26, "label": "PERSON", "text": "Nele Brückner", "score": 0.9999117851257324, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 38, "end": 63, "label": "ORGANIZATION", "text": "Westhaven Water Authority", "score": 0.9946510195732117, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 84, "end": 97, "label": "PERSON", "text": "Nele Brückner", "score": 0.9998905658721924, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 110, "end": 118, "label": "CITY", "text": "Würzburg", "score": 0.9998799562454224, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 3, "end": 11, "label": "CITY", "text": "Würzburg", "uncovered_positions": [] }, { "start": 13, "end": 26, "label": "PERSON", "text": "Nele Brückner", "uncovered_positions": [] }, { "start": 38, "end": 63, "label": "ORGANIZATION", "text": "Westhaven Water Authority", "uncovered_positions": [] }, { "start": 84, "end": 97, "label": "PERSON", "text": "Nele Brückner", "uncovered_positions": [] }, { "start": 110, "end": 118, "label": "CITY", "text": "Würzburg", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-066", "scenario": null, "predictions": [ { "start": 6, "end": 20, "label": "PERSON", "text": "Nele-\nBrückner", "score": 0.9997313618659973, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "dehyphenation-v1" }, { "start": 26, "end": 36, "label": "CITY", "text": "Würz-\nburg", "score": 0.9999507665634155, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "dehyphenation-v1" }, { "start": 50, "end": 75, "label": "ORGANIZATION", "text": "Westhaven Water Authority", "score": 0.9986609220504761, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "dehyphenation-v1" } ], "combined_predictions": [ { "start": 6, "end": 20, "label": "PERSON", "text": "Nele-\nBrückner", "score": 0.9997313618659973, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "dehyphenation-v1" }, { "start": 26, "end": 36, "label": "CITY", "text": "Würz-\nburg", "score": 0.9999507665634155, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "dehyphenation-v1" }, { "start": 50, "end": 75, "label": "ORGANIZATION", "text": "Westhaven Water Authority", "score": 0.9986609220504761, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "dehyphenation-v1" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 6, "end": 20, "label": "PERSON", "text": "Nele-\nBrückner", "uncovered_positions": [] }, { "start": 26, "end": 36, "label": "CITY", "text": "Würz-\nburg", "uncovered_positions": [] }, { "start": 50, "end": 75, "label": "ORGANIZATION", "text": "Westhaven Water Authority", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-067", "scenario": null, "predictions": [ { "start": 16, "end": 29, "label": "PERSON", "text": "Nele Brückner", "score": 0.9996700286865234, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 41, "end": 49, "label": "CITY", "text": "Würzburg", "score": 0.9998364448547363, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 63, "end": 88, "label": "ORGANIZATION", "text": "Westhaven Water Authority", "score": 0.9963690042495728, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 16, "end": 29, "label": "PERSON", "text": "Nele Brückner", "score": 0.9996700286865234, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 41, "end": 49, "label": "CITY", "text": "Würzburg", "score": 0.9998364448547363, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 63, "end": 88, "label": "ORGANIZATION", "text": "Westhaven Water Authority", "score": 0.9963690042495728, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 16, "end": 29, "label": "PERSON", "text": "Nele Brückner", "uncovered_positions": [] }, { "start": 41, "end": 49, "label": "CITY", "text": "Würzburg", "uncovered_positions": [] }, { "start": 63, "end": 88, "label": "ORGANIZATION", "text": "Westhaven Water Authority", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-068", "scenario": null, "predictions": [ { "start": 10, "end": 20, "label": "RELIGION", "text": "katholisch", "score": 0.8750774264335632, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 47, "end": 72, "label": "ORGANIZATION", "text": "Westhaven Water Authority", "score": 0.9934499859809875, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 86, "end": 99, "label": "PERSON", "text": "Nele Brückner", "score": 0.9986261129379272, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 10, "end": 20, "label": "RELIGION", "text": "katholisch", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" }, { "start": 47, "end": 72, "label": "ORGANIZATION", "text": "Westhaven Water Authority", "score": 0.9934499859809875, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 86, "end": 99, "label": "PERSON", "text": "Nele Brückner", "score": 0.9986261129379272, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 10, "end": 20, "label": "RELIGION", "text": "katholisch", "uncovered_positions": [] }, { "start": 47, "end": 72, "label": "ORGANIZATION", "text": "Westhaven Water Authority", "uncovered_positions": [] }, { "start": 86, "end": 99, "label": "PERSON", "text": "Nele Brückner", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-069", "scenario": null, "predictions": [ { "start": 101, "end": 107, "label": "PERSON", "text": "person", "score": 0.7072301506996155, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 109, "end": 114, "label": "CITY", "text": "place", "score": 0.7869006395339966, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 116, "end": 128, "label": "ORGANIZATION", "text": "organization", "score": 0.844225287437439, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 40, "end": 64, "label": "RELIGION", "text": "no religious affiliation", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" }, { "start": 101, "end": 107, "label": "PERSON", "text": "person", "score": 0.7072301506996155, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 109, "end": 114, "label": "CITY", "text": "place", "score": 0.7869006395339966, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 116, "end": 128, "label": "ORGANIZATION", "text": "organization", "score": 0.844225287437439, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [ [ 40, 64, "RELIGION" ], [ 101, 107, "PERSON" ], [ 109, 114, "CITY" ], [ 116, 128, "ORGANIZATION" ] ], "false_negatives": [], "exact_match": false, "error_code": null, "canaries": [], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-070", "scenario": null, "predictions": [ { "start": 10, "end": 25, "label": "PERSON", "text": "Silas Hawthorne", "score": 0.9949835538864136, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 39, "end": 61, "label": "ORGANIZATION", "text": "Juniper Legal Services", "score": 0.9961141347885132, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 65, "end": 73, "label": "CITY", "text": "Portland", "score": 0.9993465542793274, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 10, "end": 25, "label": "PERSON", "text": "Silas Hawthorne", "score": 0.9949835538864136, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 39, "end": 61, "label": "ORGANIZATION", "text": "Juniper Legal Services", "score": 0.9961141347885132, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 65, "end": 73, "label": "CITY", "text": "Portland", "score": 0.9993465542793274, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 10, "end": 25, "label": "PERSON", "text": "Silas Hawthorne", "uncovered_positions": [] }, { "start": 39, "end": 61, "label": "ORGANIZATION", "text": "Juniper Legal Services", "uncovered_positions": [] }, { "start": 65, "end": 73, "label": "CITY", "text": "Portland", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-071", "scenario": null, "predictions": [ { "start": 12, "end": 27, "label": "PERSON", "text": "Silas Hawthorne", "score": 0.9974763989448547, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 39, "end": 61, "label": "ORGANIZATION", "text": "Juniper Legal Services", "score": 0.9980837106704712, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 67, "end": 75, "label": "CITY", "text": "Portland", "score": 0.9996472597122192, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 12, "end": 27, "label": "PERSON", "text": "Silas Hawthorne", "score": 0.9974763989448547, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 39, "end": 61, "label": "ORGANIZATION", "text": "Juniper Legal Services", "score": 0.9980837106704712, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 67, "end": 75, "label": "CITY", "text": "Portland", "score": 0.9996472597122192, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 12, "end": 27, "label": "PERSON", "text": "Silas Hawthorne", "uncovered_positions": [] }, { "start": 39, "end": 61, "label": "ORGANIZATION", "text": "Juniper Legal Services", "uncovered_positions": [] }, { "start": 67, "end": 75, "label": "CITY", "text": "Portland", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-072", "scenario": null, "predictions": [ { "start": 0, "end": 15, "label": "PERSON", "text": "Silas Hawthorne", "score": 0.9946703314781189, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 22, "end": 33, "label": "RELIGION", "text": "evangelisch", "score": 0.9349200129508972, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 61, "end": 91, "label": "EMAIL", "text": "case108.review@example.invalid", "score": 0.9081872701644897, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 102, "end": 116, "label": "PHONE", "text": "+49 541 730108", "score": 0.9933851361274719, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 0, "end": 15, "label": "PERSON", "text": "Silas Hawthorne", "score": 0.9946703314781189, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 22, "end": 33, "label": "RELIGION", "text": "evangelisch", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" }, { "start": 61, "end": 91, "label": "EMAIL", "text": "case108.review@example.invalid", "score": null, "source": "deterministic", "model_revision": "contact-patterns-v1", "normalization": "identity" }, { "start": 102, "end": 116, "label": "PHONE", "text": "+49 541 730108", "score": null, "source": "deterministic", "model_revision": "contact-patterns-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 0, "end": 15, "label": "PERSON", "text": "Silas Hawthorne", "uncovered_positions": [] }, { "start": 22, "end": 33, "label": "RELIGION", "text": "evangelisch", "uncovered_positions": [] }, { "start": 61, "end": 91, "label": "EMAIL", "text": "case108.review@example.invalid", "uncovered_positions": [] }, { "start": 102, "end": 116, "label": "PHONE", "text": "+49 541 730108", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-073", "scenario": null, "predictions": [ { "start": 0, "end": 15, "label": "PERSON", "text": "Silas Hawthorne", "score": 0.9950461387634277, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 33, "end": 46, "label": "RELIGION", "text": "Protestantism", "score": 0.9982935786247253, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 72, "end": 102, "label": "EMAIL", "text": "case108.review@example.invalid", "score": 0.9170582890510559, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 111, "end": 125, "label": "PHONE", "text": "+49 541 730108", "score": 0.9616659283638, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 0, "end": 15, "label": "PERSON", "text": "Silas Hawthorne", "score": 0.9950461387634277, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 33, "end": 46, "label": "RELIGION", "text": "Protestantism", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" }, { "start": 72, "end": 102, "label": "EMAIL", "text": "case108.review@example.invalid", "score": null, "source": "deterministic", "model_revision": "contact-patterns-v1", "normalization": "identity" }, { "start": 111, "end": 125, "label": "PHONE", "text": "+49 541 730108", "score": null, "source": "deterministic", "model_revision": "contact-patterns-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 0, "end": 15, "label": "PERSON", "text": "Silas Hawthorne", "uncovered_positions": [] }, { "start": 33, "end": 46, "label": "RELIGION", "text": "Protestantism", "uncovered_positions": [] }, { "start": 72, "end": 102, "label": "EMAIL", "text": "case108.review@example.invalid", "uncovered_positions": [] }, { "start": 111, "end": 125, "label": "PHONE", "text": "+49 541 730108", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-074", "scenario": null, "predictions": [ { "start": 0, "end": 15, "label": "PERSON", "text": "Silas Hawthorne", "score": 0.9976925849914551, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 24, "end": 32, "label": "CITY", "text": "Portland", "score": 0.9988584518432617, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 35, "end": 50, "label": "PERSON", "text": "Silas Hawthorne", "score": 0.9983211159706116, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 76, "end": 98, "label": "ORGANIZATION", "text": "Juniper Legal Services", "score": 0.9957102537155151, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 0, "end": 15, "label": "PERSON", "text": "Silas Hawthorne", "score": 0.9976925849914551, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 24, "end": 32, "label": "CITY", "text": "Portland", "score": 0.9988584518432617, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 35, "end": 50, "label": "PERSON", "text": "Silas Hawthorne", "score": 0.9983211159706116, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 76, "end": 98, "label": "ORGANIZATION", "text": "Juniper Legal Services", "score": 0.9957102537155151, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 0, "end": 15, "label": "PERSON", "text": "Silas Hawthorne", "uncovered_positions": [] }, { "start": 24, "end": 32, "label": "CITY", "text": "Portland", "uncovered_positions": [] }, { "start": 35, "end": 50, "label": "PERSON", "text": "Silas Hawthorne", "uncovered_positions": [] }, { "start": 76, "end": 98, "label": "ORGANIZATION", "text": "Juniper Legal Services", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-075", "scenario": null, "predictions": [ { "start": 3, "end": 11, "label": "CITY", "text": "Portland", "score": 0.9997417330741882, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 13, "end": 28, "label": "PERSON", "text": "Silas Hawthorne", "score": 0.9993771910667419, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 40, "end": 62, "label": "ORGANIZATION", "text": "Juniper Legal Services", "score": 0.9973406195640564, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 83, "end": 98, "label": "PERSON", "text": "Silas Hawthorne", "score": 0.9994988441467285, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 111, "end": 119, "label": "CITY", "text": "Portland", "score": 0.9996170997619629, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 3, "end": 11, "label": "CITY", "text": "Portland", "score": 0.9997417330741882, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 13, "end": 28, "label": "PERSON", "text": "Silas Hawthorne", "score": 0.9993771910667419, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 40, "end": 62, "label": "ORGANIZATION", "text": "Juniper Legal Services", "score": 0.9973406195640564, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 83, "end": 98, "label": "PERSON", "text": "Silas Hawthorne", "score": 0.9994988441467285, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 111, "end": 119, "label": "CITY", "text": "Portland", "score": 0.9996170997619629, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 3, "end": 11, "label": "CITY", "text": "Portland", "uncovered_positions": [] }, { "start": 13, "end": 28, "label": "PERSON", "text": "Silas Hawthorne", "uncovered_positions": [] }, { "start": 40, "end": 62, "label": "ORGANIZATION", "text": "Juniper Legal Services", "uncovered_positions": [] }, { "start": 83, "end": 98, "label": "PERSON", "text": "Silas Hawthorne", "uncovered_positions": [] }, { "start": 111, "end": 119, "label": "CITY", "text": "Portland", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-076", "scenario": null, "predictions": [ { "start": 6, "end": 22, "label": "PERSON", "text": "Silas-\nHawthorne", "score": 0.9994223117828369, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "dehyphenation-v1" }, { "start": 28, "end": 38, "label": "CITY", "text": "Port-\nland", "score": 0.9999568462371826, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "dehyphenation-v1" }, { "start": 52, "end": 74, "label": "ORGANIZATION", "text": "Juniper Legal Services", "score": 0.9987984895706177, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "dehyphenation-v1" } ], "combined_predictions": [ { "start": 6, "end": 22, "label": "PERSON", "text": "Silas-\nHawthorne", "score": 0.9994223117828369, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "dehyphenation-v1" }, { "start": 28, "end": 38, "label": "CITY", "text": "Port-\nland", "score": 0.9999568462371826, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "dehyphenation-v1" }, { "start": 52, "end": 74, "label": "ORGANIZATION", "text": "Juniper Legal Services", "score": 0.9987984895706177, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "dehyphenation-v1" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 6, "end": 22, "label": "PERSON", "text": "Silas-\nHawthorne", "uncovered_positions": [] }, { "start": 28, "end": 38, "label": "CITY", "text": "Port-\nland", "uncovered_positions": [] }, { "start": 52, "end": 74, "label": "ORGANIZATION", "text": "Juniper Legal Services", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-077", "scenario": null, "predictions": [ { "start": 16, "end": 31, "label": "PERSON", "text": "S1las Hawth0rne", "score": 0.9958475828170776, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 43, "end": 51, "label": "CITY", "text": "P0rtl4nd", "score": 0.9920943379402161, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 65, "end": 87, "label": "ORGANIZATION", "text": "Juniper Legal Services", "score": 0.9983524084091187, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 16, "end": 31, "label": "PERSON", "text": "S1las Hawth0rne", "score": 0.9958475828170776, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 43, "end": 51, "label": "CITY", "text": "P0rtl4nd", "score": 0.9920943379402161, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 65, "end": 87, "label": "ORGANIZATION", "text": "Juniper Legal Services", "score": 0.9983524084091187, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 16, "end": 31, "label": "PERSON", "text": "S1las Hawth0rne", "uncovered_positions": [] }, { "start": 43, "end": 51, "label": "CITY", "text": "P0rtl4nd", "uncovered_positions": [] }, { "start": 65, "end": 87, "label": "ORGANIZATION", "text": "Juniper Legal Services", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-078", "scenario": null, "predictions": [ { "start": 10, "end": 21, "label": "RELIGION", "text": "evangelisch", "score": 0.6795831918716431, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 48, "end": 70, "label": "ORGANIZATION", "text": "Juniper Legal Services", "score": 0.9773863554000854, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 84, "end": 99, "label": "PERSON", "text": "Silas Hawthorne", "score": 0.9956725835800171, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 10, "end": 21, "label": "RELIGION", "text": "evangelisch", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" }, { "start": 48, "end": 70, "label": "ORGANIZATION", "text": "Juniper Legal Services", "score": 0.9773863554000854, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 84, "end": 99, "label": "PERSON", "text": "Silas Hawthorne", "score": 0.9956725835800171, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 10, "end": 21, "label": "RELIGION", "text": "evangelisch", "uncovered_positions": [] }, { "start": 48, "end": 70, "label": "ORGANIZATION", "text": "Juniper Legal Services", "uncovered_positions": [] }, { "start": 84, "end": 99, "label": "PERSON", "text": "Silas Hawthorne", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-079", "scenario": null, "predictions": [ { "start": 101, "end": 107, "label": "PERSON", "text": "person", "score": 0.711972177028656, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 109, "end": 114, "label": "CITY", "text": "place", "score": 0.7886146903038025, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 116, "end": 128, "label": "ORGANIZATION", "text": "organization", "score": 0.8439294695854187, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 40, "end": 64, "label": "RELIGION", "text": "no religious affiliation", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" }, { "start": 101, "end": 107, "label": "PERSON", "text": "person", "score": 0.711972177028656, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 109, "end": 114, "label": "CITY", "text": "place", "score": 0.7886146903038025, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 116, "end": 128, "label": "ORGANIZATION", "text": "organization", "score": 0.8439294695854187, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [ [ 40, 64, "RELIGION" ], [ 101, 107, "PERSON" ], [ 109, 114, "CITY" ], [ 116, 128, "ORGANIZATION" ] ], "false_negatives": [], "exact_match": false, "error_code": null, "canaries": [], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-080", "scenario": null, "predictions": [ { "start": 10, "end": 25, "label": "PERSON", "text": "Alva Rosenqvist", "score": 0.998843789100647, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 39, "end": 61, "label": "ORGANIZATION", "text": "Maple Ridge University", "score": 0.9975813627243042, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 65, "end": 71, "label": "CITY", "text": "Bergen", "score": 0.9971677660942078, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 10, "end": 25, "label": "PERSON", "text": "Alva Rosenqvist", "score": 0.998843789100647, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 39, "end": 61, "label": "ORGANIZATION", "text": "Maple Ridge University", "score": 0.9975813627243042, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 65, "end": 71, "label": "CITY", "text": "Bergen", "score": 0.9971677660942078, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 10, "end": 25, "label": "PERSON", "text": "Alva Rosenqvist", "uncovered_positions": [] }, { "start": 39, "end": 61, "label": "ORGANIZATION", "text": "Maple Ridge University", "uncovered_positions": [] }, { "start": 65, "end": 71, "label": "CITY", "text": "Bergen", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-081", "scenario": null, "predictions": [ { "start": 12, "end": 27, "label": "PERSON", "text": "Alva Rosenqvist", "score": 0.9987525939941406, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 39, "end": 61, "label": "ORGANIZATION", "text": "Maple Ridge University", "score": 0.9985166192054749, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 67, "end": 73, "label": "CITY", "text": "Bergen", "score": 0.9985432624816895, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 12, "end": 27, "label": "PERSON", "text": "Alva Rosenqvist", "score": 0.9987525939941406, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 39, "end": 61, "label": "ORGANIZATION", "text": "Maple Ridge University", "score": 0.9985166192054749, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 67, "end": 73, "label": "CITY", "text": "Bergen", "score": 0.9985432624816895, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 12, "end": 27, "label": "PERSON", "text": "Alva Rosenqvist", "uncovered_positions": [] }, { "start": 39, "end": 61, "label": "ORGANIZATION", "text": "Maple Ridge University", "uncovered_positions": [] }, { "start": 67, "end": 73, "label": "CITY", "text": "Bergen", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-082", "scenario": null, "predictions": [ { "start": 0, "end": 15, "label": "PERSON", "text": "Alva Rosenqvist", "score": 0.9994316697120667, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 22, "end": 29, "label": "RELIGION", "text": "jüdisch", "score": 0.9905497431755066, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 57, "end": 87, "label": "EMAIL", "text": "case109.review@example.invalid", "score": 0.9014635682106018, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 98, "end": 112, "label": "PHONE", "text": "+49 541 730109", "score": 0.993009090423584, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 0, "end": 15, "label": "PERSON", "text": "Alva Rosenqvist", "score": 0.9994316697120667, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 22, "end": 29, "label": "RELIGION", "text": "jüdisch", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" }, { "start": 57, "end": 87, "label": "EMAIL", "text": "case109.review@example.invalid", "score": null, "source": "deterministic", "model_revision": "contact-patterns-v1", "normalization": "identity" }, { "start": 98, "end": 112, "label": "PHONE", "text": "+49 541 730109", "score": null, "source": "deterministic", "model_revision": "contact-patterns-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 0, "end": 15, "label": "PERSON", "text": "Alva Rosenqvist", "uncovered_positions": [] }, { "start": 22, "end": 29, "label": "RELIGION", "text": "jüdisch", "uncovered_positions": [] }, { "start": 57, "end": 87, "label": "EMAIL", "text": "case109.review@example.invalid", "uncovered_positions": [] }, { "start": 98, "end": 112, "label": "PHONE", "text": "+49 541 730109", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-083", "scenario": null, "predictions": [ { "start": 0, "end": 15, "label": "PERSON", "text": "Alva Rosenqvist", "score": 0.9990854263305664, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 33, "end": 40, "label": "RELIGION", "text": "Judaism", "score": 0.9997567534446716, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 66, "end": 96, "label": "EMAIL", "text": "case109.review@example.invalid", "score": 0.9070643186569214, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 105, "end": 119, "label": "PHONE", "text": "+49 541 730109", "score": 0.9656404256820679, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 0, "end": 15, "label": "PERSON", "text": "Alva Rosenqvist", "score": 0.9990854263305664, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 33, "end": 40, "label": "RELIGION", "text": "Judaism", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" }, { "start": 66, "end": 96, "label": "EMAIL", "text": "case109.review@example.invalid", "score": null, "source": "deterministic", "model_revision": "contact-patterns-v1", "normalization": "identity" }, { "start": 105, "end": 119, "label": "PHONE", "text": "+49 541 730109", "score": null, "source": "deterministic", "model_revision": "contact-patterns-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 0, "end": 15, "label": "PERSON", "text": "Alva Rosenqvist", "uncovered_positions": [] }, { "start": 33, "end": 40, "label": "RELIGION", "text": "Judaism", "uncovered_positions": [] }, { "start": 66, "end": 96, "label": "EMAIL", "text": "case109.review@example.invalid", "uncovered_positions": [] }, { "start": 105, "end": 119, "label": "PHONE", "text": "+49 541 730109", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-084", "scenario": null, "predictions": [ { "start": 0, "end": 15, "label": "PERSON", "text": "Alva Rosenqvist", "score": 0.99967360496521, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 24, "end": 30, "label": "CITY", "text": "Bergen", "score": 0.9969269633293152, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 33, "end": 48, "label": "PERSON", "text": "Alva Rosenqvist", "score": 0.9995853304862976, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 74, "end": 96, "label": "ORGANIZATION", "text": "Maple Ridge University", "score": 0.9907796382904053, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 0, "end": 15, "label": "PERSON", "text": "Alva Rosenqvist", "score": 0.99967360496521, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 24, "end": 30, "label": "CITY", "text": "Bergen", "score": 0.9969269633293152, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 33, "end": 48, "label": "PERSON", "text": "Alva Rosenqvist", "score": 0.9995853304862976, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 74, "end": 96, "label": "ORGANIZATION", "text": "Maple Ridge University", "score": 0.9907796382904053, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 0, "end": 15, "label": "PERSON", "text": "Alva Rosenqvist", "uncovered_positions": [] }, { "start": 24, "end": 30, "label": "CITY", "text": "Bergen", "uncovered_positions": [] }, { "start": 33, "end": 48, "label": "PERSON", "text": "Alva Rosenqvist", "uncovered_positions": [] }, { "start": 74, "end": 96, "label": "ORGANIZATION", "text": "Maple Ridge University", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-085", "scenario": null, "predictions": [ { "start": 3, "end": 9, "label": "CITY", "text": "Bergen", "score": 0.9995262622833252, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 11, "end": 26, "label": "PERSON", "text": "Alva Rosenqvist", "score": 0.9998084902763367, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 38, "end": 60, "label": "ORGANIZATION", "text": "Maple Ridge University", "score": 0.99769526720047, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 81, "end": 96, "label": "PERSON", "text": "Alva Rosenqvist", "score": 0.999796450138092, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 109, "end": 115, "label": "CITY", "text": "Bergen", "score": 0.9991483688354492, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 3, "end": 9, "label": "CITY", "text": "Bergen", "score": 0.9995262622833252, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 11, "end": 26, "label": "PERSON", "text": "Alva Rosenqvist", "score": 0.9998084902763367, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 38, "end": 60, "label": "ORGANIZATION", "text": "Maple Ridge University", "score": 0.99769526720047, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 81, "end": 96, "label": "PERSON", "text": "Alva Rosenqvist", "score": 0.999796450138092, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 109, "end": 115, "label": "CITY", "text": "Bergen", "score": 0.9991483688354492, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 3, "end": 9, "label": "CITY", "text": "Bergen", "uncovered_positions": [] }, { "start": 11, "end": 26, "label": "PERSON", "text": "Alva Rosenqvist", "uncovered_positions": [] }, { "start": 38, "end": 60, "label": "ORGANIZATION", "text": "Maple Ridge University", "uncovered_positions": [] }, { "start": 81, "end": 96, "label": "PERSON", "text": "Alva Rosenqvist", "uncovered_positions": [] }, { "start": 109, "end": 115, "label": "CITY", "text": "Bergen", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-086", "scenario": null, "predictions": [ { "start": 6, "end": 22, "label": "PERSON", "text": "Alva-\nRosenqvist", "score": 0.9996846914291382, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "dehyphenation-v1" }, { "start": 28, "end": 36, "label": "CITY", "text": "Ber-\ngen", "score": 0.9998655319213867, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "dehyphenation-v1" }, { "start": 50, "end": 72, "label": "ORGANIZATION", "text": "Maple Ridge University", "score": 0.9983516931533813, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "dehyphenation-v1" } ], "combined_predictions": [ { "start": 6, "end": 22, "label": "PERSON", "text": "Alva-\nRosenqvist", "score": 0.9996846914291382, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "dehyphenation-v1" }, { "start": 28, "end": 36, "label": "CITY", "text": "Ber-\ngen", "score": 0.9998655319213867, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "dehyphenation-v1" }, { "start": 50, "end": 72, "label": "ORGANIZATION", "text": "Maple Ridge University", "score": 0.9983516931533813, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "dehyphenation-v1" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 6, "end": 22, "label": "PERSON", "text": "Alva-\nRosenqvist", "uncovered_positions": [] }, { "start": 28, "end": 36, "label": "CITY", "text": "Ber-\ngen", "uncovered_positions": [] }, { "start": 50, "end": 72, "label": "ORGANIZATION", "text": "Maple Ridge University", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-087", "scenario": null, "predictions": [ { "start": 16, "end": 31, "label": "PERSON", "text": "Alva R0senqv1st", "score": 0.951907753944397, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 43, "end": 49, "label": "CITY", "text": "Bergen", "score": 0.9994907379150391, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 63, "end": 85, "label": "ORGANIZATION", "text": "Maple Ridge University", "score": 0.9973485469818115, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 16, "end": 31, "label": "PERSON", "text": "Alva R0senqv1st", "score": 0.951907753944397, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 43, "end": 49, "label": "CITY", "text": "Bergen", "score": 0.9994907379150391, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 63, "end": 85, "label": "ORGANIZATION", "text": "Maple Ridge University", "score": 0.9973485469818115, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 16, "end": 31, "label": "PERSON", "text": "Alva R0senqv1st", "uncovered_positions": [] }, { "start": 43, "end": 49, "label": "CITY", "text": "Bergen", "uncovered_positions": [] }, { "start": 63, "end": 85, "label": "ORGANIZATION", "text": "Maple Ridge University", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-088", "scenario": null, "predictions": [ { "start": 10, "end": 17, "label": "RELIGION", "text": "jüdisch", "score": 0.8295804262161255, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 44, "end": 66, "label": "ORGANIZATION", "text": "Maple Ridge University", "score": 0.9811441898345947, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 80, "end": 95, "label": "PERSON", "text": "Alva Rosenqvist", "score": 0.998078465461731, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 10, "end": 17, "label": "RELIGION", "text": "jüdisch", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" }, { "start": 44, "end": 66, "label": "ORGANIZATION", "text": "Maple Ridge University", "score": 0.9811441898345947, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 80, "end": 95, "label": "PERSON", "text": "Alva Rosenqvist", "score": 0.998078465461731, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 10, "end": 17, "label": "RELIGION", "text": "jüdisch", "uncovered_positions": [] }, { "start": 44, "end": 66, "label": "ORGANIZATION", "text": "Maple Ridge University", "uncovered_positions": [] }, { "start": 80, "end": 95, "label": "PERSON", "text": "Alva Rosenqvist", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-089", "scenario": null, "predictions": [ { "start": 101, "end": 107, "label": "PERSON", "text": "person", "score": 0.7157840132713318, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 109, "end": 114, "label": "CITY", "text": "place", "score": 0.7889829277992249, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 116, "end": 128, "label": "ORGANIZATION", "text": "organization", "score": 0.8459369540214539, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 40, "end": 64, "label": "RELIGION", "text": "no religious affiliation", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" }, { "start": 101, "end": 107, "label": "PERSON", "text": "person", "score": 0.7157840132713318, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 109, "end": 114, "label": "CITY", "text": "place", "score": 0.7889829277992249, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 116, "end": 128, "label": "ORGANIZATION", "text": "organization", "score": 0.8459369540214539, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [ [ 40, 64, "RELIGION" ], [ 101, 107, "PERSON" ], [ 109, 114, "CITY" ], [ 116, 128, "ORGANIZATION" ] ], "false_negatives": [], "exact_match": false, "error_code": null, "canaries": [], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-090", "scenario": null, "predictions": [ { "start": 10, "end": 27, "label": "PERSON", "text": "Theo Merriweather", "score": 0.9981582760810852, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 41, "end": 59, "label": "ORGANIZATION", "text": "Polar Systems Ltd.", "score": 0.9988139867782593, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 63, "end": 67, "label": "CITY", "text": "York", "score": 0.9996792078018188, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 10, "end": 27, "label": "PERSON", "text": "Theo Merriweather", "score": 0.9981582760810852, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 41, "end": 59, "label": "ORGANIZATION", "text": "Polar Systems Ltd.", "score": 0.9988139867782593, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 63, "end": 67, "label": "CITY", "text": "York", "score": 0.9996792078018188, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 10, "end": 27, "label": "PERSON", "text": "Theo Merriweather", "uncovered_positions": [] }, { "start": 41, "end": 59, "label": "ORGANIZATION", "text": "Polar Systems Ltd.", "uncovered_positions": [] }, { "start": 63, "end": 67, "label": "CITY", "text": "York", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-091", "scenario": null, "predictions": [ { "start": 12, "end": 29, "label": "PERSON", "text": "Theo Merriweather", "score": 0.9984655380249023, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 41, "end": 59, "label": "ORGANIZATION", "text": "Polar Systems Ltd.", "score": 0.998237133026123, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 65, "end": 69, "label": "CITY", "text": "York", "score": 0.9996891021728516, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 12, "end": 29, "label": "PERSON", "text": "Theo Merriweather", "score": 0.9984655380249023, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 41, "end": 59, "label": "ORGANIZATION", "text": "Polar Systems Ltd.", "score": 0.998237133026123, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 65, "end": 69, "label": "CITY", "text": "York", "score": 0.9996891021728516, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 12, "end": 29, "label": "PERSON", "text": "Theo Merriweather", "uncovered_positions": [] }, { "start": 41, "end": 59, "label": "ORGANIZATION", "text": "Polar Systems Ltd.", "uncovered_positions": [] }, { "start": 65, "end": 69, "label": "CITY", "text": "York", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-092", "scenario": null, "predictions": [ { "start": 0, "end": 17, "label": "PERSON", "text": "Theo Merriweather", "score": 0.9836243391036987, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 24, "end": 34, "label": "RELIGION", "text": "muslimisch", "score": 0.9906712770462036, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 62, "end": 92, "label": "EMAIL", "text": "case110.review@example.invalid", "score": 0.9017051458358765, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 103, "end": 117, "label": "PHONE", "text": "+49 541 730110", "score": 0.9932438731193542, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 0, "end": 17, "label": "PERSON", "text": "Theo Merriweather", "score": 0.9836243391036987, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 24, "end": 34, "label": "RELIGION", "text": "muslimisch", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" }, { "start": 62, "end": 92, "label": "EMAIL", "text": "case110.review@example.invalid", "score": null, "source": "deterministic", "model_revision": "contact-patterns-v1", "normalization": "identity" }, { "start": 103, "end": 117, "label": "PHONE", "text": "+49 541 730110", "score": null, "source": "deterministic", "model_revision": "contact-patterns-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 0, "end": 17, "label": "PERSON", "text": "Theo Merriweather", "uncovered_positions": [] }, { "start": 24, "end": 34, "label": "RELIGION", "text": "muslimisch", "uncovered_positions": [] }, { "start": 62, "end": 92, "label": "EMAIL", "text": "case110.review@example.invalid", "uncovered_positions": [] }, { "start": 103, "end": 117, "label": "PHONE", "text": "+49 541 730110", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-093", "scenario": null, "predictions": [ { "start": 0, "end": 17, "label": "PERSON", "text": "Theo Merriweather", "score": 0.9977192282676697, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 35, "end": 40, "label": "RELIGION", "text": "Islam", "score": 0.999936580657959, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 66, "end": 96, "label": "EMAIL", "text": "case110.review@example.invalid", "score": 0.8946349620819092, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 105, "end": 119, "label": "PHONE", "text": "+49 541 730110", "score": 0.9599586129188538, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 0, "end": 17, "label": "PERSON", "text": "Theo Merriweather", "score": 0.9977192282676697, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 35, "end": 40, "label": "RELIGION", "text": "Islam", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" }, { "start": 66, "end": 96, "label": "EMAIL", "text": "case110.review@example.invalid", "score": null, "source": "deterministic", "model_revision": "contact-patterns-v1", "normalization": "identity" }, { "start": 105, "end": 119, "label": "PHONE", "text": "+49 541 730110", "score": null, "source": "deterministic", "model_revision": "contact-patterns-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 0, "end": 17, "label": "PERSON", "text": "Theo Merriweather", "uncovered_positions": [] }, { "start": 35, "end": 40, "label": "RELIGION", "text": "Islam", "uncovered_positions": [] }, { "start": 66, "end": 96, "label": "EMAIL", "text": "case110.review@example.invalid", "uncovered_positions": [] }, { "start": 105, "end": 119, "label": "PHONE", "text": "+49 541 730110", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-094", "scenario": null, "predictions": [ { "start": 0, "end": 17, "label": "PERSON", "text": "Theo Merriweather", "score": 0.9987108707427979, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 26, "end": 30, "label": "CITY", "text": "York", "score": 0.9983842372894287, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 33, "end": 50, "label": "PERSON", "text": "Theo Merriweather", "score": 0.9984440207481384, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 76, "end": 94, "label": "ORGANIZATION", "text": "Polar Systems Ltd.", "score": 0.9534488320350647, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 0, "end": 17, "label": "PERSON", "text": "Theo Merriweather", "score": 0.9987108707427979, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 26, "end": 30, "label": "CITY", "text": "York", "score": 0.9983842372894287, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 33, "end": 50, "label": "PERSON", "text": "Theo Merriweather", "score": 0.9984440207481384, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 76, "end": 94, "label": "ORGANIZATION", "text": "Polar Systems Ltd.", "score": 0.9534488320350647, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 0, "end": 17, "label": "PERSON", "text": "Theo Merriweather", "uncovered_positions": [] }, { "start": 26, "end": 30, "label": "CITY", "text": "York", "uncovered_positions": [] }, { "start": 33, "end": 50, "label": "PERSON", "text": "Theo Merriweather", "uncovered_positions": [] }, { "start": 76, "end": 94, "label": "ORGANIZATION", "text": "Polar Systems Ltd.", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-095", "scenario": null, "predictions": [ { "start": 3, "end": 7, "label": "CITY", "text": "York", "score": 0.9998012185096741, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 9, "end": 26, "label": "PERSON", "text": "Theo Merriweather", "score": 0.9998037219047546, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 38, "end": 56, "label": "ORGANIZATION", "text": "Polar Systems Ltd.", "score": 0.9765823483467102, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 77, "end": 94, "label": "PERSON", "text": "Theo Merriweather", "score": 0.9997954964637756, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 107, "end": 111, "label": "CITY", "text": "York", "score": 0.9997724890708923, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 3, "end": 7, "label": "CITY", "text": "York", "score": 0.9998012185096741, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 9, "end": 26, "label": "PERSON", "text": "Theo Merriweather", "score": 0.9998037219047546, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 38, "end": 56, "label": "ORGANIZATION", "text": "Polar Systems Ltd.", "score": 0.9765823483467102, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 77, "end": 94, "label": "PERSON", "text": "Theo Merriweather", "score": 0.9997954964637756, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 107, "end": 111, "label": "CITY", "text": "York", "score": 0.9997724890708923, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 3, "end": 7, "label": "CITY", "text": "York", "uncovered_positions": [] }, { "start": 9, "end": 26, "label": "PERSON", "text": "Theo Merriweather", "uncovered_positions": [] }, { "start": 38, "end": 56, "label": "ORGANIZATION", "text": "Polar Systems Ltd.", "uncovered_positions": [] }, { "start": 77, "end": 94, "label": "PERSON", "text": "Theo Merriweather", "uncovered_positions": [] }, { "start": 107, "end": 111, "label": "CITY", "text": "York", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-096", "scenario": null, "predictions": [ { "start": 6, "end": 24, "label": "PERSON", "text": "Theo-\nMerriweather", "score": 0.9998553991317749, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "dehyphenation-v1" }, { "start": 30, "end": 36, "label": "CITY", "text": "Yo-\nrk", "score": 0.9999533891677856, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "dehyphenation-v1" }, { "start": 50, "end": 68, "label": "ORGANIZATION", "text": "Polar Systems Ltd.", "score": 0.9765849709510803, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "dehyphenation-v1" } ], "combined_predictions": [ { "start": 6, "end": 24, "label": "PERSON", "text": "Theo-\nMerriweather", "score": 0.9998553991317749, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "dehyphenation-v1" }, { "start": 30, "end": 36, "label": "CITY", "text": "Yo-\nrk", "score": 0.9999533891677856, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "dehyphenation-v1" }, { "start": 50, "end": 68, "label": "ORGANIZATION", "text": "Polar Systems Ltd.", "score": 0.9765849709510803, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "dehyphenation-v1" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 6, "end": 24, "label": "PERSON", "text": "Theo-\nMerriweather", "uncovered_positions": [] }, { "start": 30, "end": 36, "label": "CITY", "text": "Yo-\nrk", "uncovered_positions": [] }, { "start": 50, "end": 68, "label": "ORGANIZATION", "text": "Polar Systems Ltd.", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-097", "scenario": null, "predictions": [ { "start": 16, "end": 33, "label": "PERSON", "text": "The0 Merr1weather", "score": 0.9711146950721741, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 45, "end": 49, "label": "CITY", "text": "Y0rk", "score": 0.9964478015899658, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 63, "end": 81, "label": "ORGANIZATION", "text": "Polar Systems Ltd.", "score": 0.9427483677864075, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 16, "end": 33, "label": "PERSON", "text": "The0 Merr1weather", "score": 0.9711146950721741, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 45, "end": 49, "label": "CITY", "text": "Y0rk", "score": 0.9964478015899658, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 63, "end": 81, "label": "ORGANIZATION", "text": "Polar Systems Ltd.", "score": 0.9427483677864075, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 16, "end": 33, "label": "PERSON", "text": "The0 Merr1weather", "uncovered_positions": [] }, { "start": 45, "end": 49, "label": "CITY", "text": "Y0rk", "uncovered_positions": [] }, { "start": 63, "end": 81, "label": "ORGANIZATION", "text": "Polar Systems Ltd.", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-098", "scenario": null, "predictions": [ { "start": 10, "end": 20, "label": "RELIGION", "text": "muslimisch", "score": 0.8173997402191162, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 47, "end": 65, "label": "ORGANIZATION", "text": "Polar Systems Ltd.", "score": 0.990338146686554, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 79, "end": 96, "label": "PERSON", "text": "Theo Merriweather", "score": 0.9961032867431641, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 10, "end": 20, "label": "RELIGION", "text": "muslimisch", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" }, { "start": 47, "end": 65, "label": "ORGANIZATION", "text": "Polar Systems Ltd.", "score": 0.990338146686554, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 79, "end": 96, "label": "PERSON", "text": "Theo Merriweather", "score": 0.9961032867431641, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 10, "end": 20, "label": "RELIGION", "text": "muslimisch", "uncovered_positions": [] }, { "start": 47, "end": 65, "label": "ORGANIZATION", "text": "Polar Systems Ltd.", "uncovered_positions": [] }, { "start": 79, "end": 96, "label": "PERSON", "text": "Theo Merriweather", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-099", "scenario": null, "predictions": [ { "start": 101, "end": 107, "label": "PERSON", "text": "person", "score": 0.7109703421592712, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 109, "end": 114, "label": "CITY", "text": "place", "score": 0.7852903604507446, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 116, "end": 128, "label": "ORGANIZATION", "text": "organization", "score": 0.8414570093154907, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 40, "end": 64, "label": "RELIGION", "text": "no religious affiliation", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" }, { "start": 101, "end": 107, "label": "PERSON", "text": "person", "score": 0.7109703421592712, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 109, "end": 114, "label": "CITY", "text": "place", "score": 0.7852903604507446, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 116, "end": 128, "label": "ORGANIZATION", "text": "organization", "score": 0.8414570093154907, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [ [ 40, 64, "RELIGION" ], [ 101, 107, "PERSON" ], [ 109, 114, "CITY" ], [ 116, 128, "ORGANIZATION" ] ], "false_negatives": [], "exact_match": false, "error_code": null, "canaries": [], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-100", "scenario": null, "predictions": [ { "start": 10, "end": 25, "label": "PERSON", "text": "Maja Kirschbaum", "score": 0.9991471767425537, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 39, "end": 57, "label": "ORGANIZATION", "text": "Sonnenbogen Verein", "score": 0.9799805283546448, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 61, "end": 69, "label": "CITY", "text": "Lüneburg", "score": 0.9990435242652893, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 10, "end": 25, "label": "PERSON", "text": "Maja Kirschbaum", "score": 0.9991471767425537, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 39, "end": 57, "label": "ORGANIZATION", "text": "Sonnenbogen Verein", "score": 0.9799805283546448, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 61, "end": 69, "label": "CITY", "text": "Lüneburg", "score": 0.9990435242652893, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 10, "end": 25, "label": "PERSON", "text": "Maja Kirschbaum", "uncovered_positions": [] }, { "start": 39, "end": 57, "label": "ORGANIZATION", "text": "Sonnenbogen Verein", "uncovered_positions": [] }, { "start": 61, "end": 69, "label": "CITY", "text": "Lüneburg", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-101", "scenario": null, "predictions": [ { "start": 12, "end": 27, "label": "PERSON", "text": "Maja Kirschbaum", "score": 0.9968947172164917, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 39, "end": 57, "label": "ORGANIZATION", "text": "Sonnenbogen Verein", "score": 0.9929682612419128, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 63, "end": 71, "label": "CITY", "text": "Lüneburg", "score": 0.9971613883972168, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 12, "end": 27, "label": "PERSON", "text": "Maja Kirschbaum", "score": 0.9968947172164917, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 39, "end": 57, "label": "ORGANIZATION", "text": "Sonnenbogen Verein", "score": 0.9929682612419128, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 63, "end": 71, "label": "CITY", "text": "Lüneburg", "score": 0.9971613883972168, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 12, "end": 27, "label": "PERSON", "text": "Maja Kirschbaum", "uncovered_positions": [] }, { "start": 39, "end": 57, "label": "ORGANIZATION", "text": "Sonnenbogen Verein", "uncovered_positions": [] }, { "start": 63, "end": 71, "label": "CITY", "text": "Lüneburg", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-102", "scenario": null, "predictions": [ { "start": 0, "end": 15, "label": "PERSON", "text": "Maja Kirschbaum", "score": 0.998397171497345, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 64, "end": 94, "label": "EMAIL", "text": "case111.review@example.invalid", "score": 0.8911723494529724, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 105, "end": 119, "label": "PHONE", "text": "+49 541 730111", "score": 0.9943364262580872, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 0, "end": 15, "label": "PERSON", "text": "Maja Kirschbaum", "score": 0.998397171497345, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 22, "end": 36, "label": "RELIGION", "text": "konfessionslos", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" }, { "start": 64, "end": 94, "label": "EMAIL", "text": "case111.review@example.invalid", "score": null, "source": "deterministic", "model_revision": "contact-patterns-v1", "normalization": "identity" }, { "start": 105, "end": 119, "label": "PHONE", "text": "+49 541 730111", "score": null, "source": "deterministic", "model_revision": "contact-patterns-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 0, "end": 15, "label": "PERSON", "text": "Maja Kirschbaum", "uncovered_positions": [] }, { "start": 22, "end": 36, "label": "RELIGION", "text": "konfessionslos", "uncovered_positions": [] }, { "start": 64, "end": 94, "label": "EMAIL", "text": "case111.review@example.invalid", "uncovered_positions": [] }, { "start": 105, "end": 119, "label": "PHONE", "text": "+49 541 730111", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-103", "scenario": null, "predictions": [ { "start": 0, "end": 15, "label": "PERSON", "text": "Maja Kirschbaum", "score": 0.9988783001899719, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 33, "end": 40, "label": "RELIGION", "text": "atheism", "score": 0.99891197681427, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 66, "end": 96, "label": "EMAIL", "text": "case111.review@example.invalid", "score": 0.8873010277748108, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 105, "end": 119, "label": "PHONE", "text": "+49 541 730111", "score": 0.9670958518981934, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 0, "end": 15, "label": "PERSON", "text": "Maja Kirschbaum", "score": 0.9988783001899719, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 33, "end": 40, "label": "RELIGION", "text": "atheism", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" }, { "start": 66, "end": 96, "label": "EMAIL", "text": "case111.review@example.invalid", "score": null, "source": "deterministic", "model_revision": "contact-patterns-v1", "normalization": "identity" }, { "start": 105, "end": 119, "label": "PHONE", "text": "+49 541 730111", "score": null, "source": "deterministic", "model_revision": "contact-patterns-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 0, "end": 15, "label": "PERSON", "text": "Maja Kirschbaum", "uncovered_positions": [] }, { "start": 33, "end": 40, "label": "RELIGION", "text": "atheism", "uncovered_positions": [] }, { "start": 66, "end": 96, "label": "EMAIL", "text": "case111.review@example.invalid", "uncovered_positions": [] }, { "start": 105, "end": 119, "label": "PHONE", "text": "+49 541 730111", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-104", "scenario": null, "predictions": [ { "start": 0, "end": 15, "label": "PERSON", "text": "Maja Kirschbaum", "score": 0.9995399713516235, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 24, "end": 32, "label": "CITY", "text": "Lüneburg", "score": 0.9943020343780518, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 35, "end": 50, "label": "PERSON", "text": "Maja Kirschbaum", "score": 0.9996929168701172, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 76, "end": 94, "label": "ORGANIZATION", "text": "Sonnenbogen Verein", "score": 0.9872075915336609, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 0, "end": 15, "label": "PERSON", "text": "Maja Kirschbaum", "score": 0.9995399713516235, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 24, "end": 32, "label": "CITY", "text": "Lüneburg", "score": 0.9943020343780518, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 35, "end": 50, "label": "PERSON", "text": "Maja Kirschbaum", "score": 0.9996929168701172, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 76, "end": 94, "label": "ORGANIZATION", "text": "Sonnenbogen Verein", "score": 0.9872075915336609, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 0, "end": 15, "label": "PERSON", "text": "Maja Kirschbaum", "uncovered_positions": [] }, { "start": 24, "end": 32, "label": "CITY", "text": "Lüneburg", "uncovered_positions": [] }, { "start": 35, "end": 50, "label": "PERSON", "text": "Maja Kirschbaum", "uncovered_positions": [] }, { "start": 76, "end": 94, "label": "ORGANIZATION", "text": "Sonnenbogen Verein", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-105", "scenario": null, "predictions": [ { "start": 3, "end": 11, "label": "CITY", "text": "Lüneburg", "score": 0.9998090863227844, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 13, "end": 28, "label": "PERSON", "text": "Maja Kirschbaum", "score": 0.9998596906661987, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 40, "end": 58, "label": "ORGANIZATION", "text": "Sonnenbogen Verein", "score": 0.9954612851142883, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 79, "end": 94, "label": "PERSON", "text": "Maja Kirschbaum", "score": 0.9997907280921936, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 107, "end": 115, "label": "CITY", "text": "Lüneburg", "score": 0.9995126724243164, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 3, "end": 11, "label": "CITY", "text": "Lüneburg", "score": 0.9998090863227844, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 13, "end": 28, "label": "PERSON", "text": "Maja Kirschbaum", "score": 0.9998596906661987, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 40, "end": 58, "label": "ORGANIZATION", "text": "Sonnenbogen Verein", "score": 0.9954612851142883, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 79, "end": 94, "label": "PERSON", "text": "Maja Kirschbaum", "score": 0.9997907280921936, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 107, "end": 115, "label": "CITY", "text": "Lüneburg", "score": 0.9995126724243164, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 3, "end": 11, "label": "CITY", "text": "Lüneburg", "uncovered_positions": [] }, { "start": 13, "end": 28, "label": "PERSON", "text": "Maja Kirschbaum", "uncovered_positions": [] }, { "start": 40, "end": 58, "label": "ORGANIZATION", "text": "Sonnenbogen Verein", "uncovered_positions": [] }, { "start": 79, "end": 94, "label": "PERSON", "text": "Maja Kirschbaum", "uncovered_positions": [] }, { "start": 107, "end": 115, "label": "CITY", "text": "Lüneburg", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-106", "scenario": null, "predictions": [ { "start": 6, "end": 22, "label": "PERSON", "text": "Maja-\nKirschbaum", "score": 0.999875545501709, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "dehyphenation-v1" }, { "start": 28, "end": 38, "label": "CITY", "text": "Lüne-\nburg", "score": 0.9999223947525024, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "dehyphenation-v1" }, { "start": 52, "end": 70, "label": "ORGANIZATION", "text": "Sonnenbogen Verein", "score": 0.9918258190155029, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "dehyphenation-v1" } ], "combined_predictions": [ { "start": 6, "end": 22, "label": "PERSON", "text": "Maja-\nKirschbaum", "score": 0.999875545501709, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "dehyphenation-v1" }, { "start": 28, "end": 38, "label": "CITY", "text": "Lüne-\nburg", "score": 0.9999223947525024, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "dehyphenation-v1" }, { "start": 52, "end": 70, "label": "ORGANIZATION", "text": "Sonnenbogen Verein", "score": 0.9918258190155029, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "dehyphenation-v1" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 6, "end": 22, "label": "PERSON", "text": "Maja-\nKirschbaum", "uncovered_positions": [] }, { "start": 28, "end": 38, "label": "CITY", "text": "Lüne-\nburg", "uncovered_positions": [] }, { "start": 52, "end": 70, "label": "ORGANIZATION", "text": "Sonnenbogen Verein", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-107", "scenario": null, "predictions": [ { "start": 16, "end": 31, "label": "PERSON", "text": "Maja K1rschbaum", "score": 0.9977808594703674, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 43, "end": 51, "label": "CITY", "text": "Lüneburg", "score": 0.9994350075721741, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 65, "end": 83, "label": "ORGANIZATION", "text": "Sonnenbogen Verein", "score": 0.9930818676948547, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 16, "end": 31, "label": "PERSON", "text": "Maja K1rschbaum", "score": 0.9977808594703674, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 43, "end": 51, "label": "CITY", "text": "Lüneburg", "score": 0.9994350075721741, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 65, "end": 83, "label": "ORGANIZATION", "text": "Sonnenbogen Verein", "score": 0.9930818676948547, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 16, "end": 31, "label": "PERSON", "text": "Maja K1rschbaum", "uncovered_positions": [] }, { "start": 43, "end": 51, "label": "CITY", "text": "Lüneburg", "uncovered_positions": [] }, { "start": 65, "end": 83, "label": "ORGANIZATION", "text": "Sonnenbogen Verein", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-108", "scenario": null, "predictions": [ { "start": 51, "end": 69, "label": "ORGANIZATION", "text": "Sonnenbogen Verein", "score": 0.9550279378890991, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 83, "end": 98, "label": "PERSON", "text": "Maja Kirschbaum", "score": 0.9982492923736572, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 10, "end": 24, "label": "RELIGION", "text": "konfessionslos", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" }, { "start": 51, "end": 69, "label": "ORGANIZATION", "text": "Sonnenbogen Verein", "score": 0.9550279378890991, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 83, "end": 98, "label": "PERSON", "text": "Maja Kirschbaum", "score": 0.9982492923736572, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 10, "end": 24, "label": "RELIGION", "text": "konfessionslos", "uncovered_positions": [] }, { "start": 51, "end": 69, "label": "ORGANIZATION", "text": "Sonnenbogen Verein", "uncovered_positions": [] }, { "start": 83, "end": 98, "label": "PERSON", "text": "Maja Kirschbaum", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-109", "scenario": null, "predictions": [ { "start": 101, "end": 107, "label": "PERSON", "text": "person", "score": 0.6989420056343079, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 109, "end": 114, "label": "CITY", "text": "place", "score": 0.7754619121551514, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 116, "end": 128, "label": "ORGANIZATION", "text": "organization", "score": 0.8311129808425903, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 40, "end": 64, "label": "RELIGION", "text": "no religious affiliation", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" }, { "start": 101, "end": 107, "label": "PERSON", "text": "person", "score": 0.6989420056343079, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 109, "end": 114, "label": "CITY", "text": "place", "score": 0.7754619121551514, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 116, "end": 128, "label": "ORGANIZATION", "text": "organization", "score": 0.8311129808425903, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [ [ 40, 64, "RELIGION" ], [ 101, 107, "PERSON" ], [ 109, 114, "CITY" ], [ 116, 128, "ORGANIZATION" ] ], "false_negatives": [], "exact_match": false, "error_code": null, "canaries": [], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-110", "scenario": null, "predictions": [ { "start": 10, "end": 24, "label": "PERSON", "text": "Ronan Whitmore", "score": 0.9985572695732117, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 38, "end": 56, "label": "ORGANIZATION", "text": "Beacon Valley Labs", "score": 0.9976409673690796, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 60, "end": 64, "label": "CITY", "text": "Bath", "score": 0.9994964599609375, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 10, "end": 24, "label": "PERSON", "text": "Ronan Whitmore", "score": 0.9985572695732117, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 38, "end": 56, "label": "ORGANIZATION", "text": "Beacon Valley Labs", "score": 0.9976409673690796, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 60, "end": 64, "label": "CITY", "text": "Bath", "score": 0.9994964599609375, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 10, "end": 24, "label": "PERSON", "text": "Ronan Whitmore", "uncovered_positions": [] }, { "start": 38, "end": 56, "label": "ORGANIZATION", "text": "Beacon Valley Labs", "uncovered_positions": [] }, { "start": 60, "end": 64, "label": "CITY", "text": "Bath", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-111", "scenario": null, "predictions": [ { "start": 12, "end": 26, "label": "PERSON", "text": "Ronan Whitmore", "score": 0.998796820640564, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 38, "end": 56, "label": "ORGANIZATION", "text": "Beacon Valley Labs", "score": 0.9980693459510803, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 62, "end": 66, "label": "CITY", "text": "Bath", "score": 0.9995183944702148, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 12, "end": 26, "label": "PERSON", "text": "Ronan Whitmore", "score": 0.998796820640564, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 38, "end": 56, "label": "ORGANIZATION", "text": "Beacon Valley Labs", "score": 0.9980693459510803, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 62, "end": 66, "label": "CITY", "text": "Bath", "score": 0.9995183944702148, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 12, "end": 26, "label": "PERSON", "text": "Ronan Whitmore", "uncovered_positions": [] }, { "start": 38, "end": 56, "label": "ORGANIZATION", "text": "Beacon Valley Labs", "uncovered_positions": [] }, { "start": 62, "end": 66, "label": "CITY", "text": "Bath", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-112", "scenario": null, "predictions": [ { "start": 0, "end": 14, "label": "PERSON", "text": "Ronan Whitmore", "score": 0.99787437915802, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 21, "end": 31, "label": "RELIGION", "text": "christlich", "score": 0.9895966649055481, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 59, "end": 89, "label": "EMAIL", "text": "case112.review@example.invalid", "score": 0.9079257249832153, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 100, "end": 114, "label": "PHONE", "text": "+49 541 730112", "score": 0.9931344985961914, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 0, "end": 14, "label": "PERSON", "text": "Ronan Whitmore", "score": 0.99787437915802, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 21, "end": 31, "label": "RELIGION", "text": "christlich", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" }, { "start": 59, "end": 89, "label": "EMAIL", "text": "case112.review@example.invalid", "score": null, "source": "deterministic", "model_revision": "contact-patterns-v1", "normalization": "identity" }, { "start": 100, "end": 114, "label": "PHONE", "text": "+49 541 730112", "score": null, "source": "deterministic", "model_revision": "contact-patterns-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 0, "end": 14, "label": "PERSON", "text": "Ronan Whitmore", "uncovered_positions": [] }, { "start": 21, "end": 31, "label": "RELIGION", "text": "christlich", "uncovered_positions": [] }, { "start": 59, "end": 89, "label": "EMAIL", "text": "case112.review@example.invalid", "uncovered_positions": [] }, { "start": 100, "end": 114, "label": "PHONE", "text": "+49 541 730112", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-113", "scenario": null, "predictions": [ { "start": 0, "end": 14, "label": "PERSON", "text": "Ronan Whitmore", "score": 0.9984022974967957, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 32, "end": 44, "label": "RELIGION", "text": "Christianity", "score": 0.9992650151252747, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 70, "end": 100, "label": "EMAIL", "text": "case112.review@example.invalid", "score": 0.9013476371765137, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 109, "end": 123, "label": "PHONE", "text": "+49 541 730112", "score": 0.9611164331436157, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 0, "end": 14, "label": "PERSON", "text": "Ronan Whitmore", "score": 0.9984022974967957, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 32, "end": 44, "label": "RELIGION", "text": "Christianity", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" }, { "start": 70, "end": 100, "label": "EMAIL", "text": "case112.review@example.invalid", "score": null, "source": "deterministic", "model_revision": "contact-patterns-v1", "normalization": "identity" }, { "start": 109, "end": 123, "label": "PHONE", "text": "+49 541 730112", "score": null, "source": "deterministic", "model_revision": "contact-patterns-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 0, "end": 14, "label": "PERSON", "text": "Ronan Whitmore", "uncovered_positions": [] }, { "start": 32, "end": 44, "label": "RELIGION", "text": "Christianity", "uncovered_positions": [] }, { "start": 70, "end": 100, "label": "EMAIL", "text": "case112.review@example.invalid", "uncovered_positions": [] }, { "start": 109, "end": 123, "label": "PHONE", "text": "+49 541 730112", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-114", "scenario": null, "predictions": [ { "start": 0, "end": 14, "label": "PERSON", "text": "Ronan Whitmore", "score": 0.9992198944091797, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 23, "end": 27, "label": "CITY", "text": "Bath", "score": 0.9989506006240845, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 30, "end": 44, "label": "PERSON", "text": "Ronan Whitmore", "score": 0.9990702271461487, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 70, "end": 88, "label": "ORGANIZATION", "text": "Beacon Valley Labs", "score": 0.9951003193855286, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 0, "end": 14, "label": "PERSON", "text": "Ronan Whitmore", "score": 0.9992198944091797, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 23, "end": 27, "label": "CITY", "text": "Bath", "score": 0.9989506006240845, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 30, "end": 44, "label": "PERSON", "text": "Ronan Whitmore", "score": 0.9990702271461487, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 70, "end": 88, "label": "ORGANIZATION", "text": "Beacon Valley Labs", "score": 0.9951003193855286, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 0, "end": 14, "label": "PERSON", "text": "Ronan Whitmore", "uncovered_positions": [] }, { "start": 23, "end": 27, "label": "CITY", "text": "Bath", "uncovered_positions": [] }, { "start": 30, "end": 44, "label": "PERSON", "text": "Ronan Whitmore", "uncovered_positions": [] }, { "start": 70, "end": 88, "label": "ORGANIZATION", "text": "Beacon Valley Labs", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-115", "scenario": null, "predictions": [ { "start": 3, "end": 7, "label": "CITY", "text": "Bath", "score": 0.9997978806495667, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 9, "end": 23, "label": "PERSON", "text": "Ronan Whitmore", "score": 0.999768078327179, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 35, "end": 53, "label": "ORGANIZATION", "text": "Beacon Valley Labs", "score": 0.9973915815353394, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 74, "end": 88, "label": "PERSON", "text": "Ronan Whitmore", "score": 0.9997625946998596, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 101, "end": 105, "label": "CITY", "text": "Bath", "score": 0.9997105002403259, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 3, "end": 7, "label": "CITY", "text": "Bath", "score": 0.9997978806495667, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 9, "end": 23, "label": "PERSON", "text": "Ronan Whitmore", "score": 0.999768078327179, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 35, "end": 53, "label": "ORGANIZATION", "text": "Beacon Valley Labs", "score": 0.9973915815353394, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 74, "end": 88, "label": "PERSON", "text": "Ronan Whitmore", "score": 0.9997625946998596, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 101, "end": 105, "label": "CITY", "text": "Bath", "score": 0.9997105002403259, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 3, "end": 7, "label": "CITY", "text": "Bath", "uncovered_positions": [] }, { "start": 9, "end": 23, "label": "PERSON", "text": "Ronan Whitmore", "uncovered_positions": [] }, { "start": 35, "end": 53, "label": "ORGANIZATION", "text": "Beacon Valley Labs", "uncovered_positions": [] }, { "start": 74, "end": 88, "label": "PERSON", "text": "Ronan Whitmore", "uncovered_positions": [] }, { "start": 101, "end": 105, "label": "CITY", "text": "Bath", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-116", "scenario": null, "predictions": [ { "start": 6, "end": 21, "label": "PERSON", "text": "Ronan-\nWhitmore", "score": 0.9997969269752502, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "dehyphenation-v1" }, { "start": 27, "end": 33, "label": "CITY", "text": "Ba-\nth", "score": 0.9999775886535645, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "dehyphenation-v1" }, { "start": 47, "end": 65, "label": "ORGANIZATION", "text": "Beacon Valley Labs", "score": 0.9984259605407715, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "dehyphenation-v1" } ], "combined_predictions": [ { "start": 6, "end": 21, "label": "PERSON", "text": "Ronan-\nWhitmore", "score": 0.9997969269752502, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "dehyphenation-v1" }, { "start": 27, "end": 33, "label": "CITY", "text": "Ba-\nth", "score": 0.9999775886535645, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "dehyphenation-v1" }, { "start": 47, "end": 65, "label": "ORGANIZATION", "text": "Beacon Valley Labs", "score": 0.9984259605407715, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "dehyphenation-v1" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 6, "end": 21, "label": "PERSON", "text": "Ronan-\nWhitmore", "uncovered_positions": [] }, { "start": 27, "end": 33, "label": "CITY", "text": "Ba-\nth", "uncovered_positions": [] }, { "start": 47, "end": 65, "label": "ORGANIZATION", "text": "Beacon Valley Labs", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-117", "scenario": null, "predictions": [ { "start": 16, "end": 30, "label": "PERSON", "text": "R0nan Wh1tm0re", "score": 0.9801152944564819, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 42, "end": 46, "label": "CITY", "text": "B4th", "score": 0.9872779846191406, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 60, "end": 78, "label": "ORGANIZATION", "text": "Beacon Valley Labs", "score": 0.9977201819419861, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 16, "end": 30, "label": "PERSON", "text": "R0nan Wh1tm0re", "score": 0.9801152944564819, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 42, "end": 46, "label": "CITY", "text": "B4th", "score": 0.9872779846191406, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 60, "end": 78, "label": "ORGANIZATION", "text": "Beacon Valley Labs", "score": 0.9977201819419861, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 16, "end": 30, "label": "PERSON", "text": "R0nan Wh1tm0re", "uncovered_positions": [] }, { "start": 42, "end": 46, "label": "CITY", "text": "B4th", "uncovered_positions": [] }, { "start": 60, "end": 78, "label": "ORGANIZATION", "text": "Beacon Valley Labs", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-118", "scenario": null, "predictions": [ { "start": 10, "end": 20, "label": "RELIGION", "text": "christlich", "score": 0.7355935573577881, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 47, "end": 65, "label": "ORGANIZATION", "text": "Beacon Valley Labs", "score": 0.9870188236236572, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 79, "end": 93, "label": "PERSON", "text": "Ronan Whitmore", "score": 0.9990026354789734, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 10, "end": 20, "label": "RELIGION", "text": "christlich", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" }, { "start": 47, "end": 65, "label": "ORGANIZATION", "text": "Beacon Valley Labs", "score": 0.9870188236236572, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 79, "end": 93, "label": "PERSON", "text": "Ronan Whitmore", "score": 0.9990026354789734, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 10, "end": 20, "label": "RELIGION", "text": "christlich", "uncovered_positions": [] }, { "start": 47, "end": 65, "label": "ORGANIZATION", "text": "Beacon Valley Labs", "uncovered_positions": [] }, { "start": 79, "end": 93, "label": "PERSON", "text": "Ronan Whitmore", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-119", "scenario": null, "predictions": [ { "start": 101, "end": 107, "label": "PERSON", "text": "person", "score": 0.7034855484962463, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 109, "end": 114, "label": "CITY", "text": "place", "score": 0.7791854739189148, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 116, "end": 128, "label": "ORGANIZATION", "text": "organization", "score": 0.8351515531539917, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 40, "end": 64, "label": "RELIGION", "text": "no religious affiliation", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" }, { "start": 101, "end": 107, "label": "PERSON", "text": "person", "score": 0.7034855484962463, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 109, "end": 114, "label": "CITY", "text": "place", "score": 0.7791854739189148, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 116, "end": 128, "label": "ORGANIZATION", "text": "organization", "score": 0.8351515531539917, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [ [ 40, 64, "RELIGION" ], [ 101, 107, "PERSON" ], [ 109, 114, "CITY" ], [ 116, 128, "ORGANIZATION" ] ], "false_negatives": [], "exact_match": false, "error_code": null, "canaries": [], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" } ], "automated_release_eligible": false, "recognition_qualified": false }, "development03d": { "schema_version": "development-candidate-outcomes/v1", "accuracy": { "documents": 160, "ner_entity_metrics": { "PERSON": { "tp": 21, "fp": 9, "fn": 1, "precision": 0.7, "recall": 0.9545454545454546, "f1": 0.8076923076923077 }, "CITY": { "tp": 8, "fp": 3, "fn": 0, "precision": 0.7272727272727273, "recall": 1.0, "f1": 0.8421052631578948 }, "RELIGION": { "tp": 108, "fp": 6, "fn": 42, "precision": 0.9473684210526315, "recall": 0.72, "f1": 0.8181818181818181 }, "EMAIL": { "tp": 3, "fp": 0, "fn": 1, "precision": 1.0, "recall": 0.75, "f1": 0.8571428571428571 }, "PHONE": { "tp": 2, "fp": 0, "fn": 0, "precision": 1.0, "recall": 1.0, "f1": 1.0 }, "ORGANIZATION": { "tp": 5, "fp": 14, "fn": 1, "precision": 0.2631578947368421, "recall": 0.8333333333333334, "f1": 0.39999999999999997 } }, "combined_entity_metrics": { "PERSON": { "tp": 20, "fp": 5, "fn": 2, "precision": 0.8, "recall": 0.9090909090909091, "f1": 0.8510638297872342 }, "CITY": { "tp": 7, "fp": 2, "fn": 1, "precision": 0.7777777777777778, "recall": 0.875, "f1": 0.823529411764706 }, "RELIGION": { "tp": 137, "fp": 3, "fn": 13, "precision": 0.9785714285714285, "recall": 0.9133333333333333, "f1": 0.9448275862068967 }, "EMAIL": { "tp": 3, "fp": 0, "fn": 1, "precision": 1.0, "recall": 0.75, "f1": 0.8571428571428571 }, "PHONE": { "tp": 2, "fp": 0, "fn": 0, "precision": 1.0, "recall": 1.0, "f1": 1.0 }, "ORGANIZATION": { "tp": 5, "fp": 12, "fn": 1, "precision": 0.29411764705882354, "recall": 0.8333333333333334, "f1": 0.4347826086956522 } }, "document_exact_match_rate": 0.825, "groups": { "de": { "documents": 80, "exact_matches": 60, "false_negatives": 14 }, "en": { "documents": 80, "exact_matches": 72, "false_negatives": 4 }, "ocr_noise": { "documents": 16, "exact_matches": 10, "false_negatives": 4 } }, "blocked_documents": 5, "failure_codes": { "ambiguous_entity_overlap": 5 }, "automated_release_eligible": false, "uncovered_source_canaries_including_incomplete": { "PERSON": 1, "CITY": 1, "RELIGION": 9, "EMAIL": 1, "PHONE": 0, "ORGANIZATION": 1 }, "literal_presence_or_incomplete": { "PERSON": 1, "CITY": 1, "RELIGION": 8, "EMAIL": 1, "PHONE": 0, "ORGANIZATION": 0 }, "partially_covered_source_canaries_including_incomplete": { "PERSON": 0, "CITY": 0, "RELIGION": 1, "EMAIL": 0, "PHONE": 0, "ORGANIZATION": 1 }, "fully_uncovered_source_canaries_including_incomplete": { "PERSON": 1, "CITY": 1, "RELIGION": 8, "EMAIL": 1, "PHONE": 0, "ORGANIZATION": 0 }, "all_document_source_coverage": { "PERSON": { "covered": 258, "total": 270 }, "CITY": { "covered": 38, "total": 42 }, "RELIGION": { "covered": 1412, "total": 1510 }, "EMAIL": { "covered": 58, "total": 76 }, "PHONE": { "covered": 23, "total": 23 }, "ORGANIZATION": { "covered": 101, "total": 108 } }, "conservative_accuracy_gate_passed": false, "scope": "all source spans; incomplete processing counts as missing, not leaked output", "deterministic_replay_latency_ms": { "median": 0.08834699838189408, "max": 3.8897630001883954 } }, "candidate_outcomes": { "complete_no_measured_leakage": 151, "candidate_partial_mask": 2, "candidate_fully_unmasked": 2, "blocked_conflicting_spans": 5, "blocked_model_runtime": 0, "other_incomplete": 0 }, "candidate_return_rate": 0.96875, "safe_abstention_rate": 0.03125, "incomplete_rate": 0.03125, "no_measured_leakage_rate_among_returned": 0.9741935483870968, "candidate_partial_canaries": { "PERSON": 0, "CITY": 0, "RELIGION": 1, "EMAIL": 0, "PHONE": 0, "ORGANIZATION": 1 }, "candidate_fully_unmasked_canaries": { "PERSON": 0, "CITY": 0, "RELIGION": 2, "EMAIL": 0, "PHONE": 0, "ORGANIZATION": 0 }, "returned_candidate_entity_metrics": { "PERSON": { "tp": 20, "fp": 5, "fn": 1, "precision": 0.8, "recall": 0.9523809523809523, "f1": 0.8695652173913043 }, "CITY": { "tp": 7, "fp": 2, "fn": 0, "precision": 0.7777777777777778, "recall": 1.0, "f1": 0.8750000000000001 }, "RELIGION": { "tp": 137, "fp": 3, "fn": 7, "precision": 0.9785714285714285, "recall": 0.9513888888888888, "f1": 0.9647887323943661 }, "EMAIL": { "tp": 3, "fp": 0, "fn": 0, "precision": 1.0, "recall": 1.0, "f1": 1.0 }, "PHONE": { "tp": 2, "fp": 0, "fn": 0, "precision": 1.0, "recall": 1.0, "f1": 1.0 }, "ORGANIZATION": { "tp": 5, "fp": 12, "fn": 1, "precision": 0.29411764705882354, "recall": 0.8333333333333334, "f1": 0.4347826086956522 } }, "per_case": [ { "case_id": "holdout03d-001", "scenario": "affiliation", "predictions": [ { "start": 24, "end": 34, "label": "PERSON", "text": "Bekenntnis", "score": 0.5389757752418518, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 35, "end": 53, "label": "RELIGION", "text": "römisch-katholisch", "score": 0.843971312046051, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 24, "end": 34, "label": "PERSON", "text": "Bekenntnis", "score": 0.5389757752418518, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 35, "end": 53, "label": "RELIGION", "text": "römisch-katholisch", "score": 0.843971312046051, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [ [ 24, 34, "PERSON" ] ], "false_negatives": [], "exact_match": false, "error_code": null, "canaries": [ { "start": 35, "end": 53, "label": "RELIGION", "text": "römisch-katholisch", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-002", "scenario": "affiliation", "predictions": [ { "start": 35, "end": 49, "label": "RELIGION", "text": "Roman Catholic", "score": 0.9411200881004333, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 35, "end": 49, "label": "RELIGION", "text": "Roman Catholic", "score": 0.9411200881004333, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 35, "end": 49, "label": "RELIGION", "text": "Roman Catholic", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-003", "scenario": "affiliation", "predictions": [ { "start": 0, "end": 14, "label": "ORGANIZATION", "text": "Tabea Hornfels", "score": 0.9000996351242065, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 45, "end": 55, "label": "RELIGION", "text": "sunnitisch", "score": 0.6417439579963684, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 0, "end": 14, "label": "ORGANIZATION", "text": "Tabea Hornfels", "score": 0.9000996351242065, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 45, "end": 55, "label": "RELIGION", "text": "sunnitisch", "score": 0.6417439579963684, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [ [ 0, 14, "ORGANIZATION" ] ], "false_negatives": [ [ 0, 14, "PERSON" ] ], "exact_match": false, "error_code": null, "canaries": [ { "start": 0, "end": 14, "label": "PERSON", "text": "Tabea Hornfels", "uncovered_positions": [] }, { "start": 45, "end": 55, "label": "RELIGION", "text": "sunnitisch", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-004", "scenario": "affiliation", "predictions": [ { "start": 0, "end": 14, "label": "PERSON", "text": "Ewan Fairbrook", "score": 0.9914917349815369, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 29, "end": 40, "label": "RELIGION", "text": "Shia Muslim", "score": 0.7597324848175049, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 0, "end": 14, "label": "PERSON", "text": "Ewan Fairbrook", "score": 0.9914917349815369, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 29, "end": 40, "label": "RELIGION", "text": "Shia Muslim", "score": 0.7597324848175049, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 0, "end": 14, "label": "PERSON", "text": "Ewan Fairbrook", "uncovered_positions": [] }, { "start": 29, "end": 40, "label": "RELIGION", "text": "Shia Muslim", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-005", "scenario": "affiliation", "predictions": [ { "start": 27, "end": 37, "label": "RELIGION", "text": "Jesidentum", "score": 0.7354034185409546, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 27, "end": 37, "label": "RELIGION", "text": "Jesidentum", "score": 0.7354034185409546, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 27, "end": 37, "label": "RELIGION", "text": "Jesidentum", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-006", "scenario": "affiliation", "predictions": [ { "start": 24, "end": 30, "label": "ORGANIZATION", "text": "Quaker", "score": 0.9522815346717834, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 24, "end": 30, "label": "ORGANIZATION", "text": "Quaker", "score": 0.9522815346717834, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [ [ 24, 30, "ORGANIZATION" ] ], "false_negatives": [ [ 24, 30, "RELIGION" ] ], "exact_match": false, "error_code": null, "canaries": [ { "start": 24, "end": 30, "label": "RELIGION", "text": "Quaker", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-007", "scenario": "affiliation", "predictions": [ { "start": 26, "end": 35, "label": "RELIGION", "text": "Sikhismus", "score": 0.9770961403846741, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 26, "end": 35, "label": "RELIGION", "text": "Sikhismus", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 26, "end": 35, "label": "RELIGION", "text": "Sikhismus", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-008", "scenario": "affiliation", "predictions": [ { "start": 27, "end": 35, "label": "RELIGION", "text": "Hinduism", "score": 0.9994035959243774, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 27, "end": 35, "label": "RELIGION", "text": "Hinduism", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 27, "end": 35, "label": "RELIGION", "text": "Hinduism", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-009", "scenario": "affiliation", "predictions": [ { "start": 44, "end": 52, "label": "RELIGION", "text": "Judentum", "score": 0.918600857257843, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 44, "end": 52, "label": "RELIGION", "text": "Judentum", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 44, "end": 52, "label": "RELIGION", "text": "Judentum", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-010", "scenario": "affiliation", "predictions": [ { "start": 38, "end": 46, "label": "RELIGION", "text": "Buddhism", "score": 0.9944268465042114, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 38, "end": 46, "label": "RELIGION", "text": "Buddhism", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 38, "end": 46, "label": "RELIGION", "text": "Buddhism", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-011", "scenario": "affiliation", "predictions": [ { "start": 46, "end": 51, "label": "RELIGION", "text": "Bahai", "score": 0.8567683100700378, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 46, "end": 51, "label": "RELIGION", "text": "Bahai", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 46, "end": 51, "label": "RELIGION", "text": "Bahai", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-012", "scenario": "affiliation", "predictions": [ { "start": 46, "end": 53, "label": "RELIGION", "text": "Jainism", "score": 0.9846257567405701, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 46, "end": 53, "label": "RELIGION", "text": "Jainism", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 46, "end": 53, "label": "RELIGION", "text": "Jainism", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-013", "scenario": "affiliation", "predictions": [ { "start": 31, "end": 53, "label": "RELIGION", "text": "evangelisch-lutherisch", "score": 0.7786831259727478, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 31, "end": 53, "label": "RELIGION", "text": "evangelisch-lutherisch", "score": 0.7786831259727478, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 31, "end": 53, "label": "RELIGION", "text": "evangelisch-lutherisch", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-014", "scenario": "affiliation", "predictions": [ { "start": 51, "end": 72, "label": "RELIGION", "text": "Seventh-day Adventist", "score": 0.7101534605026245, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 51, "end": 72, "label": "RELIGION", "text": "Seventh-day Adventist", "score": 0.7101534605026245, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 51, "end": 72, "label": "RELIGION", "text": "Seventh-day Adventist", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-015", "scenario": "affiliation", "predictions": [ { "start": 0, "end": 14, "label": "PERSON", "text": "Raban Moosfeld", "score": 0.9412690997123718, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 32, "end": 43, "label": "RELIGION", "text": "Christentum", "score": 0.9764376282691956, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 0, "end": 14, "label": "PERSON", "text": "Raban Moosfeld", "score": 0.9412690997123718, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 32, "end": 43, "label": "RELIGION", "text": "Christentum", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 0, "end": 14, "label": "PERSON", "text": "Raban Moosfeld", "uncovered_positions": [] }, { "start": 32, "end": 43, "label": "RELIGION", "text": "Christentum", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-016", "scenario": "affiliation", "predictions": [ { "start": 0, "end": 14, "label": "PERSON", "text": "Iona Crestwell", "score": 0.9930251836776733, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 41, "end": 55, "label": "RELIGION", "text": "Zoroastrianism", "score": 0.9812561869621277, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 0, "end": 14, "label": "PERSON", "text": "Iona Crestwell", "score": 0.9930251836776733, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 41, "end": 55, "label": "RELIGION", "text": "Zoroastrianism", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 0, "end": 14, "label": "PERSON", "text": "Iona Crestwell", "uncovered_positions": [] }, { "start": 41, "end": 55, "label": "RELIGION", "text": "Zoroastrianism", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-017", "scenario": "nonaffiliation", "predictions": [], "combined_predictions": [ { "start": 38, "end": 53, "label": "RELIGION", "text": "konfessionsfrei", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 38, "end": 53, "label": "RELIGION", "text": "konfessionsfrei", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-018", "scenario": "nonaffiliation", "predictions": [], "combined_predictions": [ { "start": 18, "end": 42, "label": "RELIGION", "text": "no religious affiliation", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 18, "end": 42, "label": "RELIGION", "text": "no religious affiliation", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-019", "scenario": "nonaffiliation", "predictions": [], "combined_predictions": [ { "start": 27, "end": 39, "label": "RELIGION", "text": "religionslos", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 27, "end": 39, "label": "RELIGION", "text": "religionslos", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-020", "scenario": "nonaffiliation", "predictions": [ { "start": 43, "end": 50, "label": "RELIGION", "text": "atheist", "score": 0.6892726421356201, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 43, "end": 50, "label": "RELIGION", "text": "atheist", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 43, "end": 50, "label": "RELIGION", "text": "atheist", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-021", "scenario": "nonaffiliation", "predictions": [], "combined_predictions": [ { "start": 47, "end": 57, "label": "RELIGION", "text": "agnostisch", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 47, "end": 57, "label": "RELIGION", "text": "agnostisch", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-022", "scenario": "nonaffiliation", "predictions": [], "combined_predictions": [ { "start": 36, "end": 44, "label": "RELIGION", "text": "agnostic", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 36, "end": 44, "label": "RELIGION", "text": "agnostic", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-023", "scenario": "nonaffiliation", "predictions": [ { "start": 34, "end": 44, "label": "RELIGION", "text": "Konfession", "score": 0.8056620955467224, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 29, "end": 44, "label": "RELIGION", "text": "ohne Konfession", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 29, "end": 44, "label": "RELIGION", "text": "ohne Konfession", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-024", "scenario": "nonaffiliation", "predictions": [ { "start": 24, "end": 36, "label": "RELIGION", "text": "nonreligious", "score": 0.6173449158668518, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 24, "end": 36, "label": "RELIGION", "text": "nonreligious", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 24, "end": 36, "label": "RELIGION", "text": "nonreligious", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-025", "scenario": "nonaffiliation", "predictions": [ { "start": 27, "end": 35, "label": "RELIGION", "text": "Religion", "score": 0.6249727010726929, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 27, "end": 35, "label": "RELIGION", "text": "Religion", "score": 0.6249727010726929, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [ [ 27, 35, "RELIGION" ] ], "false_negatives": [ [ 21, 35, "RELIGION" ] ], "exact_match": false, "error_code": null, "canaries": [ { "start": 21, "end": 35, "label": "RELIGION", "text": "keine Religion", "uncovered_positions": [ 21, 22, 23, 24, 25 ] } ], "candidate_returned": true, "candidate_canaries": [ { "label": "RELIGION", "start": 21, "end": 35, "kind": "partial_mask" } ], "outcome": "candidate_partial_mask" }, { "case_id": "holdout03d-026", "scenario": "nonaffiliation", "predictions": [ { "start": 0, "end": 11, "label": "ORGANIZATION", "text": "Affiliation", "score": 0.6462724208831787, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 0, "end": 11, "label": "ORGANIZATION", "text": "Affiliation", "score": 0.6462724208831787, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [ [ 0, 11, "ORGANIZATION" ] ], "false_negatives": [ [ 27, 51, "RELIGION" ] ], "exact_match": false, "error_code": null, "canaries": [ { "start": 27, "end": 51, "label": "RELIGION", "text": "religiously unaffiliated", "uncovered_positions": [ 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50 ] } ], "candidate_returned": true, "candidate_canaries": [ { "label": "RELIGION", "start": 27, "end": 51, "kind": "fully_unmasked" } ], "outcome": "candidate_fully_unmasked" }, { "case_id": "holdout03d-027", "scenario": "nonaffiliation", "predictions": [], "combined_predictions": [ { "start": 32, "end": 46, "label": "RELIGION", "text": "konfessionslos", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 32, "end": 46, "label": "RELIGION", "text": "konfessionslos", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-028", "scenario": "nonaffiliation", "predictions": [ { "start": 17, "end": 24, "label": "RELIGION", "text": "atheism", "score": 0.8736236095428467, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 17, "end": 24, "label": "RELIGION", "text": "atheism", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 17, "end": 24, "label": "RELIGION", "text": "atheism", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-029", "scenario": "nonaffiliation", "predictions": [ { "start": 21, "end": 30, "label": "RELIGION", "text": "Atheismus", "score": 0.7562140226364136, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 21, "end": 30, "label": "RELIGION", "text": "Atheismus", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 21, "end": 30, "label": "RELIGION", "text": "Atheismus", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-030", "scenario": "nonaffiliation", "predictions": [], "combined_predictions": [ { "start": 29, "end": 40, "label": "RELIGION", "text": "no religion", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 29, "end": 40, "label": "RELIGION", "text": "no religion", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-031", "scenario": "nonaffiliation", "predictions": [], "combined_predictions": [ { "start": 24, "end": 37, "label": "RELIGION", "text": "Agnostizismus", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 24, "end": 37, "label": "RELIGION", "text": "Agnostizismus", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-032", "scenario": "nonaffiliation", "predictions": [], "combined_predictions": [ { "start": 29, "end": 42, "label": "RELIGION", "text": "non-religious", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 29, "end": 42, "label": "RELIGION", "text": "non-religious", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-033", "scenario": "belief_names", "predictions": [ { "start": 4, "end": 11, "label": "ORGANIZATION", "text": "Glossar", "score": 0.5172879099845886, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 22, "end": 30, "label": "RELIGION", "text": "Daoismus", "score": 0.6094017624855042, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 4, "end": 11, "label": "ORGANIZATION", "text": "Glossar", "score": 0.5172879099845886, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 22, "end": 30, "label": "RELIGION", "text": "Daoismus", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [ [ 4, 11, "ORGANIZATION" ] ], "false_negatives": [], "exact_match": false, "error_code": null, "canaries": [ { "start": 22, "end": 30, "label": "RELIGION", "text": "Daoismus", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-034", "scenario": "belief_names", "predictions": [ { "start": 32, "end": 41, "label": "RELIGION", "text": "Shintoism", "score": 0.9690232872962952, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 32, "end": 41, "label": "RELIGION", "text": "Shintoism", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 32, "end": 41, "label": "RELIGION", "text": "Shintoism", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-035", "scenario": "belief_names", "predictions": [ { "start": 40, "end": 45, "label": "RELIGION", "text": "Islam", "score": 0.9760571718215942, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 40, "end": 45, "label": "RELIGION", "text": "Islam", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 40, "end": 45, "label": "RELIGION", "text": "Islam", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-036", "scenario": "belief_names", "predictions": [ { "start": 27, "end": 34, "label": "RELIGION", "text": "Judaism", "score": 0.9818722605705261, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 27, "end": 34, "label": "RELIGION", "text": "Judaism", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 27, "end": 34, "label": "RELIGION", "text": "Judaism", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-037", "scenario": "belief_names", "predictions": [ { "start": 29, "end": 39, "label": "RELIGION", "text": "Hinduismus", "score": 0.9524589776992798, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 44, "end": 54, "label": "RELIGION", "text": "Buddhismus", "score": 0.9651915431022644, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 29, "end": 39, "label": "RELIGION", "text": "Hinduismus", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" }, { "start": 44, "end": 54, "label": "RELIGION", "text": "Buddhismus", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 29, "end": 39, "label": "RELIGION", "text": "Hinduismus", "uncovered_positions": [] }, { "start": 44, "end": 54, "label": "RELIGION", "text": "Buddhismus", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-038", "scenario": "belief_names", "predictions": [ { "start": 22, "end": 34, "label": "RELIGION", "text": "Christianity", "score": 0.9961896538734436, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 40, "end": 46, "label": "RELIGION", "text": "Taoism", "score": 0.9618191719055176, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 22, "end": 34, "label": "RELIGION", "text": "Christianity", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" }, { "start": 40, "end": 46, "label": "RELIGION", "text": "Taoism", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 22, "end": 34, "label": "RELIGION", "text": "Christianity", "uncovered_positions": [] }, { "start": 40, "end": 46, "label": "RELIGION", "text": "Taoism", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-039", "scenario": "belief_names", "predictions": [ { "start": 0, "end": 10, "label": "ORGANIZATION", "text": "Das Museum", "score": 0.8367996215820312, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 43, "end": 52, "label": "RELIGION", "text": "Jainismus", "score": 0.907336413860321, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 0, "end": 10, "label": "ORGANIZATION", "text": "Das Museum", "score": 0.8367996215820312, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 43, "end": 52, "label": "RELIGION", "text": "Jainismus", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [ [ 0, 10, "ORGANIZATION" ] ], "false_negatives": [], "exact_match": false, "error_code": null, "canaries": [ { "start": 43, "end": 52, "label": "RELIGION", "text": "Jainismus", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-040", "scenario": "belief_names", "predictions": [ { "start": 44, "end": 51, "label": "RELIGION", "text": "Sikhism", "score": 0.984706461429596, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 44, "end": 51, "label": "RELIGION", "text": "Sikhism", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 44, "end": 51, "label": "RELIGION", "text": "Sikhism", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-041", "scenario": "belief_names", "predictions": [ { "start": 22, "end": 35, "label": "RELIGION", "text": "Zoroastrismus", "score": 0.8952449560165405, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 42, "end": 50, "label": "PERSON", "text": "Personen", "score": 0.7890533804893494, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 22, "end": 35, "label": "RELIGION", "text": "Zoroastrismus", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" }, { "start": 42, "end": 50, "label": "PERSON", "text": "Personen", "score": 0.7890533804893494, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [ [ 42, 50, "PERSON" ] ], "false_negatives": [], "exact_match": false, "error_code": null, "canaries": [ { "start": 22, "end": 35, "label": "RELIGION", "text": "Zoroastrismus", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-042", "scenario": "belief_names", "predictions": [ { "start": 24, "end": 30, "label": "CITY", "text": "Baháʼí", "score": 0.7278924584388733, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 31, "end": 36, "label": "RELIGION", "text": "Faith", "score": 0.5462192296981812, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 24, "end": 36, "label": "RELIGION", "text": "Baháʼí Faith", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 24, "end": 36, "label": "RELIGION", "text": "Baháʼí Faith", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-043", "scenario": "belief_names", "predictions": [ { "start": 4, "end": 13, "label": "ORGANIZATION", "text": "Übersicht", "score": 0.5883280038833618, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 24, "end": 37, "label": "RELIGION", "text": "Katholizismus", "score": 0.6876825094223022, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 4, "end": 13, "label": "ORGANIZATION", "text": "Übersicht", "score": 0.5883280038833618, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 24, "end": 37, "label": "RELIGION", "text": "Katholizismus", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [ [ 4, 13, "ORGANIZATION" ] ], "false_negatives": [], "exact_match": false, "error_code": null, "canaries": [ { "start": 24, "end": 37, "label": "RELIGION", "text": "Katholizismus", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-044", "scenario": "belief_names", "predictions": [ { "start": 41, "end": 54, "label": "RELIGION", "text": "Protestantism", "score": 0.9522858262062073, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 41, "end": 54, "label": "RELIGION", "text": "Protestantism", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 41, "end": 54, "label": "RELIGION", "text": "Protestantism", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-045", "scenario": "belief_names", "predictions": [ { "start": 24, "end": 34, "label": "RELIGION", "text": "Orthodoxie", "score": 0.8815929293632507, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 24, "end": 34, "label": "RELIGION", "text": "Orthodoxie", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 24, "end": 34, "label": "RELIGION", "text": "Orthodoxie", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-046", "scenario": "belief_names", "predictions": [ { "start": 30, "end": 51, "label": "RELIGION", "text": "Orthodox Christianity", "score": 0.9181906580924988, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 30, "end": 51, "label": "RELIGION", "text": "Orthodox Christianity", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 30, "end": 51, "label": "RELIGION", "text": "Orthodox Christianity", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-047", "scenario": "belief_names", "predictions": [ { "start": 27, "end": 36, "label": "RELIGION", "text": "Sikhismus", "score": 0.9480417370796204, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 41, "end": 50, "label": "RELIGION", "text": "Jainismus", "score": 0.9625227451324463, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 27, "end": 36, "label": "RELIGION", "text": "Sikhismus", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" }, { "start": 41, "end": 50, "label": "RELIGION", "text": "Jainismus", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 27, "end": 36, "label": "RELIGION", "text": "Sikhismus", "uncovered_positions": [] }, { "start": 41, "end": 50, "label": "RELIGION", "text": "Jainismus", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-048", "scenario": "belief_names", "predictions": [ { "start": 4, "end": 16, "label": "ORGANIZATION", "text": "encyclopedia", "score": 0.740946352481842, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 42, "end": 54, "label": "RELIGION", "text": "Confucianism", "score": 0.9735894203186035, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 4, "end": 16, "label": "ORGANIZATION", "text": "encyclopedia", "score": 0.740946352481842, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 42, "end": 54, "label": "RELIGION", "text": "Confucianism", "score": 0.9735894203186035, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [ [ 4, 16, "ORGANIZATION" ] ], "false_negatives": [], "exact_match": false, "error_code": null, "canaries": [ { "start": 42, "end": 54, "label": "RELIGION", "text": "Confucianism", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-049", "scenario": "general_topics", "predictions": [ { "start": 38, "end": 43, "label": "RELIGION", "text": "Islam", "score": 0.9990894794464111, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 38, "end": 43, "label": "RELIGION", "text": "Islam", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 38, "end": 43, "label": "RELIGION", "text": "Islam", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-050", "scenario": "general_topics", "predictions": [ { "start": 33, "end": 41, "label": "RELIGION", "text": "Hinduism", "score": 0.9996596574783325, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 33, "end": 41, "label": "RELIGION", "text": "Hinduism", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 33, "end": 41, "label": "RELIGION", "text": "Hinduism", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-051", "scenario": "general_topics", "predictions": [ { "start": 23, "end": 34, "label": "RELIGION", "text": "Christentum", "score": 0.9296461343765259, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 45, "end": 56, "label": "CITY", "text": "Mittelalter", "score": 0.600753903388977, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 23, "end": 34, "label": "RELIGION", "text": "Christentum", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" }, { "start": 45, "end": 56, "label": "CITY", "text": "Mittelalter", "score": 0.600753903388977, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [ [ 45, 56, "CITY" ] ], "false_negatives": [], "exact_match": false, "error_code": null, "canaries": [ { "start": 23, "end": 34, "label": "RELIGION", "text": "Christentum", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-052", "scenario": "general_topics", "predictions": [ { "start": 28, "end": 36, "label": "RELIGION", "text": "Buddhism", "score": 0.9992027878761292, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 28, "end": 36, "label": "RELIGION", "text": "Buddhism", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 28, "end": 36, "label": "RELIGION", "text": "Buddhism", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-053", "scenario": "general_topics", "predictions": [ { "start": 37, "end": 45, "label": "RELIGION", "text": "Judentum", "score": 0.9257993698120117, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 37, "end": 45, "label": "RELIGION", "text": "Judentum", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 37, "end": 45, "label": "RELIGION", "text": "Judentum", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-054", "scenario": "general_topics", "predictions": [ { "start": 20, "end": 27, "label": "RELIGION", "text": "Jainism", "score": 0.9963538646697998, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 20, "end": 27, "label": "RELIGION", "text": "Jainism", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 20, "end": 27, "label": "RELIGION", "text": "Jainism", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-055", "scenario": "general_topics", "predictions": [ { "start": 24, "end": 29, "label": "PERSON", "text": "Bahai", "score": 0.9998281002044678, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [], "false_positives": [], "false_negatives": [ [ 24, 29, "RELIGION" ] ], "exact_match": false, "error_code": "ambiguous_entity_overlap", "canaries": [ { "start": 24, "end": 29, "label": "RELIGION", "text": "Bahai", "uncovered_positions": [ 24, 25, 26, 27, 28 ] } ], "candidate_returned": false, "candidate_canaries": [], "outcome": "blocked_conflicting_spans" }, { "case_id": "holdout03d-056", "scenario": "general_topics", "predictions": [ { "start": 25, "end": 32, "label": "RELIGION", "text": "Sikhism", "score": 0.9963860511779785, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 25, "end": 32, "label": "RELIGION", "text": "Sikhism", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 25, "end": 32, "label": "RELIGION", "text": "Sikhism", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-057", "scenario": "general_topics", "predictions": [ { "start": 20, "end": 29, "label": "RELIGION", "text": "Atheismus", "score": 0.5075995326042175, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 20, "end": 29, "label": "RELIGION", "text": "Atheismus", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 20, "end": 29, "label": "RELIGION", "text": "Atheismus", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-058", "scenario": "general_topics", "predictions": [ { "start": 19, "end": 26, "label": "RELIGION", "text": "atheism", "score": 0.9609501957893372, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 31, "end": 42, "label": "RELIGION", "text": "agnosticism", "score": 0.9465921521186829, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 19, "end": 26, "label": "RELIGION", "text": "atheism", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" }, { "start": 31, "end": 42, "label": "RELIGION", "text": "agnosticism", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 19, "end": 26, "label": "RELIGION", "text": "atheism", "uncovered_positions": [] }, { "start": 31, "end": 42, "label": "RELIGION", "text": "agnosticism", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-059", "scenario": "general_topics", "predictions": [ { "start": 30, "end": 41, "label": "RELIGION", "text": "Shintoismus", "score": 0.9589027762413025, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 30, "end": 41, "label": "RELIGION", "text": "Shintoismus", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 30, "end": 41, "label": "RELIGION", "text": "Shintoismus", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-060", "scenario": "general_topics", "predictions": [ { "start": 23, "end": 29, "label": "RELIGION", "text": "Taoism", "score": 0.9697747826576233, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 23, "end": 29, "label": "RELIGION", "text": "Taoism", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 23, "end": 29, "label": "RELIGION", "text": "Taoism", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-061", "scenario": "general_topics", "predictions": [ { "start": 30, "end": 45, "label": "RELIGION", "text": "Protestantismus", "score": 0.6991519927978516, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 30, "end": 45, "label": "RELIGION", "text": "Protestantismus", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 30, "end": 45, "label": "RELIGION", "text": "Protestantismus", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-062", "scenario": "general_topics", "predictions": [ { "start": 25, "end": 36, "label": "RELIGION", "text": "Catholicism", "score": 0.9966902732849121, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 25, "end": 36, "label": "RELIGION", "text": "Catholicism", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 25, "end": 36, "label": "RELIGION", "text": "Catholicism", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-063", "scenario": "general_topics", "predictions": [ { "start": 30, "end": 44, "label": "PERSON", "text": "konfessionslos", "score": 0.5395556092262268, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [], "false_positives": [], "false_negatives": [ [ 30, 44, "RELIGION" ] ], "exact_match": false, "error_code": "ambiguous_entity_overlap", "canaries": [ { "start": 30, "end": 44, "label": "RELIGION", "text": "konfessionslos", "uncovered_positions": [ 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43 ] } ], "candidate_returned": false, "candidate_canaries": [], "outcome": "blocked_conflicting_spans" }, { "case_id": "holdout03d-064", "scenario": "general_topics", "predictions": [ { "start": 29, "end": 42, "label": "RELIGION", "text": "non-religious", "score": 0.5220276713371277, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 29, "end": 42, "label": "RELIGION", "text": "non-religious", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 29, "end": 42, "label": "RELIGION", "text": "non-religious", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-065", "scenario": "negation_quotation", "predictions": [ { "start": 21, "end": 24, "label": "PERSON", "text": "Ich", "score": 0.7314160466194153, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 35, "end": 45, "label": "RELIGION", "text": "muslimisch", "score": 0.8888826966285706, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 21, "end": 24, "label": "PERSON", "text": "Ich", "score": 0.7314160466194153, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 35, "end": 45, "label": "RELIGION", "text": "muslimisch", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [ [ 21, 24, "PERSON" ] ], "false_negatives": [], "exact_match": false, "error_code": null, "canaries": [ { "start": 35, "end": 45, "label": "RELIGION", "text": "muslimisch", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-066", "scenario": "negation_quotation", "predictions": [ { "start": 28, "end": 37, "label": "RELIGION", "text": "Christian", "score": 0.931776762008667, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 28, "end": 37, "label": "RELIGION", "text": "Christian", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 28, "end": 37, "label": "RELIGION", "text": "Christian", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-067", "scenario": "negation_quotation", "predictions": [ { "start": 29, "end": 39, "label": "RELIGION", "text": "katholisch", "score": 0.8747991323471069, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 29, "end": 39, "label": "RELIGION", "text": "katholisch", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 29, "end": 39, "label": "RELIGION", "text": "katholisch", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-068", "scenario": "negation_quotation", "predictions": [], "combined_predictions": [ { "start": 35, "end": 42, "label": "RELIGION", "text": "atheist", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 35, "end": 42, "label": "RELIGION", "text": "atheist", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-069", "scenario": "negation_quotation", "predictions": [ { "start": 10, "end": 22, "label": "RELIGION", "text": "buddhistisch", "score": 0.78475022315979, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 10, "end": 22, "label": "RELIGION", "text": "buddhistisch", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 10, "end": 22, "label": "RELIGION", "text": "buddhistisch", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-070", "scenario": "negation_quotation", "predictions": [ { "start": 1, "end": 7, "label": "RELIGION", "text": "Jewish", "score": 0.922519862651825, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 1, "end": 7, "label": "RELIGION", "text": "Jewish", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 1, "end": 7, "label": "RELIGION", "text": "Jewish", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-071", "scenario": "negation_quotation", "predictions": [ { "start": 40, "end": 52, "label": "RELIGION", "text": "hinduistisch", "score": 0.7200407981872559, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 40, "end": 52, "label": "RELIGION", "text": "hinduistisch", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 40, "end": 52, "label": "RELIGION", "text": "hinduistisch", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-072", "scenario": "negation_quotation", "predictions": [ { "start": 37, "end": 43, "label": "RELIGION", "text": "Muslim", "score": 0.995964527130127, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 37, "end": 43, "label": "RELIGION", "text": "Muslim", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 37, "end": 43, "label": "RELIGION", "text": "Muslim", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-073", "scenario": "negation_quotation", "predictions": [], "combined_predictions": [ { "start": 1, "end": 16, "label": "RELIGION", "text": "Ohne Konfession", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 1, "end": 16, "label": "RELIGION", "text": "Ohne Konfession", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-074", "scenario": "negation_quotation", "predictions": [ { "start": 30, "end": 38, "label": "RELIGION", "text": "religion", "score": 0.5185865163803101, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 27, "end": 38, "label": "RELIGION", "text": "no religion", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 27, "end": 38, "label": "RELIGION", "text": "no religion", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-075", "scenario": "negation_quotation", "predictions": [], "combined_predictions": [ { "start": 6, "end": 17, "label": "RELIGION", "text": "evangelisch", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" }, { "start": 27, "end": 37, "label": "RELIGION", "text": "agnostisch", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 6, "end": 17, "label": "RELIGION", "text": "evangelisch", "uncovered_positions": [] }, { "start": 27, "end": 37, "label": "RELIGION", "text": "agnostisch", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-076", "scenario": "negation_quotation", "predictions": [ { "start": 18, "end": 26, "label": "RELIGION", "text": "Catholic", "score": 0.9851877689361572, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 30, "end": 42, "label": "RELIGION", "text": "nonreligious", "score": 0.5536500811576843, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 18, "end": 26, "label": "RELIGION", "text": "Catholic", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" }, { "start": 30, "end": 42, "label": "RELIGION", "text": "nonreligious", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 18, "end": 26, "label": "RELIGION", "text": "Catholic", "uncovered_positions": [] }, { "start": 30, "end": 42, "label": "RELIGION", "text": "nonreligious", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-077", "scenario": "negation_quotation", "predictions": [ { "start": 22, "end": 30, "label": "RELIGION", "text": "Judentum", "score": 0.9432740807533264, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 22, "end": 30, "label": "RELIGION", "text": "Judentum", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 22, "end": 30, "label": "RELIGION", "text": "Judentum", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-078", "scenario": "negation_quotation", "predictions": [ { "start": 23, "end": 29, "label": "PERSON", "text": "Baha'i", "score": 0.8763203620910645, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [], "false_positives": [], "false_negatives": [ [ 23, 29, "RELIGION" ] ], "exact_match": false, "error_code": "ambiguous_entity_overlap", "canaries": [ { "start": 23, "end": 29, "label": "RELIGION", "text": "Baha'i", "uncovered_positions": [ 23, 24, 25, 26, 28 ] } ], "candidate_returned": false, "candidate_canaries": [], "outcome": "blocked_conflicting_spans" }, { "case_id": "holdout03d-079", "scenario": "negation_quotation", "predictions": [ { "start": 21, "end": 32, "label": "RELIGION", "text": "islamischen", "score": 0.6762732863426208, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 21, "end": 32, "label": "RELIGION", "text": "islamischen", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 21, "end": 32, "label": "RELIGION", "text": "islamischen", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-080", "scenario": "negation_quotation", "predictions": [ { "start": 15, "end": 23, "label": "RELIGION", "text": "Buddhist", "score": 0.964024007320404, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 15, "end": 23, "label": "RELIGION", "text": "Buddhist", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 15, "end": 23, "label": "RELIGION", "text": "Buddhist", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-081", "scenario": "ambiguity", "predictions": [], "combined_predictions": [], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-082", "scenario": "ambiguity", "predictions": [ { "start": 4, "end": 12, "label": "RELIGION", "text": "catholic", "score": 0.6429972648620605, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 4, "end": 12, "label": "RELIGION", "text": "catholic", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [ [ 4, 12, "RELIGION" ] ], "false_negatives": [], "exact_match": false, "error_code": null, "canaries": [], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-083", "scenario": "ambiguity", "predictions": [], "combined_predictions": [], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-084", "scenario": "ambiguity", "predictions": [ { "start": 3, "end": 14, "label": "RELIGION", "text": "evangelical", "score": 0.8391301035881042, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 3, "end": 14, "label": "RELIGION", "text": "evangelical", "score": 0.8391301035881042, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [ [ 3, 14, "RELIGION" ] ], "false_negatives": [], "exact_match": false, "error_code": null, "canaries": [], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-085", "scenario": "ambiguity", "predictions": [ { "start": 18, "end": 26, "label": "RELIGION", "text": "orthodox", "score": 0.8430137634277344, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 18, "end": 26, "label": "RELIGION", "text": "orthodox", "score": 0.8430137634277344, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 18, "end": 26, "label": "RELIGION", "text": "orthodox", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-086", "scenario": "ambiguity", "predictions": [ { "start": 52, "end": 60, "label": "RELIGION", "text": "Orthodox", "score": 0.8877473473548889, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 52, "end": 60, "label": "RELIGION", "text": "Orthodox", "score": 0.8877473473548889, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 52, "end": 60, "label": "RELIGION", "text": "Orthodox", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-087", "scenario": "ambiguity", "predictions": [ { "start": 12, "end": 23, "label": "RELIGION", "text": "evangelisch", "score": 0.7735995650291443, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 25, "end": 36, "label": "ORGANIZATION", "text": "Algorithmus", "score": 0.5926805734634399, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 12, "end": 23, "label": "RELIGION", "text": "evangelisch", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" }, { "start": 25, "end": 36, "label": "ORGANIZATION", "text": "Algorithmus", "score": 0.5926805734634399, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [ [ 25, 36, "ORGANIZATION" ] ], "false_negatives": [], "exact_match": false, "error_code": null, "canaries": [ { "start": 12, "end": 23, "label": "RELIGION", "text": "evangelisch", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-088", "scenario": "ambiguity", "predictions": [ { "start": 26, "end": 37, "label": "RELIGION", "text": "Evangelical", "score": 0.9244890809059143, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 26, "end": 37, "label": "RELIGION", "text": "Evangelical", "score": 0.9244890809059143, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 26, "end": 37, "label": "RELIGION", "text": "Evangelical", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-089", "scenario": "ambiguity", "predictions": [], "combined_predictions": [], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-090", "scenario": "ambiguity", "predictions": [], "combined_predictions": [], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-091", "scenario": "ambiguity", "predictions": [ { "start": 17, "end": 27, "label": "RELIGION", "text": "christlich", "score": 0.9146808981895447, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 48, "end": 60, "label": "PERSON", "text": "Personenname", "score": 0.6654636263847351, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 17, "end": 27, "label": "RELIGION", "text": "christlich", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" }, { "start": 48, "end": 60, "label": "PERSON", "text": "Personenname", "score": 0.6654636263847351, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [ [ 48, 60, "PERSON" ] ], "false_negatives": [], "exact_match": false, "error_code": null, "canaries": [ { "start": 17, "end": 27, "label": "RELIGION", "text": "christlich", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-092", "scenario": "ambiguity", "predictions": [ { "start": 22, "end": 31, "label": "RELIGION", "text": "Christian", "score": 0.9971898198127747, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 22, "end": 31, "label": "RELIGION", "text": "Christian", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 22, "end": 31, "label": "RELIGION", "text": "Christian", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-093", "scenario": "ambiguity", "predictions": [], "combined_predictions": [], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-094", "scenario": "ambiguity", "predictions": [], "combined_predictions": [], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-095", "scenario": "ambiguity", "predictions": [ { "start": 26, "end": 33, "label": "RELIGION", "text": "jüdisch", "score": 0.9290997982025146, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 26, "end": 33, "label": "RELIGION", "text": "jüdisch", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 26, "end": 33, "label": "RELIGION", "text": "jüdisch", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-096", "scenario": "ambiguity", "predictions": [ { "start": 44, "end": 52, "label": "RELIGION", "text": "Reformed", "score": 0.8552035689353943, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 44, "end": 52, "label": "RELIGION", "text": "Reformed", "score": 0.8552035689353943, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 44, "end": 52, "label": "RELIGION", "text": "Reformed", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-097", "scenario": "ocr_noise", "predictions": [ { "start": 9, "end": 13, "label": "CITY", "text": "laut", "score": 0.6887308955192566, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 20, "end": 30, "label": "RELIGION", "text": "Buddh1smus", "score": 0.8085448145866394, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 9, "end": 13, "label": "CITY", "text": "laut", "score": 0.6887308955192566, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 20, "end": 30, "label": "RELIGION", "text": "Buddh1smus", "score": 0.8085448145866394, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [ [ 9, 13, "CITY" ] ], "false_negatives": [], "exact_match": false, "error_code": null, "canaries": [ { "start": 20, "end": 30, "label": "RELIGION", "text": "Buddh1smus", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-098", "scenario": "ocr_noise", "predictions": [ { "start": 31, "end": 43, "label": "RELIGION", "text": "Christlanity", "score": 0.5787172317504883, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 31, "end": 43, "label": "RELIGION", "text": "Christlanity", "score": 0.5787172317504883, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 31, "end": 43, "label": "RELIGION", "text": "Christlanity", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-099", "scenario": "ocr_noise", "predictions": [], "combined_predictions": [ { "start": 24, "end": 40, "label": "RELIGION", "text": "konfessions-\nlos", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "dehyphenation-v1" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 24, "end": 40, "label": "RELIGION", "text": "konfessions-\nlos", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-100", "scenario": "ocr_noise", "predictions": [], "combined_predictions": [ { "start": 21, "end": 35, "label": "RELIGION", "text": "non-\nreligious", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "dehyphenation-v1" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 21, "end": 35, "label": "RELIGION", "text": "non-\nreligious", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-101", "scenario": "ocr_noise", "predictions": [ { "start": 11, "end": 21, "label": "RELIGION", "text": "katho1isch", "score": 0.7328035831451416, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 11, "end": 21, "label": "RELIGION", "text": "katho1isch", "score": 0.7328035831451416, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 11, "end": 21, "label": "RELIGION", "text": "katho1isch", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-102", "scenario": "ocr_noise", "predictions": [ { "start": 24, "end": 33, "label": "RELIGION", "text": "Hinduisrn", "score": 0.7377975583076477, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 24, "end": 33, "label": "RELIGION", "text": "Hinduisrn", "score": 0.7377975583076477, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 24, "end": 33, "label": "RELIGION", "text": "Hinduisrn", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-103", "scenario": "ocr_noise", "predictions": [], "combined_predictions": [ { "start": 18, "end": 30, "label": "RELIGION", "text": "agnos-\ntisch", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "dehyphenation-v1" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 18, "end": 30, "label": "RELIGION", "text": "agnos-\ntisch", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-104", "scenario": "ocr_noise", "predictions": [ { "start": 25, "end": 34, "label": "RELIGION", "text": "Juda-\nism", "score": 0.984998881816864, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "dehyphenation-v1" } ], "combined_predictions": [ { "start": 25, "end": 34, "label": "RELIGION", "text": "Juda-\nism", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "dehyphenation-v1" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 25, "end": 34, "label": "RELIGION", "text": "Juda-\nism", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-105", "scenario": "ocr_noise", "predictions": [ { "start": 19, "end": 24, "label": "PERSON", "text": "lslam", "score": 0.8412295579910278, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 19, "end": 24, "label": "PERSON", "text": "lslam", "score": 0.8412295579910278, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [ [ 19, 24, "PERSON" ] ], "false_negatives": [ [ 19, 24, "RELIGION" ] ], "exact_match": false, "error_code": null, "canaries": [ { "start": 19, "end": 24, "label": "RELIGION", "text": "lslam", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-106", "scenario": "ocr_noise", "predictions": [ { "start": 4, "end": 7, "label": "ORGANIZATION", "text": "OCR", "score": 0.993300199508667, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 18, "end": 25, "label": "RELIGION", "text": "S1khism", "score": 0.9311596751213074, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 4, "end": 7, "label": "ORGANIZATION", "text": "OCR", "score": 0.993300199508667, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 18, "end": 25, "label": "RELIGION", "text": "S1khism", "score": 0.9311596751213074, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [ [ 4, 7, "ORGANIZATION" ] ], "false_negatives": [], "exact_match": false, "error_code": null, "canaries": [ { "start": 18, "end": 25, "label": "RELIGION", "text": "S1khism", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-107", "scenario": "ocr_noise", "predictions": [ { "start": 35, "end": 49, "label": "RELIGION", "text": "buddhis-\ntisch", "score": 0.7703148126602173, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "dehyphenation-v1" } ], "combined_predictions": [ { "start": 35, "end": 49, "label": "RELIGION", "text": "buddhis-\ntisch", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "dehyphenation-v1" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 35, "end": 49, "label": "RELIGION", "text": "buddhis-\ntisch", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-108", "scenario": "ocr_noise", "predictions": [ { "start": 27, "end": 43, "label": "RELIGION", "text": "Zoroas-\ntrianism", "score": 0.9521776437759399, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "dehyphenation-v1" } ], "combined_predictions": [ { "start": 27, "end": 43, "label": "RELIGION", "text": "Zoroas-\ntrianism", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "dehyphenation-v1" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 27, "end": 43, "label": "RELIGION", "text": "Zoroas-\ntrianism", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-109", "scenario": "ocr_noise", "predictions": [ { "start": 14, "end": 27, "label": "PERSON", "text": "Fenja Torwald", "score": 0.9993094205856323, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 29, "end": 49, "label": "ORGANIZATION", "text": "Bekenntnis Judenturn", "score": 0.8337629437446594, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 14, "end": 27, "label": "PERSON", "text": "Fenja Torwald", "score": 0.9993094205856323, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 29, "end": 49, "label": "ORGANIZATION", "text": "Bekenntnis Judenturn", "score": 0.8337629437446594, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [ [ 29, 49, "ORGANIZATION" ] ], "false_negatives": [ [ 40, 49, "RELIGION" ] ], "exact_match": false, "error_code": null, "canaries": [ { "start": 14, "end": 27, "label": "PERSON", "text": "Fenja Torwald", "uncovered_positions": [] }, { "start": 40, "end": 49, "label": "RELIGION", "text": "Judenturn", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-110", "scenario": "ocr_noise", "predictions": [ { "start": 21, "end": 33, "label": "PERSON", "text": "Orin Wexford", "score": 0.9491236209869385, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 38, "end": 49, "label": "RELIGION", "text": "Cath0licism", "score": 0.8634129166603088, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 21, "end": 33, "label": "PERSON", "text": "Orin Wexford", "score": 0.9491236209869385, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 38, "end": 49, "label": "RELIGION", "text": "Cath0licism", "score": 0.8634129166603088, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 21, "end": 33, "label": "PERSON", "text": "Orin Wexford", "uncovered_positions": [] }, { "start": 38, "end": 49, "label": "RELIGION", "text": "Cath0licism", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-111", "scenario": "ocr_noise", "predictions": [], "combined_predictions": [], "false_positives": [], "false_negatives": [ [ 13, 28, "RELIGION" ] ], "exact_match": false, "error_code": null, "canaries": [ { "start": 13, "end": 28, "label": "RELIGION", "text": "konfesslonsfrei", "uncovered_positions": [ 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27 ] } ], "candidate_returned": true, "candidate_canaries": [ { "label": "RELIGION", "start": 13, "end": 28, "kind": "fully_unmasked" } ], "outcome": "candidate_fully_unmasked" }, { "case_id": "holdout03d-112", "scenario": "ocr_noise", "predictions": [ { "start": 21, "end": 29, "label": "ORGANIZATION", "text": "agn0stic", "score": 0.8037384748458862, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 21, "end": 29, "label": "ORGANIZATION", "text": "agn0stic", "score": 0.8037384748458862, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [ [ 21, 29, "ORGANIZATION" ] ], "false_negatives": [ [ 21, 29, "RELIGION" ] ], "exact_match": false, "error_code": null, "canaries": [ { "start": 21, "end": 29, "label": "RELIGION", "text": "agn0stic", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-113", "scenario": "multiple_entities", "predictions": [ { "start": 0, "end": 14, "label": "PERSON", "text": "Silja Dornbach", "score": 0.9992048144340515, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 19, "end": 26, "label": "CITY", "text": "Rostock", "score": 0.999563992023468, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 34, "end": 42, "label": "RELIGION", "text": "Judentum", "score": 0.9391108751296997, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 48, "end": 66, "label": "ORGANIZATION", "text": "Morgenkai Institut", "score": 0.9742982983589172, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 0, "end": 14, "label": "PERSON", "text": "Silja Dornbach", "score": 0.9992048144340515, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 19, "end": 26, "label": "CITY", "text": "Rostock", "score": 0.999563992023468, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 34, "end": 42, "label": "RELIGION", "text": "Judentum", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" }, { "start": 48, "end": 66, "label": "ORGANIZATION", "text": "Morgenkai Institut", "score": 0.9742982983589172, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 0, "end": 14, "label": "PERSON", "text": "Silja Dornbach", "uncovered_positions": [] }, { "start": 19, "end": 26, "label": "CITY", "text": "Rostock", "uncovered_positions": [] }, { "start": 34, "end": 42, "label": "RELIGION", "text": "Judentum", "uncovered_positions": [] }, { "start": 48, "end": 66, "label": "ORGANIZATION", "text": "Morgenkai Institut", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-114", "scenario": "multiple_entities", "predictions": [ { "start": 0, "end": 15, "label": "PERSON", "text": "Nolan Briarwood", "score": 0.9830585718154907, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 19, "end": 23, "label": "CITY", "text": "York", "score": 0.9994452595710754, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 30, "end": 37, "label": "RELIGION", "text": "Jainism", "score": 0.9943238496780396, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 45, "end": 62, "label": "ORGANIZATION", "text": "Cobalt Quay Trust", "score": 0.9601506590843201, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 0, "end": 15, "label": "PERSON", "text": "Nolan Briarwood", "score": 0.9830585718154907, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 19, "end": 23, "label": "CITY", "text": "York", "score": 0.9994452595710754, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 30, "end": 37, "label": "RELIGION", "text": "Jainism", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" }, { "start": 45, "end": 62, "label": "ORGANIZATION", "text": "Cobalt Quay Trust", "score": 0.9601506590843201, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 0, "end": 15, "label": "PERSON", "text": "Nolan Briarwood", "uncovered_positions": [] }, { "start": 19, "end": 23, "label": "CITY", "text": "York", "uncovered_positions": [] }, { "start": 30, "end": 37, "label": "RELIGION", "text": "Jainism", "uncovered_positions": [] }, { "start": 45, "end": 62, "label": "ORGANIZATION", "text": "Cobalt Quay Trust", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-115", "scenario": "multiple_entities", "predictions": [ { "start": 0, "end": 22, "label": "ORGANIZATION", "text": "Bernsteinpfad Stiftung", "score": 0.9246055483818054, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 32, "end": 47, "label": "PERSON", "text": "Henrik Wiesenau", "score": 0.9986317753791809, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 0, "end": 22, "label": "ORGANIZATION", "text": "Bernsteinpfad Stiftung", "score": 0.9246055483818054, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 32, "end": 47, "label": "PERSON", "text": "Henrik Wiesenau", "score": 0.9986317753791809, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 52, "end": 66, "label": "RELIGION", "text": "konfessionslos", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 0, "end": 22, "label": "ORGANIZATION", "text": "Bernsteinpfad Stiftung", "uncovered_positions": [] }, { "start": 32, "end": 47, "label": "PERSON", "text": "Henrik Wiesenau", "uncovered_positions": [] }, { "start": 52, "end": 66, "label": "RELIGION", "text": "konfessionslos", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-116", "scenario": "multiple_entities", "predictions": [ { "start": 4, "end": 26, "label": "ORGANIZATION", "text": "Silver Estuary Council", "score": 0.8049675822257996, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 34, "end": 47, "label": "PERSON", "text": "Ada Merefield", "score": 0.9995853304862976, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 4, "end": 26, "label": "ORGANIZATION", "text": "Silver Estuary Council", "score": 0.8049675822257996, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 34, "end": 47, "label": "PERSON", "text": "Ada Merefield", "score": 0.9995853304862976, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 50, "end": 57, "label": "RELIGION", "text": "atheism", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 4, "end": 26, "label": "ORGANIZATION", "text": "Silver Estuary Council", "uncovered_positions": [] }, { "start": 34, "end": 47, "label": "PERSON", "text": "Ada Merefield", "uncovered_positions": [] }, { "start": 50, "end": 57, "label": "RELIGION", "text": "atheism", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-117", "scenario": "multiple_entities", "predictions": [ { "start": 0, "end": 12, "label": "PERSON", "text": "Ida Talbrück", "score": 0.9952776432037354, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 19, "end": 24, "label": "RELIGION", "text": "Islam", "score": 0.9999401569366455, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 35, "end": 58, "label": "EMAIL", "text": "ida.tal@example.invalid", "score": 0.8027673959732056, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 0, "end": 12, "label": "PERSON", "text": "Ida Talbrück", "score": 0.9952776432037354, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 19, "end": 24, "label": "RELIGION", "text": "Islam", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" }, { "start": 35, "end": 58, "label": "EMAIL", "text": "ida.tal@example.invalid", "score": null, "source": "deterministic", "model_revision": "contact-patterns-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 0, "end": 12, "label": "PERSON", "text": "Ida Talbrück", "uncovered_positions": [] }, { "start": 19, "end": 24, "label": "RELIGION", "text": "Islam", "uncovered_positions": [] }, { "start": 35, "end": 58, "label": "EMAIL", "text": "ida.tal@example.invalid", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-118", "scenario": "multiple_entities", "predictions": [ { "start": 0, "end": 13, "label": "PERSON", "text": "Lyle Ashcombe", "score": 0.9961810111999512, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 20, "end": 28, "label": "RELIGION", "text": "Buddhism", "score": 0.9979920387268066, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 0, "end": 13, "label": "PERSON", "text": "Lyle Ashcombe", "score": 0.9961810111999512, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 20, "end": 28, "label": "RELIGION", "text": "Buddhism", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" }, { "start": 38, "end": 60, "label": "EMAIL", "text": "lyle.a@example.invalid", "score": null, "source": "deterministic", "model_revision": "contact-patterns-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 0, "end": 13, "label": "PERSON", "text": "Lyle Ashcombe", "uncovered_positions": [] }, { "start": 20, "end": 28, "label": "RELIGION", "text": "Buddhism", "uncovered_positions": [] }, { "start": 38, "end": 60, "label": "EMAIL", "text": "lyle.a@example.invalid", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-119", "scenario": "multiple_entities", "predictions": [ { "start": 13, "end": 27, "label": "PERSON", "text": "Arvid Sandborn", "score": 0.998414158821106, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 29, "end": 43, "label": "PHONE", "text": "+49 381 482701", "score": 0.9627817869186401, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 13, "end": 27, "label": "PERSON", "text": "Arvid Sandborn", "score": 0.998414158821106, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 29, "end": 43, "label": "PHONE", "text": "+49 381 482701", "score": null, "source": "deterministic", "model_revision": "contact-patterns-v1", "normalization": "identity" }, { "start": 52, "end": 64, "label": "RELIGION", "text": "religionslos", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 13, "end": 27, "label": "PERSON", "text": "Arvid Sandborn", "uncovered_positions": [] }, { "start": 29, "end": 43, "label": "PHONE", "text": "+49 381 482701", "uncovered_positions": [] }, { "start": 52, "end": 64, "label": "RELIGION", "text": "religionslos", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-120", "scenario": "multiple_entities", "predictions": [ { "start": 5, "end": 17, "label": "PERSON", "text": "Mira Fenwick", "score": 0.9872230291366577, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 21, "end": 36, "label": "PHONE", "text": "+44 1904 582701", "score": 0.9610803127288818, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 45, "end": 53, "label": "RELIGION", "text": "Hinduism", "score": 0.9806439876556396, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 5, "end": 17, "label": "PERSON", "text": "Mira Fenwick", "score": 0.9872230291366577, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 21, "end": 36, "label": "PHONE", "text": "+44 1904 582701", "score": null, "source": "deterministic", "model_revision": "contact-patterns-v1", "normalization": "identity" }, { "start": 45, "end": 53, "label": "RELIGION", "text": "Hinduism", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 5, "end": 17, "label": "PERSON", "text": "Mira Fenwick", "uncovered_positions": [] }, { "start": 21, "end": 36, "label": "PHONE", "text": "+44 1904 582701", "uncovered_positions": [] }, { "start": 45, "end": 53, "label": "RELIGION", "text": "Hinduism", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-121", "scenario": "multiple_entities", "predictions": [ { "start": 0, "end": 6, "label": "CITY", "text": "Erfurt", "score": 0.9716014862060547, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 8, "end": 21, "label": "PERSON", "text": "Svea Morgenau", "score": 0.8673108220100403, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 33, "end": 43, "label": "RELIGION", "text": "katholisch", "score": 0.7634422779083252, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 47, "end": 57, "label": "RELIGION", "text": "agnostisch", "score": 0.5659558773040771, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 0, "end": 6, "label": "CITY", "text": "Erfurt", "score": 0.9716014862060547, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 8, "end": 21, "label": "PERSON", "text": "Svea Morgenau", "score": 0.8673108220100403, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 33, "end": 43, "label": "RELIGION", "text": "katholisch", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" }, { "start": 47, "end": 57, "label": "RELIGION", "text": "agnostisch", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 0, "end": 6, "label": "CITY", "text": "Erfurt", "uncovered_positions": [] }, { "start": 8, "end": 21, "label": "PERSON", "text": "Svea Morgenau", "uncovered_positions": [] }, { "start": 33, "end": 43, "label": "RELIGION", "text": "katholisch", "uncovered_positions": [] }, { "start": 47, "end": 57, "label": "RELIGION", "text": "agnostisch", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-122", "scenario": "multiple_entities", "predictions": [ { "start": 0, "end": 7, "label": "CITY", "text": "Bristol", "score": 0.9915996789932251, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 9, "end": 22, "label": "PERSON", "text": "Owen Thornley", "score": 0.9985686540603638, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 32, "end": 44, "label": "RELIGION", "text": "Christianity", "score": 0.9914804697036743, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 0, "end": 7, "label": "CITY", "text": "Bristol", "score": 0.9915996789932251, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 9, "end": 22, "label": "PERSON", "text": "Owen Thornley", "score": 0.9985686540603638, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 32, "end": 44, "label": "RELIGION", "text": "Christianity", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" }, { "start": 50, "end": 61, "label": "RELIGION", "text": "no religion", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 0, "end": 7, "label": "CITY", "text": "Bristol", "uncovered_positions": [] }, { "start": 9, "end": 22, "label": "PERSON", "text": "Owen Thornley", "uncovered_positions": [] }, { "start": 32, "end": 44, "label": "RELIGION", "text": "Christianity", "uncovered_positions": [] }, { "start": 50, "end": 61, "label": "RELIGION", "text": "no religion", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-123", "scenario": "multiple_entities", "predictions": [ { "start": 0, "end": 13, "label": "PERSON", "text": "Jona Falkenau", "score": 0.9965296387672424, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 18, "end": 31, "label": "PERSON", "text": "Jona Falkenau", "score": 0.9980782270431519, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 47, "end": 56, "label": "RELIGION", "text": "Sikhismus", "score": 0.9455693364143372, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 0, "end": 13, "label": "PERSON", "text": "Jona Falkenau", "score": 0.9965296387672424, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 18, "end": 31, "label": "PERSON", "text": "Jona Falkenau", "score": 0.9980782270431519, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 47, "end": 56, "label": "RELIGION", "text": "Sikhismus", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 0, "end": 13, "label": "PERSON", "text": "Jona Falkenau", "uncovered_positions": [] }, { "start": 18, "end": 31, "label": "PERSON", "text": "Jona Falkenau", "uncovered_positions": [] }, { "start": 47, "end": 56, "label": "RELIGION", "text": "Sikhismus", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-124", "scenario": "multiple_entities", "predictions": [ { "start": 0, "end": 12, "label": "PERSON", "text": "Esme Rowanby", "score": 0.9983696341514587, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 22, "end": 28, "label": "RELIGION", "text": "Taoism", "score": 0.9475066661834717, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 42, "end": 54, "label": "PERSON", "text": "Esme Rowanby", "score": 0.9983136653900146, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 0, "end": 12, "label": "PERSON", "text": "Esme Rowanby", "score": 0.9983696341514587, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 22, "end": 28, "label": "RELIGION", "text": "Taoism", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" }, { "start": 42, "end": 54, "label": "PERSON", "text": "Esme Rowanby", "score": 0.9983136653900146, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 0, "end": 12, "label": "PERSON", "text": "Esme Rowanby", "uncovered_positions": [] }, { "start": 22, "end": 28, "label": "RELIGION", "text": "Taoism", "uncovered_positions": [] }, { "start": 42, "end": 54, "label": "PERSON", "text": "Esme Rowanby", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-125", "scenario": "multiple_entities", "predictions": [ { "start": 23, "end": 33, "label": "ORGANIZATION", "text": "Hafenbogen", "score": 0.7308627963066101, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 43, "end": 47, "label": "CITY", "text": "Kiel", "score": 0.9980584979057312, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 49, "end": 60, "label": "RELIGION", "text": "evangelisch", "score": 0.7706614136695862, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 23, "end": 33, "label": "ORGANIZATION", "text": "Hafenbogen", "score": 0.7308627963066101, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 43, "end": 47, "label": "CITY", "text": "Kiel", "score": 0.9980584979057312, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 49, "end": 60, "label": "RELIGION", "text": "evangelisch", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [ [ 23, 33, "ORGANIZATION" ] ], "false_negatives": [ [ 23, 41, "ORGANIZATION" ] ], "exact_match": false, "error_code": null, "canaries": [ { "start": 23, "end": 41, "label": "ORGANIZATION", "text": "Hafenbogen Archivs", "uncovered_positions": [ 34, 35, 36, 37, 38, 39, 40 ] }, { "start": 43, "end": 47, "label": "CITY", "text": "Kiel", "uncovered_positions": [] }, { "start": 49, "end": 60, "label": "RELIGION", "text": "evangelisch", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [ { "label": "ORGANIZATION", "start": 23, "end": 41, "kind": "partial_mask" } ], "outcome": "candidate_partial_mask" }, { "case_id": "holdout03d-126", "scenario": "multiple_entities", "predictions": [ { "start": 2, "end": 22, "label": "ORGANIZATION", "text": "Maple Tide Institute", "score": 0.9682912230491638, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 35, "end": 41, "label": "CITY", "text": "Durham", "score": 0.99836665391922, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 51, "end": 58, "label": "RELIGION", "text": "Judaism", "score": 0.9933406710624695, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 2, "end": 22, "label": "ORGANIZATION", "text": "Maple Tide Institute", "score": 0.9682912230491638, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 35, "end": 41, "label": "CITY", "text": "Durham", "score": 0.99836665391922, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 51, "end": 58, "label": "RELIGION", "text": "Judaism", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 2, "end": 22, "label": "ORGANIZATION", "text": "Maple Tide Institute", "uncovered_positions": [] }, { "start": 35, "end": 41, "label": "CITY", "text": "Durham", "uncovered_positions": [] }, { "start": 51, "end": 58, "label": "RELIGION", "text": "Judaism", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-127", "scenario": "multiple_entities", "predictions": [ { "start": 0, "end": 13, "label": "PERSON", "text": "Liv Hohenrain", "score": 0.9968641400337219, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 20, "end": 24, "label": "CITY", "text": "Bonn", "score": 0.9716768860816956, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 29, "end": 34, "label": "PERSON", "text": "Bahai", "score": 0.8550848364830017, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 36, "end": 57, "label": "EMAIL", "text": "liv.h@example.invalid", "score": 0.7995248436927795, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [], "false_positives": [], "false_negatives": [ [ 0, 13, "PERSON" ], [ 20, 24, "CITY" ], [ 29, 34, "RELIGION" ], [ 36, 57, "EMAIL" ] ], "exact_match": false, "error_code": "ambiguous_entity_overlap", "canaries": [ { "start": 0, "end": 13, "label": "PERSON", "text": "Liv Hohenrain", "uncovered_positions": [ 0, 1, 2, 4, 5, 6, 7, 8, 9, 10, 11, 12 ] }, { "start": 20, "end": 24, "label": "CITY", "text": "Bonn", "uncovered_positions": [ 20, 21, 22, 23 ] }, { "start": 29, "end": 34, "label": "RELIGION", "text": "Bahai", "uncovered_positions": [ 29, 30, 31, 32, 33 ] }, { "start": 36, "end": 57, "label": "EMAIL", "text": "liv.h@example.invalid", "uncovered_positions": [ 36, 37, 38, 40, 42, 43, 44, 45, 46, 47, 48, 50, 51, 52, 53, 54, 55, 56 ] } ], "candidate_returned": false, "candidate_canaries": [], "outcome": "blocked_conflicting_spans" }, { "case_id": "holdout03d-128", "scenario": "multiple_entities", "predictions": [ { "start": 0, "end": 14, "label": "PERSON", "text": "Reed Alderwick", "score": 0.98525071144104, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 16, "end": 20, "label": "CITY", "text": "Bath", "score": 0.9997182488441467, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 22, "end": 33, "label": "RELIGION", "text": "agnosticism", "score": 0.9372888803482056, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 35, "end": 57, "label": "EMAIL", "text": "reed.a@example.invalid", "score": 0.644429087638855, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 0, "end": 14, "label": "PERSON", "text": "Reed Alderwick", "score": 0.98525071144104, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 16, "end": 20, "label": "CITY", "text": "Bath", "score": 0.9997182488441467, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 22, "end": 33, "label": "RELIGION", "text": "agnosticism", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" }, { "start": 35, "end": 57, "label": "EMAIL", "text": "reed.a@example.invalid", "score": null, "source": "deterministic", "model_revision": "contact-patterns-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 0, "end": 14, "label": "PERSON", "text": "Reed Alderwick", "uncovered_positions": [] }, { "start": 16, "end": 20, "label": "CITY", "text": "Bath", "uncovered_positions": [] }, { "start": 22, "end": 33, "label": "RELIGION", "text": "agnosticism", "uncovered_positions": [] }, { "start": 35, "end": 57, "label": "EMAIL", "text": "reed.a@example.invalid", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-129", "scenario": "formats_repetition", "predictions": [ { "start": 15, "end": 20, "label": "RELIGION", "text": "Islam", "score": 0.9793844223022461, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 15, "end": 20, "label": "RELIGION", "text": "Islam", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 15, "end": 20, "label": "RELIGION", "text": "Islam", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-130", "scenario": "formats_repetition", "predictions": [ { "start": 11, "end": 19, "label": "RELIGION", "text": "Buddhism", "score": 0.9502246379852295, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 11, "end": 19, "label": "RELIGION", "text": "Buddhism", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 11, "end": 19, "label": "RELIGION", "text": "Buddhism", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-131", "scenario": "formats_repetition", "predictions": [], "combined_predictions": [ { "start": 9, "end": 24, "label": "RELIGION", "text": "konfessionsfrei", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" }, { "start": 35, "end": 50, "label": "RELIGION", "text": "konfessionsfrei", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 9, "end": 24, "label": "RELIGION", "text": "konfessionsfrei", "uncovered_positions": [] }, { "start": 35, "end": 50, "label": "RELIGION", "text": "konfessionsfrei", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-132", "scenario": "formats_repetition", "predictions": [], "combined_predictions": [ { "start": 9, "end": 16, "label": "RELIGION", "text": "atheist", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" }, { "start": 27, "end": 34, "label": "RELIGION", "text": "atheist", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 9, "end": 16, "label": "RELIGION", "text": "atheist", "uncovered_positions": [] }, { "start": 27, "end": 34, "label": "RELIGION", "text": "atheist", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-133", "scenario": "formats_repetition", "predictions": [ { "start": 9, "end": 17, "label": "RELIGION", "text": "Judentum", "score": 0.9829030632972717, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 9, "end": 17, "label": "RELIGION", "text": "Judentum", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 9, "end": 17, "label": "RELIGION", "text": "Judentum", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-134", "scenario": "formats_repetition", "predictions": [ { "start": 7, "end": 13, "label": "RELIGION", "text": "Shinto", "score": 0.9690591096878052, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 7, "end": 13, "label": "RELIGION", "text": "Shinto", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 7, "end": 13, "label": "RELIGION", "text": "Shinto", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-135", "scenario": "formats_repetition", "predictions": [ { "start": 18, "end": 31, "label": "RELIGION", "text": "Agnostizismus", "score": 0.5336430072784424, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 18, "end": 31, "label": "RELIGION", "text": "Agnostizismus", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 18, "end": 31, "label": "RELIGION", "text": "Agnostizismus", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-136", "scenario": "formats_repetition", "predictions": [ { "start": 10, "end": 17, "label": "RELIGION", "text": "Sikhism", "score": 0.973417341709137, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 10, "end": 17, "label": "RELIGION", "text": "Sikhism", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 10, "end": 17, "label": "RELIGION", "text": "Sikhism", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-137", "scenario": "formats_repetition", "predictions": [ { "start": 11, "end": 24, "label": "ORGANIZATION", "text": "Katholizismus", "score": 0.6781677603721619, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 35, "end": 48, "label": "ORGANIZATION", "text": "Katholizismus", "score": 0.73296058177948, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [], "false_positives": [], "false_negatives": [ [ 11, 24, "RELIGION" ], [ 35, 48, "RELIGION" ] ], "exact_match": false, "error_code": "ambiguous_entity_overlap", "canaries": [ { "start": 11, "end": 24, "label": "RELIGION", "text": "Katholizismus", "uncovered_positions": [ 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23 ] }, { "start": 35, "end": 48, "label": "RELIGION", "text": "Katholizismus", "uncovered_positions": [ 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47 ] } ], "candidate_returned": false, "candidate_canaries": [], "outcome": "blocked_conflicting_spans" }, { "case_id": "holdout03d-138", "scenario": "formats_repetition", "predictions": [ { "start": 7, "end": 15, "label": "RELIGION", "text": "Hinduism", "score": 0.9410762786865234, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 26, "end": 34, "label": "RELIGION", "text": "Hinduism", "score": 0.9376835227012634, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 7, "end": 15, "label": "RELIGION", "text": "Hinduism", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" }, { "start": 26, "end": 34, "label": "RELIGION", "text": "Hinduism", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 7, "end": 15, "label": "RELIGION", "text": "Hinduism", "uncovered_positions": [] }, { "start": 26, "end": 34, "label": "RELIGION", "text": "Hinduism", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-139", "scenario": "formats_repetition", "predictions": [], "combined_predictions": [ { "start": 13, "end": 28, "label": "RELIGION", "text": "ohne Konfession", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 13, "end": 28, "label": "RELIGION", "text": "ohne Konfession", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-140", "scenario": "formats_repetition", "predictions": [], "combined_predictions": [ { "start": 11, "end": 24, "label": "RELIGION", "text": "non-religious", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 11, "end": 24, "label": "RELIGION", "text": "non-religious", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-141", "scenario": "formats_repetition", "predictions": [ { "start": 0, "end": 11, "label": "RELIGION", "text": "Christentum", "score": 0.8982365727424622, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 14, "end": 23, "label": "RELIGION", "text": "Atheismus", "score": 0.7216019034385681, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 0, "end": 11, "label": "RELIGION", "text": "Christentum", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" }, { "start": 14, "end": 23, "label": "RELIGION", "text": "Atheismus", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 0, "end": 11, "label": "RELIGION", "text": "Christentum", "uncovered_positions": [] }, { "start": 14, "end": 23, "label": "RELIGION", "text": "Atheismus", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-142", "scenario": "formats_repetition", "predictions": [ { "start": 0, "end": 12, "label": "RELIGION", "text": "Christianity", "score": 0.9415823221206665, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 15, "end": 26, "label": "RELIGION", "text": "agnosticism", "score": 0.61921626329422, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 0, "end": 12, "label": "RELIGION", "text": "Christianity", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" }, { "start": 15, "end": 26, "label": "RELIGION", "text": "agnosticism", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 0, "end": 12, "label": "RELIGION", "text": "Christianity", "uncovered_positions": [] }, { "start": 15, "end": 26, "label": "RELIGION", "text": "agnosticism", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-143", "scenario": "formats_repetition", "predictions": [ { "start": 11, "end": 21, "label": "RELIGION", "text": "Buddhismus", "score": 0.7847886085510254, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 11, "end": 21, "label": "RELIGION", "text": "Buddhismus", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 11, "end": 21, "label": "RELIGION", "text": "Buddhismus", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-144", "scenario": "formats_repetition", "predictions": [ { "start": 9, "end": 16, "label": "RELIGION", "text": "Jainism", "score": 0.7793956995010376, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 9, "end": 16, "label": "RELIGION", "text": "Jainism", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 9, "end": 16, "label": "RELIGION", "text": "Jainism", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-145", "scenario": "negative_controls", "predictions": [], "combined_predictions": [], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-146", "scenario": "negative_controls", "predictions": [], "combined_predictions": [], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-147", "scenario": "negative_controls", "predictions": [], "combined_predictions": [], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-148", "scenario": "negative_controls", "predictions": [], "combined_predictions": [], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-149", "scenario": "negative_controls", "predictions": [], "combined_predictions": [], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-150", "scenario": "negative_controls", "predictions": [], "combined_predictions": [], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-151", "scenario": "negative_controls", "predictions": [], "combined_predictions": [], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-152", "scenario": "negative_controls", "predictions": [], "combined_predictions": [], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-153", "scenario": "negative_controls", "predictions": [], "combined_predictions": [], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-154", "scenario": "negative_controls", "predictions": [], "combined_predictions": [], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-155", "scenario": "negative_controls", "predictions": [], "combined_predictions": [], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-156", "scenario": "negative_controls", "predictions": [], "combined_predictions": [], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-157", "scenario": "negative_controls", "predictions": [], "combined_predictions": [], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-158", "scenario": "negative_controls", "predictions": [], "combined_predictions": [], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-159", "scenario": "negative_controls", "predictions": [], "combined_predictions": [], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-160", "scenario": "negative_controls", "predictions": [], "combined_predictions": [], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" } ], "automated_release_eligible": false, "recognition_qualified": false } }, "contextual_status": { "qualification": { "schema_version": "development-candidate-outcomes/v1", "accuracy": { "documents": 40, "ner_entity_metrics": { "PERSON": { "tp": 18, "fp": 3, "fn": 2, "precision": 0.8571428571428571, "recall": 0.9, "f1": 0.8780487804878048 }, "CITY": { "tp": 16, "fp": 2, "fn": 1, "precision": 0.8888888888888888, "recall": 0.9411764705882353, "f1": 0.9142857142857143 }, "RELIGION": { "tp": 6, "fp": 0, "fn": 0, "precision": 1.0, "recall": 1.0, "f1": 1.0 }, "EMAIL": { "tp": 1, "fp": 0, "fn": 1, "precision": 1.0, "recall": 0.5, "f1": 0.6666666666666666 }, "PHONE": { "tp": 1, "fp": 0, "fn": 0, "precision": 1.0, "recall": 1.0, "f1": 1.0 }, "ORGANIZATION": { "tp": 1, "fp": 2, "fn": 1, "precision": 0.3333333333333333, "recall": 0.5, "f1": 0.4 } }, "combined_entity_metrics": { "PERSON": { "tp": 18, "fp": 3, "fn": 2, "precision": 0.8571428571428571, "recall": 0.9, "f1": 0.8780487804878048 }, "CITY": { "tp": 16, "fp": 1, "fn": 1, "precision": 0.9411764705882353, "recall": 0.9411764705882353, "f1": 0.9411764705882353 }, "RELIGION": { "tp": 6, "fp": 0, "fn": 0, "precision": 1.0, "recall": 1.0, "f1": 1.0 }, "EMAIL": { "tp": 2, "fp": 0, "fn": 0, "precision": 1.0, "recall": 1.0, "f1": 1.0 }, "PHONE": { "tp": 1, "fp": 0, "fn": 0, "precision": 1.0, "recall": 1.0, "f1": 1.0 }, "ORGANIZATION": { "tp": 1, "fp": 2, "fn": 1, "precision": 0.3333333333333333, "recall": 0.5, "f1": 0.4 } }, "document_exact_match_rate": 0.875, "groups": { "de": { "documents": 10, "exact_matches": 8, "false_negatives": 1 }, "en": { "documents": 29, "exact_matches": 26, "false_negatives": 3 }, "mixed": { "documents": 1, "exact_matches": 1, "false_negatives": 0 }, "ocr_noise": { "documents": 4, "exact_matches": 2, "false_negatives": 2 } }, "blocked_documents": 0, "failure_codes": {}, "automated_release_eligible": false, "uncovered_source_canaries_including_incomplete": { "PERSON": 0, "CITY": 0, "RELIGION": 0, "EMAIL": 0, "PHONE": 0, "ORGANIZATION": 0 }, "literal_presence_or_incomplete": { "PERSON": 0, "CITY": 0, "RELIGION": 0, "EMAIL": 0, "PHONE": 0, "ORGANIZATION": 0 }, "partially_covered_source_canaries_including_incomplete": { "PERSON": 0, "CITY": 0, "RELIGION": 0, "EMAIL": 0, "PHONE": 0, "ORGANIZATION": 0 }, "fully_uncovered_source_canaries_including_incomplete": { "PERSON": 0, "CITY": 0, "RELIGION": 0, "EMAIL": 0, "PHONE": 0, "ORGANIZATION": 0 }, "all_document_source_coverage": { "PERSON": { "covered": 208, "total": 208 }, "CITY": { "covered": 118, "total": 118 }, "RELIGION": { "covered": 50, "total": 50 }, "EMAIL": { "covered": 27, "total": 27 }, "PHONE": { "covered": 7, "total": 7 }, "ORGANIZATION": { "covered": 11, "total": 11 } }, "conservative_accuracy_gate_passed": true, "scope": "all source spans; incomplete processing counts as missing, not leaked output", "deterministic_replay_latency_ms": { "median": 0.08865800191415474, "max": 4.73125500138849 } }, "candidate_outcomes": { "complete_no_measured_leakage": 40, "candidate_partial_mask": 0, "candidate_fully_unmasked": 0, "blocked_conflicting_spans": 0, "blocked_model_runtime": 0, "other_incomplete": 0 }, "candidate_return_rate": 1.0, "safe_abstention_rate": 0.0, "incomplete_rate": 0.0, "no_measured_leakage_rate_among_returned": 1.0, "candidate_partial_canaries": { "PERSON": 0, "CITY": 0, "RELIGION": 0, "EMAIL": 0, "PHONE": 0, "ORGANIZATION": 0 }, "candidate_fully_unmasked_canaries": { "PERSON": 0, "CITY": 0, "RELIGION": 0, "EMAIL": 0, "PHONE": 0, "ORGANIZATION": 0 }, "returned_candidate_entity_metrics": { "PERSON": { "tp": 18, "fp": 3, "fn": 2, "precision": 0.8571428571428571, "recall": 0.9, "f1": 0.8780487804878048 }, "CITY": { "tp": 16, "fp": 1, "fn": 1, "precision": 0.9411764705882353, "recall": 0.9411764705882353, "f1": 0.9411764705882353 }, "RELIGION": { "tp": 6, "fp": 0, "fn": 0, "precision": 1.0, "recall": 1.0, "f1": 1.0 }, "EMAIL": { "tp": 2, "fp": 0, "fn": 0, "precision": 1.0, "recall": 1.0, "f1": 1.0 }, "PHONE": { "tp": 1, "fp": 0, "fn": 0, "precision": 1.0, "recall": 1.0, "f1": 1.0 }, "ORGANIZATION": { "tp": 1, "fp": 2, "fn": 1, "precision": 0.3333333333333333, "recall": 0.5, "f1": 0.4 } }, "per_case": [ { "case_id": "baseline-00", "scenario": null, "predictions": [ { "start": 0, "end": 16, "label": "PERSON", "text": "Lina Wolkenstein", "score": 0.9969016313552856, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 26, "end": 32, "label": "CITY", "text": "Berlin", "score": 0.9999364614486694, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 0, "end": 16, "label": "PERSON", "text": "Lina Wolkenstein", "score": 0.9969016313552856, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 26, "end": 32, "label": "CITY", "text": "Berlin", "score": 0.9999364614486694, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 0, "end": 16, "label": "PERSON", "text": "Lina Wolkenstein", "uncovered_positions": [] }, { "start": 26, "end": 32, "label": "CITY", "text": "Berlin", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "baseline-01", "scenario": null, "predictions": [ { "start": 0, "end": 11, "label": "PERSON", "text": "Hans Müller", "score": 0.9912292957305908, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 0, "end": 11, "label": "PERSON", "text": "Hans Müller", "score": 0.9912292957305908, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 0, "end": 11, "label": "PERSON", "text": "Hans Müller", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "baseline-02", "scenario": null, "predictions": [ { "start": 13, "end": 30, "label": "PERSON", "text": "Dr. Klaus Schmidt", "score": 0.9196021556854248, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 13, "end": 30, "label": "PERSON", "text": "Dr. Klaus Schmidt", "score": 0.9196021556854248, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [ [ 13, 30, "PERSON" ] ], "false_negatives": [ [ 17, 30, "PERSON" ] ], "exact_match": false, "error_code": null, "canaries": [ { "start": 17, "end": 30, "label": "PERSON", "text": "Klaus Schmidt", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "baseline-03", "scenario": null, "predictions": [ { "start": 0, "end": 17, "label": "PERSON", "text": "Friedrich Wilhelm", "score": 0.9821144938468933, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 0, "end": 17, "label": "PERSON", "text": "Friedrich Wilhelm", "score": 0.9821144938468933, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 0, "end": 17, "label": "PERSON", "text": "Friedrich Wilhelm", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "baseline-04", "scenario": null, "predictions": [ { "start": 15, "end": 35, "label": "PERSON", "text": "Anne-Marie Schneider", "score": 0.981816291809082, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 15, "end": 35, "label": "PERSON", "text": "Anne-Marie Schneider", "score": 0.981816291809082, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 15, "end": 35, "label": "PERSON", "text": "Anne-Marie Schneider", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "baseline-05", "scenario": null, "predictions": [ { "start": 0, "end": 8, "label": "PERSON", "text": "John Doe", "score": 0.9945249557495117, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 0, "end": 8, "label": "PERSON", "text": "John Doe", "score": 0.9945249557495117, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 0, "end": 8, "label": "PERSON", "text": "John Doe", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "baseline-06", "scenario": null, "predictions": [ { "start": 0, "end": 5, "label": "PERSON", "text": "Alice", "score": 0.9985244870185852, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 10, "end": 13, "label": "PERSON", "text": "Bob", "score": 0.9991118311882019, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 26, "end": 31, "label": "CITY", "text": "store", "score": 0.5467662811279297, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 0, "end": 5, "label": "PERSON", "text": "Alice", "score": 0.9985244870185852, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 10, "end": 13, "label": "PERSON", "text": "Bob", "score": 0.9991118311882019, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 26, "end": 31, "label": "CITY", "text": "store", "score": 0.5467662811279297, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [ [ 26, 31, "CITY" ] ], "false_negatives": [], "exact_match": false, "error_code": null, "canaries": [ { "start": 0, "end": 5, "label": "PERSON", "text": "Alice", "uncovered_positions": [] }, { "start": 10, "end": 13, "label": "PERSON", "text": "Bob", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "baseline-07", "scenario": null, "predictions": [ { "start": 3, "end": 15, "label": "PERSON", "text": "Mary Johnson", "score": 0.9960722923278809, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 3, "end": 15, "label": "PERSON", "text": "Mary Johnson", "score": 0.9960722923278809, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 3, "end": 15, "label": "PERSON", "text": "Mary Johnson", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "baseline-08", "scenario": null, "predictions": [ { "start": 17, "end": 35, "label": "PERSON", "text": "Charles Montgomery", "score": 0.9641899466514587, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 17, "end": 35, "label": "PERSON", "text": "Charles Montgomery", "score": 0.9641899466514587, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 17, "end": 35, "label": "PERSON", "text": "Charles Montgomery", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "baseline-09", "scenario": null, "predictions": [ { "start": 0, "end": 16, "label": "PERSON", "text": "Elizabeth Taylor", "score": 0.9948871731758118, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 0, "end": 16, "label": "PERSON", "text": "Elizabeth Taylor", "score": 0.9948871731758118, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 0, "end": 16, "label": "PERSON", "text": "Elizabeth Taylor", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "baseline-10", "scenario": null, "predictions": [ { "start": 10, "end": 18, "label": "CITY", "text": "New York", "score": 0.9997181296348572, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 10, "end": 18, "label": "CITY", "text": "New York", "score": 0.9997181296348572, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 10, "end": 18, "label": "CITY", "text": "New York", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "baseline-11", "scenario": null, "predictions": [ { "start": 13, "end": 20, "label": "CITY", "text": "München", "score": 0.9988282322883606, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 13, "end": 20, "label": "CITY", "text": "München", "score": 0.9988282322883606, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 13, "end": 20, "label": "CITY", "text": "München", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "baseline-12", "scenario": null, "predictions": [ { "start": 0, "end": 5, "label": "CITY", "text": "Paris", "score": 0.9994774460792542, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 0, "end": 5, "label": "CITY", "text": "Paris", "score": 0.9994774460792542, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 0, "end": 5, "label": "CITY", "text": "Paris", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "baseline-13", "scenario": null, "predictions": [ { "start": 9, "end": 18, "label": "CITY", "text": "Frankfurt", "score": 0.9987022876739502, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 24, "end": 29, "label": "CITY", "text": "Tokyo", "score": 0.9982670545578003, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 9, "end": 18, "label": "CITY", "text": "Frankfurt", "score": 0.9987022876739502, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 24, "end": 29, "label": "CITY", "text": "Tokyo", "score": 0.9982670545578003, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 9, "end": 18, "label": "CITY", "text": "Frankfurt", "uncovered_positions": [] }, { "start": 24, "end": 29, "label": "CITY", "text": "Tokyo", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "baseline-14", "scenario": null, "predictions": [ { "start": 11, "end": 22, "label": "CITY", "text": "Los Angeles", "score": 0.9923924207687378, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 24, "end": 34, "label": "CITY", "text": "California", "score": 0.9569051861763, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 11, "end": 22, "label": "CITY", "text": "Los Angeles", "score": 0.9923924207687378, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 24, "end": 34, "label": "CITY", "text": "California", "score": 0.9569051861763, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 11, "end": 22, "label": "CITY", "text": "Los Angeles", "uncovered_positions": [] }, { "start": 24, "end": 34, "label": "CITY", "text": "California", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "baseline-15", "scenario": null, "predictions": [ { "start": 32, "end": 43, "label": "RELIGION", "text": "evangelisch", "score": 0.8318354487419128, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 32, "end": 43, "label": "RELIGION", "text": "evangelisch", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 32, "end": 43, "label": "RELIGION", "text": "evangelisch", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "baseline-16", "scenario": null, "predictions": [ { "start": 0, "end": 2, "label": "PERSON", "text": "Er", "score": 0.6436470746994019, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 7, "end": 17, "label": "RELIGION", "text": "katholisch", "score": 0.7684664726257324, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 0, "end": 2, "label": "PERSON", "text": "Er", "score": 0.6436470746994019, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 7, "end": 17, "label": "RELIGION", "text": "katholisch", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [ [ 0, 2, "PERSON" ] ], "false_negatives": [], "exact_match": false, "error_code": null, "canaries": [ { "start": 7, "end": 17, "label": "RELIGION", "text": "katholisch", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "baseline-17", "scenario": null, "predictions": [ { "start": 14, "end": 21, "label": "RELIGION", "text": "Judaism", "score": 0.9957650899887085, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 14, "end": 21, "label": "RELIGION", "text": "Judaism", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 14, "end": 21, "label": "RELIGION", "text": "Judaism", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "baseline-18", "scenario": null, "predictions": [ { "start": 15, "end": 21, "label": "RELIGION", "text": "Muslim", "score": 0.966416597366333, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 15, "end": 21, "label": "RELIGION", "text": "Muslim", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 15, "end": 21, "label": "RELIGION", "text": "Muslim", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "baseline-19", "scenario": null, "predictions": [ { "start": 0, "end": 8, "label": "RELIGION", "text": "Buddhist", "score": 0.7523731589317322, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 0, "end": 8, "label": "RELIGION", "text": "Buddhist", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 0, "end": 8, "label": "RELIGION", "text": "Buddhist", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "baseline-20", "scenario": null, "predictions": [ { "start": 0, "end": 8, "label": "PERSON", "text": "John Doe", "score": 0.9968609809875488, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 14, "end": 22, "label": "CITY", "text": "New York", "score": 0.9980940222740173, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 27, "end": 43, "label": "PERSON", "text": "Lina Wolkenstein", "score": 0.9969224333763123, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 47, "end": 53, "label": "CITY", "text": "Berlin", "score": 0.9998062252998352, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 0, "end": 8, "label": "PERSON", "text": "John Doe", "score": 0.9968609809875488, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 14, "end": 22, "label": "CITY", "text": "New York", "score": 0.9980940222740173, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 27, "end": 43, "label": "PERSON", "text": "Lina Wolkenstein", "score": 0.9969224333763123, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 47, "end": 53, "label": "CITY", "text": "Berlin", "score": 0.9998062252998352, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 0, "end": 8, "label": "PERSON", "text": "John Doe", "uncovered_positions": [] }, { "start": 14, "end": 22, "label": "CITY", "text": "New York", "uncovered_positions": [] }, { "start": 27, "end": 43, "label": "PERSON", "text": "Lina Wolkenstein", "uncovered_positions": [] }, { "start": 47, "end": 53, "label": "CITY", "text": "Berlin", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "baseline-21", "scenario": null, "predictions": [ { "start": 0, "end": 5, "label": "PERSON", "text": "David", "score": 0.9995999932289124, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 9, "end": 17, "label": "RELIGION", "text": "Catholic", "score": 0.9957641363143921, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 31, "end": 35, "label": "CITY", "text": "Rome", "score": 0.9998972415924072, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 0, "end": 5, "label": "PERSON", "text": "David", "score": 0.9995999932289124, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 9, "end": 17, "label": "RELIGION", "text": "Catholic", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" }, { "start": 31, "end": 35, "label": "CITY", "text": "Rome", "score": 0.9998972415924072, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 0, "end": 5, "label": "PERSON", "text": "David", "uncovered_positions": [] }, { "start": 9, "end": 17, "label": "RELIGION", "text": "Catholic", "uncovered_positions": [] }, { "start": 31, "end": 35, "label": "CITY", "text": "Rome", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "baseline-22", "scenario": null, "predictions": [ { "start": 0, "end": 4, "label": "PERSON", "text": "John", "score": 0.9999161958694458, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 11, "end": 15, "label": "PERSON", "text": "John", "score": 0.9968172311782837, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 0, "end": 4, "label": "PERSON", "text": "John", "score": 0.9999161958694458, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 11, "end": 15, "label": "PERSON", "text": "John", "score": 0.9968172311782837, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 0, "end": 4, "label": "PERSON", "text": "John", "uncovered_positions": [] }, { "start": 11, "end": 15, "label": "PERSON", "text": "John", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "baseline-23", "scenario": null, "predictions": [ { "start": 0, "end": 6, "label": "CITY", "text": "Berlin", "score": 0.9994149208068848, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 11, "end": 17, "label": "CITY", "text": "Berlin", "score": 0.9993422627449036, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 0, "end": 6, "label": "CITY", "text": "Berlin", "score": 0.9994149208068848, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 11, "end": 17, "label": "CITY", "text": "Berlin", "score": 0.9993422627449036, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 0, "end": 6, "label": "CITY", "text": "Berlin", "uncovered_positions": [] }, { "start": 11, "end": 17, "label": "CITY", "text": "Berlin", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "baseline-24", "scenario": null, "predictions": [ { "start": 0, "end": 10, "label": "CITY", "text": "Washington", "score": 0.9995170831680298, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 0, "end": 10, "label": "CITY", "text": "Washington", "score": 0.9995170831680298, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 0, "end": 10, "label": "CITY", "text": "Washington", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "baseline-25", "scenario": null, "predictions": [ { "start": 0, "end": 17, "label": "PERSON", "text": "George Washington", "score": 0.9903205037117004, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 0, "end": 17, "label": "PERSON", "text": "George Washington", "score": 0.9903205037117004, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 0, "end": 17, "label": "PERSON", "text": "George Washington", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "baseline-26", "scenario": null, "predictions": [ { "start": 14, "end": 20, "label": "CITY", "text": "Jordan", "score": 0.9931551814079285, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 14, "end": 20, "label": "CITY", "text": "Jordan", "score": 0.9931551814079285, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 14, "end": 20, "label": "CITY", "text": "Jordan", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "baseline-27", "scenario": null, "predictions": [ { "start": 0, "end": 14, "label": "PERSON", "text": "Michael Jordan", "score": 0.9887990355491638, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 0, "end": 14, "label": "PERSON", "text": "Michael Jordan", "score": 0.9887990355491638, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 0, "end": 14, "label": "PERSON", "text": "Michael Jordan", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "baseline-28", "scenario": null, "predictions": [ { "start": 0, "end": 14, "label": "PERSON", "text": "Mr. Smith-\nson", "score": 0.9715586304664612, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "dehyphenation-v1" } ], "combined_predictions": [ { "start": 0, "end": 14, "label": "PERSON", "text": "Mr. Smith-\nson", "score": 0.9715586304664612, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "dehyphenation-v1" } ], "false_positives": [ [ 0, 14, "PERSON" ] ], "false_negatives": [ [ 4, 14, "PERSON" ] ], "exact_match": false, "error_code": null, "canaries": [ { "start": 4, "end": 14, "label": "PERSON", "text": "Smith-\nson", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "baseline-29", "scenario": null, "predictions": [ { "start": 0, "end": 8, "label": "CITY", "text": "Ber-\nlin", "score": 0.9711049795150757, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "dehyphenation-v1" } ], "combined_predictions": [ { "start": 0, "end": 8, "label": "CITY", "text": "Ber-\nlin", "score": 0.9711049795150757, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "dehyphenation-v1" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 0, "end": 8, "label": "CITY", "text": "Ber-\nlin", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "baseline-30", "scenario": null, "predictions": [ { "start": 0, "end": 8, "label": "PERSON", "text": "Jchn D0e", "score": 0.6583331823348999, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 0, "end": 8, "label": "PERSON", "text": "Jchn D0e", "score": 0.6583331823348999, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 0, "end": 8, "label": "PERSON", "text": "Jchn D0e", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "baseline-31", "scenario": null, "predictions": [ { "start": 11, "end": 20, "label": "ORGANIZATION", "text": "Nevv Y0rk", "score": 0.8232883810997009, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 11, "end": 20, "label": "ORGANIZATION", "text": "Nevv Y0rk", "score": 0.8232883810997009, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [ [ 11, 20, "ORGANIZATION" ] ], "false_negatives": [ [ 11, 20, "CITY" ] ], "exact_match": false, "error_code": null, "canaries": [ { "start": 11, "end": 20, "label": "CITY", "text": "Nevv Y0rk", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "baseline-32", "scenario": null, "predictions": [], "combined_predictions": [], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "baseline-33", "scenario": null, "predictions": [], "combined_predictions": [], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "baseline-34", "scenario": null, "predictions": [], "combined_predictions": [], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "baseline-35", "scenario": null, "predictions": [ { "start": 12, "end": 28, "label": "EMAIL", "text": "test@example.com", "score": 0.8803917765617371, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 12, "end": 28, "label": "EMAIL", "text": "test@example.com", "score": null, "source": "deterministic", "model_revision": "contact-patterns-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 12, "end": 28, "label": "EMAIL", "text": "test@example.com", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "baseline-36", "scenario": null, "predictions": [ { "start": 11, "end": 19, "label": "PHONE", "text": "555-0199", "score": 0.9762608408927917, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 11, "end": 19, "label": "PHONE", "text": "555-0199", "score": null, "source": "deterministic", "model_revision": "contact-patterns-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 11, "end": 19, "label": "PHONE", "text": "555-0199", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "baseline-37", "scenario": null, "predictions": [ { "start": 11, "end": 21, "label": "ORGANIZATION", "text": "Google Inc", "score": 0.7403514981269836, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 11, "end": 21, "label": "ORGANIZATION", "text": "Google Inc", "score": 0.7403514981269836, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [ [ 11, 21, "ORGANIZATION" ] ], "false_negatives": [ [ 11, 22, "ORGANIZATION" ] ], "exact_match": true, "error_code": null, "canaries": [ { "start": 11, "end": 22, "label": "ORGANIZATION", "text": "Google Inc.", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "baseline-38", "scenario": null, "predictions": [ { "start": 13, "end": 23, "label": "CITY", "text": "company.de", "score": 0.8622952103614807, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 8, "end": 23, "label": "EMAIL", "text": "info@company.de", "score": null, "source": "deterministic", "model_revision": "contact-patterns-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 8, "end": 23, "label": "EMAIL", "text": "info@company.de", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "baseline-39", "scenario": null, "predictions": [ { "start": 4, "end": 6, "label": "ORGANIZATION", "text": "UN", "score": 0.9981075525283813, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 4, "end": 6, "label": "ORGANIZATION", "text": "UN", "score": 0.9981075525283813, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 4, "end": 6, "label": "ORGANIZATION", "text": "UN", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" } ], "automated_release_eligible": false, "recognition_qualified": false }, "validation": { "schema_version": "development-candidate-outcomes/v1", "accuracy": { "documents": 10, "ner_entity_metrics": { "PERSON": { "tp": 7, "fp": 0, "fn": 0, "precision": 1.0, "recall": 1.0, "f1": 1.0 }, "CITY": { "tp": 5, "fp": 1, "fn": 0, "precision": 0.8333333333333334, "recall": 1.0, "f1": 0.9090909090909091 }, "RELIGION": { "tp": 2, "fp": 1, "fn": 0, "precision": 0.6666666666666666, "recall": 1.0, "f1": 0.8 }, "EMAIL": { "tp": 2, "fp": 0, "fn": 0, "precision": 1.0, "recall": 1.0, "f1": 1.0 }, "PHONE": { "tp": 2, "fp": 0, "fn": 0, "precision": 1.0, "recall": 1.0, "f1": 1.0 }, "ORGANIZATION": { "tp": 2, "fp": 0, "fn": 0, "precision": 1.0, "recall": 1.0, "f1": 1.0 } }, "combined_entity_metrics": { "PERSON": { "tp": 7, "fp": 0, "fn": 0, "precision": 1.0, "recall": 1.0, "f1": 1.0 }, "CITY": { "tp": 5, "fp": 1, "fn": 0, "precision": 0.8333333333333334, "recall": 1.0, "f1": 0.9090909090909091 }, "RELIGION": { "tp": 2, "fp": 1, "fn": 0, "precision": 0.6666666666666666, "recall": 1.0, "f1": 0.8 }, "EMAIL": { "tp": 2, "fp": 0, "fn": 0, "precision": 1.0, "recall": 1.0, "f1": 1.0 }, "PHONE": { "tp": 2, "fp": 0, "fn": 0, "precision": 1.0, "recall": 1.0, "f1": 1.0 }, "ORGANIZATION": { "tp": 2, "fp": 0, "fn": 0, "precision": 1.0, "recall": 1.0, "f1": 1.0 } }, "document_exact_match_rate": 0.8, "groups": { "de": { "documents": 5, "exact_matches": 4, "false_negatives": 0 }, "en": { "documents": 5, "exact_matches": 4, "false_negatives": 0 }, "ocr_noise": { "documents": 2, "exact_matches": 2, "false_negatives": 0 } }, "blocked_documents": 0, "failure_codes": {}, "automated_release_eligible": false, "uncovered_source_canaries_including_incomplete": { "PERSON": 0, "CITY": 0, "RELIGION": 0, "EMAIL": 0, "PHONE": 0, "ORGANIZATION": 0 }, "literal_presence_or_incomplete": { "PERSON": 0, "CITY": 0, "RELIGION": 0, "EMAIL": 0, "PHONE": 0, "ORGANIZATION": 0 }, "partially_covered_source_canaries_including_incomplete": { "PERSON": 0, "CITY": 0, "RELIGION": 0, "EMAIL": 0, "PHONE": 0, "ORGANIZATION": 0 }, "fully_uncovered_source_canaries_including_incomplete": { "PERSON": 0, "CITY": 0, "RELIGION": 0, "EMAIL": 0, "PHONE": 0, "ORGANIZATION": 0 }, "all_document_source_coverage": { "PERSON": { "covered": 75, "total": 75 }, "CITY": { "covered": 30, "total": 30 }, "RELIGION": { "covered": 18, "total": 18 }, "EMAIL": { "covered": 33, "total": 33 }, "PHONE": { "covered": 23, "total": 23 }, "ORGANIZATION": { "covered": 22, "total": 22 } }, "conservative_accuracy_gate_passed": true, "scope": "all source spans; incomplete processing counts as missing, not leaked output", "deterministic_replay_latency_ms": { "median": 0.13845950161339715, "max": 0.1903869997477159 } }, "candidate_outcomes": { "complete_no_measured_leakage": 10, "candidate_partial_mask": 0, "candidate_fully_unmasked": 0, "blocked_conflicting_spans": 0, "blocked_model_runtime": 0, "other_incomplete": 0 }, "candidate_return_rate": 1.0, "safe_abstention_rate": 0.0, "incomplete_rate": 0.0, "no_measured_leakage_rate_among_returned": 1.0, "candidate_partial_canaries": { "PERSON": 0, "CITY": 0, "RELIGION": 0, "EMAIL": 0, "PHONE": 0, "ORGANIZATION": 0 }, "candidate_fully_unmasked_canaries": { "PERSON": 0, "CITY": 0, "RELIGION": 0, "EMAIL": 0, "PHONE": 0, "ORGANIZATION": 0 }, "returned_candidate_entity_metrics": { "PERSON": { "tp": 7, "fp": 0, "fn": 0, "precision": 1.0, "recall": 1.0, "f1": 1.0 }, "CITY": { "tp": 5, "fp": 1, "fn": 0, "precision": 0.8333333333333334, "recall": 1.0, "f1": 0.9090909090909091 }, "RELIGION": { "tp": 2, "fp": 1, "fn": 0, "precision": 0.6666666666666666, "recall": 1.0, "f1": 0.8 }, "EMAIL": { "tp": 2, "fp": 0, "fn": 0, "precision": 1.0, "recall": 1.0, "f1": 1.0 }, "PHONE": { "tp": 2, "fp": 0, "fn": 0, "precision": 1.0, "recall": 1.0, "f1": 1.0 }, "ORGANIZATION": { "tp": 2, "fp": 0, "fn": 0, "precision": 1.0, "recall": 1.0, "f1": 1.0 } }, "per_case": [ { "case_id": "validation-00", "scenario": null, "predictions": [ { "start": 0, "end": 9, "label": "PERSON", "text": "Mara Kühn", "score": 0.9990392923355103, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 23, "end": 30, "label": "ORGANIZATION", "text": "Siemens", "score": 0.9998852014541626, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 34, "end": 41, "label": "CITY", "text": "Dresden", "score": 0.9993748068809509, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 0, "end": 9, "label": "PERSON", "text": "Mara Kühn", "score": 0.9990392923355103, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 23, "end": 30, "label": "ORGANIZATION", "text": "Siemens", "score": 0.9998852014541626, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 34, "end": 41, "label": "CITY", "text": "Dresden", "score": 0.9993748068809509, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 0, "end": 9, "label": "PERSON", "text": "Mara Kühn", "uncovered_positions": [] }, { "start": 23, "end": 30, "label": "ORGANIZATION", "text": "Siemens", "uncovered_positions": [] }, { "start": 34, "end": 41, "label": "CITY", "text": "Dresden", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "validation-01", "scenario": null, "predictions": [ { "start": 0, "end": 11, "label": "PERSON", "text": "Emma Wilson", "score": 0.9975014328956604, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 22, "end": 38, "label": "ORGANIZATION", "text": "Acme Corporation", "score": 0.9995630383491516, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 42, "end": 48, "label": "CITY", "text": "London", "score": 0.9999483823776245, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 0, "end": 11, "label": "PERSON", "text": "Emma Wilson", "score": 0.9975014328956604, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 22, "end": 38, "label": "ORGANIZATION", "text": "Acme Corporation", "score": 0.9995630383491516, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 42, "end": 48, "label": "CITY", "text": "London", "score": 0.9999483823776245, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 0, "end": 11, "label": "PERSON", "text": "Emma Wilson", "uncovered_positions": [] }, { "start": 22, "end": 38, "label": "ORGANIZATION", "text": "Acme Corporation", "uncovered_positions": [] }, { "start": 42, "end": 48, "label": "CITY", "text": "London", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "validation-02", "scenario": null, "predictions": [ { "start": 8, "end": 18, "label": "RELIGION", "text": "muslimisch", "score": 0.9497504234313965, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 29, "end": 51, "label": "EMAIL", "text": "mara.kuehn@example.org", "score": 0.9375261664390564, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 53, "end": 68, "label": "PHONE", "text": "+49 351 7654321", "score": 0.9705594182014465, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 8, "end": 18, "label": "RELIGION", "text": "muslimisch", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" }, { "start": 29, "end": 51, "label": "EMAIL", "text": "mara.kuehn@example.org", "score": null, "source": "deterministic", "model_revision": "contact-patterns-v1", "normalization": "identity" }, { "start": 53, "end": 68, "label": "PHONE", "text": "+49 351 7654321", "score": null, "source": "deterministic", "model_revision": "contact-patterns-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 8, "end": 18, "label": "RELIGION", "text": "muslimisch", "uncovered_positions": [] }, { "start": 29, "end": 51, "label": "EMAIL", "text": "mara.kuehn@example.org", "uncovered_positions": [] }, { "start": 53, "end": 68, "label": "PHONE", "text": "+49 351 7654321", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "validation-03", "scenario": null, "predictions": [ { "start": 16, "end": 24, "label": "RELIGION", "text": "Hinduism", "score": 0.9999716281890869, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 34, "end": 50, "label": "EMAIL", "text": "emma@example.net", "score": 0.903052568435669, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 54, "end": 69, "label": "PHONE", "text": "+1 212 555 0188", "score": 0.951524555683136, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 16, "end": 24, "label": "RELIGION", "text": "Hinduism", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" }, { "start": 34, "end": 50, "label": "EMAIL", "text": "emma@example.net", "score": null, "source": "deterministic", "model_revision": "contact-patterns-v1", "normalization": "identity" }, { "start": 54, "end": 69, "label": "PHONE", "text": "+1 212 555 0188", "score": null, "source": "deterministic", "model_revision": "contact-patterns-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 16, "end": 24, "label": "RELIGION", "text": "Hinduism", "uncovered_positions": [] }, { "start": 34, "end": 50, "label": "EMAIL", "text": "emma@example.net", "uncovered_positions": [] }, { "start": 54, "end": 69, "label": "PHONE", "text": "+1 212 555 0188", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "validation-04", "scenario": null, "predictions": [ { "start": 0, "end": 11, "label": "PERSON", "text": "Nora Winter", "score": 0.9975367784500122, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 20, "end": 24, "label": "CITY", "text": "Köln", "score": 0.9991357922554016, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 26, "end": 37, "label": "PERSON", "text": "Nora Winter", "score": 0.99813312292099, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 0, "end": 11, "label": "PERSON", "text": "Nora Winter", "score": 0.9975367784500122, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 20, "end": 24, "label": "CITY", "text": "Köln", "score": 0.9991357922554016, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 26, "end": 37, "label": "PERSON", "text": "Nora Winter", "score": 0.99813312292099, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 0, "end": 11, "label": "PERSON", "text": "Nora Winter", "uncovered_positions": [] }, { "start": 20, "end": 24, "label": "CITY", "text": "Köln", "uncovered_positions": [] }, { "start": 26, "end": 37, "label": "PERSON", "text": "Nora Winter", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "validation-05", "scenario": null, "predictions": [ { "start": 0, "end": 6, "label": "CITY", "text": "Jordan", "score": 0.9668874144554138, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 23, "end": 36, "label": "PERSON", "text": "Jordan Miller", "score": 0.982417643070221, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 0, "end": 6, "label": "CITY", "text": "Jordan", "score": 0.9668874144554138, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 23, "end": 36, "label": "PERSON", "text": "Jordan Miller", "score": 0.982417643070221, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [ [ 0, 6, "CITY" ] ], "false_negatives": [], "exact_match": false, "error_code": null, "canaries": [ { "start": 23, "end": 36, "label": "PERSON", "text": "Jordan Miller", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "validation-06", "scenario": null, "predictions": [ { "start": 0, "end": 8, "label": "RELIGION", "text": "Religion", "score": 0.6538923978805542, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 0, "end": 8, "label": "RELIGION", "text": "Religion", "score": 0.6538923978805542, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [ [ 0, 8, "RELIGION" ] ], "false_negatives": [], "exact_match": false, "error_code": null, "canaries": [], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "validation-07", "scenario": null, "predictions": [], "combined_predictions": [], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "validation-08", "scenario": null, "predictions": [ { "start": 5, "end": 23, "label": "PERSON", "text": "Eva-Luise Schröder", "score": 0.986770510673523, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "dehyphenation-v1" }, { "start": 33, "end": 42, "label": "CITY", "text": "Ham-\nburg", "score": 0.9998313188552856, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "dehyphenation-v1" } ], "combined_predictions": [ { "start": 5, "end": 23, "label": "PERSON", "text": "Eva-Luise Schröder", "score": 0.986770510673523, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "dehyphenation-v1" }, { "start": 33, "end": 42, "label": "CITY", "text": "Ham-\nburg", "score": 0.9998313188552856, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "dehyphenation-v1" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 5, "end": 23, "label": "PERSON", "text": "Eva-Luise Schröder", "uncovered_positions": [] }, { "start": 33, "end": 42, "label": "CITY", "text": "Ham-\nburg", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "validation-09", "scenario": null, "predictions": [ { "start": 0, "end": 10, "label": "PERSON", "text": "M4ry Br0wn", "score": 0.9766657948493958, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 20, "end": 26, "label": "CITY", "text": "L0ndon", "score": 0.996916651725769, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 0, "end": 10, "label": "PERSON", "text": "M4ry Br0wn", "score": 0.9766657948493958, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 20, "end": 26, "label": "CITY", "text": "L0ndon", "score": 0.996916651725769, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 0, "end": 10, "label": "PERSON", "text": "M4ry Br0wn", "uncovered_positions": [] }, { "start": 20, "end": 26, "label": "CITY", "text": "L0ndon", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" } ], "automated_release_eligible": false, "recognition_qualified": false }, "development03c": { "schema_version": "development-candidate-outcomes/v1", "accuracy": { "documents": 120, "ner_entity_metrics": { "PERSON": { "tp": 132, "fp": 12, "fn": 0, "precision": 0.9166666666666666, "recall": 1.0, "f1": 0.9565217391304348 }, "CITY": { "tp": 84, "fp": 13, "fn": 0, "precision": 0.865979381443299, "recall": 1.0, "f1": 0.9281767955801106 }, "RELIGION": { "tp": 34, "fp": 0, "fn": 2, "precision": 1.0, "recall": 0.9444444444444444, "f1": 0.9714285714285714 }, "EMAIL": { "tp": 23, "fp": 0, "fn": 1, "precision": 1.0, "recall": 0.9583333333333334, "f1": 0.9787234042553191 }, "PHONE": { "tp": 24, "fp": 0, "fn": 0, "precision": 1.0, "recall": 1.0, "f1": 1.0 }, "ORGANIZATION": { "tp": 84, "fp": 12, "fn": 0, "precision": 0.875, "recall": 1.0, "f1": 0.9333333333333333 } }, "combined_entity_metrics": { "PERSON": { "tp": 132, "fp": 12, "fn": 0, "precision": 0.9166666666666666, "recall": 1.0, "f1": 0.9565217391304348 }, "CITY": { "tp": 84, "fp": 12, "fn": 0, "precision": 0.875, "recall": 1.0, "f1": 0.9333333333333333 }, "RELIGION": { "tp": 36, "fp": 12, "fn": 0, "precision": 0.75, "recall": 1.0, "f1": 0.8571428571428571 }, "EMAIL": { "tp": 24, "fp": 0, "fn": 0, "precision": 1.0, "recall": 1.0, "f1": 1.0 }, "PHONE": { "tp": 24, "fp": 0, "fn": 0, "precision": 1.0, "recall": 1.0, "f1": 1.0 }, "ORGANIZATION": { "tp": 84, "fp": 12, "fn": 0, "precision": 0.875, "recall": 1.0, "f1": 0.9333333333333333 } }, "document_exact_match_rate": 0.9, "groups": { "de": { "documents": 60, "exact_matches": 60, "false_negatives": 0 }, "en": { "documents": 60, "exact_matches": 48, "false_negatives": 0 }, "ocr_noise": { "documents": 24, "exact_matches": 24, "false_negatives": 0 } }, "blocked_documents": 0, "failure_codes": {}, "automated_release_eligible": false, "uncovered_source_canaries_including_incomplete": { "PERSON": 0, "CITY": 0, "RELIGION": 0, "EMAIL": 0, "PHONE": 0, "ORGANIZATION": 0 }, "literal_presence_or_incomplete": { "PERSON": 0, "CITY": 0, "RELIGION": 0, "EMAIL": 0, "PHONE": 0, "ORGANIZATION": 0 }, "partially_covered_source_canaries_including_incomplete": { "PERSON": 0, "CITY": 0, "RELIGION": 0, "EMAIL": 0, "PHONE": 0, "ORGANIZATION": 0 }, "fully_uncovered_source_canaries_including_incomplete": { "PERSON": 0, "CITY": 0, "RELIGION": 0, "EMAIL": 0, "PHONE": 0, "ORGANIZATION": 0 }, "all_document_source_coverage": { "PERSON": { "covered": 1727, "total": 1727 }, "CITY": { "covered": 588, "total": 588 }, "RELIGION": { "covered": 358, "total": 358 }, "EMAIL": { "covered": 648, "total": 648 }, "PHONE": { "covered": 264, "total": 264 }, "ORGANIZATION": { "covered": 1631, "total": 1631 } }, "conservative_accuracy_gate_passed": true, "scope": "all source spans; incomplete processing counts as missing, not leaked output", "deterministic_replay_latency_ms": { "median": 0.2151320004486479, "max": 3.7445589987328276 } }, "candidate_outcomes": { "complete_no_measured_leakage": 120, "candidate_partial_mask": 0, "candidate_fully_unmasked": 0, "blocked_conflicting_spans": 0, "blocked_model_runtime": 0, "other_incomplete": 0 }, "candidate_return_rate": 1.0, "safe_abstention_rate": 0.0, "incomplete_rate": 0.0, "no_measured_leakage_rate_among_returned": 1.0, "candidate_partial_canaries": { "PERSON": 0, "CITY": 0, "RELIGION": 0, "EMAIL": 0, "PHONE": 0, "ORGANIZATION": 0 }, "candidate_fully_unmasked_canaries": { "PERSON": 0, "CITY": 0, "RELIGION": 0, "EMAIL": 0, "PHONE": 0, "ORGANIZATION": 0 }, "returned_candidate_entity_metrics": { "PERSON": { "tp": 132, "fp": 12, "fn": 0, "precision": 0.9166666666666666, "recall": 1.0, "f1": 0.9565217391304348 }, "CITY": { "tp": 84, "fp": 12, "fn": 0, "precision": 0.875, "recall": 1.0, "f1": 0.9333333333333333 }, "RELIGION": { "tp": 36, "fp": 12, "fn": 0, "precision": 0.75, "recall": 1.0, "f1": 0.8571428571428571 }, "EMAIL": { "tp": 24, "fp": 0, "fn": 0, "precision": 1.0, "recall": 1.0, "f1": 1.0 }, "PHONE": { "tp": 24, "fp": 0, "fn": 0, "precision": 1.0, "recall": 1.0, "f1": 1.0 }, "ORGANIZATION": { "tp": 84, "fp": 12, "fn": 0, "precision": 0.875, "recall": 1.0, "f1": 0.9333333333333333 } }, "per_case": [ { "case_id": "holdout03c-000", "scenario": null, "predictions": [ { "start": 10, "end": 25, "label": "PERSON", "text": "Saskia Dornfeld", "score": 0.9995438456535339, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 39, "end": 63, "label": "ORGANIZATION", "text": "Aster Research Institute", "score": 0.9974908828735352, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 67, "end": 76, "label": "CITY", "text": "Osnabrück", "score": 0.999496579170227, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 10, "end": 25, "label": "PERSON", "text": "Saskia Dornfeld", "score": 0.9995438456535339, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 39, "end": 63, "label": "ORGANIZATION", "text": "Aster Research Institute", "score": 0.9974908828735352, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 67, "end": 76, "label": "CITY", "text": "Osnabrück", "score": 0.999496579170227, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 10, "end": 25, "label": "PERSON", "text": "Saskia Dornfeld", "uncovered_positions": [] }, { "start": 39, "end": 63, "label": "ORGANIZATION", "text": "Aster Research Institute", "uncovered_positions": [] }, { "start": 67, "end": 76, "label": "CITY", "text": "Osnabrück", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-001", "scenario": null, "predictions": [ { "start": 12, "end": 27, "label": "PERSON", "text": "Saskia Dornfeld", "score": 0.9990620017051697, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 39, "end": 63, "label": "ORGANIZATION", "text": "Aster Research Institute", "score": 0.997700035572052, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 69, "end": 78, "label": "CITY", "text": "Osnabrück", "score": 0.9995871186256409, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 12, "end": 27, "label": "PERSON", "text": "Saskia Dornfeld", "score": 0.9990620017051697, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 39, "end": 63, "label": "ORGANIZATION", "text": "Aster Research Institute", "score": 0.997700035572052, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 69, "end": 78, "label": "CITY", "text": "Osnabrück", "score": 0.9995871186256409, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 12, "end": 27, "label": "PERSON", "text": "Saskia Dornfeld", "uncovered_positions": [] }, { "start": 39, "end": 63, "label": "ORGANIZATION", "text": "Aster Research Institute", "uncovered_positions": [] }, { "start": 69, "end": 78, "label": "CITY", "text": "Osnabrück", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-002", "scenario": null, "predictions": [ { "start": 0, "end": 15, "label": "PERSON", "text": "Saskia Dornfeld", "score": 0.9997074007987976, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 22, "end": 33, "label": "RELIGION", "text": "sikhistisch", "score": 0.9321887493133545, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 61, "end": 91, "label": "EMAIL", "text": "case101.review@example.invalid", "score": 0.9007220268249512, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 102, "end": 116, "label": "PHONE", "text": "+49 541 730101", "score": 0.9946468472480774, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 0, "end": 15, "label": "PERSON", "text": "Saskia Dornfeld", "score": 0.9997074007987976, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 22, "end": 33, "label": "RELIGION", "text": "sikhistisch", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" }, { "start": 61, "end": 91, "label": "EMAIL", "text": "case101.review@example.invalid", "score": null, "source": "deterministic", "model_revision": "contact-patterns-v1", "normalization": "identity" }, { "start": 102, "end": 116, "label": "PHONE", "text": "+49 541 730101", "score": null, "source": "deterministic", "model_revision": "contact-patterns-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 0, "end": 15, "label": "PERSON", "text": "Saskia Dornfeld", "uncovered_positions": [] }, { "start": 22, "end": 33, "label": "RELIGION", "text": "sikhistisch", "uncovered_positions": [] }, { "start": 61, "end": 91, "label": "EMAIL", "text": "case101.review@example.invalid", "uncovered_positions": [] }, { "start": 102, "end": 116, "label": "PHONE", "text": "+49 541 730101", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-003", "scenario": null, "predictions": [ { "start": 0, "end": 15, "label": "PERSON", "text": "Saskia Dornfeld", "score": 0.9992882609367371, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 33, "end": 40, "label": "RELIGION", "text": "Sikhism", "score": 0.9966249465942383, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 66, "end": 96, "label": "EMAIL", "text": "case101.review@example.invalid", "score": 0.908982515335083, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 105, "end": 119, "label": "PHONE", "text": "+49 541 730101", "score": 0.9701294898986816, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 0, "end": 15, "label": "PERSON", "text": "Saskia Dornfeld", "score": 0.9992882609367371, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 33, "end": 40, "label": "RELIGION", "text": "Sikhism", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" }, { "start": 66, "end": 96, "label": "EMAIL", "text": "case101.review@example.invalid", "score": null, "source": "deterministic", "model_revision": "contact-patterns-v1", "normalization": "identity" }, { "start": 105, "end": 119, "label": "PHONE", "text": "+49 541 730101", "score": null, "source": "deterministic", "model_revision": "contact-patterns-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 0, "end": 15, "label": "PERSON", "text": "Saskia Dornfeld", "uncovered_positions": [] }, { "start": 33, "end": 40, "label": "RELIGION", "text": "Sikhism", "uncovered_positions": [] }, { "start": 66, "end": 96, "label": "EMAIL", "text": "case101.review@example.invalid", "uncovered_positions": [] }, { "start": 105, "end": 119, "label": "PHONE", "text": "+49 541 730101", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-004", "scenario": null, "predictions": [ { "start": 0, "end": 15, "label": "PERSON", "text": "Saskia Dornfeld", "score": 0.9996077418327332, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 24, "end": 33, "label": "CITY", "text": "Osnabrück", "score": 0.998096764087677, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 36, "end": 51, "label": "PERSON", "text": "Saskia Dornfeld", "score": 0.9998266100883484, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 77, "end": 101, "label": "ORGANIZATION", "text": "Aster Research Institute", "score": 0.9952711462974548, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 0, "end": 15, "label": "PERSON", "text": "Saskia Dornfeld", "score": 0.9996077418327332, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 24, "end": 33, "label": "CITY", "text": "Osnabrück", "score": 0.998096764087677, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 36, "end": 51, "label": "PERSON", "text": "Saskia Dornfeld", "score": 0.9998266100883484, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 77, "end": 101, "label": "ORGANIZATION", "text": "Aster Research Institute", "score": 0.9952711462974548, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 0, "end": 15, "label": "PERSON", "text": "Saskia Dornfeld", "uncovered_positions": [] }, { "start": 24, "end": 33, "label": "CITY", "text": "Osnabrück", "uncovered_positions": [] }, { "start": 36, "end": 51, "label": "PERSON", "text": "Saskia Dornfeld", "uncovered_positions": [] }, { "start": 77, "end": 101, "label": "ORGANIZATION", "text": "Aster Research Institute", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-005", "scenario": null, "predictions": [ { "start": 3, "end": 12, "label": "CITY", "text": "Osnabrück", "score": 0.9999159574508667, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 14, "end": 29, "label": "PERSON", "text": "Saskia Dornfeld", "score": 0.9998455047607422, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 41, "end": 65, "label": "ORGANIZATION", "text": "Aster Research Institute", "score": 0.9969690442085266, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 86, "end": 101, "label": "PERSON", "text": "Saskia Dornfeld", "score": 0.9999145269393921, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 114, "end": 123, "label": "CITY", "text": "Osnabrück", "score": 0.999828577041626, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 3, "end": 12, "label": "CITY", "text": "Osnabrück", "score": 0.9999159574508667, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 14, "end": 29, "label": "PERSON", "text": "Saskia Dornfeld", "score": 0.9998455047607422, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 41, "end": 65, "label": "ORGANIZATION", "text": "Aster Research Institute", "score": 0.9969690442085266, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 86, "end": 101, "label": "PERSON", "text": "Saskia Dornfeld", "score": 0.9999145269393921, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 114, "end": 123, "label": "CITY", "text": "Osnabrück", "score": 0.999828577041626, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 3, "end": 12, "label": "CITY", "text": "Osnabrück", "uncovered_positions": [] }, { "start": 14, "end": 29, "label": "PERSON", "text": "Saskia Dornfeld", "uncovered_positions": [] }, { "start": 41, "end": 65, "label": "ORGANIZATION", "text": "Aster Research Institute", "uncovered_positions": [] }, { "start": 86, "end": 101, "label": "PERSON", "text": "Saskia Dornfeld", "uncovered_positions": [] }, { "start": 114, "end": 123, "label": "CITY", "text": "Osnabrück", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-006", "scenario": null, "predictions": [ { "start": 6, "end": 22, "label": "PERSON", "text": "Saskia-\nDornfeld", "score": 0.9996308088302612, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "dehyphenation-v1" }, { "start": 28, "end": 39, "label": "CITY", "text": "Osna-\nbrück", "score": 0.9999524354934692, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "dehyphenation-v1" }, { "start": 53, "end": 77, "label": "ORGANIZATION", "text": "Aster Research Institute", "score": 0.9986594915390015, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "dehyphenation-v1" } ], "combined_predictions": [ { "start": 6, "end": 22, "label": "PERSON", "text": "Saskia-\nDornfeld", "score": 0.9996308088302612, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "dehyphenation-v1" }, { "start": 28, "end": 39, "label": "CITY", "text": "Osna-\nbrück", "score": 0.9999524354934692, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "dehyphenation-v1" }, { "start": 53, "end": 77, "label": "ORGANIZATION", "text": "Aster Research Institute", "score": 0.9986594915390015, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "dehyphenation-v1" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 6, "end": 22, "label": "PERSON", "text": "Saskia-\nDornfeld", "uncovered_positions": [] }, { "start": 28, "end": 39, "label": "CITY", "text": "Osna-\nbrück", "uncovered_positions": [] }, { "start": 53, "end": 77, "label": "ORGANIZATION", "text": "Aster Research Institute", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-007", "scenario": null, "predictions": [ { "start": 16, "end": 31, "label": "PERSON", "text": "Sask1a D0rnfeld", "score": 0.9964213371276855, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 43, "end": 52, "label": "CITY", "text": "Osn4brück", "score": 0.999607503414154, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 66, "end": 90, "label": "ORGANIZATION", "text": "Aster Research Institute", "score": 0.9983013868331909, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 16, "end": 31, "label": "PERSON", "text": "Sask1a D0rnfeld", "score": 0.9964213371276855, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 43, "end": 52, "label": "CITY", "text": "Osn4brück", "score": 0.999607503414154, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 66, "end": 90, "label": "ORGANIZATION", "text": "Aster Research Institute", "score": 0.9983013868331909, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 16, "end": 31, "label": "PERSON", "text": "Sask1a D0rnfeld", "uncovered_positions": [] }, { "start": 43, "end": 52, "label": "CITY", "text": "Osn4brück", "uncovered_positions": [] }, { "start": 66, "end": 90, "label": "ORGANIZATION", "text": "Aster Research Institute", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-008", "scenario": null, "predictions": [ { "start": 10, "end": 21, "label": "RELIGION", "text": "sikhistisch", "score": 0.820220947265625, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 48, "end": 72, "label": "ORGANIZATION", "text": "Aster Research Institute", "score": 0.983711302280426, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 86, "end": 101, "label": "PERSON", "text": "Saskia Dornfeld", "score": 0.9986269474029541, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 10, "end": 21, "label": "RELIGION", "text": "sikhistisch", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" }, { "start": 48, "end": 72, "label": "ORGANIZATION", "text": "Aster Research Institute", "score": 0.983711302280426, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 86, "end": 101, "label": "PERSON", "text": "Saskia Dornfeld", "score": 0.9986269474029541, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 10, "end": 21, "label": "RELIGION", "text": "sikhistisch", "uncovered_positions": [] }, { "start": 48, "end": 72, "label": "ORGANIZATION", "text": "Aster Research Institute", "uncovered_positions": [] }, { "start": 86, "end": 101, "label": "PERSON", "text": "Saskia Dornfeld", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-009", "scenario": null, "predictions": [ { "start": 101, "end": 107, "label": "PERSON", "text": "person", "score": 0.7040723562240601, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 109, "end": 114, "label": "CITY", "text": "place", "score": 0.7776683568954468, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 116, "end": 128, "label": "ORGANIZATION", "text": "organization", "score": 0.8319209218025208, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 40, "end": 64, "label": "RELIGION", "text": "no religious affiliation", "score": null, "source": "deterministic", "model_revision": "religious-status-v1", "normalization": "identity" }, { "start": 101, "end": 107, "label": "PERSON", "text": "person", "score": 0.7040723562240601, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 109, "end": 114, "label": "CITY", "text": "place", "score": 0.7776683568954468, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 116, "end": 128, "label": "ORGANIZATION", "text": "organization", "score": 0.8319209218025208, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [ [ 40, 64, "RELIGION" ], [ 101, 107, "PERSON" ], [ 109, 114, "CITY" ], [ 116, 128, "ORGANIZATION" ] ], "false_negatives": [], "exact_match": false, "error_code": null, "canaries": [], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-010", "scenario": null, "predictions": [ { "start": 10, "end": 21, "label": "PERSON", "text": "Emil Farber", "score": 0.9987824559211731, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 35, "end": 56, "label": "ORGANIZATION", "text": "Northbridge Analytics", "score": 0.9969447255134583, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 60, "end": 67, "label": "CITY", "text": "Potsdam", "score": 0.9994106292724609, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 10, "end": 21, "label": "PERSON", "text": "Emil Farber", "score": 0.9987824559211731, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 35, "end": 56, "label": "ORGANIZATION", "text": "Northbridge Analytics", "score": 0.9969447255134583, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 60, "end": 67, "label": "CITY", "text": "Potsdam", "score": 0.9994106292724609, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 10, "end": 21, "label": "PERSON", "text": "Emil Farber", "uncovered_positions": [] }, { "start": 35, "end": 56, "label": "ORGANIZATION", "text": "Northbridge Analytics", "uncovered_positions": [] }, { "start": 60, "end": 67, "label": "CITY", "text": "Potsdam", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-011", "scenario": null, "predictions": [ { "start": 12, "end": 23, "label": "PERSON", "text": "Emil Farber", "score": 0.9989939332008362, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 35, "end": 56, "label": "ORGANIZATION", "text": "Northbridge Analytics", "score": 0.9984185695648193, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 62, "end": 69, "label": "CITY", "text": "Potsdam", "score": 0.9994024038314819, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 12, "end": 23, "label": "PERSON", "text": "Emil Farber", "score": 0.9989939332008362, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 35, "end": 56, "label": "ORGANIZATION", "text": "Northbridge Analytics", "score": 0.9984185695648193, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 62, "end": 69, "label": "CITY", "text": "Potsdam", "score": 0.9994024038314819, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 12, "end": 23, "label": "PERSON", "text": "Emil Farber", "uncovered_positions": [] }, { "start": 35, "end": 56, "label": "ORGANIZATION", "text": "Northbridge Analytics", "uncovered_positions": [] }, { "start": 62, "end": 69, "label": "CITY", "text": "Potsdam", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-012", "scenario": null, "predictions": [ { "start": 0, "end": 11, "label": "PERSON", "text": "Emil Farber", "score": 0.9991351962089539, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 18, "end": 26, "label": "RELIGION", "text": "orthodox", "score": 0.9795414805412292, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 54, "end": 84, "label": "EMAIL", "text": "case102.review@example.invalid", "score": 0.9129301309585571, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 95, "end": 109, "label": "PHONE", "text": "+49 541 730102", "score": 0.9935047626495361, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 0, "end": 11, "label": "PERSON", "text": "Emil Farber", "score": 0.9991351962089539, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 18, "end": 26, "label": "RELIGION", "text": "orthodox", "score": 0.9795414805412292, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 54, "end": 84, "label": "EMAIL", "text": "case102.review@example.invalid", "score": null, "source": "deterministic", "model_revision": "contact-patterns-v1", "normalization": "identity" }, { "start": 95, "end": 109, "label": "PHONE", "text": "+49 541 730102", "score": null, "source": "deterministic", "model_revision": "contact-patterns-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 0, "end": 11, "label": "PERSON", "text": "Emil Farber", "uncovered_positions": [] }, { "start": 18, "end": 26, "label": "RELIGION", "text": "orthodox", "uncovered_positions": [] }, { "start": 54, "end": 84, "label": "EMAIL", "text": "case102.review@example.invalid", "uncovered_positions": [] }, { "start": 95, "end": 109, "label": "PHONE", "text": "+49 541 730102", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-013", "scenario": null, "predictions": [ { "start": 0, "end": 11, "label": "PERSON", "text": "Emil Farber", "score": 0.9992079138755798, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 29, "end": 50, "label": "RELIGION", "text": "Orthodox Christianity", "score": 0.9964632391929626, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 76, "end": 106, "label": "EMAIL", "text": "case102.review@example.invalid", "score": 0.9135122895240784, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 115, "end": 129, "label": "PHONE", "text": "+49 541 730102", "score": 0.9687923192977905, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 0, "end": 11, "label": "PERSON", "text": "Emil Farber", "score": 0.9992079138755798, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 29, "end": 50, "label": "RELIGION", "text": "Orthodox Christianity", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" }, { "start": 76, "end": 106, "label": "EMAIL", "text": "case102.review@example.invalid", "score": null, "source": "deterministic", "model_revision": "contact-patterns-v1", "normalization": "identity" }, { "start": 115, "end": 129, "label": "PHONE", "text": "+49 541 730102", "score": null, "source": "deterministic", "model_revision": "contact-patterns-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 0, "end": 11, "label": "PERSON", "text": "Emil Farber", "uncovered_positions": [] }, { "start": 29, "end": 50, "label": "RELIGION", "text": "Orthodox Christianity", "uncovered_positions": [] }, { "start": 76, "end": 106, "label": "EMAIL", "text": "case102.review@example.invalid", "uncovered_positions": [] }, { "start": 115, "end": 129, "label": "PHONE", "text": "+49 541 730102", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-014", "scenario": null, "predictions": [ { "start": 0, "end": 11, "label": "PERSON", "text": "Emil Farber", "score": 0.9997215867042542, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 20, "end": 27, "label": "CITY", "text": "Potsdam", "score": 0.999298095703125, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 30, "end": 41, "label": "PERSON", "text": "Emil Farber", "score": 0.9996708631515503, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 67, "end": 88, "label": "ORGANIZATION", "text": "Northbridge Analytics", "score": 0.9961026906967163, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 0, "end": 11, "label": "PERSON", "text": "Emil Farber", "score": 0.9997215867042542, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 20, "end": 27, "label": "CITY", "text": "Potsdam", "score": 0.999298095703125, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 30, "end": 41, "label": "PERSON", "text": "Emil Farber", "score": 0.9996708631515503, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 67, "end": 88, "label": "ORGANIZATION", "text": "Northbridge Analytics", "score": 0.9961026906967163, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 0, "end": 11, "label": "PERSON", "text": "Emil Farber", "uncovered_positions": [] }, { "start": 20, "end": 27, "label": "CITY", "text": "Potsdam", "uncovered_positions": [] }, { "start": 30, "end": 41, "label": "PERSON", "text": "Emil Farber", "uncovered_positions": [] }, { "start": 67, "end": 88, "label": "ORGANIZATION", "text": "Northbridge Analytics", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-015", "scenario": null, "predictions": [ { "start": 3, "end": 10, "label": "CITY", "text": "Potsdam", "score": 0.9998461008071899, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 12, "end": 23, "label": "PERSON", "text": "Emil Farber", "score": 0.9998921155929565, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 35, "end": 56, "label": "ORGANIZATION", "text": "Northbridge Analytics", "score": 0.9982470273971558, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 77, "end": 88, "label": "PERSON", "text": "Emil Farber", "score": 0.9998248219490051, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 101, "end": 108, "label": "CITY", "text": "Potsdam", "score": 0.9998860359191895, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 3, "end": 10, "label": "CITY", "text": "Potsdam", "score": 0.9998461008071899, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 12, "end": 23, "label": "PERSON", "text": "Emil Farber", "score": 0.9998921155929565, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 35, "end": 56, "label": "ORGANIZATION", "text": "Northbridge Analytics", "score": 0.9982470273971558, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 77, "end": 88, "label": "PERSON", "text": "Emil Farber", "score": 0.9998248219490051, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 101, "end": 108, "label": "CITY", "text": "Potsdam", "score": 0.9998860359191895, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 3, "end": 10, "label": "CITY", "text": "Potsdam", "uncovered_positions": [] }, { "start": 12, "end": 23, "label": "PERSON", "text": "Emil Farber", "uncovered_positions": [] }, { "start": 35, "end": 56, "label": "ORGANIZATION", "text": "Northbridge Analytics", "uncovered_positions": [] }, { "start": 77, "end": 88, "label": "PERSON", "text": "Emil Farber", "uncovered_positions": [] }, { "start": 101, "end": 108, "label": "CITY", "text": "Potsdam", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-016", "scenario": null, "predictions": [ { "start": 6, "end": 18, "label": "PERSON", "text": "Emil-\nFarber", "score": 0.9998793601989746, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "dehyphenation-v1" }, { "start": 24, "end": 33, "label": "CITY", "text": "Pot-\nsdam", "score": 0.9999624490737915, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "dehyphenation-v1" }, { "start": 47, "end": 68, "label": "ORGANIZATION", "text": "Northbridge Analytics", "score": 0.997786283493042, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "dehyphenation-v1" } ], "combined_predictions": [ { "start": 6, "end": 18, "label": "PERSON", "text": "Emil-\nFarber", "score": 0.9998793601989746, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "dehyphenation-v1" }, { "start": 24, "end": 33, "label": "CITY", "text": "Pot-\nsdam", "score": 0.9999624490737915, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "dehyphenation-v1" }, { "start": 47, "end": 68, "label": "ORGANIZATION", "text": "Northbridge Analytics", "score": 0.997786283493042, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "dehyphenation-v1" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 6, "end": 18, "label": "PERSON", "text": "Emil-\nFarber", "uncovered_positions": [] }, { "start": 24, "end": 33, "label": "CITY", "text": "Pot-\nsdam", "uncovered_positions": [] }, { "start": 47, "end": 68, "label": "ORGANIZATION", "text": "Northbridge Analytics", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-017", "scenario": null, "predictions": [ { "start": 16, "end": 27, "label": "PERSON", "text": "Em1l Farber", "score": 0.9948354959487915, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 39, "end": 46, "label": "CITY", "text": "P0tsd4m", "score": 0.9718403220176697, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 60, "end": 81, "label": "ORGANIZATION", "text": "Northbridge Analytics", "score": 0.9991218447685242, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 16, "end": 27, "label": "PERSON", "text": "Em1l Farber", "score": 0.9948354959487915, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 39, "end": 46, "label": "CITY", "text": "P0tsd4m", "score": 0.9718403220176697, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 60, "end": 81, "label": "ORGANIZATION", "text": "Northbridge Analytics", "score": 0.9991218447685242, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 16, "end": 27, "label": "PERSON", "text": "Em1l Farber", "uncovered_positions": [] }, { "start": 39, "end": 46, "label": "CITY", "text": "P0tsd4m", "uncovered_positions": [] }, { "start": 60, "end": 81, "label": "ORGANIZATION", "text": "Northbridge Analytics", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-018", "scenario": null, "predictions": [ { "start": 10, "end": 18, "label": "RELIGION", "text": "orthodox", "score": 0.8374165296554565, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 45, "end": 66, "label": "ORGANIZATION", "text": "Northbridge Analytics", "score": 0.9889349341392517, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 80, "end": 91, "label": "PERSON", "text": "Emil Farber", "score": 0.9984270334243774, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 10, "end": 18, "label": "RELIGION", "text": "orthodox", "score": 0.8374165296554565, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 45, "end": 66, "label": "ORGANIZATION", "text": "Northbridge Analytics", "score": 0.9889349341392517, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 80, "end": 91, "label": "PERSON", "text": "Emil Farber", "score": 0.9984270334243774, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 10, "end": 18, "label": "RELIGION", "text": "orthodox", "uncovered_positions": [] }, { "start": 45, "end": 66, "label": "ORGANIZATION", "text": "Northbridge Analytics", "uncovered_positions": [] }, { "start": 80, "end": 91, "label": "PERSON", "text": "Emil Farber", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-019", "scenario": null, "predictions": [ { "start": 101, "end": 107, "label": "PERSON", "text": "person", "score": 0.7027997374534607, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 109, "end": 114, "label": "CITY", "text": "place", "score": 0.7789004445075989, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 116, "end": 128, "label": "ORGANIZATION", "text": "organization", "score": 0.8365603685379028, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 40, "end": 64, "label": "RELIGION", "text": "no religious affiliation", "score": null, "source": "deterministic", "model_revision": "religious-status-v1", "normalization": "identity" }, { "start": 101, "end": 107, "label": "PERSON", "text": "person", "score": 0.7027997374534607, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 109, "end": 114, "label": "CITY", "text": "place", "score": 0.7789004445075989, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 116, "end": 128, "label": "ORGANIZATION", "text": "organization", "score": 0.8365603685379028, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [ [ 40, 64, "RELIGION" ], [ 101, 107, "PERSON" ], [ 109, 114, "CITY" ], [ 116, 128, "ORGANIZATION" ] ], "false_negatives": [], "exact_match": false, "error_code": null, "canaries": [], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-020", "scenario": null, "predictions": [ { "start": 10, "end": 25, "label": "PERSON", "text": "Cora Lindenfels", "score": 0.9973998069763184, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 39, "end": 62, "label": "ORGANIZATION", "text": "Amber Health Foundation", "score": 0.9973830580711365, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 66, "end": 72, "label": "CITY", "text": "Dundee", "score": 0.9992653727531433, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 10, "end": 25, "label": "PERSON", "text": "Cora Lindenfels", "score": 0.9973998069763184, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 39, "end": 62, "label": "ORGANIZATION", "text": "Amber Health Foundation", "score": 0.9973830580711365, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 66, "end": 72, "label": "CITY", "text": "Dundee", "score": 0.9992653727531433, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 10, "end": 25, "label": "PERSON", "text": "Cora Lindenfels", "uncovered_positions": [] }, { "start": 39, "end": 62, "label": "ORGANIZATION", "text": "Amber Health Foundation", "uncovered_positions": [] }, { "start": 66, "end": 72, "label": "CITY", "text": "Dundee", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-021", "scenario": null, "predictions": [ { "start": 12, "end": 27, "label": "PERSON", "text": "Cora Lindenfels", "score": 0.997750461101532, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 39, "end": 62, "label": "ORGANIZATION", "text": "Amber Health Foundation", "score": 0.9969223141670227, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 68, "end": 74, "label": "CITY", "text": "Dundee", "score": 0.9994328618049622, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 12, "end": 27, "label": "PERSON", "text": "Cora Lindenfels", "score": 0.997750461101532, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 39, "end": 62, "label": "ORGANIZATION", "text": "Amber Health Foundation", "score": 0.9969223141670227, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 68, "end": 74, "label": "CITY", "text": "Dundee", "score": 0.9994328618049622, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 12, "end": 27, "label": "PERSON", "text": "Cora Lindenfels", "uncovered_positions": [] }, { "start": 39, "end": 62, "label": "ORGANIZATION", "text": "Amber Health Foundation", "uncovered_positions": [] }, { "start": 68, "end": 74, "label": "CITY", "text": "Dundee", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-022", "scenario": null, "predictions": [ { "start": 0, "end": 15, "label": "PERSON", "text": "Cora Lindenfels", "score": 0.9937719702720642, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 22, "end": 33, "label": "RELIGION", "text": "jainistisch", "score": 0.9889543056488037, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 61, "end": 91, "label": "EMAIL", "text": "case103.review@example.invalid", "score": 0.9086615443229675, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 102, "end": 116, "label": "PHONE", "text": "+49 541 730103", "score": 0.9945864677429199, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 0, "end": 15, "label": "PERSON", "text": "Cora Lindenfels", "score": 0.9937719702720642, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 22, "end": 33, "label": "RELIGION", "text": "jainistisch", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" }, { "start": 61, "end": 91, "label": "EMAIL", "text": "case103.review@example.invalid", "score": null, "source": "deterministic", "model_revision": "contact-patterns-v1", "normalization": "identity" }, { "start": 102, "end": 116, "label": "PHONE", "text": "+49 541 730103", "score": null, "source": "deterministic", "model_revision": "contact-patterns-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 0, "end": 15, "label": "PERSON", "text": "Cora Lindenfels", "uncovered_positions": [] }, { "start": 22, "end": 33, "label": "RELIGION", "text": "jainistisch", "uncovered_positions": [] }, { "start": 61, "end": 91, "label": "EMAIL", "text": "case103.review@example.invalid", "uncovered_positions": [] }, { "start": 102, "end": 116, "label": "PHONE", "text": "+49 541 730103", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-023", "scenario": null, "predictions": [ { "start": 0, "end": 15, "label": "PERSON", "text": "Cora Lindenfels", "score": 0.9916738867759705, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 33, "end": 40, "label": "RELIGION", "text": "Jainism", "score": 0.9980966448783875, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 66, "end": 96, "label": "EMAIL", "text": "case103.review@example.invalid", "score": 0.9247252345085144, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 105, "end": 119, "label": "PHONE", "text": "+49 541 730103", "score": 0.9735237956047058, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 0, "end": 15, "label": "PERSON", "text": "Cora Lindenfels", "score": 0.9916738867759705, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 33, "end": 40, "label": "RELIGION", "text": "Jainism", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" }, { "start": 66, "end": 96, "label": "EMAIL", "text": "case103.review@example.invalid", "score": null, "source": "deterministic", "model_revision": "contact-patterns-v1", "normalization": "identity" }, { "start": 105, "end": 119, "label": "PHONE", "text": "+49 541 730103", "score": null, "source": "deterministic", "model_revision": "contact-patterns-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 0, "end": 15, "label": "PERSON", "text": "Cora Lindenfels", "uncovered_positions": [] }, { "start": 33, "end": 40, "label": "RELIGION", "text": "Jainism", "uncovered_positions": [] }, { "start": 66, "end": 96, "label": "EMAIL", "text": "case103.review@example.invalid", "uncovered_positions": [] }, { "start": 105, "end": 119, "label": "PHONE", "text": "+49 541 730103", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-024", "scenario": null, "predictions": [ { "start": 0, "end": 15, "label": "PERSON", "text": "Cora Lindenfels", "score": 0.9980031847953796, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 24, "end": 30, "label": "CITY", "text": "Dundee", "score": 0.9969285130500793, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 33, "end": 48, "label": "PERSON", "text": "Cora Lindenfels", "score": 0.9981155395507812, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 74, "end": 97, "label": "ORGANIZATION", "text": "Amber Health Foundation", "score": 0.9936736822128296, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 0, "end": 15, "label": "PERSON", "text": "Cora Lindenfels", "score": 0.9980031847953796, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 24, "end": 30, "label": "CITY", "text": "Dundee", "score": 0.9969285130500793, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 33, "end": 48, "label": "PERSON", "text": "Cora Lindenfels", "score": 0.9981155395507812, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 74, "end": 97, "label": "ORGANIZATION", "text": "Amber Health Foundation", "score": 0.9936736822128296, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 0, "end": 15, "label": "PERSON", "text": "Cora Lindenfels", "uncovered_positions": [] }, { "start": 24, "end": 30, "label": "CITY", "text": "Dundee", "uncovered_positions": [] }, { "start": 33, "end": 48, "label": "PERSON", "text": "Cora Lindenfels", "uncovered_positions": [] }, { "start": 74, "end": 97, "label": "ORGANIZATION", "text": "Amber Health Foundation", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-025", "scenario": null, "predictions": [ { "start": 3, "end": 9, "label": "CITY", "text": "Dundee", "score": 0.9998049139976501, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 11, "end": 26, "label": "PERSON", "text": "Cora Lindenfels", "score": 0.9993300437927246, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 38, "end": 61, "label": "ORGANIZATION", "text": "Amber Health Foundation", "score": 0.9960338473320007, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 82, "end": 97, "label": "PERSON", "text": "Cora Lindenfels", "score": 0.9993378520011902, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 110, "end": 116, "label": "CITY", "text": "Dundee", "score": 0.9997878670692444, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 3, "end": 9, "label": "CITY", "text": "Dundee", "score": 0.9998049139976501, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 11, "end": 26, "label": "PERSON", "text": "Cora Lindenfels", "score": 0.9993300437927246, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 38, "end": 61, "label": "ORGANIZATION", "text": "Amber Health Foundation", "score": 0.9960338473320007, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 82, "end": 97, "label": "PERSON", "text": "Cora Lindenfels", "score": 0.9993378520011902, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 110, "end": 116, "label": "CITY", "text": "Dundee", "score": 0.9997878670692444, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 3, "end": 9, "label": "CITY", "text": "Dundee", "uncovered_positions": [] }, { "start": 11, "end": 26, "label": "PERSON", "text": "Cora Lindenfels", "uncovered_positions": [] }, { "start": 38, "end": 61, "label": "ORGANIZATION", "text": "Amber Health Foundation", "uncovered_positions": [] }, { "start": 82, "end": 97, "label": "PERSON", "text": "Cora Lindenfels", "uncovered_positions": [] }, { "start": 110, "end": 116, "label": "CITY", "text": "Dundee", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-026", "scenario": null, "predictions": [ { "start": 6, "end": 22, "label": "PERSON", "text": "Cora-\nLindenfels", "score": 0.997488260269165, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "dehyphenation-v1" }, { "start": 28, "end": 36, "label": "CITY", "text": "Dun-\ndee", "score": 0.999882698059082, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "dehyphenation-v1" }, { "start": 50, "end": 73, "label": "ORGANIZATION", "text": "Amber Health Foundation", "score": 0.9973509311676025, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "dehyphenation-v1" } ], "combined_predictions": [ { "start": 6, "end": 22, "label": "PERSON", "text": "Cora-\nLindenfels", "score": 0.997488260269165, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "dehyphenation-v1" }, { "start": 28, "end": 36, "label": "CITY", "text": "Dun-\ndee", "score": 0.999882698059082, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "dehyphenation-v1" }, { "start": 50, "end": 73, "label": "ORGANIZATION", "text": "Amber Health Foundation", "score": 0.9973509311676025, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "dehyphenation-v1" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 6, "end": 22, "label": "PERSON", "text": "Cora-\nLindenfels", "uncovered_positions": [] }, { "start": 28, "end": 36, "label": "CITY", "text": "Dun-\ndee", "uncovered_positions": [] }, { "start": 50, "end": 73, "label": "ORGANIZATION", "text": "Amber Health Foundation", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-027", "scenario": null, "predictions": [ { "start": 16, "end": 31, "label": "PERSON", "text": "C0ra L1ndenfels", "score": 0.9803281426429749, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 43, "end": 49, "label": "CITY", "text": "Dundee", "score": 0.9996110796928406, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 63, "end": 86, "label": "ORGANIZATION", "text": "Amber Health Foundation", "score": 0.9959501028060913, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 16, "end": 31, "label": "PERSON", "text": "C0ra L1ndenfels", "score": 0.9803281426429749, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 43, "end": 49, "label": "CITY", "text": "Dundee", "score": 0.9996110796928406, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 63, "end": 86, "label": "ORGANIZATION", "text": "Amber Health Foundation", "score": 0.9959501028060913, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 16, "end": 31, "label": "PERSON", "text": "C0ra L1ndenfels", "uncovered_positions": [] }, { "start": 43, "end": 49, "label": "CITY", "text": "Dundee", "uncovered_positions": [] }, { "start": 63, "end": 86, "label": "ORGANIZATION", "text": "Amber Health Foundation", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-028", "scenario": null, "predictions": [ { "start": 10, "end": 21, "label": "RELIGION", "text": "jainistisch", "score": 0.7595014572143555, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 48, "end": 71, "label": "ORGANIZATION", "text": "Amber Health Foundation", "score": 0.9929428100585938, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 85, "end": 100, "label": "PERSON", "text": "Cora Lindenfels", "score": 0.9964152574539185, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 10, "end": 21, "label": "RELIGION", "text": "jainistisch", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" }, { "start": 48, "end": 71, "label": "ORGANIZATION", "text": "Amber Health Foundation", "score": 0.9929428100585938, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 85, "end": 100, "label": "PERSON", "text": "Cora Lindenfels", "score": 0.9964152574539185, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 10, "end": 21, "label": "RELIGION", "text": "jainistisch", "uncovered_positions": [] }, { "start": 48, "end": 71, "label": "ORGANIZATION", "text": "Amber Health Foundation", "uncovered_positions": [] }, { "start": 85, "end": 100, "label": "PERSON", "text": "Cora Lindenfels", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-029", "scenario": null, "predictions": [ { "start": 101, "end": 107, "label": "PERSON", "text": "person", "score": 0.7094536423683167, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 109, "end": 114, "label": "CITY", "text": "place", "score": 0.7874830365180969, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 116, "end": 128, "label": "ORGANIZATION", "text": "organization", "score": 0.8446553945541382, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 40, "end": 64, "label": "RELIGION", "text": "no religious affiliation", "score": null, "source": "deterministic", "model_revision": "religious-status-v1", "normalization": "identity" }, { "start": 101, "end": 107, "label": "PERSON", "text": "person", "score": 0.7094536423683167, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 109, "end": 114, "label": "CITY", "text": "place", "score": 0.7874830365180969, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 116, "end": 128, "label": "ORGANIZATION", "text": "organization", "score": 0.8446553945541382, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [ [ 40, 64, "RELIGION" ], [ 101, 107, "PERSON" ], [ 109, 114, "CITY" ], [ 116, 128, "ORGANIZATION" ] ], "false_negatives": [], "exact_match": false, "error_code": null, "canaries": [], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-030", "scenario": null, "predictions": [ { "start": 10, "end": 22, "label": "PERSON", "text": "Jasper Keene", "score": 0.997222900390625, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 36, "end": 57, "label": "ORGANIZATION", "text": "Silver Finch Robotics", "score": 0.9982925057411194, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 61, "end": 69, "label": "CITY", "text": "Aberdeen", "score": 0.9969948530197144, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 10, "end": 22, "label": "PERSON", "text": "Jasper Keene", "score": 0.997222900390625, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 36, "end": 57, "label": "ORGANIZATION", "text": "Silver Finch Robotics", "score": 0.9982925057411194, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 61, "end": 69, "label": "CITY", "text": "Aberdeen", "score": 0.9969948530197144, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 10, "end": 22, "label": "PERSON", "text": "Jasper Keene", "uncovered_positions": [] }, { "start": 36, "end": 57, "label": "ORGANIZATION", "text": "Silver Finch Robotics", "uncovered_positions": [] }, { "start": 61, "end": 69, "label": "CITY", "text": "Aberdeen", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-031", "scenario": null, "predictions": [ { "start": 12, "end": 24, "label": "PERSON", "text": "Jasper Keene", "score": 0.9982971549034119, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 36, "end": 57, "label": "ORGANIZATION", "text": "Silver Finch Robotics", "score": 0.9989097118377686, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 63, "end": 71, "label": "CITY", "text": "Aberdeen", "score": 0.9978512525558472, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 12, "end": 24, "label": "PERSON", "text": "Jasper Keene", "score": 0.9982971549034119, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 36, "end": 57, "label": "ORGANIZATION", "text": "Silver Finch Robotics", "score": 0.9989097118377686, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 63, "end": 71, "label": "CITY", "text": "Aberdeen", "score": 0.9978512525558472, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 12, "end": 24, "label": "PERSON", "text": "Jasper Keene", "uncovered_positions": [] }, { "start": 36, "end": 57, "label": "ORGANIZATION", "text": "Silver Finch Robotics", "uncovered_positions": [] }, { "start": 63, "end": 71, "label": "CITY", "text": "Aberdeen", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-032", "scenario": null, "predictions": [ { "start": 0, "end": 12, "label": "PERSON", "text": "Jasper Keene", "score": 0.9964504241943359, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 19, "end": 31, "label": "RELIGION", "text": "hinduistisch", "score": 0.9917290806770325, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 59, "end": 89, "label": "EMAIL", "text": "case104.review@example.invalid", "score": 0.9179330468177795, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 100, "end": 114, "label": "PHONE", "text": "+49 541 730104", "score": 0.9929746389389038, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 0, "end": 12, "label": "PERSON", "text": "Jasper Keene", "score": 0.9964504241943359, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 19, "end": 31, "label": "RELIGION", "text": "hinduistisch", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" }, { "start": 59, "end": 89, "label": "EMAIL", "text": "case104.review@example.invalid", "score": null, "source": "deterministic", "model_revision": "contact-patterns-v1", "normalization": "identity" }, { "start": 100, "end": 114, "label": "PHONE", "text": "+49 541 730104", "score": null, "source": "deterministic", "model_revision": "contact-patterns-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 0, "end": 12, "label": "PERSON", "text": "Jasper Keene", "uncovered_positions": [] }, { "start": 19, "end": 31, "label": "RELIGION", "text": "hinduistisch", "uncovered_positions": [] }, { "start": 59, "end": 89, "label": "EMAIL", "text": "case104.review@example.invalid", "uncovered_positions": [] }, { "start": 100, "end": 114, "label": "PHONE", "text": "+49 541 730104", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-033", "scenario": null, "predictions": [ { "start": 0, "end": 12, "label": "PERSON", "text": "Jasper Keene", "score": 0.9973819851875305, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 30, "end": 38, "label": "RELIGION", "text": "Hinduism", "score": 0.9998874664306641, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 64, "end": 94, "label": "EMAIL", "text": "case104.review@example.invalid", "score": 0.9190782308578491, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 103, "end": 117, "label": "PHONE", "text": "+49 541 730104", "score": 0.9675639271736145, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 0, "end": 12, "label": "PERSON", "text": "Jasper Keene", "score": 0.9973819851875305, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 30, "end": 38, "label": "RELIGION", "text": "Hinduism", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" }, { "start": 64, "end": 94, "label": "EMAIL", "text": "case104.review@example.invalid", "score": null, "source": "deterministic", "model_revision": "contact-patterns-v1", "normalization": "identity" }, { "start": 103, "end": 117, "label": "PHONE", "text": "+49 541 730104", "score": null, "source": "deterministic", "model_revision": "contact-patterns-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 0, "end": 12, "label": "PERSON", "text": "Jasper Keene", "uncovered_positions": [] }, { "start": 30, "end": 38, "label": "RELIGION", "text": "Hinduism", "uncovered_positions": [] }, { "start": 64, "end": 94, "label": "EMAIL", "text": "case104.review@example.invalid", "uncovered_positions": [] }, { "start": 103, "end": 117, "label": "PHONE", "text": "+49 541 730104", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-034", "scenario": null, "predictions": [ { "start": 0, "end": 12, "label": "PERSON", "text": "Jasper Keene", "score": 0.9992870688438416, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 21, "end": 29, "label": "CITY", "text": "Aberdeen", "score": 0.9939911961555481, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 32, "end": 44, "label": "PERSON", "text": "Jasper Keene", "score": 0.9992408752441406, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 70, "end": 91, "label": "ORGANIZATION", "text": "Silver Finch Robotics", "score": 0.9981598258018494, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 0, "end": 12, "label": "PERSON", "text": "Jasper Keene", "score": 0.9992870688438416, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 21, "end": 29, "label": "CITY", "text": "Aberdeen", "score": 0.9939911961555481, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 32, "end": 44, "label": "PERSON", "text": "Jasper Keene", "score": 0.9992408752441406, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 70, "end": 91, "label": "ORGANIZATION", "text": "Silver Finch Robotics", "score": 0.9981598258018494, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 0, "end": 12, "label": "PERSON", "text": "Jasper Keene", "uncovered_positions": [] }, { "start": 21, "end": 29, "label": "CITY", "text": "Aberdeen", "uncovered_positions": [] }, { "start": 32, "end": 44, "label": "PERSON", "text": "Jasper Keene", "uncovered_positions": [] }, { "start": 70, "end": 91, "label": "ORGANIZATION", "text": "Silver Finch Robotics", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-035", "scenario": null, "predictions": [ { "start": 3, "end": 11, "label": "CITY", "text": "Aberdeen", "score": 0.9991244673728943, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 13, "end": 25, "label": "PERSON", "text": "Jasper Keene", "score": 0.9997237324714661, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 37, "end": 58, "label": "ORGANIZATION", "text": "Silver Finch Robotics", "score": 0.9979164004325867, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 79, "end": 91, "label": "PERSON", "text": "Jasper Keene", "score": 0.9997982382774353, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 104, "end": 112, "label": "CITY", "text": "Aberdeen", "score": 0.9984569549560547, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 3, "end": 11, "label": "CITY", "text": "Aberdeen", "score": 0.9991244673728943, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 13, "end": 25, "label": "PERSON", "text": "Jasper Keene", "score": 0.9997237324714661, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 37, "end": 58, "label": "ORGANIZATION", "text": "Silver Finch Robotics", "score": 0.9979164004325867, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 79, "end": 91, "label": "PERSON", "text": "Jasper Keene", "score": 0.9997982382774353, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 104, "end": 112, "label": "CITY", "text": "Aberdeen", "score": 0.9984569549560547, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 3, "end": 11, "label": "CITY", "text": "Aberdeen", "uncovered_positions": [] }, { "start": 13, "end": 25, "label": "PERSON", "text": "Jasper Keene", "uncovered_positions": [] }, { "start": 37, "end": 58, "label": "ORGANIZATION", "text": "Silver Finch Robotics", "uncovered_positions": [] }, { "start": 79, "end": 91, "label": "PERSON", "text": "Jasper Keene", "uncovered_positions": [] }, { "start": 104, "end": 112, "label": "CITY", "text": "Aberdeen", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-036", "scenario": null, "predictions": [ { "start": 6, "end": 19, "label": "PERSON", "text": "Jasper-\nKeene", "score": 0.9998306035995483, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "dehyphenation-v1" }, { "start": 25, "end": 35, "label": "CITY", "text": "Aber-\ndeen", "score": 0.9998304843902588, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "dehyphenation-v1" }, { "start": 49, "end": 70, "label": "ORGANIZATION", "text": "Silver Finch Robotics", "score": 0.9984242916107178, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "dehyphenation-v1" } ], "combined_predictions": [ { "start": 6, "end": 19, "label": "PERSON", "text": "Jasper-\nKeene", "score": 0.9998306035995483, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "dehyphenation-v1" }, { "start": 25, "end": 35, "label": "CITY", "text": "Aber-\ndeen", "score": 0.9998304843902588, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "dehyphenation-v1" }, { "start": 49, "end": 70, "label": "ORGANIZATION", "text": "Silver Finch Robotics", "score": 0.9984242916107178, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "dehyphenation-v1" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 6, "end": 19, "label": "PERSON", "text": "Jasper-\nKeene", "uncovered_positions": [] }, { "start": 25, "end": 35, "label": "CITY", "text": "Aber-\ndeen", "uncovered_positions": [] }, { "start": 49, "end": 70, "label": "ORGANIZATION", "text": "Silver Finch Robotics", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-037", "scenario": null, "predictions": [ { "start": 16, "end": 28, "label": "PERSON", "text": "Jasper Keene", "score": 0.9981817007064819, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 40, "end": 48, "label": "CITY", "text": "Aberdeen", "score": 0.999523401260376, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 62, "end": 83, "label": "ORGANIZATION", "text": "Silver Finch Robotics", "score": 0.9983817338943481, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 16, "end": 28, "label": "PERSON", "text": "Jasper Keene", "score": 0.9981817007064819, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 40, "end": 48, "label": "CITY", "text": "Aberdeen", "score": 0.999523401260376, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 62, "end": 83, "label": "ORGANIZATION", "text": "Silver Finch Robotics", "score": 0.9983817338943481, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 16, "end": 28, "label": "PERSON", "text": "Jasper Keene", "uncovered_positions": [] }, { "start": 40, "end": 48, "label": "CITY", "text": "Aberdeen", "uncovered_positions": [] }, { "start": 62, "end": 83, "label": "ORGANIZATION", "text": "Silver Finch Robotics", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-038", "scenario": null, "predictions": [ { "start": 10, "end": 22, "label": "RELIGION", "text": "hinduistisch", "score": 0.8530606627464294, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 49, "end": 70, "label": "ORGANIZATION", "text": "Silver Finch Robotics", "score": 0.991775631904602, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 84, "end": 96, "label": "PERSON", "text": "Jasper Keene", "score": 0.9976971745491028, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 10, "end": 22, "label": "RELIGION", "text": "hinduistisch", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" }, { "start": 49, "end": 70, "label": "ORGANIZATION", "text": "Silver Finch Robotics", "score": 0.991775631904602, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 84, "end": 96, "label": "PERSON", "text": "Jasper Keene", "score": 0.9976971745491028, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 10, "end": 22, "label": "RELIGION", "text": "hinduistisch", "uncovered_positions": [] }, { "start": 49, "end": 70, "label": "ORGANIZATION", "text": "Silver Finch Robotics", "uncovered_positions": [] }, { "start": 84, "end": 96, "label": "PERSON", "text": "Jasper Keene", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-039", "scenario": null, "predictions": [ { "start": 101, "end": 107, "label": "PERSON", "text": "person", "score": 0.7161614894866943, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 109, "end": 114, "label": "CITY", "text": "place", "score": 0.7881067395210266, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 116, "end": 128, "label": "ORGANIZATION", "text": "organization", "score": 0.847603976726532, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 40, "end": 64, "label": "RELIGION", "text": "no religious affiliation", "score": null, "source": "deterministic", "model_revision": "religious-status-v1", "normalization": "identity" }, { "start": 101, "end": 107, "label": "PERSON", "text": "person", "score": 0.7161614894866943, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 109, "end": 114, "label": "CITY", "text": "place", "score": 0.7881067395210266, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 116, "end": 128, "label": "ORGANIZATION", "text": "organization", "score": 0.847603976726532, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [ [ 40, 64, "RELIGION" ], [ 101, 107, "PERSON" ], [ 109, 114, "CITY" ], [ 116, 128, "ORGANIZATION" ] ], "false_negatives": [], "exact_match": false, "error_code": null, "canaries": [], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-040", "scenario": null, "predictions": [ { "start": 10, "end": 23, "label": "PERSON", "text": "Frida Östberg", "score": 0.9995619654655457, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 37, "end": 59, "label": "ORGANIZATION", "text": "Blue Cedar Cooperative", "score": 0.9986785054206848, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 63, "end": 70, "label": "CITY", "text": "Uppsala", "score": 0.9995537400245667, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 10, "end": 23, "label": "PERSON", "text": "Frida Östberg", "score": 0.9995619654655457, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 37, "end": 59, "label": "ORGANIZATION", "text": "Blue Cedar Cooperative", "score": 0.9986785054206848, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 63, "end": 70, "label": "CITY", "text": "Uppsala", "score": 0.9995537400245667, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 10, "end": 23, "label": "PERSON", "text": "Frida Östberg", "uncovered_positions": [] }, { "start": 37, "end": 59, "label": "ORGANIZATION", "text": "Blue Cedar Cooperative", "uncovered_positions": [] }, { "start": 63, "end": 70, "label": "CITY", "text": "Uppsala", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-041", "scenario": null, "predictions": [ { "start": 12, "end": 25, "label": "PERSON", "text": "Frida Östberg", "score": 0.9993315935134888, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 37, "end": 59, "label": "ORGANIZATION", "text": "Blue Cedar Cooperative", "score": 0.9984630346298218, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 65, "end": 72, "label": "CITY", "text": "Uppsala", "score": 0.9996377229690552, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 12, "end": 25, "label": "PERSON", "text": "Frida Östberg", "score": 0.9993315935134888, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 37, "end": 59, "label": "ORGANIZATION", "text": "Blue Cedar Cooperative", "score": 0.9984630346298218, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 65, "end": 72, "label": "CITY", "text": "Uppsala", "score": 0.9996377229690552, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 12, "end": 25, "label": "PERSON", "text": "Frida Östberg", "uncovered_positions": [] }, { "start": 37, "end": 59, "label": "ORGANIZATION", "text": "Blue Cedar Cooperative", "uncovered_positions": [] }, { "start": 65, "end": 72, "label": "CITY", "text": "Uppsala", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-042", "scenario": null, "predictions": [ { "start": 0, "end": 13, "label": "PERSON", "text": "Frida Östberg", "score": 0.9994117021560669, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 20, "end": 25, "label": "RELIGION", "text": "bahai", "score": 0.9852348566055298, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 53, "end": 83, "label": "EMAIL", "text": "case105.review@example.invalid", "score": 0.8997676968574524, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 94, "end": 108, "label": "PHONE", "text": "+49 541 730105", "score": 0.9945896863937378, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 0, "end": 13, "label": "PERSON", "text": "Frida Östberg", "score": 0.9994117021560669, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 20, "end": 25, "label": "RELIGION", "text": "bahai", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" }, { "start": 53, "end": 83, "label": "EMAIL", "text": "case105.review@example.invalid", "score": null, "source": "deterministic", "model_revision": "contact-patterns-v1", "normalization": "identity" }, { "start": 94, "end": 108, "label": "PHONE", "text": "+49 541 730105", "score": null, "source": "deterministic", "model_revision": "contact-patterns-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 0, "end": 13, "label": "PERSON", "text": "Frida Östberg", "uncovered_positions": [] }, { "start": 20, "end": 25, "label": "RELIGION", "text": "bahai", "uncovered_positions": [] }, { "start": 53, "end": 83, "label": "EMAIL", "text": "case105.review@example.invalid", "uncovered_positions": [] }, { "start": 94, "end": 108, "label": "PHONE", "text": "+49 541 730105", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-043", "scenario": null, "predictions": [ { "start": 0, "end": 13, "label": "PERSON", "text": "Frida Östberg", "score": 0.9995088577270508, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 31, "end": 43, "label": "RELIGION", "text": "Baháʼí Faith", "score": 0.9641181826591492, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 69, "end": 99, "label": "EMAIL", "text": "case105.review@example.invalid", "score": 0.9057281017303467, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 108, "end": 122, "label": "PHONE", "text": "+49 541 730105", "score": 0.9755640625953674, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 0, "end": 13, "label": "PERSON", "text": "Frida Östberg", "score": 0.9995088577270508, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 31, "end": 43, "label": "RELIGION", "text": "Baháʼí Faith", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" }, { "start": 69, "end": 99, "label": "EMAIL", "text": "case105.review@example.invalid", "score": null, "source": "deterministic", "model_revision": "contact-patterns-v1", "normalization": "identity" }, { "start": 108, "end": 122, "label": "PHONE", "text": "+49 541 730105", "score": null, "source": "deterministic", "model_revision": "contact-patterns-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 0, "end": 13, "label": "PERSON", "text": "Frida Östberg", "uncovered_positions": [] }, { "start": 31, "end": 43, "label": "RELIGION", "text": "Baháʼí Faith", "uncovered_positions": [] }, { "start": 69, "end": 99, "label": "EMAIL", "text": "case105.review@example.invalid", "uncovered_positions": [] }, { "start": 108, "end": 122, "label": "PHONE", "text": "+49 541 730105", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-044", "scenario": null, "predictions": [ { "start": 0, "end": 13, "label": "PERSON", "text": "Frida Östberg", "score": 0.9998176693916321, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 22, "end": 29, "label": "CITY", "text": "Uppsala", "score": 0.9994066953659058, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 32, "end": 45, "label": "PERSON", "text": "Frida Östberg", "score": 0.999852180480957, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 71, "end": 93, "label": "ORGANIZATION", "text": "Blue Cedar Cooperative", "score": 0.9967365860939026, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 0, "end": 13, "label": "PERSON", "text": "Frida Östberg", "score": 0.9998176693916321, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 22, "end": 29, "label": "CITY", "text": "Uppsala", "score": 0.9994066953659058, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 32, "end": 45, "label": "PERSON", "text": "Frida Östberg", "score": 0.999852180480957, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 71, "end": 93, "label": "ORGANIZATION", "text": "Blue Cedar Cooperative", "score": 0.9967365860939026, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 0, "end": 13, "label": "PERSON", "text": "Frida Östberg", "uncovered_positions": [] }, { "start": 22, "end": 29, "label": "CITY", "text": "Uppsala", "uncovered_positions": [] }, { "start": 32, "end": 45, "label": "PERSON", "text": "Frida Östberg", "uncovered_positions": [] }, { "start": 71, "end": 93, "label": "ORGANIZATION", "text": "Blue Cedar Cooperative", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-045", "scenario": null, "predictions": [ { "start": 3, "end": 10, "label": "CITY", "text": "Uppsala", "score": 0.9998781681060791, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 12, "end": 25, "label": "PERSON", "text": "Frida Östberg", "score": 0.9998877048492432, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 37, "end": 59, "label": "ORGANIZATION", "text": "Blue Cedar Cooperative", "score": 0.9978963136672974, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 80, "end": 93, "label": "PERSON", "text": "Frida Östberg", "score": 0.9998670816421509, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 106, "end": 113, "label": "CITY", "text": "Uppsala", "score": 0.9998447895050049, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 3, "end": 10, "label": "CITY", "text": "Uppsala", "score": 0.9998781681060791, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 12, "end": 25, "label": "PERSON", "text": "Frida Östberg", "score": 0.9998877048492432, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 37, "end": 59, "label": "ORGANIZATION", "text": "Blue Cedar Cooperative", "score": 0.9978963136672974, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 80, "end": 93, "label": "PERSON", "text": "Frida Östberg", "score": 0.9998670816421509, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 106, "end": 113, "label": "CITY", "text": "Uppsala", "score": 0.9998447895050049, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 3, "end": 10, "label": "CITY", "text": "Uppsala", "uncovered_positions": [] }, { "start": 12, "end": 25, "label": "PERSON", "text": "Frida Östberg", "uncovered_positions": [] }, { "start": 37, "end": 59, "label": "ORGANIZATION", "text": "Blue Cedar Cooperative", "uncovered_positions": [] }, { "start": 80, "end": 93, "label": "PERSON", "text": "Frida Östberg", "uncovered_positions": [] }, { "start": 106, "end": 113, "label": "CITY", "text": "Uppsala", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-046", "scenario": null, "predictions": [ { "start": 6, "end": 20, "label": "PERSON", "text": "Frida-\nÖstberg", "score": 0.9996815919876099, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "dehyphenation-v1" }, { "start": 26, "end": 35, "label": "CITY", "text": "Upp-\nsala", "score": 0.9999419450759888, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "dehyphenation-v1" }, { "start": 49, "end": 71, "label": "ORGANIZATION", "text": "Blue Cedar Cooperative", "score": 0.9979609251022339, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "dehyphenation-v1" } ], "combined_predictions": [ { "start": 6, "end": 20, "label": "PERSON", "text": "Frida-\nÖstberg", "score": 0.9996815919876099, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "dehyphenation-v1" }, { "start": 26, "end": 35, "label": "CITY", "text": "Upp-\nsala", "score": 0.9999419450759888, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "dehyphenation-v1" }, { "start": 49, "end": 71, "label": "ORGANIZATION", "text": "Blue Cedar Cooperative", "score": 0.9979609251022339, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "dehyphenation-v1" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 6, "end": 20, "label": "PERSON", "text": "Frida-\nÖstberg", "uncovered_positions": [] }, { "start": 26, "end": 35, "label": "CITY", "text": "Upp-\nsala", "uncovered_positions": [] }, { "start": 49, "end": 71, "label": "ORGANIZATION", "text": "Blue Cedar Cooperative", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-047", "scenario": null, "predictions": [ { "start": 16, "end": 29, "label": "PERSON", "text": "Fr1da Östberg", "score": 0.9966031312942505, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 41, "end": 48, "label": "CITY", "text": "Upps4l4", "score": 0.990062415599823, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 62, "end": 84, "label": "ORGANIZATION", "text": "Blue Cedar Cooperative", "score": 0.9982581734657288, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 16, "end": 29, "label": "PERSON", "text": "Fr1da Östberg", "score": 0.9966031312942505, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 41, "end": 48, "label": "CITY", "text": "Upps4l4", "score": 0.990062415599823, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 62, "end": 84, "label": "ORGANIZATION", "text": "Blue Cedar Cooperative", "score": 0.9982581734657288, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 16, "end": 29, "label": "PERSON", "text": "Fr1da Östberg", "uncovered_positions": [] }, { "start": 41, "end": 48, "label": "CITY", "text": "Upps4l4", "uncovered_positions": [] }, { "start": 62, "end": 84, "label": "ORGANIZATION", "text": "Blue Cedar Cooperative", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-048", "scenario": null, "predictions": [ { "start": 10, "end": 15, "label": "RELIGION", "text": "bahai", "score": 0.8015578985214233, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 42, "end": 64, "label": "ORGANIZATION", "text": "Blue Cedar Cooperative", "score": 0.9946749210357666, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 78, "end": 91, "label": "PERSON", "text": "Frida Östberg", "score": 0.9991986155509949, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 10, "end": 15, "label": "RELIGION", "text": "bahai", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" }, { "start": 42, "end": 64, "label": "ORGANIZATION", "text": "Blue Cedar Cooperative", "score": 0.9946749210357666, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 78, "end": 91, "label": "PERSON", "text": "Frida Östberg", "score": 0.9991986155509949, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 10, "end": 15, "label": "RELIGION", "text": "bahai", "uncovered_positions": [] }, { "start": 42, "end": 64, "label": "ORGANIZATION", "text": "Blue Cedar Cooperative", "uncovered_positions": [] }, { "start": 78, "end": 91, "label": "PERSON", "text": "Frida Östberg", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-049", "scenario": null, "predictions": [ { "start": 101, "end": 107, "label": "PERSON", "text": "person", "score": 0.7069818377494812, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 109, "end": 114, "label": "CITY", "text": "place", "score": 0.785081684589386, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 116, "end": 128, "label": "ORGANIZATION", "text": "organization", "score": 0.8447502255439758, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 40, "end": 64, "label": "RELIGION", "text": "no religious affiliation", "score": null, "source": "deterministic", "model_revision": "religious-status-v1", "normalization": "identity" }, { "start": 101, "end": 107, "label": "PERSON", "text": "person", "score": 0.7069818377494812, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 109, "end": 114, "label": "CITY", "text": "place", "score": 0.785081684589386, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 116, "end": 128, "label": "ORGANIZATION", "text": "organization", "score": 0.8447502255439758, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [ [ 40, 64, "RELIGION" ], [ 101, 107, "PERSON" ], [ 109, 114, "CITY" ], [ 116, 128, "ORGANIZATION" ] ], "false_negatives": [], "exact_match": false, "error_code": null, "canaries": [], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-050", "scenario": null, "predictions": [ { "start": 10, "end": 24, "label": "PERSON", "text": "Owen Fairchild", "score": 0.9981813430786133, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 38, "end": 60, "label": "ORGANIZATION", "text": "Oakwell Public Library", "score": 0.9969713687896729, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 64, "end": 73, "label": "CITY", "text": "Leicester", "score": 0.999505877494812, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 10, "end": 24, "label": "PERSON", "text": "Owen Fairchild", "score": 0.9981813430786133, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 38, "end": 60, "label": "ORGANIZATION", "text": "Oakwell Public Library", "score": 0.9969713687896729, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 64, "end": 73, "label": "CITY", "text": "Leicester", "score": 0.999505877494812, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 10, "end": 24, "label": "PERSON", "text": "Owen Fairchild", "uncovered_positions": [] }, { "start": 38, "end": 60, "label": "ORGANIZATION", "text": "Oakwell Public Library", "uncovered_positions": [] }, { "start": 64, "end": 73, "label": "CITY", "text": "Leicester", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-051", "scenario": null, "predictions": [ { "start": 12, "end": 26, "label": "PERSON", "text": "Owen Fairchild", "score": 0.9984040856361389, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 38, "end": 60, "label": "ORGANIZATION", "text": "Oakwell Public Library", "score": 0.9967267513275146, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 66, "end": 75, "label": "CITY", "text": "Leicester", "score": 0.9997547268867493, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 12, "end": 26, "label": "PERSON", "text": "Owen Fairchild", "score": 0.9984040856361389, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 38, "end": 60, "label": "ORGANIZATION", "text": "Oakwell Public Library", "score": 0.9967267513275146, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 66, "end": 75, "label": "CITY", "text": "Leicester", "score": 0.9997547268867493, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 12, "end": 26, "label": "PERSON", "text": "Owen Fairchild", "uncovered_positions": [] }, { "start": 38, "end": 60, "label": "ORGANIZATION", "text": "Oakwell Public Library", "uncovered_positions": [] }, { "start": 66, "end": 75, "label": "CITY", "text": "Leicester", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-052", "scenario": null, "predictions": [ { "start": 0, "end": 14, "label": "PERSON", "text": "Owen Fairchild", "score": 0.9966481328010559, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 21, "end": 33, "label": "RELIGION", "text": "buddhistisch", "score": 0.9927167296409607, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 61, "end": 91, "label": "EMAIL", "text": "case106.review@example.invalid", "score": 0.9082765579223633, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 102, "end": 116, "label": "PHONE", "text": "+49 541 730106", "score": 0.9930872321128845, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 0, "end": 14, "label": "PERSON", "text": "Owen Fairchild", "score": 0.9966481328010559, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 21, "end": 33, "label": "RELIGION", "text": "buddhistisch", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" }, { "start": 61, "end": 91, "label": "EMAIL", "text": "case106.review@example.invalid", "score": null, "source": "deterministic", "model_revision": "contact-patterns-v1", "normalization": "identity" }, { "start": 102, "end": 116, "label": "PHONE", "text": "+49 541 730106", "score": null, "source": "deterministic", "model_revision": "contact-patterns-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 0, "end": 14, "label": "PERSON", "text": "Owen Fairchild", "uncovered_positions": [] }, { "start": 21, "end": 33, "label": "RELIGION", "text": "buddhistisch", "uncovered_positions": [] }, { "start": 61, "end": 91, "label": "EMAIL", "text": "case106.review@example.invalid", "uncovered_positions": [] }, { "start": 102, "end": 116, "label": "PHONE", "text": "+49 541 730106", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-053", "scenario": null, "predictions": [ { "start": 0, "end": 14, "label": "PERSON", "text": "Owen Fairchild", "score": 0.9962939620018005, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 32, "end": 40, "label": "RELIGION", "text": "Buddhism", "score": 0.9998692274093628, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 66, "end": 96, "label": "EMAIL", "text": "case106.review@example.invalid", "score": 0.9211816191673279, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 105, "end": 119, "label": "PHONE", "text": "+49 541 730106", "score": 0.9646264910697937, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 0, "end": 14, "label": "PERSON", "text": "Owen Fairchild", "score": 0.9962939620018005, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 32, "end": 40, "label": "RELIGION", "text": "Buddhism", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" }, { "start": 66, "end": 96, "label": "EMAIL", "text": "case106.review@example.invalid", "score": null, "source": "deterministic", "model_revision": "contact-patterns-v1", "normalization": "identity" }, { "start": 105, "end": 119, "label": "PHONE", "text": "+49 541 730106", "score": null, "source": "deterministic", "model_revision": "contact-patterns-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 0, "end": 14, "label": "PERSON", "text": "Owen Fairchild", "uncovered_positions": [] }, { "start": 32, "end": 40, "label": "RELIGION", "text": "Buddhism", "uncovered_positions": [] }, { "start": 66, "end": 96, "label": "EMAIL", "text": "case106.review@example.invalid", "uncovered_positions": [] }, { "start": 105, "end": 119, "label": "PHONE", "text": "+49 541 730106", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-054", "scenario": null, "predictions": [ { "start": 0, "end": 14, "label": "PERSON", "text": "Owen Fairchild", "score": 0.9986757636070251, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 23, "end": 32, "label": "CITY", "text": "Leicester", "score": 0.9993718266487122, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 35, "end": 49, "label": "PERSON", "text": "Owen Fairchild", "score": 0.9993817806243896, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 75, "end": 97, "label": "ORGANIZATION", "text": "Oakwell Public Library", "score": 0.9926812052726746, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 0, "end": 14, "label": "PERSON", "text": "Owen Fairchild", "score": 0.9986757636070251, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 23, "end": 32, "label": "CITY", "text": "Leicester", "score": 0.9993718266487122, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 35, "end": 49, "label": "PERSON", "text": "Owen Fairchild", "score": 0.9993817806243896, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 75, "end": 97, "label": "ORGANIZATION", "text": "Oakwell Public Library", "score": 0.9926812052726746, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 0, "end": 14, "label": "PERSON", "text": "Owen Fairchild", "uncovered_positions": [] }, { "start": 23, "end": 32, "label": "CITY", "text": "Leicester", "uncovered_positions": [] }, { "start": 35, "end": 49, "label": "PERSON", "text": "Owen Fairchild", "uncovered_positions": [] }, { "start": 75, "end": 97, "label": "ORGANIZATION", "text": "Oakwell Public Library", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-055", "scenario": null, "predictions": [ { "start": 3, "end": 12, "label": "CITY", "text": "Leicester", "score": 0.9998503923416138, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 14, "end": 28, "label": "PERSON", "text": "Owen Fairchild", "score": 0.999387264251709, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 40, "end": 62, "label": "ORGANIZATION", "text": "Oakwell Public Library", "score": 0.9959281086921692, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 83, "end": 97, "label": "PERSON", "text": "Owen Fairchild", "score": 0.9997162222862244, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 110, "end": 119, "label": "CITY", "text": "Leicester", "score": 0.9999486207962036, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 3, "end": 12, "label": "CITY", "text": "Leicester", "score": 0.9998503923416138, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 14, "end": 28, "label": "PERSON", "text": "Owen Fairchild", "score": 0.999387264251709, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 40, "end": 62, "label": "ORGANIZATION", "text": "Oakwell Public Library", "score": 0.9959281086921692, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 83, "end": 97, "label": "PERSON", "text": "Owen Fairchild", "score": 0.9997162222862244, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 110, "end": 119, "label": "CITY", "text": "Leicester", "score": 0.9999486207962036, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 3, "end": 12, "label": "CITY", "text": "Leicester", "uncovered_positions": [] }, { "start": 14, "end": 28, "label": "PERSON", "text": "Owen Fairchild", "uncovered_positions": [] }, { "start": 40, "end": 62, "label": "ORGANIZATION", "text": "Oakwell Public Library", "uncovered_positions": [] }, { "start": 83, "end": 97, "label": "PERSON", "text": "Owen Fairchild", "uncovered_positions": [] }, { "start": 110, "end": 119, "label": "CITY", "text": "Leicester", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-056", "scenario": null, "predictions": [ { "start": 6, "end": 21, "label": "PERSON", "text": "Owen-\nFairchild", "score": 0.9996999502182007, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "dehyphenation-v1" }, { "start": 27, "end": 38, "label": "CITY", "text": "Leic-\nester", "score": 0.9999775886535645, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "dehyphenation-v1" }, { "start": 52, "end": 74, "label": "ORGANIZATION", "text": "Oakwell Public Library", "score": 0.9970940351486206, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "dehyphenation-v1" } ], "combined_predictions": [ { "start": 6, "end": 21, "label": "PERSON", "text": "Owen-\nFairchild", "score": 0.9996999502182007, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "dehyphenation-v1" }, { "start": 27, "end": 38, "label": "CITY", "text": "Leic-\nester", "score": 0.9999775886535645, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "dehyphenation-v1" }, { "start": 52, "end": 74, "label": "ORGANIZATION", "text": "Oakwell Public Library", "score": 0.9970940351486206, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "dehyphenation-v1" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 6, "end": 21, "label": "PERSON", "text": "Owen-\nFairchild", "uncovered_positions": [] }, { "start": 27, "end": 38, "label": "CITY", "text": "Leic-\nester", "uncovered_positions": [] }, { "start": 52, "end": 74, "label": "ORGANIZATION", "text": "Oakwell Public Library", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-057", "scenario": null, "predictions": [ { "start": 16, "end": 30, "label": "PERSON", "text": "Owen Fa1rch1ld", "score": 0.9971356391906738, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 42, "end": 51, "label": "CITY", "text": "Leicester", "score": 0.9998725652694702, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 65, "end": 87, "label": "ORGANIZATION", "text": "Oakwell Public Library", "score": 0.9962441921234131, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 16, "end": 30, "label": "PERSON", "text": "Owen Fa1rch1ld", "score": 0.9971356391906738, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 42, "end": 51, "label": "CITY", "text": "Leicester", "score": 0.9998725652694702, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 65, "end": 87, "label": "ORGANIZATION", "text": "Oakwell Public Library", "score": 0.9962441921234131, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 16, "end": 30, "label": "PERSON", "text": "Owen Fa1rch1ld", "uncovered_positions": [] }, { "start": 42, "end": 51, "label": "CITY", "text": "Leicester", "uncovered_positions": [] }, { "start": 65, "end": 87, "label": "ORGANIZATION", "text": "Oakwell Public Library", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-058", "scenario": null, "predictions": [ { "start": 10, "end": 22, "label": "RELIGION", "text": "buddhistisch", "score": 0.8650351166725159, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 49, "end": 71, "label": "ORGANIZATION", "text": "Oakwell Public Library", "score": 0.9895272254943848, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 85, "end": 99, "label": "PERSON", "text": "Owen Fairchild", "score": 0.9970881342887878, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 10, "end": 22, "label": "RELIGION", "text": "buddhistisch", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" }, { "start": 49, "end": 71, "label": "ORGANIZATION", "text": "Oakwell Public Library", "score": 0.9895272254943848, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 85, "end": 99, "label": "PERSON", "text": "Owen Fairchild", "score": 0.9970881342887878, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 10, "end": 22, "label": "RELIGION", "text": "buddhistisch", "uncovered_positions": [] }, { "start": 49, "end": 71, "label": "ORGANIZATION", "text": "Oakwell Public Library", "uncovered_positions": [] }, { "start": 85, "end": 99, "label": "PERSON", "text": "Owen Fairchild", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-059", "scenario": null, "predictions": [ { "start": 101, "end": 107, "label": "PERSON", "text": "person", "score": 0.7122743725776672, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 109, "end": 114, "label": "CITY", "text": "place", "score": 0.7866309881210327, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 116, "end": 128, "label": "ORGANIZATION", "text": "organization", "score": 0.8471381068229675, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 40, "end": 64, "label": "RELIGION", "text": "no religious affiliation", "score": null, "source": "deterministic", "model_revision": "religious-status-v1", "normalization": "identity" }, { "start": 101, "end": 107, "label": "PERSON", "text": "person", "score": 0.7122743725776672, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 109, "end": 114, "label": "CITY", "text": "place", "score": 0.7866309881210327, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 116, "end": 128, "label": "ORGANIZATION", "text": "organization", "score": 0.8471381068229675, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [ [ 40, 64, "RELIGION" ], [ 101, 107, "PERSON" ], [ 109, 114, "CITY" ], [ 116, 128, "ORGANIZATION" ] ], "false_negatives": [], "exact_match": false, "error_code": null, "canaries": [], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-060", "scenario": null, "predictions": [ { "start": 10, "end": 23, "label": "PERSON", "text": "Nele Brückner", "score": 0.9995220899581909, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 37, "end": 62, "label": "ORGANIZATION", "text": "Westhaven Water Authority", "score": 0.9957793951034546, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 66, "end": 74, "label": "CITY", "text": "Würzburg", "score": 0.9995125532150269, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 10, "end": 23, "label": "PERSON", "text": "Nele Brückner", "score": 0.9995220899581909, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 37, "end": 62, "label": "ORGANIZATION", "text": "Westhaven Water Authority", "score": 0.9957793951034546, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 66, "end": 74, "label": "CITY", "text": "Würzburg", "score": 0.9995125532150269, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 10, "end": 23, "label": "PERSON", "text": "Nele Brückner", "uncovered_positions": [] }, { "start": 37, "end": 62, "label": "ORGANIZATION", "text": "Westhaven Water Authority", "uncovered_positions": [] }, { "start": 66, "end": 74, "label": "CITY", "text": "Würzburg", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-061", "scenario": null, "predictions": [ { "start": 12, "end": 25, "label": "PERSON", "text": "Nele Brückner", "score": 0.9993897676467896, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 37, "end": 62, "label": "ORGANIZATION", "text": "Westhaven Water Authority", "score": 0.9940574169158936, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 68, "end": 76, "label": "CITY", "text": "Würzburg", "score": 0.9994066953659058, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 12, "end": 25, "label": "PERSON", "text": "Nele Brückner", "score": 0.9993897676467896, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 37, "end": 62, "label": "ORGANIZATION", "text": "Westhaven Water Authority", "score": 0.9940574169158936, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 68, "end": 76, "label": "CITY", "text": "Würzburg", "score": 0.9994066953659058, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 12, "end": 25, "label": "PERSON", "text": "Nele Brückner", "uncovered_positions": [] }, { "start": 37, "end": 62, "label": "ORGANIZATION", "text": "Westhaven Water Authority", "uncovered_positions": [] }, { "start": 68, "end": 76, "label": "CITY", "text": "Würzburg", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-062", "scenario": null, "predictions": [ { "start": 0, "end": 13, "label": "PERSON", "text": "Nele Brückner", "score": 0.9983347058296204, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 20, "end": 30, "label": "RELIGION", "text": "katholisch", "score": 0.9748141169548035, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 73, "end": 88, "label": "CITY", "text": "example.invalid", "score": 0.9023637175559998, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 99, "end": 113, "label": "PHONE", "text": "+49 541 730107", "score": 0.9952542781829834, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 0, "end": 13, "label": "PERSON", "text": "Nele Brückner", "score": 0.9983347058296204, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 20, "end": 30, "label": "RELIGION", "text": "katholisch", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" }, { "start": 58, "end": 88, "label": "EMAIL", "text": "case107.review@example.invalid", "score": null, "source": "deterministic", "model_revision": "contact-patterns-v1", "normalization": "identity" }, { "start": 99, "end": 113, "label": "PHONE", "text": "+49 541 730107", "score": null, "source": "deterministic", "model_revision": "contact-patterns-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 0, "end": 13, "label": "PERSON", "text": "Nele Brückner", "uncovered_positions": [] }, { "start": 20, "end": 30, "label": "RELIGION", "text": "katholisch", "uncovered_positions": [] }, { "start": 58, "end": 88, "label": "EMAIL", "text": "case107.review@example.invalid", "uncovered_positions": [] }, { "start": 99, "end": 113, "label": "PHONE", "text": "+49 541 730107", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-063", "scenario": null, "predictions": [ { "start": 0, "end": 13, "label": "PERSON", "text": "Nele Brückner", "score": 0.9967458248138428, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 31, "end": 42, "label": "RELIGION", "text": "Catholicism", "score": 0.9980359673500061, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 68, "end": 98, "label": "EMAIL", "text": "case107.review@example.invalid", "score": 0.9019798040390015, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 107, "end": 121, "label": "PHONE", "text": "+49 541 730107", "score": 0.9717420339584351, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 0, "end": 13, "label": "PERSON", "text": "Nele Brückner", "score": 0.9967458248138428, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 31, "end": 42, "label": "RELIGION", "text": "Catholicism", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" }, { "start": 68, "end": 98, "label": "EMAIL", "text": "case107.review@example.invalid", "score": null, "source": "deterministic", "model_revision": "contact-patterns-v1", "normalization": "identity" }, { "start": 107, "end": 121, "label": "PHONE", "text": "+49 541 730107", "score": null, "source": "deterministic", "model_revision": "contact-patterns-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 0, "end": 13, "label": "PERSON", "text": "Nele Brückner", "uncovered_positions": [] }, { "start": 31, "end": 42, "label": "RELIGION", "text": "Catholicism", "uncovered_positions": [] }, { "start": 68, "end": 98, "label": "EMAIL", "text": "case107.review@example.invalid", "uncovered_positions": [] }, { "start": 107, "end": 121, "label": "PHONE", "text": "+49 541 730107", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-064", "scenario": null, "predictions": [ { "start": 0, "end": 13, "label": "PERSON", "text": "Nele Brückner", "score": 0.9998051524162292, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 22, "end": 30, "label": "CITY", "text": "Würzburg", "score": 0.9990678429603577, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 33, "end": 46, "label": "PERSON", "text": "Nele Brückner", "score": 0.9998308420181274, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 72, "end": 97, "label": "ORGANIZATION", "text": "Westhaven Water Authority", "score": 0.9937987923622131, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 0, "end": 13, "label": "PERSON", "text": "Nele Brückner", "score": 0.9998051524162292, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 22, "end": 30, "label": "CITY", "text": "Würzburg", "score": 0.9990678429603577, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 33, "end": 46, "label": "PERSON", "text": "Nele Brückner", "score": 0.9998308420181274, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 72, "end": 97, "label": "ORGANIZATION", "text": "Westhaven Water Authority", "score": 0.9937987923622131, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 0, "end": 13, "label": "PERSON", "text": "Nele Brückner", "uncovered_positions": [] }, { "start": 22, "end": 30, "label": "CITY", "text": "Würzburg", "uncovered_positions": [] }, { "start": 33, "end": 46, "label": "PERSON", "text": "Nele Brückner", "uncovered_positions": [] }, { "start": 72, "end": 97, "label": "ORGANIZATION", "text": "Westhaven Water Authority", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-065", "scenario": null, "predictions": [ { "start": 3, "end": 11, "label": "CITY", "text": "Würzburg", "score": 0.9997674822807312, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 13, "end": 26, "label": "PERSON", "text": "Nele Brückner", "score": 0.9999117851257324, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 38, "end": 63, "label": "ORGANIZATION", "text": "Westhaven Water Authority", "score": 0.9946510195732117, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 84, "end": 97, "label": "PERSON", "text": "Nele Brückner", "score": 0.9998905658721924, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 110, "end": 118, "label": "CITY", "text": "Würzburg", "score": 0.9998799562454224, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 3, "end": 11, "label": "CITY", "text": "Würzburg", "score": 0.9997674822807312, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 13, "end": 26, "label": "PERSON", "text": "Nele Brückner", "score": 0.9999117851257324, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 38, "end": 63, "label": "ORGANIZATION", "text": "Westhaven Water Authority", "score": 0.9946510195732117, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 84, "end": 97, "label": "PERSON", "text": "Nele Brückner", "score": 0.9998905658721924, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 110, "end": 118, "label": "CITY", "text": "Würzburg", "score": 0.9998799562454224, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 3, "end": 11, "label": "CITY", "text": "Würzburg", "uncovered_positions": [] }, { "start": 13, "end": 26, "label": "PERSON", "text": "Nele Brückner", "uncovered_positions": [] }, { "start": 38, "end": 63, "label": "ORGANIZATION", "text": "Westhaven Water Authority", "uncovered_positions": [] }, { "start": 84, "end": 97, "label": "PERSON", "text": "Nele Brückner", "uncovered_positions": [] }, { "start": 110, "end": 118, "label": "CITY", "text": "Würzburg", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-066", "scenario": null, "predictions": [ { "start": 6, "end": 20, "label": "PERSON", "text": "Nele-\nBrückner", "score": 0.9997313618659973, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "dehyphenation-v1" }, { "start": 26, "end": 36, "label": "CITY", "text": "Würz-\nburg", "score": 0.9999507665634155, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "dehyphenation-v1" }, { "start": 50, "end": 75, "label": "ORGANIZATION", "text": "Westhaven Water Authority", "score": 0.9986609220504761, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "dehyphenation-v1" } ], "combined_predictions": [ { "start": 6, "end": 20, "label": "PERSON", "text": "Nele-\nBrückner", "score": 0.9997313618659973, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "dehyphenation-v1" }, { "start": 26, "end": 36, "label": "CITY", "text": "Würz-\nburg", "score": 0.9999507665634155, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "dehyphenation-v1" }, { "start": 50, "end": 75, "label": "ORGANIZATION", "text": "Westhaven Water Authority", "score": 0.9986609220504761, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "dehyphenation-v1" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 6, "end": 20, "label": "PERSON", "text": "Nele-\nBrückner", "uncovered_positions": [] }, { "start": 26, "end": 36, "label": "CITY", "text": "Würz-\nburg", "uncovered_positions": [] }, { "start": 50, "end": 75, "label": "ORGANIZATION", "text": "Westhaven Water Authority", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-067", "scenario": null, "predictions": [ { "start": 16, "end": 29, "label": "PERSON", "text": "Nele Brückner", "score": 0.9996700286865234, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 41, "end": 49, "label": "CITY", "text": "Würzburg", "score": 0.9998364448547363, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 63, "end": 88, "label": "ORGANIZATION", "text": "Westhaven Water Authority", "score": 0.9963690042495728, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 16, "end": 29, "label": "PERSON", "text": "Nele Brückner", "score": 0.9996700286865234, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 41, "end": 49, "label": "CITY", "text": "Würzburg", "score": 0.9998364448547363, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 63, "end": 88, "label": "ORGANIZATION", "text": "Westhaven Water Authority", "score": 0.9963690042495728, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 16, "end": 29, "label": "PERSON", "text": "Nele Brückner", "uncovered_positions": [] }, { "start": 41, "end": 49, "label": "CITY", "text": "Würzburg", "uncovered_positions": [] }, { "start": 63, "end": 88, "label": "ORGANIZATION", "text": "Westhaven Water Authority", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-068", "scenario": null, "predictions": [ { "start": 10, "end": 20, "label": "RELIGION", "text": "katholisch", "score": 0.8750774264335632, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 47, "end": 72, "label": "ORGANIZATION", "text": "Westhaven Water Authority", "score": 0.9934499859809875, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 86, "end": 99, "label": "PERSON", "text": "Nele Brückner", "score": 0.9986261129379272, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 10, "end": 20, "label": "RELIGION", "text": "katholisch", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" }, { "start": 47, "end": 72, "label": "ORGANIZATION", "text": "Westhaven Water Authority", "score": 0.9934499859809875, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 86, "end": 99, "label": "PERSON", "text": "Nele Brückner", "score": 0.9986261129379272, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 10, "end": 20, "label": "RELIGION", "text": "katholisch", "uncovered_positions": [] }, { "start": 47, "end": 72, "label": "ORGANIZATION", "text": "Westhaven Water Authority", "uncovered_positions": [] }, { "start": 86, "end": 99, "label": "PERSON", "text": "Nele Brückner", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-069", "scenario": null, "predictions": [ { "start": 101, "end": 107, "label": "PERSON", "text": "person", "score": 0.7072301506996155, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 109, "end": 114, "label": "CITY", "text": "place", "score": 0.7869006395339966, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 116, "end": 128, "label": "ORGANIZATION", "text": "organization", "score": 0.844225287437439, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 40, "end": 64, "label": "RELIGION", "text": "no religious affiliation", "score": null, "source": "deterministic", "model_revision": "religious-status-v1", "normalization": "identity" }, { "start": 101, "end": 107, "label": "PERSON", "text": "person", "score": 0.7072301506996155, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 109, "end": 114, "label": "CITY", "text": "place", "score": 0.7869006395339966, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 116, "end": 128, "label": "ORGANIZATION", "text": "organization", "score": 0.844225287437439, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [ [ 40, 64, "RELIGION" ], [ 101, 107, "PERSON" ], [ 109, 114, "CITY" ], [ 116, 128, "ORGANIZATION" ] ], "false_negatives": [], "exact_match": false, "error_code": null, "canaries": [], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-070", "scenario": null, "predictions": [ { "start": 10, "end": 25, "label": "PERSON", "text": "Silas Hawthorne", "score": 0.9949835538864136, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 39, "end": 61, "label": "ORGANIZATION", "text": "Juniper Legal Services", "score": 0.9961141347885132, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 65, "end": 73, "label": "CITY", "text": "Portland", "score": 0.9993465542793274, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 10, "end": 25, "label": "PERSON", "text": "Silas Hawthorne", "score": 0.9949835538864136, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 39, "end": 61, "label": "ORGANIZATION", "text": "Juniper Legal Services", "score": 0.9961141347885132, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 65, "end": 73, "label": "CITY", "text": "Portland", "score": 0.9993465542793274, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 10, "end": 25, "label": "PERSON", "text": "Silas Hawthorne", "uncovered_positions": [] }, { "start": 39, "end": 61, "label": "ORGANIZATION", "text": "Juniper Legal Services", "uncovered_positions": [] }, { "start": 65, "end": 73, "label": "CITY", "text": "Portland", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-071", "scenario": null, "predictions": [ { "start": 12, "end": 27, "label": "PERSON", "text": "Silas Hawthorne", "score": 0.9974763989448547, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 39, "end": 61, "label": "ORGANIZATION", "text": "Juniper Legal Services", "score": 0.9980837106704712, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 67, "end": 75, "label": "CITY", "text": "Portland", "score": 0.9996472597122192, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 12, "end": 27, "label": "PERSON", "text": "Silas Hawthorne", "score": 0.9974763989448547, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 39, "end": 61, "label": "ORGANIZATION", "text": "Juniper Legal Services", "score": 0.9980837106704712, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 67, "end": 75, "label": "CITY", "text": "Portland", "score": 0.9996472597122192, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 12, "end": 27, "label": "PERSON", "text": "Silas Hawthorne", "uncovered_positions": [] }, { "start": 39, "end": 61, "label": "ORGANIZATION", "text": "Juniper Legal Services", "uncovered_positions": [] }, { "start": 67, "end": 75, "label": "CITY", "text": "Portland", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-072", "scenario": null, "predictions": [ { "start": 0, "end": 15, "label": "PERSON", "text": "Silas Hawthorne", "score": 0.9946703314781189, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 22, "end": 33, "label": "RELIGION", "text": "evangelisch", "score": 0.9349200129508972, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 61, "end": 91, "label": "EMAIL", "text": "case108.review@example.invalid", "score": 0.9081872701644897, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 102, "end": 116, "label": "PHONE", "text": "+49 541 730108", "score": 0.9933851361274719, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 0, "end": 15, "label": "PERSON", "text": "Silas Hawthorne", "score": 0.9946703314781189, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 22, "end": 33, "label": "RELIGION", "text": "evangelisch", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" }, { "start": 61, "end": 91, "label": "EMAIL", "text": "case108.review@example.invalid", "score": null, "source": "deterministic", "model_revision": "contact-patterns-v1", "normalization": "identity" }, { "start": 102, "end": 116, "label": "PHONE", "text": "+49 541 730108", "score": null, "source": "deterministic", "model_revision": "contact-patterns-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 0, "end": 15, "label": "PERSON", "text": "Silas Hawthorne", "uncovered_positions": [] }, { "start": 22, "end": 33, "label": "RELIGION", "text": "evangelisch", "uncovered_positions": [] }, { "start": 61, "end": 91, "label": "EMAIL", "text": "case108.review@example.invalid", "uncovered_positions": [] }, { "start": 102, "end": 116, "label": "PHONE", "text": "+49 541 730108", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-073", "scenario": null, "predictions": [ { "start": 0, "end": 15, "label": "PERSON", "text": "Silas Hawthorne", "score": 0.9950461387634277, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 33, "end": 46, "label": "RELIGION", "text": "Protestantism", "score": 0.9982935786247253, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 72, "end": 102, "label": "EMAIL", "text": "case108.review@example.invalid", "score": 0.9170582890510559, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 111, "end": 125, "label": "PHONE", "text": "+49 541 730108", "score": 0.9616659283638, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 0, "end": 15, "label": "PERSON", "text": "Silas Hawthorne", "score": 0.9950461387634277, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 33, "end": 46, "label": "RELIGION", "text": "Protestantism", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" }, { "start": 72, "end": 102, "label": "EMAIL", "text": "case108.review@example.invalid", "score": null, "source": "deterministic", "model_revision": "contact-patterns-v1", "normalization": "identity" }, { "start": 111, "end": 125, "label": "PHONE", "text": "+49 541 730108", "score": null, "source": "deterministic", "model_revision": "contact-patterns-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 0, "end": 15, "label": "PERSON", "text": "Silas Hawthorne", "uncovered_positions": [] }, { "start": 33, "end": 46, "label": "RELIGION", "text": "Protestantism", "uncovered_positions": [] }, { "start": 72, "end": 102, "label": "EMAIL", "text": "case108.review@example.invalid", "uncovered_positions": [] }, { "start": 111, "end": 125, "label": "PHONE", "text": "+49 541 730108", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-074", "scenario": null, "predictions": [ { "start": 0, "end": 15, "label": "PERSON", "text": "Silas Hawthorne", "score": 0.9976925849914551, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 24, "end": 32, "label": "CITY", "text": "Portland", "score": 0.9988584518432617, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 35, "end": 50, "label": "PERSON", "text": "Silas Hawthorne", "score": 0.9983211159706116, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 76, "end": 98, "label": "ORGANIZATION", "text": "Juniper Legal Services", "score": 0.9957102537155151, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 0, "end": 15, "label": "PERSON", "text": "Silas Hawthorne", "score": 0.9976925849914551, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 24, "end": 32, "label": "CITY", "text": "Portland", "score": 0.9988584518432617, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 35, "end": 50, "label": "PERSON", "text": "Silas Hawthorne", "score": 0.9983211159706116, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 76, "end": 98, "label": "ORGANIZATION", "text": "Juniper Legal Services", "score": 0.9957102537155151, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 0, "end": 15, "label": "PERSON", "text": "Silas Hawthorne", "uncovered_positions": [] }, { "start": 24, "end": 32, "label": "CITY", "text": "Portland", "uncovered_positions": [] }, { "start": 35, "end": 50, "label": "PERSON", "text": "Silas Hawthorne", "uncovered_positions": [] }, { "start": 76, "end": 98, "label": "ORGANIZATION", "text": "Juniper Legal Services", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-075", "scenario": null, "predictions": [ { "start": 3, "end": 11, "label": "CITY", "text": "Portland", "score": 0.9997417330741882, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 13, "end": 28, "label": "PERSON", "text": "Silas Hawthorne", "score": 0.9993771910667419, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 40, "end": 62, "label": "ORGANIZATION", "text": "Juniper Legal Services", "score": 0.9973406195640564, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 83, "end": 98, "label": "PERSON", "text": "Silas Hawthorne", "score": 0.9994988441467285, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 111, "end": 119, "label": "CITY", "text": "Portland", "score": 0.9996170997619629, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 3, "end": 11, "label": "CITY", "text": "Portland", "score": 0.9997417330741882, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 13, "end": 28, "label": "PERSON", "text": "Silas Hawthorne", "score": 0.9993771910667419, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 40, "end": 62, "label": "ORGANIZATION", "text": "Juniper Legal Services", "score": 0.9973406195640564, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 83, "end": 98, "label": "PERSON", "text": "Silas Hawthorne", "score": 0.9994988441467285, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 111, "end": 119, "label": "CITY", "text": "Portland", "score": 0.9996170997619629, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 3, "end": 11, "label": "CITY", "text": "Portland", "uncovered_positions": [] }, { "start": 13, "end": 28, "label": "PERSON", "text": "Silas Hawthorne", "uncovered_positions": [] }, { "start": 40, "end": 62, "label": "ORGANIZATION", "text": "Juniper Legal Services", "uncovered_positions": [] }, { "start": 83, "end": 98, "label": "PERSON", "text": "Silas Hawthorne", "uncovered_positions": [] }, { "start": 111, "end": 119, "label": "CITY", "text": "Portland", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-076", "scenario": null, "predictions": [ { "start": 6, "end": 22, "label": "PERSON", "text": "Silas-\nHawthorne", "score": 0.9994223117828369, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "dehyphenation-v1" }, { "start": 28, "end": 38, "label": "CITY", "text": "Port-\nland", "score": 0.9999568462371826, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "dehyphenation-v1" }, { "start": 52, "end": 74, "label": "ORGANIZATION", "text": "Juniper Legal Services", "score": 0.9987984895706177, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "dehyphenation-v1" } ], "combined_predictions": [ { "start": 6, "end": 22, "label": "PERSON", "text": "Silas-\nHawthorne", "score": 0.9994223117828369, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "dehyphenation-v1" }, { "start": 28, "end": 38, "label": "CITY", "text": "Port-\nland", "score": 0.9999568462371826, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "dehyphenation-v1" }, { "start": 52, "end": 74, "label": "ORGANIZATION", "text": "Juniper Legal Services", "score": 0.9987984895706177, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "dehyphenation-v1" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 6, "end": 22, "label": "PERSON", "text": "Silas-\nHawthorne", "uncovered_positions": [] }, { "start": 28, "end": 38, "label": "CITY", "text": "Port-\nland", "uncovered_positions": [] }, { "start": 52, "end": 74, "label": "ORGANIZATION", "text": "Juniper Legal Services", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-077", "scenario": null, "predictions": [ { "start": 16, "end": 31, "label": "PERSON", "text": "S1las Hawth0rne", "score": 0.9958475828170776, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 43, "end": 51, "label": "CITY", "text": "P0rtl4nd", "score": 0.9920943379402161, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 65, "end": 87, "label": "ORGANIZATION", "text": "Juniper Legal Services", "score": 0.9983524084091187, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 16, "end": 31, "label": "PERSON", "text": "S1las Hawth0rne", "score": 0.9958475828170776, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 43, "end": 51, "label": "CITY", "text": "P0rtl4nd", "score": 0.9920943379402161, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 65, "end": 87, "label": "ORGANIZATION", "text": "Juniper Legal Services", "score": 0.9983524084091187, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 16, "end": 31, "label": "PERSON", "text": "S1las Hawth0rne", "uncovered_positions": [] }, { "start": 43, "end": 51, "label": "CITY", "text": "P0rtl4nd", "uncovered_positions": [] }, { "start": 65, "end": 87, "label": "ORGANIZATION", "text": "Juniper Legal Services", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-078", "scenario": null, "predictions": [ { "start": 10, "end": 21, "label": "RELIGION", "text": "evangelisch", "score": 0.6795831918716431, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 48, "end": 70, "label": "ORGANIZATION", "text": "Juniper Legal Services", "score": 0.9773863554000854, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 84, "end": 99, "label": "PERSON", "text": "Silas Hawthorne", "score": 0.9956725835800171, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 10, "end": 21, "label": "RELIGION", "text": "evangelisch", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" }, { "start": 48, "end": 70, "label": "ORGANIZATION", "text": "Juniper Legal Services", "score": 0.9773863554000854, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 84, "end": 99, "label": "PERSON", "text": "Silas Hawthorne", "score": 0.9956725835800171, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 10, "end": 21, "label": "RELIGION", "text": "evangelisch", "uncovered_positions": [] }, { "start": 48, "end": 70, "label": "ORGANIZATION", "text": "Juniper Legal Services", "uncovered_positions": [] }, { "start": 84, "end": 99, "label": "PERSON", "text": "Silas Hawthorne", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-079", "scenario": null, "predictions": [ { "start": 101, "end": 107, "label": "PERSON", "text": "person", "score": 0.711972177028656, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 109, "end": 114, "label": "CITY", "text": "place", "score": 0.7886146903038025, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 116, "end": 128, "label": "ORGANIZATION", "text": "organization", "score": 0.8439294695854187, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 40, "end": 64, "label": "RELIGION", "text": "no religious affiliation", "score": null, "source": "deterministic", "model_revision": "religious-status-v1", "normalization": "identity" }, { "start": 101, "end": 107, "label": "PERSON", "text": "person", "score": 0.711972177028656, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 109, "end": 114, "label": "CITY", "text": "place", "score": 0.7886146903038025, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 116, "end": 128, "label": "ORGANIZATION", "text": "organization", "score": 0.8439294695854187, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [ [ 40, 64, "RELIGION" ], [ 101, 107, "PERSON" ], [ 109, 114, "CITY" ], [ 116, 128, "ORGANIZATION" ] ], "false_negatives": [], "exact_match": false, "error_code": null, "canaries": [], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-080", "scenario": null, "predictions": [ { "start": 10, "end": 25, "label": "PERSON", "text": "Alva Rosenqvist", "score": 0.998843789100647, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 39, "end": 61, "label": "ORGANIZATION", "text": "Maple Ridge University", "score": 0.9975813627243042, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 65, "end": 71, "label": "CITY", "text": "Bergen", "score": 0.9971677660942078, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 10, "end": 25, "label": "PERSON", "text": "Alva Rosenqvist", "score": 0.998843789100647, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 39, "end": 61, "label": "ORGANIZATION", "text": "Maple Ridge University", "score": 0.9975813627243042, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 65, "end": 71, "label": "CITY", "text": "Bergen", "score": 0.9971677660942078, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 10, "end": 25, "label": "PERSON", "text": "Alva Rosenqvist", "uncovered_positions": [] }, { "start": 39, "end": 61, "label": "ORGANIZATION", "text": "Maple Ridge University", "uncovered_positions": [] }, { "start": 65, "end": 71, "label": "CITY", "text": "Bergen", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-081", "scenario": null, "predictions": [ { "start": 12, "end": 27, "label": "PERSON", "text": "Alva Rosenqvist", "score": 0.9987525939941406, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 39, "end": 61, "label": "ORGANIZATION", "text": "Maple Ridge University", "score": 0.9985166192054749, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 67, "end": 73, "label": "CITY", "text": "Bergen", "score": 0.9985432624816895, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 12, "end": 27, "label": "PERSON", "text": "Alva Rosenqvist", "score": 0.9987525939941406, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 39, "end": 61, "label": "ORGANIZATION", "text": "Maple Ridge University", "score": 0.9985166192054749, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 67, "end": 73, "label": "CITY", "text": "Bergen", "score": 0.9985432624816895, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 12, "end": 27, "label": "PERSON", "text": "Alva Rosenqvist", "uncovered_positions": [] }, { "start": 39, "end": 61, "label": "ORGANIZATION", "text": "Maple Ridge University", "uncovered_positions": [] }, { "start": 67, "end": 73, "label": "CITY", "text": "Bergen", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-082", "scenario": null, "predictions": [ { "start": 0, "end": 15, "label": "PERSON", "text": "Alva Rosenqvist", "score": 0.9994316697120667, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 22, "end": 29, "label": "RELIGION", "text": "jüdisch", "score": 0.9905497431755066, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 57, "end": 87, "label": "EMAIL", "text": "case109.review@example.invalid", "score": 0.9014635682106018, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 98, "end": 112, "label": "PHONE", "text": "+49 541 730109", "score": 0.993009090423584, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 0, "end": 15, "label": "PERSON", "text": "Alva Rosenqvist", "score": 0.9994316697120667, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 22, "end": 29, "label": "RELIGION", "text": "jüdisch", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" }, { "start": 57, "end": 87, "label": "EMAIL", "text": "case109.review@example.invalid", "score": null, "source": "deterministic", "model_revision": "contact-patterns-v1", "normalization": "identity" }, { "start": 98, "end": 112, "label": "PHONE", "text": "+49 541 730109", "score": null, "source": "deterministic", "model_revision": "contact-patterns-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 0, "end": 15, "label": "PERSON", "text": "Alva Rosenqvist", "uncovered_positions": [] }, { "start": 22, "end": 29, "label": "RELIGION", "text": "jüdisch", "uncovered_positions": [] }, { "start": 57, "end": 87, "label": "EMAIL", "text": "case109.review@example.invalid", "uncovered_positions": [] }, { "start": 98, "end": 112, "label": "PHONE", "text": "+49 541 730109", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-083", "scenario": null, "predictions": [ { "start": 0, "end": 15, "label": "PERSON", "text": "Alva Rosenqvist", "score": 0.9990854263305664, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 33, "end": 40, "label": "RELIGION", "text": "Judaism", "score": 0.9997567534446716, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 66, "end": 96, "label": "EMAIL", "text": "case109.review@example.invalid", "score": 0.9070643186569214, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 105, "end": 119, "label": "PHONE", "text": "+49 541 730109", "score": 0.9656404256820679, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 0, "end": 15, "label": "PERSON", "text": "Alva Rosenqvist", "score": 0.9990854263305664, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 33, "end": 40, "label": "RELIGION", "text": "Judaism", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" }, { "start": 66, "end": 96, "label": "EMAIL", "text": "case109.review@example.invalid", "score": null, "source": "deterministic", "model_revision": "contact-patterns-v1", "normalization": "identity" }, { "start": 105, "end": 119, "label": "PHONE", "text": "+49 541 730109", "score": null, "source": "deterministic", "model_revision": "contact-patterns-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 0, "end": 15, "label": "PERSON", "text": "Alva Rosenqvist", "uncovered_positions": [] }, { "start": 33, "end": 40, "label": "RELIGION", "text": "Judaism", "uncovered_positions": [] }, { "start": 66, "end": 96, "label": "EMAIL", "text": "case109.review@example.invalid", "uncovered_positions": [] }, { "start": 105, "end": 119, "label": "PHONE", "text": "+49 541 730109", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-084", "scenario": null, "predictions": [ { "start": 0, "end": 15, "label": "PERSON", "text": "Alva Rosenqvist", "score": 0.99967360496521, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 24, "end": 30, "label": "CITY", "text": "Bergen", "score": 0.9969269633293152, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 33, "end": 48, "label": "PERSON", "text": "Alva Rosenqvist", "score": 0.9995853304862976, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 74, "end": 96, "label": "ORGANIZATION", "text": "Maple Ridge University", "score": 0.9907796382904053, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 0, "end": 15, "label": "PERSON", "text": "Alva Rosenqvist", "score": 0.99967360496521, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 24, "end": 30, "label": "CITY", "text": "Bergen", "score": 0.9969269633293152, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 33, "end": 48, "label": "PERSON", "text": "Alva Rosenqvist", "score": 0.9995853304862976, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 74, "end": 96, "label": "ORGANIZATION", "text": "Maple Ridge University", "score": 0.9907796382904053, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 0, "end": 15, "label": "PERSON", "text": "Alva Rosenqvist", "uncovered_positions": [] }, { "start": 24, "end": 30, "label": "CITY", "text": "Bergen", "uncovered_positions": [] }, { "start": 33, "end": 48, "label": "PERSON", "text": "Alva Rosenqvist", "uncovered_positions": [] }, { "start": 74, "end": 96, "label": "ORGANIZATION", "text": "Maple Ridge University", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-085", "scenario": null, "predictions": [ { "start": 3, "end": 9, "label": "CITY", "text": "Bergen", "score": 0.9995262622833252, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 11, "end": 26, "label": "PERSON", "text": "Alva Rosenqvist", "score": 0.9998084902763367, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 38, "end": 60, "label": "ORGANIZATION", "text": "Maple Ridge University", "score": 0.99769526720047, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 81, "end": 96, "label": "PERSON", "text": "Alva Rosenqvist", "score": 0.999796450138092, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 109, "end": 115, "label": "CITY", "text": "Bergen", "score": 0.9991483688354492, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 3, "end": 9, "label": "CITY", "text": "Bergen", "score": 0.9995262622833252, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 11, "end": 26, "label": "PERSON", "text": "Alva Rosenqvist", "score": 0.9998084902763367, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 38, "end": 60, "label": "ORGANIZATION", "text": "Maple Ridge University", "score": 0.99769526720047, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 81, "end": 96, "label": "PERSON", "text": "Alva Rosenqvist", "score": 0.999796450138092, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 109, "end": 115, "label": "CITY", "text": "Bergen", "score": 0.9991483688354492, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 3, "end": 9, "label": "CITY", "text": "Bergen", "uncovered_positions": [] }, { "start": 11, "end": 26, "label": "PERSON", "text": "Alva Rosenqvist", "uncovered_positions": [] }, { "start": 38, "end": 60, "label": "ORGANIZATION", "text": "Maple Ridge University", "uncovered_positions": [] }, { "start": 81, "end": 96, "label": "PERSON", "text": "Alva Rosenqvist", "uncovered_positions": [] }, { "start": 109, "end": 115, "label": "CITY", "text": "Bergen", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-086", "scenario": null, "predictions": [ { "start": 6, "end": 22, "label": "PERSON", "text": "Alva-\nRosenqvist", "score": 0.9996846914291382, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "dehyphenation-v1" }, { "start": 28, "end": 36, "label": "CITY", "text": "Ber-\ngen", "score": 0.9998655319213867, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "dehyphenation-v1" }, { "start": 50, "end": 72, "label": "ORGANIZATION", "text": "Maple Ridge University", "score": 0.9983516931533813, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "dehyphenation-v1" } ], "combined_predictions": [ { "start": 6, "end": 22, "label": "PERSON", "text": "Alva-\nRosenqvist", "score": 0.9996846914291382, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "dehyphenation-v1" }, { "start": 28, "end": 36, "label": "CITY", "text": "Ber-\ngen", "score": 0.9998655319213867, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "dehyphenation-v1" }, { "start": 50, "end": 72, "label": "ORGANIZATION", "text": "Maple Ridge University", "score": 0.9983516931533813, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "dehyphenation-v1" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 6, "end": 22, "label": "PERSON", "text": "Alva-\nRosenqvist", "uncovered_positions": [] }, { "start": 28, "end": 36, "label": "CITY", "text": "Ber-\ngen", "uncovered_positions": [] }, { "start": 50, "end": 72, "label": "ORGANIZATION", "text": "Maple Ridge University", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-087", "scenario": null, "predictions": [ { "start": 16, "end": 31, "label": "PERSON", "text": "Alva R0senqv1st", "score": 0.951907753944397, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 43, "end": 49, "label": "CITY", "text": "Bergen", "score": 0.9994907379150391, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 63, "end": 85, "label": "ORGANIZATION", "text": "Maple Ridge University", "score": 0.9973485469818115, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 16, "end": 31, "label": "PERSON", "text": "Alva R0senqv1st", "score": 0.951907753944397, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 43, "end": 49, "label": "CITY", "text": "Bergen", "score": 0.9994907379150391, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 63, "end": 85, "label": "ORGANIZATION", "text": "Maple Ridge University", "score": 0.9973485469818115, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 16, "end": 31, "label": "PERSON", "text": "Alva R0senqv1st", "uncovered_positions": [] }, { "start": 43, "end": 49, "label": "CITY", "text": "Bergen", "uncovered_positions": [] }, { "start": 63, "end": 85, "label": "ORGANIZATION", "text": "Maple Ridge University", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-088", "scenario": null, "predictions": [ { "start": 10, "end": 17, "label": "RELIGION", "text": "jüdisch", "score": 0.8295804262161255, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 44, "end": 66, "label": "ORGANIZATION", "text": "Maple Ridge University", "score": 0.9811441898345947, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 80, "end": 95, "label": "PERSON", "text": "Alva Rosenqvist", "score": 0.998078465461731, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 10, "end": 17, "label": "RELIGION", "text": "jüdisch", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" }, { "start": 44, "end": 66, "label": "ORGANIZATION", "text": "Maple Ridge University", "score": 0.9811441898345947, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 80, "end": 95, "label": "PERSON", "text": "Alva Rosenqvist", "score": 0.998078465461731, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 10, "end": 17, "label": "RELIGION", "text": "jüdisch", "uncovered_positions": [] }, { "start": 44, "end": 66, "label": "ORGANIZATION", "text": "Maple Ridge University", "uncovered_positions": [] }, { "start": 80, "end": 95, "label": "PERSON", "text": "Alva Rosenqvist", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-089", "scenario": null, "predictions": [ { "start": 101, "end": 107, "label": "PERSON", "text": "person", "score": 0.7157840132713318, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 109, "end": 114, "label": "CITY", "text": "place", "score": 0.7889829277992249, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 116, "end": 128, "label": "ORGANIZATION", "text": "organization", "score": 0.8459369540214539, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 40, "end": 64, "label": "RELIGION", "text": "no religious affiliation", "score": null, "source": "deterministic", "model_revision": "religious-status-v1", "normalization": "identity" }, { "start": 101, "end": 107, "label": "PERSON", "text": "person", "score": 0.7157840132713318, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 109, "end": 114, "label": "CITY", "text": "place", "score": 0.7889829277992249, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 116, "end": 128, "label": "ORGANIZATION", "text": "organization", "score": 0.8459369540214539, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [ [ 40, 64, "RELIGION" ], [ 101, 107, "PERSON" ], [ 109, 114, "CITY" ], [ 116, 128, "ORGANIZATION" ] ], "false_negatives": [], "exact_match": false, "error_code": null, "canaries": [], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-090", "scenario": null, "predictions": [ { "start": 10, "end": 27, "label": "PERSON", "text": "Theo Merriweather", "score": 0.9981582760810852, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 41, "end": 59, "label": "ORGANIZATION", "text": "Polar Systems Ltd.", "score": 0.9988139867782593, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 63, "end": 67, "label": "CITY", "text": "York", "score": 0.9996792078018188, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 10, "end": 27, "label": "PERSON", "text": "Theo Merriweather", "score": 0.9981582760810852, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 41, "end": 59, "label": "ORGANIZATION", "text": "Polar Systems Ltd.", "score": 0.9988139867782593, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 63, "end": 67, "label": "CITY", "text": "York", "score": 0.9996792078018188, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 10, "end": 27, "label": "PERSON", "text": "Theo Merriweather", "uncovered_positions": [] }, { "start": 41, "end": 59, "label": "ORGANIZATION", "text": "Polar Systems Ltd.", "uncovered_positions": [] }, { "start": 63, "end": 67, "label": "CITY", "text": "York", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-091", "scenario": null, "predictions": [ { "start": 12, "end": 29, "label": "PERSON", "text": "Theo Merriweather", "score": 0.9984655380249023, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 41, "end": 59, "label": "ORGANIZATION", "text": "Polar Systems Ltd.", "score": 0.998237133026123, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 65, "end": 69, "label": "CITY", "text": "York", "score": 0.9996891021728516, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 12, "end": 29, "label": "PERSON", "text": "Theo Merriweather", "score": 0.9984655380249023, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 41, "end": 59, "label": "ORGANIZATION", "text": "Polar Systems Ltd.", "score": 0.998237133026123, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 65, "end": 69, "label": "CITY", "text": "York", "score": 0.9996891021728516, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 12, "end": 29, "label": "PERSON", "text": "Theo Merriweather", "uncovered_positions": [] }, { "start": 41, "end": 59, "label": "ORGANIZATION", "text": "Polar Systems Ltd.", "uncovered_positions": [] }, { "start": 65, "end": 69, "label": "CITY", "text": "York", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-092", "scenario": null, "predictions": [ { "start": 0, "end": 17, "label": "PERSON", "text": "Theo Merriweather", "score": 0.9836243391036987, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 24, "end": 34, "label": "RELIGION", "text": "muslimisch", "score": 0.9906712770462036, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 62, "end": 92, "label": "EMAIL", "text": "case110.review@example.invalid", "score": 0.9017051458358765, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 103, "end": 117, "label": "PHONE", "text": "+49 541 730110", "score": 0.9932438731193542, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 0, "end": 17, "label": "PERSON", "text": "Theo Merriweather", "score": 0.9836243391036987, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 24, "end": 34, "label": "RELIGION", "text": "muslimisch", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" }, { "start": 62, "end": 92, "label": "EMAIL", "text": "case110.review@example.invalid", "score": null, "source": "deterministic", "model_revision": "contact-patterns-v1", "normalization": "identity" }, { "start": 103, "end": 117, "label": "PHONE", "text": "+49 541 730110", "score": null, "source": "deterministic", "model_revision": "contact-patterns-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 0, "end": 17, "label": "PERSON", "text": "Theo Merriweather", "uncovered_positions": [] }, { "start": 24, "end": 34, "label": "RELIGION", "text": "muslimisch", "uncovered_positions": [] }, { "start": 62, "end": 92, "label": "EMAIL", "text": "case110.review@example.invalid", "uncovered_positions": [] }, { "start": 103, "end": 117, "label": "PHONE", "text": "+49 541 730110", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-093", "scenario": null, "predictions": [ { "start": 0, "end": 17, "label": "PERSON", "text": "Theo Merriweather", "score": 0.9977192282676697, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 35, "end": 40, "label": "RELIGION", "text": "Islam", "score": 0.999936580657959, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 66, "end": 96, "label": "EMAIL", "text": "case110.review@example.invalid", "score": 0.8946349620819092, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 105, "end": 119, "label": "PHONE", "text": "+49 541 730110", "score": 0.9599586129188538, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 0, "end": 17, "label": "PERSON", "text": "Theo Merriweather", "score": 0.9977192282676697, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 35, "end": 40, "label": "RELIGION", "text": "Islam", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" }, { "start": 66, "end": 96, "label": "EMAIL", "text": "case110.review@example.invalid", "score": null, "source": "deterministic", "model_revision": "contact-patterns-v1", "normalization": "identity" }, { "start": 105, "end": 119, "label": "PHONE", "text": "+49 541 730110", "score": null, "source": "deterministic", "model_revision": "contact-patterns-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 0, "end": 17, "label": "PERSON", "text": "Theo Merriweather", "uncovered_positions": [] }, { "start": 35, "end": 40, "label": "RELIGION", "text": "Islam", "uncovered_positions": [] }, { "start": 66, "end": 96, "label": "EMAIL", "text": "case110.review@example.invalid", "uncovered_positions": [] }, { "start": 105, "end": 119, "label": "PHONE", "text": "+49 541 730110", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-094", "scenario": null, "predictions": [ { "start": 0, "end": 17, "label": "PERSON", "text": "Theo Merriweather", "score": 0.9987108707427979, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 26, "end": 30, "label": "CITY", "text": "York", "score": 0.9983842372894287, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 33, "end": 50, "label": "PERSON", "text": "Theo Merriweather", "score": 0.9984440207481384, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 76, "end": 94, "label": "ORGANIZATION", "text": "Polar Systems Ltd.", "score": 0.9534488320350647, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 0, "end": 17, "label": "PERSON", "text": "Theo Merriweather", "score": 0.9987108707427979, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 26, "end": 30, "label": "CITY", "text": "York", "score": 0.9983842372894287, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 33, "end": 50, "label": "PERSON", "text": "Theo Merriweather", "score": 0.9984440207481384, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 76, "end": 94, "label": "ORGANIZATION", "text": "Polar Systems Ltd.", "score": 0.9534488320350647, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 0, "end": 17, "label": "PERSON", "text": "Theo Merriweather", "uncovered_positions": [] }, { "start": 26, "end": 30, "label": "CITY", "text": "York", "uncovered_positions": [] }, { "start": 33, "end": 50, "label": "PERSON", "text": "Theo Merriweather", "uncovered_positions": [] }, { "start": 76, "end": 94, "label": "ORGANIZATION", "text": "Polar Systems Ltd.", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-095", "scenario": null, "predictions": [ { "start": 3, "end": 7, "label": "CITY", "text": "York", "score": 0.9998012185096741, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 9, "end": 26, "label": "PERSON", "text": "Theo Merriweather", "score": 0.9998037219047546, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 38, "end": 56, "label": "ORGANIZATION", "text": "Polar Systems Ltd.", "score": 0.9765823483467102, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 77, "end": 94, "label": "PERSON", "text": "Theo Merriweather", "score": 0.9997954964637756, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 107, "end": 111, "label": "CITY", "text": "York", "score": 0.9997724890708923, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 3, "end": 7, "label": "CITY", "text": "York", "score": 0.9998012185096741, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 9, "end": 26, "label": "PERSON", "text": "Theo Merriweather", "score": 0.9998037219047546, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 38, "end": 56, "label": "ORGANIZATION", "text": "Polar Systems Ltd.", "score": 0.9765823483467102, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 77, "end": 94, "label": "PERSON", "text": "Theo Merriweather", "score": 0.9997954964637756, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 107, "end": 111, "label": "CITY", "text": "York", "score": 0.9997724890708923, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 3, "end": 7, "label": "CITY", "text": "York", "uncovered_positions": [] }, { "start": 9, "end": 26, "label": "PERSON", "text": "Theo Merriweather", "uncovered_positions": [] }, { "start": 38, "end": 56, "label": "ORGANIZATION", "text": "Polar Systems Ltd.", "uncovered_positions": [] }, { "start": 77, "end": 94, "label": "PERSON", "text": "Theo Merriweather", "uncovered_positions": [] }, { "start": 107, "end": 111, "label": "CITY", "text": "York", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-096", "scenario": null, "predictions": [ { "start": 6, "end": 24, "label": "PERSON", "text": "Theo-\nMerriweather", "score": 0.9998553991317749, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "dehyphenation-v1" }, { "start": 30, "end": 36, "label": "CITY", "text": "Yo-\nrk", "score": 0.9999533891677856, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "dehyphenation-v1" }, { "start": 50, "end": 68, "label": "ORGANIZATION", "text": "Polar Systems Ltd.", "score": 0.9765849709510803, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "dehyphenation-v1" } ], "combined_predictions": [ { "start": 6, "end": 24, "label": "PERSON", "text": "Theo-\nMerriweather", "score": 0.9998553991317749, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "dehyphenation-v1" }, { "start": 30, "end": 36, "label": "CITY", "text": "Yo-\nrk", "score": 0.9999533891677856, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "dehyphenation-v1" }, { "start": 50, "end": 68, "label": "ORGANIZATION", "text": "Polar Systems Ltd.", "score": 0.9765849709510803, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "dehyphenation-v1" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 6, "end": 24, "label": "PERSON", "text": "Theo-\nMerriweather", "uncovered_positions": [] }, { "start": 30, "end": 36, "label": "CITY", "text": "Yo-\nrk", "uncovered_positions": [] }, { "start": 50, "end": 68, "label": "ORGANIZATION", "text": "Polar Systems Ltd.", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-097", "scenario": null, "predictions": [ { "start": 16, "end": 33, "label": "PERSON", "text": "The0 Merr1weather", "score": 0.9711146950721741, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 45, "end": 49, "label": "CITY", "text": "Y0rk", "score": 0.9964478015899658, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 63, "end": 81, "label": "ORGANIZATION", "text": "Polar Systems Ltd.", "score": 0.9427483677864075, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 16, "end": 33, "label": "PERSON", "text": "The0 Merr1weather", "score": 0.9711146950721741, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 45, "end": 49, "label": "CITY", "text": "Y0rk", "score": 0.9964478015899658, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 63, "end": 81, "label": "ORGANIZATION", "text": "Polar Systems Ltd.", "score": 0.9427483677864075, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 16, "end": 33, "label": "PERSON", "text": "The0 Merr1weather", "uncovered_positions": [] }, { "start": 45, "end": 49, "label": "CITY", "text": "Y0rk", "uncovered_positions": [] }, { "start": 63, "end": 81, "label": "ORGANIZATION", "text": "Polar Systems Ltd.", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-098", "scenario": null, "predictions": [ { "start": 10, "end": 20, "label": "RELIGION", "text": "muslimisch", "score": 0.8173997402191162, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 47, "end": 65, "label": "ORGANIZATION", "text": "Polar Systems Ltd.", "score": 0.990338146686554, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 79, "end": 96, "label": "PERSON", "text": "Theo Merriweather", "score": 0.9961032867431641, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 10, "end": 20, "label": "RELIGION", "text": "muslimisch", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" }, { "start": 47, "end": 65, "label": "ORGANIZATION", "text": "Polar Systems Ltd.", "score": 0.990338146686554, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 79, "end": 96, "label": "PERSON", "text": "Theo Merriweather", "score": 0.9961032867431641, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 10, "end": 20, "label": "RELIGION", "text": "muslimisch", "uncovered_positions": [] }, { "start": 47, "end": 65, "label": "ORGANIZATION", "text": "Polar Systems Ltd.", "uncovered_positions": [] }, { "start": 79, "end": 96, "label": "PERSON", "text": "Theo Merriweather", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-099", "scenario": null, "predictions": [ { "start": 101, "end": 107, "label": "PERSON", "text": "person", "score": 0.7109703421592712, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 109, "end": 114, "label": "CITY", "text": "place", "score": 0.7852903604507446, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 116, "end": 128, "label": "ORGANIZATION", "text": "organization", "score": 0.8414570093154907, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 40, "end": 64, "label": "RELIGION", "text": "no religious affiliation", "score": null, "source": "deterministic", "model_revision": "religious-status-v1", "normalization": "identity" }, { "start": 101, "end": 107, "label": "PERSON", "text": "person", "score": 0.7109703421592712, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 109, "end": 114, "label": "CITY", "text": "place", "score": 0.7852903604507446, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 116, "end": 128, "label": "ORGANIZATION", "text": "organization", "score": 0.8414570093154907, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [ [ 40, 64, "RELIGION" ], [ 101, 107, "PERSON" ], [ 109, 114, "CITY" ], [ 116, 128, "ORGANIZATION" ] ], "false_negatives": [], "exact_match": false, "error_code": null, "canaries": [], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-100", "scenario": null, "predictions": [ { "start": 10, "end": 25, "label": "PERSON", "text": "Maja Kirschbaum", "score": 0.9991471767425537, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 39, "end": 57, "label": "ORGANIZATION", "text": "Sonnenbogen Verein", "score": 0.9799805283546448, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 61, "end": 69, "label": "CITY", "text": "Lüneburg", "score": 0.9990435242652893, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 10, "end": 25, "label": "PERSON", "text": "Maja Kirschbaum", "score": 0.9991471767425537, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 39, "end": 57, "label": "ORGANIZATION", "text": "Sonnenbogen Verein", "score": 0.9799805283546448, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 61, "end": 69, "label": "CITY", "text": "Lüneburg", "score": 0.9990435242652893, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 10, "end": 25, "label": "PERSON", "text": "Maja Kirschbaum", "uncovered_positions": [] }, { "start": 39, "end": 57, "label": "ORGANIZATION", "text": "Sonnenbogen Verein", "uncovered_positions": [] }, { "start": 61, "end": 69, "label": "CITY", "text": "Lüneburg", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-101", "scenario": null, "predictions": [ { "start": 12, "end": 27, "label": "PERSON", "text": "Maja Kirschbaum", "score": 0.9968947172164917, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 39, "end": 57, "label": "ORGANIZATION", "text": "Sonnenbogen Verein", "score": 0.9929682612419128, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 63, "end": 71, "label": "CITY", "text": "Lüneburg", "score": 0.9971613883972168, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 12, "end": 27, "label": "PERSON", "text": "Maja Kirschbaum", "score": 0.9968947172164917, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 39, "end": 57, "label": "ORGANIZATION", "text": "Sonnenbogen Verein", "score": 0.9929682612419128, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 63, "end": 71, "label": "CITY", "text": "Lüneburg", "score": 0.9971613883972168, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 12, "end": 27, "label": "PERSON", "text": "Maja Kirschbaum", "uncovered_positions": [] }, { "start": 39, "end": 57, "label": "ORGANIZATION", "text": "Sonnenbogen Verein", "uncovered_positions": [] }, { "start": 63, "end": 71, "label": "CITY", "text": "Lüneburg", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-102", "scenario": null, "predictions": [ { "start": 0, "end": 15, "label": "PERSON", "text": "Maja Kirschbaum", "score": 0.998397171497345, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 64, "end": 94, "label": "EMAIL", "text": "case111.review@example.invalid", "score": 0.8911723494529724, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 105, "end": 119, "label": "PHONE", "text": "+49 541 730111", "score": 0.9943364262580872, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 0, "end": 15, "label": "PERSON", "text": "Maja Kirschbaum", "score": 0.998397171497345, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 22, "end": 36, "label": "RELIGION", "text": "konfessionslos", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" }, { "start": 64, "end": 94, "label": "EMAIL", "text": "case111.review@example.invalid", "score": null, "source": "deterministic", "model_revision": "contact-patterns-v1", "normalization": "identity" }, { "start": 105, "end": 119, "label": "PHONE", "text": "+49 541 730111", "score": null, "source": "deterministic", "model_revision": "contact-patterns-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 0, "end": 15, "label": "PERSON", "text": "Maja Kirschbaum", "uncovered_positions": [] }, { "start": 22, "end": 36, "label": "RELIGION", "text": "konfessionslos", "uncovered_positions": [] }, { "start": 64, "end": 94, "label": "EMAIL", "text": "case111.review@example.invalid", "uncovered_positions": [] }, { "start": 105, "end": 119, "label": "PHONE", "text": "+49 541 730111", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-103", "scenario": null, "predictions": [ { "start": 0, "end": 15, "label": "PERSON", "text": "Maja Kirschbaum", "score": 0.9988783001899719, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 33, "end": 40, "label": "RELIGION", "text": "atheism", "score": 0.99891197681427, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 66, "end": 96, "label": "EMAIL", "text": "case111.review@example.invalid", "score": 0.8873010277748108, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 105, "end": 119, "label": "PHONE", "text": "+49 541 730111", "score": 0.9670958518981934, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 0, "end": 15, "label": "PERSON", "text": "Maja Kirschbaum", "score": 0.9988783001899719, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 33, "end": 40, "label": "RELIGION", "text": "atheism", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" }, { "start": 66, "end": 96, "label": "EMAIL", "text": "case111.review@example.invalid", "score": null, "source": "deterministic", "model_revision": "contact-patterns-v1", "normalization": "identity" }, { "start": 105, "end": 119, "label": "PHONE", "text": "+49 541 730111", "score": null, "source": "deterministic", "model_revision": "contact-patterns-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 0, "end": 15, "label": "PERSON", "text": "Maja Kirschbaum", "uncovered_positions": [] }, { "start": 33, "end": 40, "label": "RELIGION", "text": "atheism", "uncovered_positions": [] }, { "start": 66, "end": 96, "label": "EMAIL", "text": "case111.review@example.invalid", "uncovered_positions": [] }, { "start": 105, "end": 119, "label": "PHONE", "text": "+49 541 730111", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-104", "scenario": null, "predictions": [ { "start": 0, "end": 15, "label": "PERSON", "text": "Maja Kirschbaum", "score": 0.9995399713516235, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 24, "end": 32, "label": "CITY", "text": "Lüneburg", "score": 0.9943020343780518, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 35, "end": 50, "label": "PERSON", "text": "Maja Kirschbaum", "score": 0.9996929168701172, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 76, "end": 94, "label": "ORGANIZATION", "text": "Sonnenbogen Verein", "score": 0.9872075915336609, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 0, "end": 15, "label": "PERSON", "text": "Maja Kirschbaum", "score": 0.9995399713516235, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 24, "end": 32, "label": "CITY", "text": "Lüneburg", "score": 0.9943020343780518, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 35, "end": 50, "label": "PERSON", "text": "Maja Kirschbaum", "score": 0.9996929168701172, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 76, "end": 94, "label": "ORGANIZATION", "text": "Sonnenbogen Verein", "score": 0.9872075915336609, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 0, "end": 15, "label": "PERSON", "text": "Maja Kirschbaum", "uncovered_positions": [] }, { "start": 24, "end": 32, "label": "CITY", "text": "Lüneburg", "uncovered_positions": [] }, { "start": 35, "end": 50, "label": "PERSON", "text": "Maja Kirschbaum", "uncovered_positions": [] }, { "start": 76, "end": 94, "label": "ORGANIZATION", "text": "Sonnenbogen Verein", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-105", "scenario": null, "predictions": [ { "start": 3, "end": 11, "label": "CITY", "text": "Lüneburg", "score": 0.9998090863227844, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 13, "end": 28, "label": "PERSON", "text": "Maja Kirschbaum", "score": 0.9998596906661987, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 40, "end": 58, "label": "ORGANIZATION", "text": "Sonnenbogen Verein", "score": 0.9954612851142883, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 79, "end": 94, "label": "PERSON", "text": "Maja Kirschbaum", "score": 0.9997907280921936, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 107, "end": 115, "label": "CITY", "text": "Lüneburg", "score": 0.9995126724243164, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 3, "end": 11, "label": "CITY", "text": "Lüneburg", "score": 0.9998090863227844, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 13, "end": 28, "label": "PERSON", "text": "Maja Kirschbaum", "score": 0.9998596906661987, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 40, "end": 58, "label": "ORGANIZATION", "text": "Sonnenbogen Verein", "score": 0.9954612851142883, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 79, "end": 94, "label": "PERSON", "text": "Maja Kirschbaum", "score": 0.9997907280921936, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 107, "end": 115, "label": "CITY", "text": "Lüneburg", "score": 0.9995126724243164, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 3, "end": 11, "label": "CITY", "text": "Lüneburg", "uncovered_positions": [] }, { "start": 13, "end": 28, "label": "PERSON", "text": "Maja Kirschbaum", "uncovered_positions": [] }, { "start": 40, "end": 58, "label": "ORGANIZATION", "text": "Sonnenbogen Verein", "uncovered_positions": [] }, { "start": 79, "end": 94, "label": "PERSON", "text": "Maja Kirschbaum", "uncovered_positions": [] }, { "start": 107, "end": 115, "label": "CITY", "text": "Lüneburg", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-106", "scenario": null, "predictions": [ { "start": 6, "end": 22, "label": "PERSON", "text": "Maja-\nKirschbaum", "score": 0.999875545501709, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "dehyphenation-v1" }, { "start": 28, "end": 38, "label": "CITY", "text": "Lüne-\nburg", "score": 0.9999223947525024, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "dehyphenation-v1" }, { "start": 52, "end": 70, "label": "ORGANIZATION", "text": "Sonnenbogen Verein", "score": 0.9918258190155029, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "dehyphenation-v1" } ], "combined_predictions": [ { "start": 6, "end": 22, "label": "PERSON", "text": "Maja-\nKirschbaum", "score": 0.999875545501709, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "dehyphenation-v1" }, { "start": 28, "end": 38, "label": "CITY", "text": "Lüne-\nburg", "score": 0.9999223947525024, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "dehyphenation-v1" }, { "start": 52, "end": 70, "label": "ORGANIZATION", "text": "Sonnenbogen Verein", "score": 0.9918258190155029, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "dehyphenation-v1" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 6, "end": 22, "label": "PERSON", "text": "Maja-\nKirschbaum", "uncovered_positions": [] }, { "start": 28, "end": 38, "label": "CITY", "text": "Lüne-\nburg", "uncovered_positions": [] }, { "start": 52, "end": 70, "label": "ORGANIZATION", "text": "Sonnenbogen Verein", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-107", "scenario": null, "predictions": [ { "start": 16, "end": 31, "label": "PERSON", "text": "Maja K1rschbaum", "score": 0.9977808594703674, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 43, "end": 51, "label": "CITY", "text": "Lüneburg", "score": 0.9994350075721741, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 65, "end": 83, "label": "ORGANIZATION", "text": "Sonnenbogen Verein", "score": 0.9930818676948547, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 16, "end": 31, "label": "PERSON", "text": "Maja K1rschbaum", "score": 0.9977808594703674, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 43, "end": 51, "label": "CITY", "text": "Lüneburg", "score": 0.9994350075721741, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 65, "end": 83, "label": "ORGANIZATION", "text": "Sonnenbogen Verein", "score": 0.9930818676948547, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 16, "end": 31, "label": "PERSON", "text": "Maja K1rschbaum", "uncovered_positions": [] }, { "start": 43, "end": 51, "label": "CITY", "text": "Lüneburg", "uncovered_positions": [] }, { "start": 65, "end": 83, "label": "ORGANIZATION", "text": "Sonnenbogen Verein", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-108", "scenario": null, "predictions": [ { "start": 51, "end": 69, "label": "ORGANIZATION", "text": "Sonnenbogen Verein", "score": 0.9550279378890991, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 83, "end": 98, "label": "PERSON", "text": "Maja Kirschbaum", "score": 0.9982492923736572, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 10, "end": 24, "label": "RELIGION", "text": "konfessionslos", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" }, { "start": 51, "end": 69, "label": "ORGANIZATION", "text": "Sonnenbogen Verein", "score": 0.9550279378890991, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 83, "end": 98, "label": "PERSON", "text": "Maja Kirschbaum", "score": 0.9982492923736572, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 10, "end": 24, "label": "RELIGION", "text": "konfessionslos", "uncovered_positions": [] }, { "start": 51, "end": 69, "label": "ORGANIZATION", "text": "Sonnenbogen Verein", "uncovered_positions": [] }, { "start": 83, "end": 98, "label": "PERSON", "text": "Maja Kirschbaum", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-109", "scenario": null, "predictions": [ { "start": 101, "end": 107, "label": "PERSON", "text": "person", "score": 0.6989420056343079, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 109, "end": 114, "label": "CITY", "text": "place", "score": 0.7754619121551514, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 116, "end": 128, "label": "ORGANIZATION", "text": "organization", "score": 0.8311129808425903, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 40, "end": 64, "label": "RELIGION", "text": "no religious affiliation", "score": null, "source": "deterministic", "model_revision": "religious-status-v1", "normalization": "identity" }, { "start": 101, "end": 107, "label": "PERSON", "text": "person", "score": 0.6989420056343079, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 109, "end": 114, "label": "CITY", "text": "place", "score": 0.7754619121551514, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 116, "end": 128, "label": "ORGANIZATION", "text": "organization", "score": 0.8311129808425903, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [ [ 40, 64, "RELIGION" ], [ 101, 107, "PERSON" ], [ 109, 114, "CITY" ], [ 116, 128, "ORGANIZATION" ] ], "false_negatives": [], "exact_match": false, "error_code": null, "canaries": [], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-110", "scenario": null, "predictions": [ { "start": 10, "end": 24, "label": "PERSON", "text": "Ronan Whitmore", "score": 0.9985572695732117, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 38, "end": 56, "label": "ORGANIZATION", "text": "Beacon Valley Labs", "score": 0.9976409673690796, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 60, "end": 64, "label": "CITY", "text": "Bath", "score": 0.9994964599609375, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 10, "end": 24, "label": "PERSON", "text": "Ronan Whitmore", "score": 0.9985572695732117, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 38, "end": 56, "label": "ORGANIZATION", "text": "Beacon Valley Labs", "score": 0.9976409673690796, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 60, "end": 64, "label": "CITY", "text": "Bath", "score": 0.9994964599609375, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 10, "end": 24, "label": "PERSON", "text": "Ronan Whitmore", "uncovered_positions": [] }, { "start": 38, "end": 56, "label": "ORGANIZATION", "text": "Beacon Valley Labs", "uncovered_positions": [] }, { "start": 60, "end": 64, "label": "CITY", "text": "Bath", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-111", "scenario": null, "predictions": [ { "start": 12, "end": 26, "label": "PERSON", "text": "Ronan Whitmore", "score": 0.998796820640564, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 38, "end": 56, "label": "ORGANIZATION", "text": "Beacon Valley Labs", "score": 0.9980693459510803, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 62, "end": 66, "label": "CITY", "text": "Bath", "score": 0.9995183944702148, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 12, "end": 26, "label": "PERSON", "text": "Ronan Whitmore", "score": 0.998796820640564, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 38, "end": 56, "label": "ORGANIZATION", "text": "Beacon Valley Labs", "score": 0.9980693459510803, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 62, "end": 66, "label": "CITY", "text": "Bath", "score": 0.9995183944702148, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 12, "end": 26, "label": "PERSON", "text": "Ronan Whitmore", "uncovered_positions": [] }, { "start": 38, "end": 56, "label": "ORGANIZATION", "text": "Beacon Valley Labs", "uncovered_positions": [] }, { "start": 62, "end": 66, "label": "CITY", "text": "Bath", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-112", "scenario": null, "predictions": [ { "start": 0, "end": 14, "label": "PERSON", "text": "Ronan Whitmore", "score": 0.99787437915802, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 21, "end": 31, "label": "RELIGION", "text": "christlich", "score": 0.9895966649055481, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 59, "end": 89, "label": "EMAIL", "text": "case112.review@example.invalid", "score": 0.9079257249832153, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 100, "end": 114, "label": "PHONE", "text": "+49 541 730112", "score": 0.9931344985961914, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 0, "end": 14, "label": "PERSON", "text": "Ronan Whitmore", "score": 0.99787437915802, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 21, "end": 31, "label": "RELIGION", "text": "christlich", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" }, { "start": 59, "end": 89, "label": "EMAIL", "text": "case112.review@example.invalid", "score": null, "source": "deterministic", "model_revision": "contact-patterns-v1", "normalization": "identity" }, { "start": 100, "end": 114, "label": "PHONE", "text": "+49 541 730112", "score": null, "source": "deterministic", "model_revision": "contact-patterns-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 0, "end": 14, "label": "PERSON", "text": "Ronan Whitmore", "uncovered_positions": [] }, { "start": 21, "end": 31, "label": "RELIGION", "text": "christlich", "uncovered_positions": [] }, { "start": 59, "end": 89, "label": "EMAIL", "text": "case112.review@example.invalid", "uncovered_positions": [] }, { "start": 100, "end": 114, "label": "PHONE", "text": "+49 541 730112", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-113", "scenario": null, "predictions": [ { "start": 0, "end": 14, "label": "PERSON", "text": "Ronan Whitmore", "score": 0.9984022974967957, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 32, "end": 44, "label": "RELIGION", "text": "Christianity", "score": 0.9992650151252747, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 70, "end": 100, "label": "EMAIL", "text": "case112.review@example.invalid", "score": 0.9013476371765137, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 109, "end": 123, "label": "PHONE", "text": "+49 541 730112", "score": 0.9611164331436157, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 0, "end": 14, "label": "PERSON", "text": "Ronan Whitmore", "score": 0.9984022974967957, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 32, "end": 44, "label": "RELIGION", "text": "Christianity", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" }, { "start": 70, "end": 100, "label": "EMAIL", "text": "case112.review@example.invalid", "score": null, "source": "deterministic", "model_revision": "contact-patterns-v1", "normalization": "identity" }, { "start": 109, "end": 123, "label": "PHONE", "text": "+49 541 730112", "score": null, "source": "deterministic", "model_revision": "contact-patterns-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 0, "end": 14, "label": "PERSON", "text": "Ronan Whitmore", "uncovered_positions": [] }, { "start": 32, "end": 44, "label": "RELIGION", "text": "Christianity", "uncovered_positions": [] }, { "start": 70, "end": 100, "label": "EMAIL", "text": "case112.review@example.invalid", "uncovered_positions": [] }, { "start": 109, "end": 123, "label": "PHONE", "text": "+49 541 730112", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-114", "scenario": null, "predictions": [ { "start": 0, "end": 14, "label": "PERSON", "text": "Ronan Whitmore", "score": 0.9992198944091797, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 23, "end": 27, "label": "CITY", "text": "Bath", "score": 0.9989506006240845, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 30, "end": 44, "label": "PERSON", "text": "Ronan Whitmore", "score": 0.9990702271461487, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 70, "end": 88, "label": "ORGANIZATION", "text": "Beacon Valley Labs", "score": 0.9951003193855286, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 0, "end": 14, "label": "PERSON", "text": "Ronan Whitmore", "score": 0.9992198944091797, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 23, "end": 27, "label": "CITY", "text": "Bath", "score": 0.9989506006240845, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 30, "end": 44, "label": "PERSON", "text": "Ronan Whitmore", "score": 0.9990702271461487, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 70, "end": 88, "label": "ORGANIZATION", "text": "Beacon Valley Labs", "score": 0.9951003193855286, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 0, "end": 14, "label": "PERSON", "text": "Ronan Whitmore", "uncovered_positions": [] }, { "start": 23, "end": 27, "label": "CITY", "text": "Bath", "uncovered_positions": [] }, { "start": 30, "end": 44, "label": "PERSON", "text": "Ronan Whitmore", "uncovered_positions": [] }, { "start": 70, "end": 88, "label": "ORGANIZATION", "text": "Beacon Valley Labs", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-115", "scenario": null, "predictions": [ { "start": 3, "end": 7, "label": "CITY", "text": "Bath", "score": 0.9997978806495667, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 9, "end": 23, "label": "PERSON", "text": "Ronan Whitmore", "score": 0.999768078327179, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 35, "end": 53, "label": "ORGANIZATION", "text": "Beacon Valley Labs", "score": 0.9973915815353394, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 74, "end": 88, "label": "PERSON", "text": "Ronan Whitmore", "score": 0.9997625946998596, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 101, "end": 105, "label": "CITY", "text": "Bath", "score": 0.9997105002403259, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 3, "end": 7, "label": "CITY", "text": "Bath", "score": 0.9997978806495667, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 9, "end": 23, "label": "PERSON", "text": "Ronan Whitmore", "score": 0.999768078327179, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 35, "end": 53, "label": "ORGANIZATION", "text": "Beacon Valley Labs", "score": 0.9973915815353394, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 74, "end": 88, "label": "PERSON", "text": "Ronan Whitmore", "score": 0.9997625946998596, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 101, "end": 105, "label": "CITY", "text": "Bath", "score": 0.9997105002403259, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 3, "end": 7, "label": "CITY", "text": "Bath", "uncovered_positions": [] }, { "start": 9, "end": 23, "label": "PERSON", "text": "Ronan Whitmore", "uncovered_positions": [] }, { "start": 35, "end": 53, "label": "ORGANIZATION", "text": "Beacon Valley Labs", "uncovered_positions": [] }, { "start": 74, "end": 88, "label": "PERSON", "text": "Ronan Whitmore", "uncovered_positions": [] }, { "start": 101, "end": 105, "label": "CITY", "text": "Bath", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-116", "scenario": null, "predictions": [ { "start": 6, "end": 21, "label": "PERSON", "text": "Ronan-\nWhitmore", "score": 0.9997969269752502, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "dehyphenation-v1" }, { "start": 27, "end": 33, "label": "CITY", "text": "Ba-\nth", "score": 0.9999775886535645, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "dehyphenation-v1" }, { "start": 47, "end": 65, "label": "ORGANIZATION", "text": "Beacon Valley Labs", "score": 0.9984259605407715, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "dehyphenation-v1" } ], "combined_predictions": [ { "start": 6, "end": 21, "label": "PERSON", "text": "Ronan-\nWhitmore", "score": 0.9997969269752502, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "dehyphenation-v1" }, { "start": 27, "end": 33, "label": "CITY", "text": "Ba-\nth", "score": 0.9999775886535645, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "dehyphenation-v1" }, { "start": 47, "end": 65, "label": "ORGANIZATION", "text": "Beacon Valley Labs", "score": 0.9984259605407715, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "dehyphenation-v1" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 6, "end": 21, "label": "PERSON", "text": "Ronan-\nWhitmore", "uncovered_positions": [] }, { "start": 27, "end": 33, "label": "CITY", "text": "Ba-\nth", "uncovered_positions": [] }, { "start": 47, "end": 65, "label": "ORGANIZATION", "text": "Beacon Valley Labs", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-117", "scenario": null, "predictions": [ { "start": 16, "end": 30, "label": "PERSON", "text": "R0nan Wh1tm0re", "score": 0.9801152944564819, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 42, "end": 46, "label": "CITY", "text": "B4th", "score": 0.9872779846191406, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 60, "end": 78, "label": "ORGANIZATION", "text": "Beacon Valley Labs", "score": 0.9977201819419861, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 16, "end": 30, "label": "PERSON", "text": "R0nan Wh1tm0re", "score": 0.9801152944564819, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 42, "end": 46, "label": "CITY", "text": "B4th", "score": 0.9872779846191406, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 60, "end": 78, "label": "ORGANIZATION", "text": "Beacon Valley Labs", "score": 0.9977201819419861, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 16, "end": 30, "label": "PERSON", "text": "R0nan Wh1tm0re", "uncovered_positions": [] }, { "start": 42, "end": 46, "label": "CITY", "text": "B4th", "uncovered_positions": [] }, { "start": 60, "end": 78, "label": "ORGANIZATION", "text": "Beacon Valley Labs", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-118", "scenario": null, "predictions": [ { "start": 10, "end": 20, "label": "RELIGION", "text": "christlich", "score": 0.7355935573577881, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 47, "end": 65, "label": "ORGANIZATION", "text": "Beacon Valley Labs", "score": 0.9870188236236572, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 79, "end": 93, "label": "PERSON", "text": "Ronan Whitmore", "score": 0.9990026354789734, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 10, "end": 20, "label": "RELIGION", "text": "christlich", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" }, { "start": 47, "end": 65, "label": "ORGANIZATION", "text": "Beacon Valley Labs", "score": 0.9870188236236572, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 79, "end": 93, "label": "PERSON", "text": "Ronan Whitmore", "score": 0.9990026354789734, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 10, "end": 20, "label": "RELIGION", "text": "christlich", "uncovered_positions": [] }, { "start": 47, "end": 65, "label": "ORGANIZATION", "text": "Beacon Valley Labs", "uncovered_positions": [] }, { "start": 79, "end": 93, "label": "PERSON", "text": "Ronan Whitmore", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-119", "scenario": null, "predictions": [ { "start": 101, "end": 107, "label": "PERSON", "text": "person", "score": 0.7034855484962463, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 109, "end": 114, "label": "CITY", "text": "place", "score": 0.7791854739189148, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 116, "end": 128, "label": "ORGANIZATION", "text": "organization", "score": 0.8351515531539917, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 40, "end": 64, "label": "RELIGION", "text": "no religious affiliation", "score": null, "source": "deterministic", "model_revision": "religious-status-v1", "normalization": "identity" }, { "start": 101, "end": 107, "label": "PERSON", "text": "person", "score": 0.7034855484962463, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 109, "end": 114, "label": "CITY", "text": "place", "score": 0.7791854739189148, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 116, "end": 128, "label": "ORGANIZATION", "text": "organization", "score": 0.8351515531539917, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [ [ 40, 64, "RELIGION" ], [ 101, 107, "PERSON" ], [ 109, 114, "CITY" ], [ 116, 128, "ORGANIZATION" ] ], "false_negatives": [], "exact_match": false, "error_code": null, "canaries": [], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" } ], "automated_release_eligible": false, "recognition_qualified": false }, "development03d": { "schema_version": "development-candidate-outcomes/v1", "accuracy": { "documents": 160, "ner_entity_metrics": { "PERSON": { "tp": 21, "fp": 9, "fn": 1, "precision": 0.7, "recall": 0.9545454545454546, "f1": 0.8076923076923077 }, "CITY": { "tp": 8, "fp": 3, "fn": 0, "precision": 0.7272727272727273, "recall": 1.0, "f1": 0.8421052631578948 }, "RELIGION": { "tp": 108, "fp": 6, "fn": 42, "precision": 0.9473684210526315, "recall": 0.72, "f1": 0.8181818181818181 }, "EMAIL": { "tp": 3, "fp": 0, "fn": 1, "precision": 1.0, "recall": 0.75, "f1": 0.8571428571428571 }, "PHONE": { "tp": 2, "fp": 0, "fn": 0, "precision": 1.0, "recall": 1.0, "f1": 1.0 }, "ORGANIZATION": { "tp": 5, "fp": 14, "fn": 1, "precision": 0.2631578947368421, "recall": 0.8333333333333334, "f1": 0.39999999999999997 } }, "combined_entity_metrics": { "PERSON": { "tp": 20, "fp": 5, "fn": 2, "precision": 0.8, "recall": 0.9090909090909091, "f1": 0.8510638297872342 }, "CITY": { "tp": 7, "fp": 2, "fn": 1, "precision": 0.7777777777777778, "recall": 0.875, "f1": 0.823529411764706 }, "RELIGION": { "tp": 139, "fp": 2, "fn": 11, "precision": 0.9858156028368794, "recall": 0.9266666666666666, "f1": 0.9553264604810996 }, "EMAIL": { "tp": 3, "fp": 0, "fn": 1, "precision": 1.0, "recall": 0.75, "f1": 0.8571428571428571 }, "PHONE": { "tp": 2, "fp": 0, "fn": 0, "precision": 1.0, "recall": 1.0, "f1": 1.0 }, "ORGANIZATION": { "tp": 5, "fp": 12, "fn": 1, "precision": 0.29411764705882354, "recall": 0.8333333333333334, "f1": 0.4347826086956522 } }, "document_exact_match_rate": 0.83125, "groups": { "de": { "documents": 80, "exact_matches": 61, "false_negatives": 13 }, "en": { "documents": 80, "exact_matches": 72, "false_negatives": 3 }, "ocr_noise": { "documents": 16, "exact_matches": 10, "false_negatives": 4 } }, "blocked_documents": 5, "failure_codes": { "ambiguous_entity_overlap": 5 }, "automated_release_eligible": false, "uncovered_source_canaries_including_incomplete": { "PERSON": 1, "CITY": 1, "RELIGION": 7, "EMAIL": 1, "PHONE": 0, "ORGANIZATION": 1 }, "literal_presence_or_incomplete": { "PERSON": 1, "CITY": 1, "RELIGION": 7, "EMAIL": 1, "PHONE": 0, "ORGANIZATION": 0 }, "partially_covered_source_canaries_including_incomplete": { "PERSON": 0, "CITY": 0, "RELIGION": 0, "EMAIL": 0, "PHONE": 0, "ORGANIZATION": 1 }, "fully_uncovered_source_canaries_including_incomplete": { "PERSON": 1, "CITY": 1, "RELIGION": 7, "EMAIL": 1, "PHONE": 0, "ORGANIZATION": 0 }, "all_document_source_coverage": { "PERSON": { "covered": 258, "total": 270 }, "CITY": { "covered": 38, "total": 42 }, "RELIGION": { "covered": 1440, "total": 1510 }, "EMAIL": { "covered": 58, "total": 76 }, "PHONE": { "covered": 23, "total": 23 }, "ORGANIZATION": { "covered": 101, "total": 108 } }, "conservative_accuracy_gate_passed": false, "scope": "all source spans; incomplete processing counts as missing, not leaked output", "deterministic_replay_latency_ms": { "median": 0.11389599967515096, "max": 4.427515996212605 } }, "candidate_outcomes": { "complete_no_measured_leakage": 153, "candidate_partial_mask": 1, "candidate_fully_unmasked": 1, "blocked_conflicting_spans": 5, "blocked_model_runtime": 0, "other_incomplete": 0 }, "candidate_return_rate": 0.96875, "safe_abstention_rate": 0.03125, "incomplete_rate": 0.03125, "no_measured_leakage_rate_among_returned": 0.9870967741935484, "candidate_partial_canaries": { "PERSON": 0, "CITY": 0, "RELIGION": 0, "EMAIL": 0, "PHONE": 0, "ORGANIZATION": 1 }, "candidate_fully_unmasked_canaries": { "PERSON": 0, "CITY": 0, "RELIGION": 1, "EMAIL": 0, "PHONE": 0, "ORGANIZATION": 0 }, "returned_candidate_entity_metrics": { "PERSON": { "tp": 20, "fp": 5, "fn": 1, "precision": 0.8, "recall": 0.9523809523809523, "f1": 0.8695652173913043 }, "CITY": { "tp": 7, "fp": 2, "fn": 0, "precision": 0.7777777777777778, "recall": 1.0, "f1": 0.8750000000000001 }, "RELIGION": { "tp": 139, "fp": 2, "fn": 5, "precision": 0.9858156028368794, "recall": 0.9652777777777778, "f1": 0.9754385964912281 }, "EMAIL": { "tp": 3, "fp": 0, "fn": 0, "precision": 1.0, "recall": 1.0, "f1": 1.0 }, "PHONE": { "tp": 2, "fp": 0, "fn": 0, "precision": 1.0, "recall": 1.0, "f1": 1.0 }, "ORGANIZATION": { "tp": 5, "fp": 12, "fn": 1, "precision": 0.29411764705882354, "recall": 0.8333333333333334, "f1": 0.4347826086956522 } }, "per_case": [ { "case_id": "holdout03d-001", "scenario": "affiliation", "predictions": [ { "start": 24, "end": 34, "label": "PERSON", "text": "Bekenntnis", "score": 0.5389757752418518, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 35, "end": 53, "label": "RELIGION", "text": "römisch-katholisch", "score": 0.843971312046051, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 24, "end": 34, "label": "PERSON", "text": "Bekenntnis", "score": 0.5389757752418518, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 35, "end": 53, "label": "RELIGION", "text": "römisch-katholisch", "score": 0.843971312046051, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [ [ 24, 34, "PERSON" ] ], "false_negatives": [], "exact_match": false, "error_code": null, "canaries": [ { "start": 35, "end": 53, "label": "RELIGION", "text": "römisch-katholisch", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-002", "scenario": "affiliation", "predictions": [ { "start": 35, "end": 49, "label": "RELIGION", "text": "Roman Catholic", "score": 0.9411200881004333, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 35, "end": 49, "label": "RELIGION", "text": "Roman Catholic", "score": 0.9411200881004333, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 35, "end": 49, "label": "RELIGION", "text": "Roman Catholic", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-003", "scenario": "affiliation", "predictions": [ { "start": 0, "end": 14, "label": "ORGANIZATION", "text": "Tabea Hornfels", "score": 0.9000996351242065, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 45, "end": 55, "label": "RELIGION", "text": "sunnitisch", "score": 0.6417439579963684, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 0, "end": 14, "label": "ORGANIZATION", "text": "Tabea Hornfels", "score": 0.9000996351242065, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 45, "end": 55, "label": "RELIGION", "text": "sunnitisch", "score": 0.6417439579963684, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [ [ 0, 14, "ORGANIZATION" ] ], "false_negatives": [ [ 0, 14, "PERSON" ] ], "exact_match": false, "error_code": null, "canaries": [ { "start": 0, "end": 14, "label": "PERSON", "text": "Tabea Hornfels", "uncovered_positions": [] }, { "start": 45, "end": 55, "label": "RELIGION", "text": "sunnitisch", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-004", "scenario": "affiliation", "predictions": [ { "start": 0, "end": 14, "label": "PERSON", "text": "Ewan Fairbrook", "score": 0.9914917349815369, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 29, "end": 40, "label": "RELIGION", "text": "Shia Muslim", "score": 0.7597324848175049, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 0, "end": 14, "label": "PERSON", "text": "Ewan Fairbrook", "score": 0.9914917349815369, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 29, "end": 40, "label": "RELIGION", "text": "Shia Muslim", "score": 0.7597324848175049, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 0, "end": 14, "label": "PERSON", "text": "Ewan Fairbrook", "uncovered_positions": [] }, { "start": 29, "end": 40, "label": "RELIGION", "text": "Shia Muslim", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-005", "scenario": "affiliation", "predictions": [ { "start": 27, "end": 37, "label": "RELIGION", "text": "Jesidentum", "score": 0.7354034185409546, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 27, "end": 37, "label": "RELIGION", "text": "Jesidentum", "score": 0.7354034185409546, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 27, "end": 37, "label": "RELIGION", "text": "Jesidentum", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-006", "scenario": "affiliation", "predictions": [ { "start": 24, "end": 30, "label": "ORGANIZATION", "text": "Quaker", "score": 0.9522815346717834, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 24, "end": 30, "label": "ORGANIZATION", "text": "Quaker", "score": 0.9522815346717834, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [ [ 24, 30, "ORGANIZATION" ] ], "false_negatives": [ [ 24, 30, "RELIGION" ] ], "exact_match": false, "error_code": null, "canaries": [ { "start": 24, "end": 30, "label": "RELIGION", "text": "Quaker", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-007", "scenario": "affiliation", "predictions": [ { "start": 26, "end": 35, "label": "RELIGION", "text": "Sikhismus", "score": 0.9770961403846741, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 26, "end": 35, "label": "RELIGION", "text": "Sikhismus", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 26, "end": 35, "label": "RELIGION", "text": "Sikhismus", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-008", "scenario": "affiliation", "predictions": [ { "start": 27, "end": 35, "label": "RELIGION", "text": "Hinduism", "score": 0.9994035959243774, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 27, "end": 35, "label": "RELIGION", "text": "Hinduism", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 27, "end": 35, "label": "RELIGION", "text": "Hinduism", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-009", "scenario": "affiliation", "predictions": [ { "start": 44, "end": 52, "label": "RELIGION", "text": "Judentum", "score": 0.918600857257843, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 44, "end": 52, "label": "RELIGION", "text": "Judentum", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 44, "end": 52, "label": "RELIGION", "text": "Judentum", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-010", "scenario": "affiliation", "predictions": [ { "start": 38, "end": 46, "label": "RELIGION", "text": "Buddhism", "score": 0.9944268465042114, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 38, "end": 46, "label": "RELIGION", "text": "Buddhism", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 38, "end": 46, "label": "RELIGION", "text": "Buddhism", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-011", "scenario": "affiliation", "predictions": [ { "start": 46, "end": 51, "label": "RELIGION", "text": "Bahai", "score": 0.8567683100700378, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 46, "end": 51, "label": "RELIGION", "text": "Bahai", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 46, "end": 51, "label": "RELIGION", "text": "Bahai", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-012", "scenario": "affiliation", "predictions": [ { "start": 46, "end": 53, "label": "RELIGION", "text": "Jainism", "score": 0.9846257567405701, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 46, "end": 53, "label": "RELIGION", "text": "Jainism", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 46, "end": 53, "label": "RELIGION", "text": "Jainism", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-013", "scenario": "affiliation", "predictions": [ { "start": 31, "end": 53, "label": "RELIGION", "text": "evangelisch-lutherisch", "score": 0.7786831259727478, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 31, "end": 53, "label": "RELIGION", "text": "evangelisch-lutherisch", "score": 0.7786831259727478, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 31, "end": 53, "label": "RELIGION", "text": "evangelisch-lutherisch", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-014", "scenario": "affiliation", "predictions": [ { "start": 51, "end": 72, "label": "RELIGION", "text": "Seventh-day Adventist", "score": 0.7101534605026245, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 51, "end": 72, "label": "RELIGION", "text": "Seventh-day Adventist", "score": 0.7101534605026245, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 51, "end": 72, "label": "RELIGION", "text": "Seventh-day Adventist", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-015", "scenario": "affiliation", "predictions": [ { "start": 0, "end": 14, "label": "PERSON", "text": "Raban Moosfeld", "score": 0.9412690997123718, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 32, "end": 43, "label": "RELIGION", "text": "Christentum", "score": 0.9764376282691956, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 0, "end": 14, "label": "PERSON", "text": "Raban Moosfeld", "score": 0.9412690997123718, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 32, "end": 43, "label": "RELIGION", "text": "Christentum", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 0, "end": 14, "label": "PERSON", "text": "Raban Moosfeld", "uncovered_positions": [] }, { "start": 32, "end": 43, "label": "RELIGION", "text": "Christentum", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-016", "scenario": "affiliation", "predictions": [ { "start": 0, "end": 14, "label": "PERSON", "text": "Iona Crestwell", "score": 0.9930251836776733, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 41, "end": 55, "label": "RELIGION", "text": "Zoroastrianism", "score": 0.9812561869621277, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 0, "end": 14, "label": "PERSON", "text": "Iona Crestwell", "score": 0.9930251836776733, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 41, "end": 55, "label": "RELIGION", "text": "Zoroastrianism", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 0, "end": 14, "label": "PERSON", "text": "Iona Crestwell", "uncovered_positions": [] }, { "start": 41, "end": 55, "label": "RELIGION", "text": "Zoroastrianism", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-017", "scenario": "nonaffiliation", "predictions": [], "combined_predictions": [ { "start": 38, "end": 53, "label": "RELIGION", "text": "konfessionsfrei", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 38, "end": 53, "label": "RELIGION", "text": "konfessionsfrei", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-018", "scenario": "nonaffiliation", "predictions": [], "combined_predictions": [ { "start": 18, "end": 42, "label": "RELIGION", "text": "no religious affiliation", "score": null, "source": "deterministic", "model_revision": "religious-status-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 18, "end": 42, "label": "RELIGION", "text": "no religious affiliation", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-019", "scenario": "nonaffiliation", "predictions": [], "combined_predictions": [ { "start": 27, "end": 39, "label": "RELIGION", "text": "religionslos", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 27, "end": 39, "label": "RELIGION", "text": "religionslos", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-020", "scenario": "nonaffiliation", "predictions": [ { "start": 43, "end": 50, "label": "RELIGION", "text": "atheist", "score": 0.6892726421356201, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 43, "end": 50, "label": "RELIGION", "text": "atheist", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 43, "end": 50, "label": "RELIGION", "text": "atheist", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-021", "scenario": "nonaffiliation", "predictions": [], "combined_predictions": [ { "start": 47, "end": 57, "label": "RELIGION", "text": "agnostisch", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 47, "end": 57, "label": "RELIGION", "text": "agnostisch", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-022", "scenario": "nonaffiliation", "predictions": [], "combined_predictions": [ { "start": 36, "end": 44, "label": "RELIGION", "text": "agnostic", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 36, "end": 44, "label": "RELIGION", "text": "agnostic", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-023", "scenario": "nonaffiliation", "predictions": [ { "start": 34, "end": 44, "label": "RELIGION", "text": "Konfession", "score": 0.8056620955467224, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 29, "end": 44, "label": "RELIGION", "text": "ohne Konfession", "score": null, "source": "deterministic", "model_revision": "religious-status-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 29, "end": 44, "label": "RELIGION", "text": "ohne Konfession", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-024", "scenario": "nonaffiliation", "predictions": [ { "start": 24, "end": 36, "label": "RELIGION", "text": "nonreligious", "score": 0.6173449158668518, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 24, "end": 36, "label": "RELIGION", "text": "nonreligious", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 24, "end": 36, "label": "RELIGION", "text": "nonreligious", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-025", "scenario": "nonaffiliation", "predictions": [ { "start": 27, "end": 35, "label": "RELIGION", "text": "Religion", "score": 0.6249727010726929, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 21, "end": 35, "label": "RELIGION", "text": "keine Religion", "score": null, "source": "deterministic", "model_revision": "religious-status-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 21, "end": 35, "label": "RELIGION", "text": "keine Religion", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-026", "scenario": "nonaffiliation", "predictions": [ { "start": 0, "end": 11, "label": "ORGANIZATION", "text": "Affiliation", "score": 0.6462724208831787, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 0, "end": 11, "label": "ORGANIZATION", "text": "Affiliation", "score": 0.6462724208831787, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 27, "end": 51, "label": "RELIGION", "text": "religiously unaffiliated", "score": null, "source": "deterministic", "model_revision": "religious-status-v1", "normalization": "identity" } ], "false_positives": [ [ 0, 11, "ORGANIZATION" ] ], "false_negatives": [], "exact_match": false, "error_code": null, "canaries": [ { "start": 27, "end": 51, "label": "RELIGION", "text": "religiously unaffiliated", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-027", "scenario": "nonaffiliation", "predictions": [], "combined_predictions": [ { "start": 32, "end": 46, "label": "RELIGION", "text": "konfessionslos", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 32, "end": 46, "label": "RELIGION", "text": "konfessionslos", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-028", "scenario": "nonaffiliation", "predictions": [ { "start": 17, "end": 24, "label": "RELIGION", "text": "atheism", "score": 0.8736236095428467, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 17, "end": 24, "label": "RELIGION", "text": "atheism", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 17, "end": 24, "label": "RELIGION", "text": "atheism", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-029", "scenario": "nonaffiliation", "predictions": [ { "start": 21, "end": 30, "label": "RELIGION", "text": "Atheismus", "score": 0.7562140226364136, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 21, "end": 30, "label": "RELIGION", "text": "Atheismus", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 21, "end": 30, "label": "RELIGION", "text": "Atheismus", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-030", "scenario": "nonaffiliation", "predictions": [], "combined_predictions": [ { "start": 29, "end": 40, "label": "RELIGION", "text": "no religion", "score": null, "source": "deterministic", "model_revision": "religious-status-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 29, "end": 40, "label": "RELIGION", "text": "no religion", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-031", "scenario": "nonaffiliation", "predictions": [], "combined_predictions": [ { "start": 24, "end": 37, "label": "RELIGION", "text": "Agnostizismus", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 24, "end": 37, "label": "RELIGION", "text": "Agnostizismus", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-032", "scenario": "nonaffiliation", "predictions": [], "combined_predictions": [ { "start": 29, "end": 42, "label": "RELIGION", "text": "non-religious", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 29, "end": 42, "label": "RELIGION", "text": "non-religious", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-033", "scenario": "belief_names", "predictions": [ { "start": 4, "end": 11, "label": "ORGANIZATION", "text": "Glossar", "score": 0.5172879099845886, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 22, "end": 30, "label": "RELIGION", "text": "Daoismus", "score": 0.6094017624855042, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 4, "end": 11, "label": "ORGANIZATION", "text": "Glossar", "score": 0.5172879099845886, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 22, "end": 30, "label": "RELIGION", "text": "Daoismus", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [ [ 4, 11, "ORGANIZATION" ] ], "false_negatives": [], "exact_match": false, "error_code": null, "canaries": [ { "start": 22, "end": 30, "label": "RELIGION", "text": "Daoismus", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-034", "scenario": "belief_names", "predictions": [ { "start": 32, "end": 41, "label": "RELIGION", "text": "Shintoism", "score": 0.9690232872962952, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 32, "end": 41, "label": "RELIGION", "text": "Shintoism", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 32, "end": 41, "label": "RELIGION", "text": "Shintoism", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-035", "scenario": "belief_names", "predictions": [ { "start": 40, "end": 45, "label": "RELIGION", "text": "Islam", "score": 0.9760571718215942, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 40, "end": 45, "label": "RELIGION", "text": "Islam", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 40, "end": 45, "label": "RELIGION", "text": "Islam", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-036", "scenario": "belief_names", "predictions": [ { "start": 27, "end": 34, "label": "RELIGION", "text": "Judaism", "score": 0.9818722605705261, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 27, "end": 34, "label": "RELIGION", "text": "Judaism", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 27, "end": 34, "label": "RELIGION", "text": "Judaism", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-037", "scenario": "belief_names", "predictions": [ { "start": 29, "end": 39, "label": "RELIGION", "text": "Hinduismus", "score": 0.9524589776992798, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 44, "end": 54, "label": "RELIGION", "text": "Buddhismus", "score": 0.9651915431022644, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 29, "end": 39, "label": "RELIGION", "text": "Hinduismus", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" }, { "start": 44, "end": 54, "label": "RELIGION", "text": "Buddhismus", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 29, "end": 39, "label": "RELIGION", "text": "Hinduismus", "uncovered_positions": [] }, { "start": 44, "end": 54, "label": "RELIGION", "text": "Buddhismus", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-038", "scenario": "belief_names", "predictions": [ { "start": 22, "end": 34, "label": "RELIGION", "text": "Christianity", "score": 0.9961896538734436, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 40, "end": 46, "label": "RELIGION", "text": "Taoism", "score": 0.9618191719055176, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 22, "end": 34, "label": "RELIGION", "text": "Christianity", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" }, { "start": 40, "end": 46, "label": "RELIGION", "text": "Taoism", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 22, "end": 34, "label": "RELIGION", "text": "Christianity", "uncovered_positions": [] }, { "start": 40, "end": 46, "label": "RELIGION", "text": "Taoism", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-039", "scenario": "belief_names", "predictions": [ { "start": 0, "end": 10, "label": "ORGANIZATION", "text": "Das Museum", "score": 0.8367996215820312, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 43, "end": 52, "label": "RELIGION", "text": "Jainismus", "score": 0.907336413860321, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 0, "end": 10, "label": "ORGANIZATION", "text": "Das Museum", "score": 0.8367996215820312, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 43, "end": 52, "label": "RELIGION", "text": "Jainismus", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [ [ 0, 10, "ORGANIZATION" ] ], "false_negatives": [], "exact_match": false, "error_code": null, "canaries": [ { "start": 43, "end": 52, "label": "RELIGION", "text": "Jainismus", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-040", "scenario": "belief_names", "predictions": [ { "start": 44, "end": 51, "label": "RELIGION", "text": "Sikhism", "score": 0.984706461429596, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 44, "end": 51, "label": "RELIGION", "text": "Sikhism", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 44, "end": 51, "label": "RELIGION", "text": "Sikhism", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-041", "scenario": "belief_names", "predictions": [ { "start": 22, "end": 35, "label": "RELIGION", "text": "Zoroastrismus", "score": 0.8952449560165405, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 42, "end": 50, "label": "PERSON", "text": "Personen", "score": 0.7890533804893494, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 22, "end": 35, "label": "RELIGION", "text": "Zoroastrismus", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" }, { "start": 42, "end": 50, "label": "PERSON", "text": "Personen", "score": 0.7890533804893494, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [ [ 42, 50, "PERSON" ] ], "false_negatives": [], "exact_match": false, "error_code": null, "canaries": [ { "start": 22, "end": 35, "label": "RELIGION", "text": "Zoroastrismus", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-042", "scenario": "belief_names", "predictions": [ { "start": 24, "end": 30, "label": "CITY", "text": "Baháʼí", "score": 0.7278924584388733, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 31, "end": 36, "label": "RELIGION", "text": "Faith", "score": 0.5462192296981812, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 24, "end": 36, "label": "RELIGION", "text": "Baháʼí Faith", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 24, "end": 36, "label": "RELIGION", "text": "Baháʼí Faith", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-043", "scenario": "belief_names", "predictions": [ { "start": 4, "end": 13, "label": "ORGANIZATION", "text": "Übersicht", "score": 0.5883280038833618, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 24, "end": 37, "label": "RELIGION", "text": "Katholizismus", "score": 0.6876825094223022, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 4, "end": 13, "label": "ORGANIZATION", "text": "Übersicht", "score": 0.5883280038833618, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 24, "end": 37, "label": "RELIGION", "text": "Katholizismus", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [ [ 4, 13, "ORGANIZATION" ] ], "false_negatives": [], "exact_match": false, "error_code": null, "canaries": [ { "start": 24, "end": 37, "label": "RELIGION", "text": "Katholizismus", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-044", "scenario": "belief_names", "predictions": [ { "start": 41, "end": 54, "label": "RELIGION", "text": "Protestantism", "score": 0.9522858262062073, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 41, "end": 54, "label": "RELIGION", "text": "Protestantism", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 41, "end": 54, "label": "RELIGION", "text": "Protestantism", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-045", "scenario": "belief_names", "predictions": [ { "start": 24, "end": 34, "label": "RELIGION", "text": "Orthodoxie", "score": 0.8815929293632507, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 24, "end": 34, "label": "RELIGION", "text": "Orthodoxie", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 24, "end": 34, "label": "RELIGION", "text": "Orthodoxie", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-046", "scenario": "belief_names", "predictions": [ { "start": 30, "end": 51, "label": "RELIGION", "text": "Orthodox Christianity", "score": 0.9181906580924988, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 30, "end": 51, "label": "RELIGION", "text": "Orthodox Christianity", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 30, "end": 51, "label": "RELIGION", "text": "Orthodox Christianity", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-047", "scenario": "belief_names", "predictions": [ { "start": 27, "end": 36, "label": "RELIGION", "text": "Sikhismus", "score": 0.9480417370796204, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 41, "end": 50, "label": "RELIGION", "text": "Jainismus", "score": 0.9625227451324463, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 27, "end": 36, "label": "RELIGION", "text": "Sikhismus", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" }, { "start": 41, "end": 50, "label": "RELIGION", "text": "Jainismus", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 27, "end": 36, "label": "RELIGION", "text": "Sikhismus", "uncovered_positions": [] }, { "start": 41, "end": 50, "label": "RELIGION", "text": "Jainismus", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-048", "scenario": "belief_names", "predictions": [ { "start": 4, "end": 16, "label": "ORGANIZATION", "text": "encyclopedia", "score": 0.740946352481842, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 42, "end": 54, "label": "RELIGION", "text": "Confucianism", "score": 0.9735894203186035, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 4, "end": 16, "label": "ORGANIZATION", "text": "encyclopedia", "score": 0.740946352481842, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 42, "end": 54, "label": "RELIGION", "text": "Confucianism", "score": 0.9735894203186035, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [ [ 4, 16, "ORGANIZATION" ] ], "false_negatives": [], "exact_match": false, "error_code": null, "canaries": [ { "start": 42, "end": 54, "label": "RELIGION", "text": "Confucianism", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-049", "scenario": "general_topics", "predictions": [ { "start": 38, "end": 43, "label": "RELIGION", "text": "Islam", "score": 0.9990894794464111, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 38, "end": 43, "label": "RELIGION", "text": "Islam", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 38, "end": 43, "label": "RELIGION", "text": "Islam", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-050", "scenario": "general_topics", "predictions": [ { "start": 33, "end": 41, "label": "RELIGION", "text": "Hinduism", "score": 0.9996596574783325, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 33, "end": 41, "label": "RELIGION", "text": "Hinduism", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 33, "end": 41, "label": "RELIGION", "text": "Hinduism", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-051", "scenario": "general_topics", "predictions": [ { "start": 23, "end": 34, "label": "RELIGION", "text": "Christentum", "score": 0.9296461343765259, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 45, "end": 56, "label": "CITY", "text": "Mittelalter", "score": 0.600753903388977, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 23, "end": 34, "label": "RELIGION", "text": "Christentum", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" }, { "start": 45, "end": 56, "label": "CITY", "text": "Mittelalter", "score": 0.600753903388977, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [ [ 45, 56, "CITY" ] ], "false_negatives": [], "exact_match": false, "error_code": null, "canaries": [ { "start": 23, "end": 34, "label": "RELIGION", "text": "Christentum", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-052", "scenario": "general_topics", "predictions": [ { "start": 28, "end": 36, "label": "RELIGION", "text": "Buddhism", "score": 0.9992027878761292, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 28, "end": 36, "label": "RELIGION", "text": "Buddhism", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 28, "end": 36, "label": "RELIGION", "text": "Buddhism", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-053", "scenario": "general_topics", "predictions": [ { "start": 37, "end": 45, "label": "RELIGION", "text": "Judentum", "score": 0.9257993698120117, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 37, "end": 45, "label": "RELIGION", "text": "Judentum", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 37, "end": 45, "label": "RELIGION", "text": "Judentum", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-054", "scenario": "general_topics", "predictions": [ { "start": 20, "end": 27, "label": "RELIGION", "text": "Jainism", "score": 0.9963538646697998, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 20, "end": 27, "label": "RELIGION", "text": "Jainism", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 20, "end": 27, "label": "RELIGION", "text": "Jainism", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-055", "scenario": "general_topics", "predictions": [ { "start": 24, "end": 29, "label": "PERSON", "text": "Bahai", "score": 0.9998281002044678, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [], "false_positives": [], "false_negatives": [ [ 24, 29, "RELIGION" ] ], "exact_match": false, "error_code": "ambiguous_entity_overlap", "canaries": [ { "start": 24, "end": 29, "label": "RELIGION", "text": "Bahai", "uncovered_positions": [ 24, 25, 26, 27, 28 ] } ], "candidate_returned": false, "candidate_canaries": [], "outcome": "blocked_conflicting_spans" }, { "case_id": "holdout03d-056", "scenario": "general_topics", "predictions": [ { "start": 25, "end": 32, "label": "RELIGION", "text": "Sikhism", "score": 0.9963860511779785, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 25, "end": 32, "label": "RELIGION", "text": "Sikhism", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 25, "end": 32, "label": "RELIGION", "text": "Sikhism", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-057", "scenario": "general_topics", "predictions": [ { "start": 20, "end": 29, "label": "RELIGION", "text": "Atheismus", "score": 0.5075995326042175, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 20, "end": 29, "label": "RELIGION", "text": "Atheismus", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 20, "end": 29, "label": "RELIGION", "text": "Atheismus", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-058", "scenario": "general_topics", "predictions": [ { "start": 19, "end": 26, "label": "RELIGION", "text": "atheism", "score": 0.9609501957893372, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 31, "end": 42, "label": "RELIGION", "text": "agnosticism", "score": 0.9465921521186829, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 19, "end": 26, "label": "RELIGION", "text": "atheism", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" }, { "start": 31, "end": 42, "label": "RELIGION", "text": "agnosticism", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 19, "end": 26, "label": "RELIGION", "text": "atheism", "uncovered_positions": [] }, { "start": 31, "end": 42, "label": "RELIGION", "text": "agnosticism", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-059", "scenario": "general_topics", "predictions": [ { "start": 30, "end": 41, "label": "RELIGION", "text": "Shintoismus", "score": 0.9589027762413025, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 30, "end": 41, "label": "RELIGION", "text": "Shintoismus", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 30, "end": 41, "label": "RELIGION", "text": "Shintoismus", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-060", "scenario": "general_topics", "predictions": [ { "start": 23, "end": 29, "label": "RELIGION", "text": "Taoism", "score": 0.9697747826576233, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 23, "end": 29, "label": "RELIGION", "text": "Taoism", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 23, "end": 29, "label": "RELIGION", "text": "Taoism", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-061", "scenario": "general_topics", "predictions": [ { "start": 30, "end": 45, "label": "RELIGION", "text": "Protestantismus", "score": 0.6991519927978516, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 30, "end": 45, "label": "RELIGION", "text": "Protestantismus", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 30, "end": 45, "label": "RELIGION", "text": "Protestantismus", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-062", "scenario": "general_topics", "predictions": [ { "start": 25, "end": 36, "label": "RELIGION", "text": "Catholicism", "score": 0.9966902732849121, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 25, "end": 36, "label": "RELIGION", "text": "Catholicism", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 25, "end": 36, "label": "RELIGION", "text": "Catholicism", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-063", "scenario": "general_topics", "predictions": [ { "start": 30, "end": 44, "label": "PERSON", "text": "konfessionslos", "score": 0.5395556092262268, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [], "false_positives": [], "false_negatives": [ [ 30, 44, "RELIGION" ] ], "exact_match": false, "error_code": "ambiguous_entity_overlap", "canaries": [ { "start": 30, "end": 44, "label": "RELIGION", "text": "konfessionslos", "uncovered_positions": [ 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43 ] } ], "candidate_returned": false, "candidate_canaries": [], "outcome": "blocked_conflicting_spans" }, { "case_id": "holdout03d-064", "scenario": "general_topics", "predictions": [ { "start": 29, "end": 42, "label": "RELIGION", "text": "non-religious", "score": 0.5220276713371277, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 29, "end": 42, "label": "RELIGION", "text": "non-religious", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 29, "end": 42, "label": "RELIGION", "text": "non-religious", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-065", "scenario": "negation_quotation", "predictions": [ { "start": 21, "end": 24, "label": "PERSON", "text": "Ich", "score": 0.7314160466194153, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 35, "end": 45, "label": "RELIGION", "text": "muslimisch", "score": 0.8888826966285706, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 21, "end": 24, "label": "PERSON", "text": "Ich", "score": 0.7314160466194153, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 35, "end": 45, "label": "RELIGION", "text": "muslimisch", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [ [ 21, 24, "PERSON" ] ], "false_negatives": [], "exact_match": false, "error_code": null, "canaries": [ { "start": 35, "end": 45, "label": "RELIGION", "text": "muslimisch", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-066", "scenario": "negation_quotation", "predictions": [ { "start": 28, "end": 37, "label": "RELIGION", "text": "Christian", "score": 0.931776762008667, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 28, "end": 37, "label": "RELIGION", "text": "Christian", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 28, "end": 37, "label": "RELIGION", "text": "Christian", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-067", "scenario": "negation_quotation", "predictions": [ { "start": 29, "end": 39, "label": "RELIGION", "text": "katholisch", "score": 0.8747991323471069, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 29, "end": 39, "label": "RELIGION", "text": "katholisch", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 29, "end": 39, "label": "RELIGION", "text": "katholisch", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-068", "scenario": "negation_quotation", "predictions": [], "combined_predictions": [ { "start": 35, "end": 42, "label": "RELIGION", "text": "atheist", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 35, "end": 42, "label": "RELIGION", "text": "atheist", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-069", "scenario": "negation_quotation", "predictions": [ { "start": 10, "end": 22, "label": "RELIGION", "text": "buddhistisch", "score": 0.78475022315979, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 10, "end": 22, "label": "RELIGION", "text": "buddhistisch", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 10, "end": 22, "label": "RELIGION", "text": "buddhistisch", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-070", "scenario": "negation_quotation", "predictions": [ { "start": 1, "end": 7, "label": "RELIGION", "text": "Jewish", "score": 0.922519862651825, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 1, "end": 7, "label": "RELIGION", "text": "Jewish", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 1, "end": 7, "label": "RELIGION", "text": "Jewish", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-071", "scenario": "negation_quotation", "predictions": [ { "start": 40, "end": 52, "label": "RELIGION", "text": "hinduistisch", "score": 0.7200407981872559, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 40, "end": 52, "label": "RELIGION", "text": "hinduistisch", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 40, "end": 52, "label": "RELIGION", "text": "hinduistisch", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-072", "scenario": "negation_quotation", "predictions": [ { "start": 37, "end": 43, "label": "RELIGION", "text": "Muslim", "score": 0.995964527130127, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 37, "end": 43, "label": "RELIGION", "text": "Muslim", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 37, "end": 43, "label": "RELIGION", "text": "Muslim", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-073", "scenario": "negation_quotation", "predictions": [], "combined_predictions": [ { "start": 1, "end": 16, "label": "RELIGION", "text": "Ohne Konfession", "score": null, "source": "deterministic", "model_revision": "religious-status-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 1, "end": 16, "label": "RELIGION", "text": "Ohne Konfession", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-074", "scenario": "negation_quotation", "predictions": [ { "start": 30, "end": 38, "label": "RELIGION", "text": "religion", "score": 0.5185865163803101, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 27, "end": 38, "label": "RELIGION", "text": "no religion", "score": null, "source": "deterministic", "model_revision": "religious-status-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 27, "end": 38, "label": "RELIGION", "text": "no religion", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-075", "scenario": "negation_quotation", "predictions": [], "combined_predictions": [ { "start": 6, "end": 17, "label": "RELIGION", "text": "evangelisch", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" }, { "start": 27, "end": 37, "label": "RELIGION", "text": "agnostisch", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 6, "end": 17, "label": "RELIGION", "text": "evangelisch", "uncovered_positions": [] }, { "start": 27, "end": 37, "label": "RELIGION", "text": "agnostisch", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-076", "scenario": "negation_quotation", "predictions": [ { "start": 18, "end": 26, "label": "RELIGION", "text": "Catholic", "score": 0.9851877689361572, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 30, "end": 42, "label": "RELIGION", "text": "nonreligious", "score": 0.5536500811576843, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 18, "end": 26, "label": "RELIGION", "text": "Catholic", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" }, { "start": 30, "end": 42, "label": "RELIGION", "text": "nonreligious", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 18, "end": 26, "label": "RELIGION", "text": "Catholic", "uncovered_positions": [] }, { "start": 30, "end": 42, "label": "RELIGION", "text": "nonreligious", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-077", "scenario": "negation_quotation", "predictions": [ { "start": 22, "end": 30, "label": "RELIGION", "text": "Judentum", "score": 0.9432740807533264, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 22, "end": 30, "label": "RELIGION", "text": "Judentum", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 22, "end": 30, "label": "RELIGION", "text": "Judentum", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-078", "scenario": "negation_quotation", "predictions": [ { "start": 23, "end": 29, "label": "PERSON", "text": "Baha'i", "score": 0.8763203620910645, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [], "false_positives": [], "false_negatives": [ [ 23, 29, "RELIGION" ] ], "exact_match": false, "error_code": "ambiguous_entity_overlap", "canaries": [ { "start": 23, "end": 29, "label": "RELIGION", "text": "Baha'i", "uncovered_positions": [ 23, 24, 25, 26, 28 ] } ], "candidate_returned": false, "candidate_canaries": [], "outcome": "blocked_conflicting_spans" }, { "case_id": "holdout03d-079", "scenario": "negation_quotation", "predictions": [ { "start": 21, "end": 32, "label": "RELIGION", "text": "islamischen", "score": 0.6762732863426208, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 21, "end": 32, "label": "RELIGION", "text": "islamischen", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 21, "end": 32, "label": "RELIGION", "text": "islamischen", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-080", "scenario": "negation_quotation", "predictions": [ { "start": 15, "end": 23, "label": "RELIGION", "text": "Buddhist", "score": 0.964024007320404, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 15, "end": 23, "label": "RELIGION", "text": "Buddhist", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 15, "end": 23, "label": "RELIGION", "text": "Buddhist", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-081", "scenario": "ambiguity", "predictions": [], "combined_predictions": [], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-082", "scenario": "ambiguity", "predictions": [ { "start": 4, "end": 12, "label": "RELIGION", "text": "catholic", "score": 0.6429972648620605, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 4, "end": 12, "label": "RELIGION", "text": "catholic", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [ [ 4, 12, "RELIGION" ] ], "false_negatives": [], "exact_match": false, "error_code": null, "canaries": [], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-083", "scenario": "ambiguity", "predictions": [], "combined_predictions": [], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-084", "scenario": "ambiguity", "predictions": [ { "start": 3, "end": 14, "label": "RELIGION", "text": "evangelical", "score": 0.8391301035881042, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 3, "end": 14, "label": "RELIGION", "text": "evangelical", "score": 0.8391301035881042, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [ [ 3, 14, "RELIGION" ] ], "false_negatives": [], "exact_match": false, "error_code": null, "canaries": [], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-085", "scenario": "ambiguity", "predictions": [ { "start": 18, "end": 26, "label": "RELIGION", "text": "orthodox", "score": 0.8430137634277344, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 18, "end": 26, "label": "RELIGION", "text": "orthodox", "score": 0.8430137634277344, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 18, "end": 26, "label": "RELIGION", "text": "orthodox", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-086", "scenario": "ambiguity", "predictions": [ { "start": 52, "end": 60, "label": "RELIGION", "text": "Orthodox", "score": 0.8877473473548889, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 52, "end": 60, "label": "RELIGION", "text": "Orthodox", "score": 0.8877473473548889, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 52, "end": 60, "label": "RELIGION", "text": "Orthodox", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-087", "scenario": "ambiguity", "predictions": [ { "start": 12, "end": 23, "label": "RELIGION", "text": "evangelisch", "score": 0.7735995650291443, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 25, "end": 36, "label": "ORGANIZATION", "text": "Algorithmus", "score": 0.5926805734634399, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 12, "end": 23, "label": "RELIGION", "text": "evangelisch", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" }, { "start": 25, "end": 36, "label": "ORGANIZATION", "text": "Algorithmus", "score": 0.5926805734634399, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [ [ 25, 36, "ORGANIZATION" ] ], "false_negatives": [], "exact_match": false, "error_code": null, "canaries": [ { "start": 12, "end": 23, "label": "RELIGION", "text": "evangelisch", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-088", "scenario": "ambiguity", "predictions": [ { "start": 26, "end": 37, "label": "RELIGION", "text": "Evangelical", "score": 0.9244890809059143, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 26, "end": 37, "label": "RELIGION", "text": "Evangelical", "score": 0.9244890809059143, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 26, "end": 37, "label": "RELIGION", "text": "Evangelical", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-089", "scenario": "ambiguity", "predictions": [], "combined_predictions": [], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-090", "scenario": "ambiguity", "predictions": [], "combined_predictions": [], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-091", "scenario": "ambiguity", "predictions": [ { "start": 17, "end": 27, "label": "RELIGION", "text": "christlich", "score": 0.9146808981895447, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 48, "end": 60, "label": "PERSON", "text": "Personenname", "score": 0.6654636263847351, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 17, "end": 27, "label": "RELIGION", "text": "christlich", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" }, { "start": 48, "end": 60, "label": "PERSON", "text": "Personenname", "score": 0.6654636263847351, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [ [ 48, 60, "PERSON" ] ], "false_negatives": [], "exact_match": false, "error_code": null, "canaries": [ { "start": 17, "end": 27, "label": "RELIGION", "text": "christlich", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-092", "scenario": "ambiguity", "predictions": [ { "start": 22, "end": 31, "label": "RELIGION", "text": "Christian", "score": 0.9971898198127747, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 22, "end": 31, "label": "RELIGION", "text": "Christian", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 22, "end": 31, "label": "RELIGION", "text": "Christian", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-093", "scenario": "ambiguity", "predictions": [], "combined_predictions": [], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-094", "scenario": "ambiguity", "predictions": [], "combined_predictions": [], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-095", "scenario": "ambiguity", "predictions": [ { "start": 26, "end": 33, "label": "RELIGION", "text": "jüdisch", "score": 0.9290997982025146, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 26, "end": 33, "label": "RELIGION", "text": "jüdisch", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 26, "end": 33, "label": "RELIGION", "text": "jüdisch", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-096", "scenario": "ambiguity", "predictions": [ { "start": 44, "end": 52, "label": "RELIGION", "text": "Reformed", "score": 0.8552035689353943, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 44, "end": 52, "label": "RELIGION", "text": "Reformed", "score": 0.8552035689353943, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 44, "end": 52, "label": "RELIGION", "text": "Reformed", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-097", "scenario": "ocr_noise", "predictions": [ { "start": 9, "end": 13, "label": "CITY", "text": "laut", "score": 0.6887308955192566, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 20, "end": 30, "label": "RELIGION", "text": "Buddh1smus", "score": 0.8085448145866394, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 9, "end": 13, "label": "CITY", "text": "laut", "score": 0.6887308955192566, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 20, "end": 30, "label": "RELIGION", "text": "Buddh1smus", "score": 0.8085448145866394, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [ [ 9, 13, "CITY" ] ], "false_negatives": [], "exact_match": false, "error_code": null, "canaries": [ { "start": 20, "end": 30, "label": "RELIGION", "text": "Buddh1smus", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-098", "scenario": "ocr_noise", "predictions": [ { "start": 31, "end": 43, "label": "RELIGION", "text": "Christlanity", "score": 0.5787172317504883, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 31, "end": 43, "label": "RELIGION", "text": "Christlanity", "score": 0.5787172317504883, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 31, "end": 43, "label": "RELIGION", "text": "Christlanity", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-099", "scenario": "ocr_noise", "predictions": [], "combined_predictions": [ { "start": 24, "end": 40, "label": "RELIGION", "text": "konfessions-\nlos", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "dehyphenation-v1" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 24, "end": 40, "label": "RELIGION", "text": "konfessions-\nlos", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-100", "scenario": "ocr_noise", "predictions": [], "combined_predictions": [ { "start": 21, "end": 35, "label": "RELIGION", "text": "non-\nreligious", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "dehyphenation-v1" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 21, "end": 35, "label": "RELIGION", "text": "non-\nreligious", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-101", "scenario": "ocr_noise", "predictions": [ { "start": 11, "end": 21, "label": "RELIGION", "text": "katho1isch", "score": 0.7328035831451416, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 11, "end": 21, "label": "RELIGION", "text": "katho1isch", "score": 0.7328035831451416, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 11, "end": 21, "label": "RELIGION", "text": "katho1isch", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-102", "scenario": "ocr_noise", "predictions": [ { "start": 24, "end": 33, "label": "RELIGION", "text": "Hinduisrn", "score": 0.7377975583076477, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 24, "end": 33, "label": "RELIGION", "text": "Hinduisrn", "score": 0.7377975583076477, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 24, "end": 33, "label": "RELIGION", "text": "Hinduisrn", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-103", "scenario": "ocr_noise", "predictions": [], "combined_predictions": [ { "start": 18, "end": 30, "label": "RELIGION", "text": "agnos-\ntisch", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "dehyphenation-v1" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 18, "end": 30, "label": "RELIGION", "text": "agnos-\ntisch", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-104", "scenario": "ocr_noise", "predictions": [ { "start": 25, "end": 34, "label": "RELIGION", "text": "Juda-\nism", "score": 0.984998881816864, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "dehyphenation-v1" } ], "combined_predictions": [ { "start": 25, "end": 34, "label": "RELIGION", "text": "Juda-\nism", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "dehyphenation-v1" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 25, "end": 34, "label": "RELIGION", "text": "Juda-\nism", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-105", "scenario": "ocr_noise", "predictions": [ { "start": 19, "end": 24, "label": "PERSON", "text": "lslam", "score": 0.8412295579910278, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 19, "end": 24, "label": "PERSON", "text": "lslam", "score": 0.8412295579910278, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [ [ 19, 24, "PERSON" ] ], "false_negatives": [ [ 19, 24, "RELIGION" ] ], "exact_match": false, "error_code": null, "canaries": [ { "start": 19, "end": 24, "label": "RELIGION", "text": "lslam", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-106", "scenario": "ocr_noise", "predictions": [ { "start": 4, "end": 7, "label": "ORGANIZATION", "text": "OCR", "score": 0.993300199508667, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 18, "end": 25, "label": "RELIGION", "text": "S1khism", "score": 0.9311596751213074, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 4, "end": 7, "label": "ORGANIZATION", "text": "OCR", "score": 0.993300199508667, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 18, "end": 25, "label": "RELIGION", "text": "S1khism", "score": 0.9311596751213074, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [ [ 4, 7, "ORGANIZATION" ] ], "false_negatives": [], "exact_match": false, "error_code": null, "canaries": [ { "start": 18, "end": 25, "label": "RELIGION", "text": "S1khism", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-107", "scenario": "ocr_noise", "predictions": [ { "start": 35, "end": 49, "label": "RELIGION", "text": "buddhis-\ntisch", "score": 0.7703148126602173, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "dehyphenation-v1" } ], "combined_predictions": [ { "start": 35, "end": 49, "label": "RELIGION", "text": "buddhis-\ntisch", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "dehyphenation-v1" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 35, "end": 49, "label": "RELIGION", "text": "buddhis-\ntisch", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-108", "scenario": "ocr_noise", "predictions": [ { "start": 27, "end": 43, "label": "RELIGION", "text": "Zoroas-\ntrianism", "score": 0.9521776437759399, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "dehyphenation-v1" } ], "combined_predictions": [ { "start": 27, "end": 43, "label": "RELIGION", "text": "Zoroas-\ntrianism", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "dehyphenation-v1" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 27, "end": 43, "label": "RELIGION", "text": "Zoroas-\ntrianism", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-109", "scenario": "ocr_noise", "predictions": [ { "start": 14, "end": 27, "label": "PERSON", "text": "Fenja Torwald", "score": 0.9993094205856323, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 29, "end": 49, "label": "ORGANIZATION", "text": "Bekenntnis Judenturn", "score": 0.8337629437446594, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 14, "end": 27, "label": "PERSON", "text": "Fenja Torwald", "score": 0.9993094205856323, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 29, "end": 49, "label": "ORGANIZATION", "text": "Bekenntnis Judenturn", "score": 0.8337629437446594, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [ [ 29, 49, "ORGANIZATION" ] ], "false_negatives": [ [ 40, 49, "RELIGION" ] ], "exact_match": false, "error_code": null, "canaries": [ { "start": 14, "end": 27, "label": "PERSON", "text": "Fenja Torwald", "uncovered_positions": [] }, { "start": 40, "end": 49, "label": "RELIGION", "text": "Judenturn", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-110", "scenario": "ocr_noise", "predictions": [ { "start": 21, "end": 33, "label": "PERSON", "text": "Orin Wexford", "score": 0.9491236209869385, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 38, "end": 49, "label": "RELIGION", "text": "Cath0licism", "score": 0.8634129166603088, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 21, "end": 33, "label": "PERSON", "text": "Orin Wexford", "score": 0.9491236209869385, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 38, "end": 49, "label": "RELIGION", "text": "Cath0licism", "score": 0.8634129166603088, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 21, "end": 33, "label": "PERSON", "text": "Orin Wexford", "uncovered_positions": [] }, { "start": 38, "end": 49, "label": "RELIGION", "text": "Cath0licism", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-111", "scenario": "ocr_noise", "predictions": [], "combined_predictions": [], "false_positives": [], "false_negatives": [ [ 13, 28, "RELIGION" ] ], "exact_match": false, "error_code": null, "canaries": [ { "start": 13, "end": 28, "label": "RELIGION", "text": "konfesslonsfrei", "uncovered_positions": [ 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27 ] } ], "candidate_returned": true, "candidate_canaries": [ { "label": "RELIGION", "start": 13, "end": 28, "kind": "fully_unmasked" } ], "outcome": "candidate_fully_unmasked" }, { "case_id": "holdout03d-112", "scenario": "ocr_noise", "predictions": [ { "start": 21, "end": 29, "label": "ORGANIZATION", "text": "agn0stic", "score": 0.8037384748458862, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 21, "end": 29, "label": "ORGANIZATION", "text": "agn0stic", "score": 0.8037384748458862, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [ [ 21, 29, "ORGANIZATION" ] ], "false_negatives": [ [ 21, 29, "RELIGION" ] ], "exact_match": false, "error_code": null, "canaries": [ { "start": 21, "end": 29, "label": "RELIGION", "text": "agn0stic", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-113", "scenario": "multiple_entities", "predictions": [ { "start": 0, "end": 14, "label": "PERSON", "text": "Silja Dornbach", "score": 0.9992048144340515, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 19, "end": 26, "label": "CITY", "text": "Rostock", "score": 0.999563992023468, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 34, "end": 42, "label": "RELIGION", "text": "Judentum", "score": 0.9391108751296997, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 48, "end": 66, "label": "ORGANIZATION", "text": "Morgenkai Institut", "score": 0.9742982983589172, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 0, "end": 14, "label": "PERSON", "text": "Silja Dornbach", "score": 0.9992048144340515, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 19, "end": 26, "label": "CITY", "text": "Rostock", "score": 0.999563992023468, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 34, "end": 42, "label": "RELIGION", "text": "Judentum", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" }, { "start": 48, "end": 66, "label": "ORGANIZATION", "text": "Morgenkai Institut", "score": 0.9742982983589172, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 0, "end": 14, "label": "PERSON", "text": "Silja Dornbach", "uncovered_positions": [] }, { "start": 19, "end": 26, "label": "CITY", "text": "Rostock", "uncovered_positions": [] }, { "start": 34, "end": 42, "label": "RELIGION", "text": "Judentum", "uncovered_positions": [] }, { "start": 48, "end": 66, "label": "ORGANIZATION", "text": "Morgenkai Institut", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-114", "scenario": "multiple_entities", "predictions": [ { "start": 0, "end": 15, "label": "PERSON", "text": "Nolan Briarwood", "score": 0.9830585718154907, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 19, "end": 23, "label": "CITY", "text": "York", "score": 0.9994452595710754, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 30, "end": 37, "label": "RELIGION", "text": "Jainism", "score": 0.9943238496780396, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 45, "end": 62, "label": "ORGANIZATION", "text": "Cobalt Quay Trust", "score": 0.9601506590843201, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 0, "end": 15, "label": "PERSON", "text": "Nolan Briarwood", "score": 0.9830585718154907, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 19, "end": 23, "label": "CITY", "text": "York", "score": 0.9994452595710754, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 30, "end": 37, "label": "RELIGION", "text": "Jainism", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" }, { "start": 45, "end": 62, "label": "ORGANIZATION", "text": "Cobalt Quay Trust", "score": 0.9601506590843201, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 0, "end": 15, "label": "PERSON", "text": "Nolan Briarwood", "uncovered_positions": [] }, { "start": 19, "end": 23, "label": "CITY", "text": "York", "uncovered_positions": [] }, { "start": 30, "end": 37, "label": "RELIGION", "text": "Jainism", "uncovered_positions": [] }, { "start": 45, "end": 62, "label": "ORGANIZATION", "text": "Cobalt Quay Trust", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-115", "scenario": "multiple_entities", "predictions": [ { "start": 0, "end": 22, "label": "ORGANIZATION", "text": "Bernsteinpfad Stiftung", "score": 0.9246055483818054, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 32, "end": 47, "label": "PERSON", "text": "Henrik Wiesenau", "score": 0.9986317753791809, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 0, "end": 22, "label": "ORGANIZATION", "text": "Bernsteinpfad Stiftung", "score": 0.9246055483818054, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 32, "end": 47, "label": "PERSON", "text": "Henrik Wiesenau", "score": 0.9986317753791809, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 52, "end": 66, "label": "RELIGION", "text": "konfessionslos", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 0, "end": 22, "label": "ORGANIZATION", "text": "Bernsteinpfad Stiftung", "uncovered_positions": [] }, { "start": 32, "end": 47, "label": "PERSON", "text": "Henrik Wiesenau", "uncovered_positions": [] }, { "start": 52, "end": 66, "label": "RELIGION", "text": "konfessionslos", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-116", "scenario": "multiple_entities", "predictions": [ { "start": 4, "end": 26, "label": "ORGANIZATION", "text": "Silver Estuary Council", "score": 0.8049675822257996, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 34, "end": 47, "label": "PERSON", "text": "Ada Merefield", "score": 0.9995853304862976, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 4, "end": 26, "label": "ORGANIZATION", "text": "Silver Estuary Council", "score": 0.8049675822257996, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 34, "end": 47, "label": "PERSON", "text": "Ada Merefield", "score": 0.9995853304862976, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 50, "end": 57, "label": "RELIGION", "text": "atheism", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 4, "end": 26, "label": "ORGANIZATION", "text": "Silver Estuary Council", "uncovered_positions": [] }, { "start": 34, "end": 47, "label": "PERSON", "text": "Ada Merefield", "uncovered_positions": [] }, { "start": 50, "end": 57, "label": "RELIGION", "text": "atheism", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-117", "scenario": "multiple_entities", "predictions": [ { "start": 0, "end": 12, "label": "PERSON", "text": "Ida Talbrück", "score": 0.9952776432037354, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 19, "end": 24, "label": "RELIGION", "text": "Islam", "score": 0.9999401569366455, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 35, "end": 58, "label": "EMAIL", "text": "ida.tal@example.invalid", "score": 0.8027673959732056, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 0, "end": 12, "label": "PERSON", "text": "Ida Talbrück", "score": 0.9952776432037354, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 19, "end": 24, "label": "RELIGION", "text": "Islam", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" }, { "start": 35, "end": 58, "label": "EMAIL", "text": "ida.tal@example.invalid", "score": null, "source": "deterministic", "model_revision": "contact-patterns-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 0, "end": 12, "label": "PERSON", "text": "Ida Talbrück", "uncovered_positions": [] }, { "start": 19, "end": 24, "label": "RELIGION", "text": "Islam", "uncovered_positions": [] }, { "start": 35, "end": 58, "label": "EMAIL", "text": "ida.tal@example.invalid", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-118", "scenario": "multiple_entities", "predictions": [ { "start": 0, "end": 13, "label": "PERSON", "text": "Lyle Ashcombe", "score": 0.9961810111999512, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 20, "end": 28, "label": "RELIGION", "text": "Buddhism", "score": 0.9979920387268066, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 0, "end": 13, "label": "PERSON", "text": "Lyle Ashcombe", "score": 0.9961810111999512, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 20, "end": 28, "label": "RELIGION", "text": "Buddhism", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" }, { "start": 38, "end": 60, "label": "EMAIL", "text": "lyle.a@example.invalid", "score": null, "source": "deterministic", "model_revision": "contact-patterns-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 0, "end": 13, "label": "PERSON", "text": "Lyle Ashcombe", "uncovered_positions": [] }, { "start": 20, "end": 28, "label": "RELIGION", "text": "Buddhism", "uncovered_positions": [] }, { "start": 38, "end": 60, "label": "EMAIL", "text": "lyle.a@example.invalid", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-119", "scenario": "multiple_entities", "predictions": [ { "start": 13, "end": 27, "label": "PERSON", "text": "Arvid Sandborn", "score": 0.998414158821106, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 29, "end": 43, "label": "PHONE", "text": "+49 381 482701", "score": 0.9627817869186401, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 13, "end": 27, "label": "PERSON", "text": "Arvid Sandborn", "score": 0.998414158821106, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 29, "end": 43, "label": "PHONE", "text": "+49 381 482701", "score": null, "source": "deterministic", "model_revision": "contact-patterns-v1", "normalization": "identity" }, { "start": 52, "end": 64, "label": "RELIGION", "text": "religionslos", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 13, "end": 27, "label": "PERSON", "text": "Arvid Sandborn", "uncovered_positions": [] }, { "start": 29, "end": 43, "label": "PHONE", "text": "+49 381 482701", "uncovered_positions": [] }, { "start": 52, "end": 64, "label": "RELIGION", "text": "religionslos", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-120", "scenario": "multiple_entities", "predictions": [ { "start": 5, "end": 17, "label": "PERSON", "text": "Mira Fenwick", "score": 0.9872230291366577, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 21, "end": 36, "label": "PHONE", "text": "+44 1904 582701", "score": 0.9610803127288818, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 45, "end": 53, "label": "RELIGION", "text": "Hinduism", "score": 0.9806439876556396, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 5, "end": 17, "label": "PERSON", "text": "Mira Fenwick", "score": 0.9872230291366577, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 21, "end": 36, "label": "PHONE", "text": "+44 1904 582701", "score": null, "source": "deterministic", "model_revision": "contact-patterns-v1", "normalization": "identity" }, { "start": 45, "end": 53, "label": "RELIGION", "text": "Hinduism", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 5, "end": 17, "label": "PERSON", "text": "Mira Fenwick", "uncovered_positions": [] }, { "start": 21, "end": 36, "label": "PHONE", "text": "+44 1904 582701", "uncovered_positions": [] }, { "start": 45, "end": 53, "label": "RELIGION", "text": "Hinduism", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-121", "scenario": "multiple_entities", "predictions": [ { "start": 0, "end": 6, "label": "CITY", "text": "Erfurt", "score": 0.9716014862060547, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 8, "end": 21, "label": "PERSON", "text": "Svea Morgenau", "score": 0.8673108220100403, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 33, "end": 43, "label": "RELIGION", "text": "katholisch", "score": 0.7634422779083252, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 47, "end": 57, "label": "RELIGION", "text": "agnostisch", "score": 0.5659558773040771, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 0, "end": 6, "label": "CITY", "text": "Erfurt", "score": 0.9716014862060547, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 8, "end": 21, "label": "PERSON", "text": "Svea Morgenau", "score": 0.8673108220100403, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 33, "end": 43, "label": "RELIGION", "text": "katholisch", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" }, { "start": 47, "end": 57, "label": "RELIGION", "text": "agnostisch", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 0, "end": 6, "label": "CITY", "text": "Erfurt", "uncovered_positions": [] }, { "start": 8, "end": 21, "label": "PERSON", "text": "Svea Morgenau", "uncovered_positions": [] }, { "start": 33, "end": 43, "label": "RELIGION", "text": "katholisch", "uncovered_positions": [] }, { "start": 47, "end": 57, "label": "RELIGION", "text": "agnostisch", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-122", "scenario": "multiple_entities", "predictions": [ { "start": 0, "end": 7, "label": "CITY", "text": "Bristol", "score": 0.9915996789932251, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 9, "end": 22, "label": "PERSON", "text": "Owen Thornley", "score": 0.9985686540603638, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 32, "end": 44, "label": "RELIGION", "text": "Christianity", "score": 0.9914804697036743, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 0, "end": 7, "label": "CITY", "text": "Bristol", "score": 0.9915996789932251, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 9, "end": 22, "label": "PERSON", "text": "Owen Thornley", "score": 0.9985686540603638, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 32, "end": 44, "label": "RELIGION", "text": "Christianity", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" }, { "start": 50, "end": 61, "label": "RELIGION", "text": "no religion", "score": null, "source": "deterministic", "model_revision": "religious-status-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 0, "end": 7, "label": "CITY", "text": "Bristol", "uncovered_positions": [] }, { "start": 9, "end": 22, "label": "PERSON", "text": "Owen Thornley", "uncovered_positions": [] }, { "start": 32, "end": 44, "label": "RELIGION", "text": "Christianity", "uncovered_positions": [] }, { "start": 50, "end": 61, "label": "RELIGION", "text": "no religion", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-123", "scenario": "multiple_entities", "predictions": [ { "start": 0, "end": 13, "label": "PERSON", "text": "Jona Falkenau", "score": 0.9965296387672424, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 18, "end": 31, "label": "PERSON", "text": "Jona Falkenau", "score": 0.9980782270431519, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 47, "end": 56, "label": "RELIGION", "text": "Sikhismus", "score": 0.9455693364143372, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 0, "end": 13, "label": "PERSON", "text": "Jona Falkenau", "score": 0.9965296387672424, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 18, "end": 31, "label": "PERSON", "text": "Jona Falkenau", "score": 0.9980782270431519, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 47, "end": 56, "label": "RELIGION", "text": "Sikhismus", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 0, "end": 13, "label": "PERSON", "text": "Jona Falkenau", "uncovered_positions": [] }, { "start": 18, "end": 31, "label": "PERSON", "text": "Jona Falkenau", "uncovered_positions": [] }, { "start": 47, "end": 56, "label": "RELIGION", "text": "Sikhismus", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-124", "scenario": "multiple_entities", "predictions": [ { "start": 0, "end": 12, "label": "PERSON", "text": "Esme Rowanby", "score": 0.9983696341514587, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 22, "end": 28, "label": "RELIGION", "text": "Taoism", "score": 0.9475066661834717, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 42, "end": 54, "label": "PERSON", "text": "Esme Rowanby", "score": 0.9983136653900146, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 0, "end": 12, "label": "PERSON", "text": "Esme Rowanby", "score": 0.9983696341514587, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 22, "end": 28, "label": "RELIGION", "text": "Taoism", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" }, { "start": 42, "end": 54, "label": "PERSON", "text": "Esme Rowanby", "score": 0.9983136653900146, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 0, "end": 12, "label": "PERSON", "text": "Esme Rowanby", "uncovered_positions": [] }, { "start": 22, "end": 28, "label": "RELIGION", "text": "Taoism", "uncovered_positions": [] }, { "start": 42, "end": 54, "label": "PERSON", "text": "Esme Rowanby", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-125", "scenario": "multiple_entities", "predictions": [ { "start": 23, "end": 33, "label": "ORGANIZATION", "text": "Hafenbogen", "score": 0.7308627963066101, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 43, "end": 47, "label": "CITY", "text": "Kiel", "score": 0.9980584979057312, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 49, "end": 60, "label": "RELIGION", "text": "evangelisch", "score": 0.7706614136695862, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 23, "end": 33, "label": "ORGANIZATION", "text": "Hafenbogen", "score": 0.7308627963066101, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 43, "end": 47, "label": "CITY", "text": "Kiel", "score": 0.9980584979057312, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 49, "end": 60, "label": "RELIGION", "text": "evangelisch", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [ [ 23, 33, "ORGANIZATION" ] ], "false_negatives": [ [ 23, 41, "ORGANIZATION" ] ], "exact_match": false, "error_code": null, "canaries": [ { "start": 23, "end": 41, "label": "ORGANIZATION", "text": "Hafenbogen Archivs", "uncovered_positions": [ 34, 35, 36, 37, 38, 39, 40 ] }, { "start": 43, "end": 47, "label": "CITY", "text": "Kiel", "uncovered_positions": [] }, { "start": 49, "end": 60, "label": "RELIGION", "text": "evangelisch", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [ { "label": "ORGANIZATION", "start": 23, "end": 41, "kind": "partial_mask" } ], "outcome": "candidate_partial_mask" }, { "case_id": "holdout03d-126", "scenario": "multiple_entities", "predictions": [ { "start": 2, "end": 22, "label": "ORGANIZATION", "text": "Maple Tide Institute", "score": 0.9682912230491638, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 35, "end": 41, "label": "CITY", "text": "Durham", "score": 0.99836665391922, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 51, "end": 58, "label": "RELIGION", "text": "Judaism", "score": 0.9933406710624695, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 2, "end": 22, "label": "ORGANIZATION", "text": "Maple Tide Institute", "score": 0.9682912230491638, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 35, "end": 41, "label": "CITY", "text": "Durham", "score": 0.99836665391922, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 51, "end": 58, "label": "RELIGION", "text": "Judaism", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 2, "end": 22, "label": "ORGANIZATION", "text": "Maple Tide Institute", "uncovered_positions": [] }, { "start": 35, "end": 41, "label": "CITY", "text": "Durham", "uncovered_positions": [] }, { "start": 51, "end": 58, "label": "RELIGION", "text": "Judaism", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-127", "scenario": "multiple_entities", "predictions": [ { "start": 0, "end": 13, "label": "PERSON", "text": "Liv Hohenrain", "score": 0.9968641400337219, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 20, "end": 24, "label": "CITY", "text": "Bonn", "score": 0.9716768860816956, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 29, "end": 34, "label": "PERSON", "text": "Bahai", "score": 0.8550848364830017, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 36, "end": 57, "label": "EMAIL", "text": "liv.h@example.invalid", "score": 0.7995248436927795, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [], "false_positives": [], "false_negatives": [ [ 0, 13, "PERSON" ], [ 20, 24, "CITY" ], [ 29, 34, "RELIGION" ], [ 36, 57, "EMAIL" ] ], "exact_match": false, "error_code": "ambiguous_entity_overlap", "canaries": [ { "start": 0, "end": 13, "label": "PERSON", "text": "Liv Hohenrain", "uncovered_positions": [ 0, 1, 2, 4, 5, 6, 7, 8, 9, 10, 11, 12 ] }, { "start": 20, "end": 24, "label": "CITY", "text": "Bonn", "uncovered_positions": [ 20, 21, 22, 23 ] }, { "start": 29, "end": 34, "label": "RELIGION", "text": "Bahai", "uncovered_positions": [ 29, 30, 31, 32, 33 ] }, { "start": 36, "end": 57, "label": "EMAIL", "text": "liv.h@example.invalid", "uncovered_positions": [ 36, 37, 38, 40, 42, 43, 44, 45, 46, 47, 48, 50, 51, 52, 53, 54, 55, 56 ] } ], "candidate_returned": false, "candidate_canaries": [], "outcome": "blocked_conflicting_spans" }, { "case_id": "holdout03d-128", "scenario": "multiple_entities", "predictions": [ { "start": 0, "end": 14, "label": "PERSON", "text": "Reed Alderwick", "score": 0.98525071144104, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 16, "end": 20, "label": "CITY", "text": "Bath", "score": 0.9997182488441467, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 22, "end": 33, "label": "RELIGION", "text": "agnosticism", "score": 0.9372888803482056, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 35, "end": 57, "label": "EMAIL", "text": "reed.a@example.invalid", "score": 0.644429087638855, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 0, "end": 14, "label": "PERSON", "text": "Reed Alderwick", "score": 0.98525071144104, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 16, "end": 20, "label": "CITY", "text": "Bath", "score": 0.9997182488441467, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 22, "end": 33, "label": "RELIGION", "text": "agnosticism", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" }, { "start": 35, "end": 57, "label": "EMAIL", "text": "reed.a@example.invalid", "score": null, "source": "deterministic", "model_revision": "contact-patterns-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 0, "end": 14, "label": "PERSON", "text": "Reed Alderwick", "uncovered_positions": [] }, { "start": 16, "end": 20, "label": "CITY", "text": "Bath", "uncovered_positions": [] }, { "start": 22, "end": 33, "label": "RELIGION", "text": "agnosticism", "uncovered_positions": [] }, { "start": 35, "end": 57, "label": "EMAIL", "text": "reed.a@example.invalid", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-129", "scenario": "formats_repetition", "predictions": [ { "start": 15, "end": 20, "label": "RELIGION", "text": "Islam", "score": 0.9793844223022461, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 15, "end": 20, "label": "RELIGION", "text": "Islam", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 15, "end": 20, "label": "RELIGION", "text": "Islam", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-130", "scenario": "formats_repetition", "predictions": [ { "start": 11, "end": 19, "label": "RELIGION", "text": "Buddhism", "score": 0.9502246379852295, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 11, "end": 19, "label": "RELIGION", "text": "Buddhism", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 11, "end": 19, "label": "RELIGION", "text": "Buddhism", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-131", "scenario": "formats_repetition", "predictions": [], "combined_predictions": [ { "start": 9, "end": 24, "label": "RELIGION", "text": "konfessionsfrei", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" }, { "start": 35, "end": 50, "label": "RELIGION", "text": "konfessionsfrei", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 9, "end": 24, "label": "RELIGION", "text": "konfessionsfrei", "uncovered_positions": [] }, { "start": 35, "end": 50, "label": "RELIGION", "text": "konfessionsfrei", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-132", "scenario": "formats_repetition", "predictions": [], "combined_predictions": [ { "start": 9, "end": 16, "label": "RELIGION", "text": "atheist", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" }, { "start": 27, "end": 34, "label": "RELIGION", "text": "atheist", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 9, "end": 16, "label": "RELIGION", "text": "atheist", "uncovered_positions": [] }, { "start": 27, "end": 34, "label": "RELIGION", "text": "atheist", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-133", "scenario": "formats_repetition", "predictions": [ { "start": 9, "end": 17, "label": "RELIGION", "text": "Judentum", "score": 0.9829030632972717, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 9, "end": 17, "label": "RELIGION", "text": "Judentum", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 9, "end": 17, "label": "RELIGION", "text": "Judentum", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-134", "scenario": "formats_repetition", "predictions": [ { "start": 7, "end": 13, "label": "RELIGION", "text": "Shinto", "score": 0.9690591096878052, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 7, "end": 13, "label": "RELIGION", "text": "Shinto", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 7, "end": 13, "label": "RELIGION", "text": "Shinto", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-135", "scenario": "formats_repetition", "predictions": [ { "start": 18, "end": 31, "label": "RELIGION", "text": "Agnostizismus", "score": 0.5336430072784424, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 18, "end": 31, "label": "RELIGION", "text": "Agnostizismus", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 18, "end": 31, "label": "RELIGION", "text": "Agnostizismus", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-136", "scenario": "formats_repetition", "predictions": [ { "start": 10, "end": 17, "label": "RELIGION", "text": "Sikhism", "score": 0.973417341709137, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 10, "end": 17, "label": "RELIGION", "text": "Sikhism", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 10, "end": 17, "label": "RELIGION", "text": "Sikhism", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-137", "scenario": "formats_repetition", "predictions": [ { "start": 11, "end": 24, "label": "ORGANIZATION", "text": "Katholizismus", "score": 0.6781677603721619, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 35, "end": 48, "label": "ORGANIZATION", "text": "Katholizismus", "score": 0.73296058177948, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [], "false_positives": [], "false_negatives": [ [ 11, 24, "RELIGION" ], [ 35, 48, "RELIGION" ] ], "exact_match": false, "error_code": "ambiguous_entity_overlap", "canaries": [ { "start": 11, "end": 24, "label": "RELIGION", "text": "Katholizismus", "uncovered_positions": [ 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23 ] }, { "start": 35, "end": 48, "label": "RELIGION", "text": "Katholizismus", "uncovered_positions": [ 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47 ] } ], "candidate_returned": false, "candidate_canaries": [], "outcome": "blocked_conflicting_spans" }, { "case_id": "holdout03d-138", "scenario": "formats_repetition", "predictions": [ { "start": 7, "end": 15, "label": "RELIGION", "text": "Hinduism", "score": 0.9410762786865234, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 26, "end": 34, "label": "RELIGION", "text": "Hinduism", "score": 0.9376835227012634, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 7, "end": 15, "label": "RELIGION", "text": "Hinduism", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" }, { "start": 26, "end": 34, "label": "RELIGION", "text": "Hinduism", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 7, "end": 15, "label": "RELIGION", "text": "Hinduism", "uncovered_positions": [] }, { "start": 26, "end": 34, "label": "RELIGION", "text": "Hinduism", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-139", "scenario": "formats_repetition", "predictions": [], "combined_predictions": [ { "start": 13, "end": 28, "label": "RELIGION", "text": "ohne Konfession", "score": null, "source": "deterministic", "model_revision": "religious-status-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 13, "end": 28, "label": "RELIGION", "text": "ohne Konfession", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-140", "scenario": "formats_repetition", "predictions": [], "combined_predictions": [ { "start": 11, "end": 24, "label": "RELIGION", "text": "non-religious", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 11, "end": 24, "label": "RELIGION", "text": "non-religious", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-141", "scenario": "formats_repetition", "predictions": [ { "start": 0, "end": 11, "label": "RELIGION", "text": "Christentum", "score": 0.8982365727424622, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 14, "end": 23, "label": "RELIGION", "text": "Atheismus", "score": 0.7216019034385681, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 0, "end": 11, "label": "RELIGION", "text": "Christentum", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" }, { "start": 14, "end": 23, "label": "RELIGION", "text": "Atheismus", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 0, "end": 11, "label": "RELIGION", "text": "Christentum", "uncovered_positions": [] }, { "start": 14, "end": 23, "label": "RELIGION", "text": "Atheismus", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-142", "scenario": "formats_repetition", "predictions": [ { "start": 0, "end": 12, "label": "RELIGION", "text": "Christianity", "score": 0.9415823221206665, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 15, "end": 26, "label": "RELIGION", "text": "agnosticism", "score": 0.61921626329422, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 0, "end": 12, "label": "RELIGION", "text": "Christianity", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" }, { "start": 15, "end": 26, "label": "RELIGION", "text": "agnosticism", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 0, "end": 12, "label": "RELIGION", "text": "Christianity", "uncovered_positions": [] }, { "start": 15, "end": 26, "label": "RELIGION", "text": "agnosticism", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-143", "scenario": "formats_repetition", "predictions": [ { "start": 11, "end": 21, "label": "RELIGION", "text": "Buddhismus", "score": 0.7847886085510254, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 11, "end": 21, "label": "RELIGION", "text": "Buddhismus", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 11, "end": 21, "label": "RELIGION", "text": "Buddhismus", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-144", "scenario": "formats_repetition", "predictions": [ { "start": 9, "end": 16, "label": "RELIGION", "text": "Jainism", "score": 0.7793956995010376, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 9, "end": 16, "label": "RELIGION", "text": "Jainism", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 9, "end": 16, "label": "RELIGION", "text": "Jainism", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-145", "scenario": "negative_controls", "predictions": [], "combined_predictions": [], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-146", "scenario": "negative_controls", "predictions": [], "combined_predictions": [], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-147", "scenario": "negative_controls", "predictions": [], "combined_predictions": [], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-148", "scenario": "negative_controls", "predictions": [], "combined_predictions": [], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-149", "scenario": "negative_controls", "predictions": [], "combined_predictions": [], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-150", "scenario": "negative_controls", "predictions": [], "combined_predictions": [], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-151", "scenario": "negative_controls", "predictions": [], "combined_predictions": [], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-152", "scenario": "negative_controls", "predictions": [], "combined_predictions": [], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-153", "scenario": "negative_controls", "predictions": [], "combined_predictions": [], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-154", "scenario": "negative_controls", "predictions": [], "combined_predictions": [], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-155", "scenario": "negative_controls", "predictions": [], "combined_predictions": [], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-156", "scenario": "negative_controls", "predictions": [], "combined_predictions": [], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-157", "scenario": "negative_controls", "predictions": [], "combined_predictions": [], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-158", "scenario": "negative_controls", "predictions": [], "combined_predictions": [], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-159", "scenario": "negative_controls", "predictions": [], "combined_predictions": [], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-160", "scenario": "negative_controls", "predictions": [], "combined_predictions": [], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" } ], "automated_release_eligible": false, "recognition_qualified": false } }, "status_scoped_ocr": { "qualification": { "schema_version": "development-candidate-outcomes/v1", "accuracy": { "documents": 40, "ner_entity_metrics": { "PERSON": { "tp": 18, "fp": 3, "fn": 2, "precision": 0.8571428571428571, "recall": 0.9, "f1": 0.8780487804878048 }, "CITY": { "tp": 16, "fp": 2, "fn": 1, "precision": 0.8888888888888888, "recall": 0.9411764705882353, "f1": 0.9142857142857143 }, "RELIGION": { "tp": 6, "fp": 0, "fn": 0, "precision": 1.0, "recall": 1.0, "f1": 1.0 }, "EMAIL": { "tp": 1, "fp": 0, "fn": 1, "precision": 1.0, "recall": 0.5, "f1": 0.6666666666666666 }, "PHONE": { "tp": 1, "fp": 0, "fn": 0, "precision": 1.0, "recall": 1.0, "f1": 1.0 }, "ORGANIZATION": { "tp": 1, "fp": 2, "fn": 1, "precision": 0.3333333333333333, "recall": 0.5, "f1": 0.4 } }, "combined_entity_metrics": { "PERSON": { "tp": 18, "fp": 3, "fn": 2, "precision": 0.8571428571428571, "recall": 0.9, "f1": 0.8780487804878048 }, "CITY": { "tp": 16, "fp": 1, "fn": 1, "precision": 0.9411764705882353, "recall": 0.9411764705882353, "f1": 0.9411764705882353 }, "RELIGION": { "tp": 6, "fp": 0, "fn": 0, "precision": 1.0, "recall": 1.0, "f1": 1.0 }, "EMAIL": { "tp": 2, "fp": 0, "fn": 0, "precision": 1.0, "recall": 1.0, "f1": 1.0 }, "PHONE": { "tp": 1, "fp": 0, "fn": 0, "precision": 1.0, "recall": 1.0, "f1": 1.0 }, "ORGANIZATION": { "tp": 1, "fp": 2, "fn": 1, "precision": 0.3333333333333333, "recall": 0.5, "f1": 0.4 } }, "document_exact_match_rate": 0.875, "groups": { "de": { "documents": 10, "exact_matches": 8, "false_negatives": 1 }, "en": { "documents": 29, "exact_matches": 26, "false_negatives": 3 }, "mixed": { "documents": 1, "exact_matches": 1, "false_negatives": 0 }, "ocr_noise": { "documents": 4, "exact_matches": 2, "false_negatives": 2 } }, "blocked_documents": 0, "failure_codes": {}, "automated_release_eligible": false, "uncovered_source_canaries_including_incomplete": { "PERSON": 0, "CITY": 0, "RELIGION": 0, "EMAIL": 0, "PHONE": 0, "ORGANIZATION": 0 }, "literal_presence_or_incomplete": { "PERSON": 0, "CITY": 0, "RELIGION": 0, "EMAIL": 0, "PHONE": 0, "ORGANIZATION": 0 }, "partially_covered_source_canaries_including_incomplete": { "PERSON": 0, "CITY": 0, "RELIGION": 0, "EMAIL": 0, "PHONE": 0, "ORGANIZATION": 0 }, "fully_uncovered_source_canaries_including_incomplete": { "PERSON": 0, "CITY": 0, "RELIGION": 0, "EMAIL": 0, "PHONE": 0, "ORGANIZATION": 0 }, "all_document_source_coverage": { "PERSON": { "covered": 208, "total": 208 }, "CITY": { "covered": 118, "total": 118 }, "RELIGION": { "covered": 50, "total": 50 }, "EMAIL": { "covered": 27, "total": 27 }, "PHONE": { "covered": 7, "total": 7 }, "ORGANIZATION": { "covered": 11, "total": 11 } }, "conservative_accuracy_gate_passed": true, "scope": "all source spans; incomplete processing counts as missing, not leaked output", "deterministic_replay_latency_ms": { "median": 0.08984049782156944, "max": 1.0899319968302734 } }, "candidate_outcomes": { "complete_no_measured_leakage": 40, "candidate_partial_mask": 0, "candidate_fully_unmasked": 0, "blocked_conflicting_spans": 0, "blocked_model_runtime": 0, "other_incomplete": 0 }, "candidate_return_rate": 1.0, "safe_abstention_rate": 0.0, "incomplete_rate": 0.0, "no_measured_leakage_rate_among_returned": 1.0, "candidate_partial_canaries": { "PERSON": 0, "CITY": 0, "RELIGION": 0, "EMAIL": 0, "PHONE": 0, "ORGANIZATION": 0 }, "candidate_fully_unmasked_canaries": { "PERSON": 0, "CITY": 0, "RELIGION": 0, "EMAIL": 0, "PHONE": 0, "ORGANIZATION": 0 }, "returned_candidate_entity_metrics": { "PERSON": { "tp": 18, "fp": 3, "fn": 2, "precision": 0.8571428571428571, "recall": 0.9, "f1": 0.8780487804878048 }, "CITY": { "tp": 16, "fp": 1, "fn": 1, "precision": 0.9411764705882353, "recall": 0.9411764705882353, "f1": 0.9411764705882353 }, "RELIGION": { "tp": 6, "fp": 0, "fn": 0, "precision": 1.0, "recall": 1.0, "f1": 1.0 }, "EMAIL": { "tp": 2, "fp": 0, "fn": 0, "precision": 1.0, "recall": 1.0, "f1": 1.0 }, "PHONE": { "tp": 1, "fp": 0, "fn": 0, "precision": 1.0, "recall": 1.0, "f1": 1.0 }, "ORGANIZATION": { "tp": 1, "fp": 2, "fn": 1, "precision": 0.3333333333333333, "recall": 0.5, "f1": 0.4 } }, "per_case": [ { "case_id": "baseline-00", "scenario": null, "predictions": [ { "start": 0, "end": 16, "label": "PERSON", "text": "Lina Wolkenstein", "score": 0.9969016313552856, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 26, "end": 32, "label": "CITY", "text": "Berlin", "score": 0.9999364614486694, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 0, "end": 16, "label": "PERSON", "text": "Lina Wolkenstein", "score": 0.9969016313552856, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 26, "end": 32, "label": "CITY", "text": "Berlin", "score": 0.9999364614486694, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 0, "end": 16, "label": "PERSON", "text": "Lina Wolkenstein", "uncovered_positions": [] }, { "start": 26, "end": 32, "label": "CITY", "text": "Berlin", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "baseline-01", "scenario": null, "predictions": [ { "start": 0, "end": 11, "label": "PERSON", "text": "Hans Müller", "score": 0.9912292957305908, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 0, "end": 11, "label": "PERSON", "text": "Hans Müller", "score": 0.9912292957305908, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 0, "end": 11, "label": "PERSON", "text": "Hans Müller", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "baseline-02", "scenario": null, "predictions": [ { "start": 13, "end": 30, "label": "PERSON", "text": "Dr. Klaus Schmidt", "score": 0.9196021556854248, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 13, "end": 30, "label": "PERSON", "text": "Dr. Klaus Schmidt", "score": 0.9196021556854248, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [ [ 13, 30, "PERSON" ] ], "false_negatives": [ [ 17, 30, "PERSON" ] ], "exact_match": false, "error_code": null, "canaries": [ { "start": 17, "end": 30, "label": "PERSON", "text": "Klaus Schmidt", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "baseline-03", "scenario": null, "predictions": [ { "start": 0, "end": 17, "label": "PERSON", "text": "Friedrich Wilhelm", "score": 0.9821144938468933, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 0, "end": 17, "label": "PERSON", "text": "Friedrich Wilhelm", "score": 0.9821144938468933, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 0, "end": 17, "label": "PERSON", "text": "Friedrich Wilhelm", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "baseline-04", "scenario": null, "predictions": [ { "start": 15, "end": 35, "label": "PERSON", "text": "Anne-Marie Schneider", "score": 0.981816291809082, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 15, "end": 35, "label": "PERSON", "text": "Anne-Marie Schneider", "score": 0.981816291809082, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 15, "end": 35, "label": "PERSON", "text": "Anne-Marie Schneider", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "baseline-05", "scenario": null, "predictions": [ { "start": 0, "end": 8, "label": "PERSON", "text": "John Doe", "score": 0.9945249557495117, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 0, "end": 8, "label": "PERSON", "text": "John Doe", "score": 0.9945249557495117, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 0, "end": 8, "label": "PERSON", "text": "John Doe", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "baseline-06", "scenario": null, "predictions": [ { "start": 0, "end": 5, "label": "PERSON", "text": "Alice", "score": 0.9985244870185852, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 10, "end": 13, "label": "PERSON", "text": "Bob", "score": 0.9991118311882019, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 26, "end": 31, "label": "CITY", "text": "store", "score": 0.5467662811279297, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 0, "end": 5, "label": "PERSON", "text": "Alice", "score": 0.9985244870185852, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 10, "end": 13, "label": "PERSON", "text": "Bob", "score": 0.9991118311882019, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 26, "end": 31, "label": "CITY", "text": "store", "score": 0.5467662811279297, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [ [ 26, 31, "CITY" ] ], "false_negatives": [], "exact_match": false, "error_code": null, "canaries": [ { "start": 0, "end": 5, "label": "PERSON", "text": "Alice", "uncovered_positions": [] }, { "start": 10, "end": 13, "label": "PERSON", "text": "Bob", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "baseline-07", "scenario": null, "predictions": [ { "start": 3, "end": 15, "label": "PERSON", "text": "Mary Johnson", "score": 0.9960722923278809, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 3, "end": 15, "label": "PERSON", "text": "Mary Johnson", "score": 0.9960722923278809, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 3, "end": 15, "label": "PERSON", "text": "Mary Johnson", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "baseline-08", "scenario": null, "predictions": [ { "start": 17, "end": 35, "label": "PERSON", "text": "Charles Montgomery", "score": 0.9641899466514587, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 17, "end": 35, "label": "PERSON", "text": "Charles Montgomery", "score": 0.9641899466514587, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 17, "end": 35, "label": "PERSON", "text": "Charles Montgomery", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "baseline-09", "scenario": null, "predictions": [ { "start": 0, "end": 16, "label": "PERSON", "text": "Elizabeth Taylor", "score": 0.9948871731758118, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 0, "end": 16, "label": "PERSON", "text": "Elizabeth Taylor", "score": 0.9948871731758118, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 0, "end": 16, "label": "PERSON", "text": "Elizabeth Taylor", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "baseline-10", "scenario": null, "predictions": [ { "start": 10, "end": 18, "label": "CITY", "text": "New York", "score": 0.9997181296348572, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 10, "end": 18, "label": "CITY", "text": "New York", "score": 0.9997181296348572, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 10, "end": 18, "label": "CITY", "text": "New York", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "baseline-11", "scenario": null, "predictions": [ { "start": 13, "end": 20, "label": "CITY", "text": "München", "score": 0.9988282322883606, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 13, "end": 20, "label": "CITY", "text": "München", "score": 0.9988282322883606, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 13, "end": 20, "label": "CITY", "text": "München", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "baseline-12", "scenario": null, "predictions": [ { "start": 0, "end": 5, "label": "CITY", "text": "Paris", "score": 0.9994774460792542, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 0, "end": 5, "label": "CITY", "text": "Paris", "score": 0.9994774460792542, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 0, "end": 5, "label": "CITY", "text": "Paris", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "baseline-13", "scenario": null, "predictions": [ { "start": 9, "end": 18, "label": "CITY", "text": "Frankfurt", "score": 0.9987022876739502, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 24, "end": 29, "label": "CITY", "text": "Tokyo", "score": 0.9982670545578003, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 9, "end": 18, "label": "CITY", "text": "Frankfurt", "score": 0.9987022876739502, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 24, "end": 29, "label": "CITY", "text": "Tokyo", "score": 0.9982670545578003, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 9, "end": 18, "label": "CITY", "text": "Frankfurt", "uncovered_positions": [] }, { "start": 24, "end": 29, "label": "CITY", "text": "Tokyo", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "baseline-14", "scenario": null, "predictions": [ { "start": 11, "end": 22, "label": "CITY", "text": "Los Angeles", "score": 0.9923924207687378, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 24, "end": 34, "label": "CITY", "text": "California", "score": 0.9569051861763, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 11, "end": 22, "label": "CITY", "text": "Los Angeles", "score": 0.9923924207687378, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 24, "end": 34, "label": "CITY", "text": "California", "score": 0.9569051861763, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 11, "end": 22, "label": "CITY", "text": "Los Angeles", "uncovered_positions": [] }, { "start": 24, "end": 34, "label": "CITY", "text": "California", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "baseline-15", "scenario": null, "predictions": [ { "start": 32, "end": 43, "label": "RELIGION", "text": "evangelisch", "score": 0.8318354487419128, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 32, "end": 43, "label": "RELIGION", "text": "evangelisch", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 32, "end": 43, "label": "RELIGION", "text": "evangelisch", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "baseline-16", "scenario": null, "predictions": [ { "start": 0, "end": 2, "label": "PERSON", "text": "Er", "score": 0.6436470746994019, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 7, "end": 17, "label": "RELIGION", "text": "katholisch", "score": 0.7684664726257324, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 0, "end": 2, "label": "PERSON", "text": "Er", "score": 0.6436470746994019, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 7, "end": 17, "label": "RELIGION", "text": "katholisch", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [ [ 0, 2, "PERSON" ] ], "false_negatives": [], "exact_match": false, "error_code": null, "canaries": [ { "start": 7, "end": 17, "label": "RELIGION", "text": "katholisch", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "baseline-17", "scenario": null, "predictions": [ { "start": 14, "end": 21, "label": "RELIGION", "text": "Judaism", "score": 0.9957650899887085, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 14, "end": 21, "label": "RELIGION", "text": "Judaism", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 14, "end": 21, "label": "RELIGION", "text": "Judaism", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "baseline-18", "scenario": null, "predictions": [ { "start": 15, "end": 21, "label": "RELIGION", "text": "Muslim", "score": 0.966416597366333, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 15, "end": 21, "label": "RELIGION", "text": "Muslim", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 15, "end": 21, "label": "RELIGION", "text": "Muslim", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "baseline-19", "scenario": null, "predictions": [ { "start": 0, "end": 8, "label": "RELIGION", "text": "Buddhist", "score": 0.7523731589317322, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 0, "end": 8, "label": "RELIGION", "text": "Buddhist", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 0, "end": 8, "label": "RELIGION", "text": "Buddhist", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "baseline-20", "scenario": null, "predictions": [ { "start": 0, "end": 8, "label": "PERSON", "text": "John Doe", "score": 0.9968609809875488, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 14, "end": 22, "label": "CITY", "text": "New York", "score": 0.9980940222740173, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 27, "end": 43, "label": "PERSON", "text": "Lina Wolkenstein", "score": 0.9969224333763123, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 47, "end": 53, "label": "CITY", "text": "Berlin", "score": 0.9998062252998352, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 0, "end": 8, "label": "PERSON", "text": "John Doe", "score": 0.9968609809875488, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 14, "end": 22, "label": "CITY", "text": "New York", "score": 0.9980940222740173, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 27, "end": 43, "label": "PERSON", "text": "Lina Wolkenstein", "score": 0.9969224333763123, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 47, "end": 53, "label": "CITY", "text": "Berlin", "score": 0.9998062252998352, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 0, "end": 8, "label": "PERSON", "text": "John Doe", "uncovered_positions": [] }, { "start": 14, "end": 22, "label": "CITY", "text": "New York", "uncovered_positions": [] }, { "start": 27, "end": 43, "label": "PERSON", "text": "Lina Wolkenstein", "uncovered_positions": [] }, { "start": 47, "end": 53, "label": "CITY", "text": "Berlin", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "baseline-21", "scenario": null, "predictions": [ { "start": 0, "end": 5, "label": "PERSON", "text": "David", "score": 0.9995999932289124, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 9, "end": 17, "label": "RELIGION", "text": "Catholic", "score": 0.9957641363143921, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 31, "end": 35, "label": "CITY", "text": "Rome", "score": 0.9998972415924072, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 0, "end": 5, "label": "PERSON", "text": "David", "score": 0.9995999932289124, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 9, "end": 17, "label": "RELIGION", "text": "Catholic", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" }, { "start": 31, "end": 35, "label": "CITY", "text": "Rome", "score": 0.9998972415924072, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 0, "end": 5, "label": "PERSON", "text": "David", "uncovered_positions": [] }, { "start": 9, "end": 17, "label": "RELIGION", "text": "Catholic", "uncovered_positions": [] }, { "start": 31, "end": 35, "label": "CITY", "text": "Rome", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "baseline-22", "scenario": null, "predictions": [ { "start": 0, "end": 4, "label": "PERSON", "text": "John", "score": 0.9999161958694458, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 11, "end": 15, "label": "PERSON", "text": "John", "score": 0.9968172311782837, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 0, "end": 4, "label": "PERSON", "text": "John", "score": 0.9999161958694458, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 11, "end": 15, "label": "PERSON", "text": "John", "score": 0.9968172311782837, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 0, "end": 4, "label": "PERSON", "text": "John", "uncovered_positions": [] }, { "start": 11, "end": 15, "label": "PERSON", "text": "John", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "baseline-23", "scenario": null, "predictions": [ { "start": 0, "end": 6, "label": "CITY", "text": "Berlin", "score": 0.9994149208068848, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 11, "end": 17, "label": "CITY", "text": "Berlin", "score": 0.9993422627449036, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 0, "end": 6, "label": "CITY", "text": "Berlin", "score": 0.9994149208068848, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 11, "end": 17, "label": "CITY", "text": "Berlin", "score": 0.9993422627449036, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 0, "end": 6, "label": "CITY", "text": "Berlin", "uncovered_positions": [] }, { "start": 11, "end": 17, "label": "CITY", "text": "Berlin", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "baseline-24", "scenario": null, "predictions": [ { "start": 0, "end": 10, "label": "CITY", "text": "Washington", "score": 0.9995170831680298, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 0, "end": 10, "label": "CITY", "text": "Washington", "score": 0.9995170831680298, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 0, "end": 10, "label": "CITY", "text": "Washington", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "baseline-25", "scenario": null, "predictions": [ { "start": 0, "end": 17, "label": "PERSON", "text": "George Washington", "score": 0.9903205037117004, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 0, "end": 17, "label": "PERSON", "text": "George Washington", "score": 0.9903205037117004, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 0, "end": 17, "label": "PERSON", "text": "George Washington", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "baseline-26", "scenario": null, "predictions": [ { "start": 14, "end": 20, "label": "CITY", "text": "Jordan", "score": 0.9931551814079285, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 14, "end": 20, "label": "CITY", "text": "Jordan", "score": 0.9931551814079285, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 14, "end": 20, "label": "CITY", "text": "Jordan", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "baseline-27", "scenario": null, "predictions": [ { "start": 0, "end": 14, "label": "PERSON", "text": "Michael Jordan", "score": 0.9887990355491638, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 0, "end": 14, "label": "PERSON", "text": "Michael Jordan", "score": 0.9887990355491638, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 0, "end": 14, "label": "PERSON", "text": "Michael Jordan", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "baseline-28", "scenario": null, "predictions": [ { "start": 0, "end": 14, "label": "PERSON", "text": "Mr. Smith-\nson", "score": 0.9715586304664612, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "dehyphenation-v1" } ], "combined_predictions": [ { "start": 0, "end": 14, "label": "PERSON", "text": "Mr. Smith-\nson", "score": 0.9715586304664612, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "dehyphenation-v1" } ], "false_positives": [ [ 0, 14, "PERSON" ] ], "false_negatives": [ [ 4, 14, "PERSON" ] ], "exact_match": false, "error_code": null, "canaries": [ { "start": 4, "end": 14, "label": "PERSON", "text": "Smith-\nson", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "baseline-29", "scenario": null, "predictions": [ { "start": 0, "end": 8, "label": "CITY", "text": "Ber-\nlin", "score": 0.9711049795150757, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "dehyphenation-v1" } ], "combined_predictions": [ { "start": 0, "end": 8, "label": "CITY", "text": "Ber-\nlin", "score": 0.9711049795150757, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "dehyphenation-v1" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 0, "end": 8, "label": "CITY", "text": "Ber-\nlin", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "baseline-30", "scenario": null, "predictions": [ { "start": 0, "end": 8, "label": "PERSON", "text": "Jchn D0e", "score": 0.6583331823348999, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 0, "end": 8, "label": "PERSON", "text": "Jchn D0e", "score": 0.6583331823348999, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 0, "end": 8, "label": "PERSON", "text": "Jchn D0e", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "baseline-31", "scenario": null, "predictions": [ { "start": 11, "end": 20, "label": "ORGANIZATION", "text": "Nevv Y0rk", "score": 0.8232883810997009, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 11, "end": 20, "label": "ORGANIZATION", "text": "Nevv Y0rk", "score": 0.8232883810997009, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [ [ 11, 20, "ORGANIZATION" ] ], "false_negatives": [ [ 11, 20, "CITY" ] ], "exact_match": false, "error_code": null, "canaries": [ { "start": 11, "end": 20, "label": "CITY", "text": "Nevv Y0rk", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "baseline-32", "scenario": null, "predictions": [], "combined_predictions": [], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "baseline-33", "scenario": null, "predictions": [], "combined_predictions": [], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "baseline-34", "scenario": null, "predictions": [], "combined_predictions": [], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "baseline-35", "scenario": null, "predictions": [ { "start": 12, "end": 28, "label": "EMAIL", "text": "test@example.com", "score": 0.8803917765617371, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 12, "end": 28, "label": "EMAIL", "text": "test@example.com", "score": null, "source": "deterministic", "model_revision": "contact-patterns-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 12, "end": 28, "label": "EMAIL", "text": "test@example.com", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "baseline-36", "scenario": null, "predictions": [ { "start": 11, "end": 19, "label": "PHONE", "text": "555-0199", "score": 0.9762608408927917, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 11, "end": 19, "label": "PHONE", "text": "555-0199", "score": null, "source": "deterministic", "model_revision": "contact-patterns-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 11, "end": 19, "label": "PHONE", "text": "555-0199", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "baseline-37", "scenario": null, "predictions": [ { "start": 11, "end": 21, "label": "ORGANIZATION", "text": "Google Inc", "score": 0.7403514981269836, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 11, "end": 21, "label": "ORGANIZATION", "text": "Google Inc", "score": 0.7403514981269836, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [ [ 11, 21, "ORGANIZATION" ] ], "false_negatives": [ [ 11, 22, "ORGANIZATION" ] ], "exact_match": true, "error_code": null, "canaries": [ { "start": 11, "end": 22, "label": "ORGANIZATION", "text": "Google Inc.", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "baseline-38", "scenario": null, "predictions": [ { "start": 13, "end": 23, "label": "CITY", "text": "company.de", "score": 0.8622952103614807, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 8, "end": 23, "label": "EMAIL", "text": "info@company.de", "score": null, "source": "deterministic", "model_revision": "contact-patterns-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 8, "end": 23, "label": "EMAIL", "text": "info@company.de", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "baseline-39", "scenario": null, "predictions": [ { "start": 4, "end": 6, "label": "ORGANIZATION", "text": "UN", "score": 0.9981075525283813, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 4, "end": 6, "label": "ORGANIZATION", "text": "UN", "score": 0.9981075525283813, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 4, "end": 6, "label": "ORGANIZATION", "text": "UN", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" } ], "automated_release_eligible": false, "recognition_qualified": false }, "validation": { "schema_version": "development-candidate-outcomes/v1", "accuracy": { "documents": 10, "ner_entity_metrics": { "PERSON": { "tp": 7, "fp": 0, "fn": 0, "precision": 1.0, "recall": 1.0, "f1": 1.0 }, "CITY": { "tp": 5, "fp": 1, "fn": 0, "precision": 0.8333333333333334, "recall": 1.0, "f1": 0.9090909090909091 }, "RELIGION": { "tp": 2, "fp": 1, "fn": 0, "precision": 0.6666666666666666, "recall": 1.0, "f1": 0.8 }, "EMAIL": { "tp": 2, "fp": 0, "fn": 0, "precision": 1.0, "recall": 1.0, "f1": 1.0 }, "PHONE": { "tp": 2, "fp": 0, "fn": 0, "precision": 1.0, "recall": 1.0, "f1": 1.0 }, "ORGANIZATION": { "tp": 2, "fp": 0, "fn": 0, "precision": 1.0, "recall": 1.0, "f1": 1.0 } }, "combined_entity_metrics": { "PERSON": { "tp": 7, "fp": 0, "fn": 0, "precision": 1.0, "recall": 1.0, "f1": 1.0 }, "CITY": { "tp": 5, "fp": 1, "fn": 0, "precision": 0.8333333333333334, "recall": 1.0, "f1": 0.9090909090909091 }, "RELIGION": { "tp": 2, "fp": 1, "fn": 0, "precision": 0.6666666666666666, "recall": 1.0, "f1": 0.8 }, "EMAIL": { "tp": 2, "fp": 0, "fn": 0, "precision": 1.0, "recall": 1.0, "f1": 1.0 }, "PHONE": { "tp": 2, "fp": 0, "fn": 0, "precision": 1.0, "recall": 1.0, "f1": 1.0 }, "ORGANIZATION": { "tp": 2, "fp": 0, "fn": 0, "precision": 1.0, "recall": 1.0, "f1": 1.0 } }, "document_exact_match_rate": 0.8, "groups": { "de": { "documents": 5, "exact_matches": 4, "false_negatives": 0 }, "en": { "documents": 5, "exact_matches": 4, "false_negatives": 0 }, "ocr_noise": { "documents": 2, "exact_matches": 2, "false_negatives": 0 } }, "blocked_documents": 0, "failure_codes": {}, "automated_release_eligible": false, "uncovered_source_canaries_including_incomplete": { "PERSON": 0, "CITY": 0, "RELIGION": 0, "EMAIL": 0, "PHONE": 0, "ORGANIZATION": 0 }, "literal_presence_or_incomplete": { "PERSON": 0, "CITY": 0, "RELIGION": 0, "EMAIL": 0, "PHONE": 0, "ORGANIZATION": 0 }, "partially_covered_source_canaries_including_incomplete": { "PERSON": 0, "CITY": 0, "RELIGION": 0, "EMAIL": 0, "PHONE": 0, "ORGANIZATION": 0 }, "fully_uncovered_source_canaries_including_incomplete": { "PERSON": 0, "CITY": 0, "RELIGION": 0, "EMAIL": 0, "PHONE": 0, "ORGANIZATION": 0 }, "all_document_source_coverage": { "PERSON": { "covered": 75, "total": 75 }, "CITY": { "covered": 30, "total": 30 }, "RELIGION": { "covered": 18, "total": 18 }, "EMAIL": { "covered": 33, "total": 33 }, "PHONE": { "covered": 23, "total": 23 }, "ORGANIZATION": { "covered": 22, "total": 22 } }, "conservative_accuracy_gate_passed": true, "scope": "all source spans; incomplete processing counts as missing, not leaked output", "deterministic_replay_latency_ms": { "median": 0.14301050032372586, "max": 3.4064589999616146 } }, "candidate_outcomes": { "complete_no_measured_leakage": 10, "candidate_partial_mask": 0, "candidate_fully_unmasked": 0, "blocked_conflicting_spans": 0, "blocked_model_runtime": 0, "other_incomplete": 0 }, "candidate_return_rate": 1.0, "safe_abstention_rate": 0.0, "incomplete_rate": 0.0, "no_measured_leakage_rate_among_returned": 1.0, "candidate_partial_canaries": { "PERSON": 0, "CITY": 0, "RELIGION": 0, "EMAIL": 0, "PHONE": 0, "ORGANIZATION": 0 }, "candidate_fully_unmasked_canaries": { "PERSON": 0, "CITY": 0, "RELIGION": 0, "EMAIL": 0, "PHONE": 0, "ORGANIZATION": 0 }, "returned_candidate_entity_metrics": { "PERSON": { "tp": 7, "fp": 0, "fn": 0, "precision": 1.0, "recall": 1.0, "f1": 1.0 }, "CITY": { "tp": 5, "fp": 1, "fn": 0, "precision": 0.8333333333333334, "recall": 1.0, "f1": 0.9090909090909091 }, "RELIGION": { "tp": 2, "fp": 1, "fn": 0, "precision": 0.6666666666666666, "recall": 1.0, "f1": 0.8 }, "EMAIL": { "tp": 2, "fp": 0, "fn": 0, "precision": 1.0, "recall": 1.0, "f1": 1.0 }, "PHONE": { "tp": 2, "fp": 0, "fn": 0, "precision": 1.0, "recall": 1.0, "f1": 1.0 }, "ORGANIZATION": { "tp": 2, "fp": 0, "fn": 0, "precision": 1.0, "recall": 1.0, "f1": 1.0 } }, "per_case": [ { "case_id": "validation-00", "scenario": null, "predictions": [ { "start": 0, "end": 9, "label": "PERSON", "text": "Mara Kühn", "score": 0.9990392923355103, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 23, "end": 30, "label": "ORGANIZATION", "text": "Siemens", "score": 0.9998852014541626, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 34, "end": 41, "label": "CITY", "text": "Dresden", "score": 0.9993748068809509, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 0, "end": 9, "label": "PERSON", "text": "Mara Kühn", "score": 0.9990392923355103, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 23, "end": 30, "label": "ORGANIZATION", "text": "Siemens", "score": 0.9998852014541626, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 34, "end": 41, "label": "CITY", "text": "Dresden", "score": 0.9993748068809509, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 0, "end": 9, "label": "PERSON", "text": "Mara Kühn", "uncovered_positions": [] }, { "start": 23, "end": 30, "label": "ORGANIZATION", "text": "Siemens", "uncovered_positions": [] }, { "start": 34, "end": 41, "label": "CITY", "text": "Dresden", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "validation-01", "scenario": null, "predictions": [ { "start": 0, "end": 11, "label": "PERSON", "text": "Emma Wilson", "score": 0.9975014328956604, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 22, "end": 38, "label": "ORGANIZATION", "text": "Acme Corporation", "score": 0.9995630383491516, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 42, "end": 48, "label": "CITY", "text": "London", "score": 0.9999483823776245, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 0, "end": 11, "label": "PERSON", "text": "Emma Wilson", "score": 0.9975014328956604, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 22, "end": 38, "label": "ORGANIZATION", "text": "Acme Corporation", "score": 0.9995630383491516, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 42, "end": 48, "label": "CITY", "text": "London", "score": 0.9999483823776245, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 0, "end": 11, "label": "PERSON", "text": "Emma Wilson", "uncovered_positions": [] }, { "start": 22, "end": 38, "label": "ORGANIZATION", "text": "Acme Corporation", "uncovered_positions": [] }, { "start": 42, "end": 48, "label": "CITY", "text": "London", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "validation-02", "scenario": null, "predictions": [ { "start": 8, "end": 18, "label": "RELIGION", "text": "muslimisch", "score": 0.9497504234313965, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 29, "end": 51, "label": "EMAIL", "text": "mara.kuehn@example.org", "score": 0.9375261664390564, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 53, "end": 68, "label": "PHONE", "text": "+49 351 7654321", "score": 0.9705594182014465, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 8, "end": 18, "label": "RELIGION", "text": "muslimisch", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" }, { "start": 29, "end": 51, "label": "EMAIL", "text": "mara.kuehn@example.org", "score": null, "source": "deterministic", "model_revision": "contact-patterns-v1", "normalization": "identity" }, { "start": 53, "end": 68, "label": "PHONE", "text": "+49 351 7654321", "score": null, "source": "deterministic", "model_revision": "contact-patterns-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 8, "end": 18, "label": "RELIGION", "text": "muslimisch", "uncovered_positions": [] }, { "start": 29, "end": 51, "label": "EMAIL", "text": "mara.kuehn@example.org", "uncovered_positions": [] }, { "start": 53, "end": 68, "label": "PHONE", "text": "+49 351 7654321", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "validation-03", "scenario": null, "predictions": [ { "start": 16, "end": 24, "label": "RELIGION", "text": "Hinduism", "score": 0.9999716281890869, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 34, "end": 50, "label": "EMAIL", "text": "emma@example.net", "score": 0.903052568435669, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 54, "end": 69, "label": "PHONE", "text": "+1 212 555 0188", "score": 0.951524555683136, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 16, "end": 24, "label": "RELIGION", "text": "Hinduism", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" }, { "start": 34, "end": 50, "label": "EMAIL", "text": "emma@example.net", "score": null, "source": "deterministic", "model_revision": "contact-patterns-v1", "normalization": "identity" }, { "start": 54, "end": 69, "label": "PHONE", "text": "+1 212 555 0188", "score": null, "source": "deterministic", "model_revision": "contact-patterns-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 16, "end": 24, "label": "RELIGION", "text": "Hinduism", "uncovered_positions": [] }, { "start": 34, "end": 50, "label": "EMAIL", "text": "emma@example.net", "uncovered_positions": [] }, { "start": 54, "end": 69, "label": "PHONE", "text": "+1 212 555 0188", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "validation-04", "scenario": null, "predictions": [ { "start": 0, "end": 11, "label": "PERSON", "text": "Nora Winter", "score": 0.9975367784500122, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 20, "end": 24, "label": "CITY", "text": "Köln", "score": 0.9991357922554016, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 26, "end": 37, "label": "PERSON", "text": "Nora Winter", "score": 0.99813312292099, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 0, "end": 11, "label": "PERSON", "text": "Nora Winter", "score": 0.9975367784500122, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 20, "end": 24, "label": "CITY", "text": "Köln", "score": 0.9991357922554016, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 26, "end": 37, "label": "PERSON", "text": "Nora Winter", "score": 0.99813312292099, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 0, "end": 11, "label": "PERSON", "text": "Nora Winter", "uncovered_positions": [] }, { "start": 20, "end": 24, "label": "CITY", "text": "Köln", "uncovered_positions": [] }, { "start": 26, "end": 37, "label": "PERSON", "text": "Nora Winter", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "validation-05", "scenario": null, "predictions": [ { "start": 0, "end": 6, "label": "CITY", "text": "Jordan", "score": 0.9668874144554138, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 23, "end": 36, "label": "PERSON", "text": "Jordan Miller", "score": 0.982417643070221, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 0, "end": 6, "label": "CITY", "text": "Jordan", "score": 0.9668874144554138, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 23, "end": 36, "label": "PERSON", "text": "Jordan Miller", "score": 0.982417643070221, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [ [ 0, 6, "CITY" ] ], "false_negatives": [], "exact_match": false, "error_code": null, "canaries": [ { "start": 23, "end": 36, "label": "PERSON", "text": "Jordan Miller", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "validation-06", "scenario": null, "predictions": [ { "start": 0, "end": 8, "label": "RELIGION", "text": "Religion", "score": 0.6538923978805542, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 0, "end": 8, "label": "RELIGION", "text": "Religion", "score": 0.6538923978805542, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [ [ 0, 8, "RELIGION" ] ], "false_negatives": [], "exact_match": false, "error_code": null, "canaries": [], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "validation-07", "scenario": null, "predictions": [], "combined_predictions": [], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "validation-08", "scenario": null, "predictions": [ { "start": 5, "end": 23, "label": "PERSON", "text": "Eva-Luise Schröder", "score": 0.986770510673523, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "dehyphenation-v1" }, { "start": 33, "end": 42, "label": "CITY", "text": "Ham-\nburg", "score": 0.9998313188552856, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "dehyphenation-v1" } ], "combined_predictions": [ { "start": 5, "end": 23, "label": "PERSON", "text": "Eva-Luise Schröder", "score": 0.986770510673523, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "dehyphenation-v1" }, { "start": 33, "end": 42, "label": "CITY", "text": "Ham-\nburg", "score": 0.9998313188552856, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "dehyphenation-v1" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 5, "end": 23, "label": "PERSON", "text": "Eva-Luise Schröder", "uncovered_positions": [] }, { "start": 33, "end": 42, "label": "CITY", "text": "Ham-\nburg", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "validation-09", "scenario": null, "predictions": [ { "start": 0, "end": 10, "label": "PERSON", "text": "M4ry Br0wn", "score": 0.9766657948493958, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 20, "end": 26, "label": "CITY", "text": "L0ndon", "score": 0.996916651725769, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 0, "end": 10, "label": "PERSON", "text": "M4ry Br0wn", "score": 0.9766657948493958, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 20, "end": 26, "label": "CITY", "text": "L0ndon", "score": 0.996916651725769, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 0, "end": 10, "label": "PERSON", "text": "M4ry Br0wn", "uncovered_positions": [] }, { "start": 20, "end": 26, "label": "CITY", "text": "L0ndon", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" } ], "automated_release_eligible": false, "recognition_qualified": false }, "development03c": { "schema_version": "development-candidate-outcomes/v1", "accuracy": { "documents": 120, "ner_entity_metrics": { "PERSON": { "tp": 132, "fp": 12, "fn": 0, "precision": 0.9166666666666666, "recall": 1.0, "f1": 0.9565217391304348 }, "CITY": { "tp": 84, "fp": 13, "fn": 0, "precision": 0.865979381443299, "recall": 1.0, "f1": 0.9281767955801106 }, "RELIGION": { "tp": 34, "fp": 0, "fn": 2, "precision": 1.0, "recall": 0.9444444444444444, "f1": 0.9714285714285714 }, "EMAIL": { "tp": 23, "fp": 0, "fn": 1, "precision": 1.0, "recall": 0.9583333333333334, "f1": 0.9787234042553191 }, "PHONE": { "tp": 24, "fp": 0, "fn": 0, "precision": 1.0, "recall": 1.0, "f1": 1.0 }, "ORGANIZATION": { "tp": 84, "fp": 12, "fn": 0, "precision": 0.875, "recall": 1.0, "f1": 0.9333333333333333 } }, "combined_entity_metrics": { "PERSON": { "tp": 132, "fp": 12, "fn": 0, "precision": 0.9166666666666666, "recall": 1.0, "f1": 0.9565217391304348 }, "CITY": { "tp": 84, "fp": 12, "fn": 0, "precision": 0.875, "recall": 1.0, "f1": 0.9333333333333333 }, "RELIGION": { "tp": 36, "fp": 12, "fn": 0, "precision": 0.75, "recall": 1.0, "f1": 0.8571428571428571 }, "EMAIL": { "tp": 24, "fp": 0, "fn": 0, "precision": 1.0, "recall": 1.0, "f1": 1.0 }, "PHONE": { "tp": 24, "fp": 0, "fn": 0, "precision": 1.0, "recall": 1.0, "f1": 1.0 }, "ORGANIZATION": { "tp": 84, "fp": 12, "fn": 0, "precision": 0.875, "recall": 1.0, "f1": 0.9333333333333333 } }, "document_exact_match_rate": 0.9, "groups": { "de": { "documents": 60, "exact_matches": 60, "false_negatives": 0 }, "en": { "documents": 60, "exact_matches": 48, "false_negatives": 0 }, "ocr_noise": { "documents": 24, "exact_matches": 24, "false_negatives": 0 } }, "blocked_documents": 0, "failure_codes": {}, "automated_release_eligible": false, "uncovered_source_canaries_including_incomplete": { "PERSON": 0, "CITY": 0, "RELIGION": 0, "EMAIL": 0, "PHONE": 0, "ORGANIZATION": 0 }, "literal_presence_or_incomplete": { "PERSON": 0, "CITY": 0, "RELIGION": 0, "EMAIL": 0, "PHONE": 0, "ORGANIZATION": 0 }, "partially_covered_source_canaries_including_incomplete": { "PERSON": 0, "CITY": 0, "RELIGION": 0, "EMAIL": 0, "PHONE": 0, "ORGANIZATION": 0 }, "fully_uncovered_source_canaries_including_incomplete": { "PERSON": 0, "CITY": 0, "RELIGION": 0, "EMAIL": 0, "PHONE": 0, "ORGANIZATION": 0 }, "all_document_source_coverage": { "PERSON": { "covered": 1727, "total": 1727 }, "CITY": { "covered": 588, "total": 588 }, "RELIGION": { "covered": 358, "total": 358 }, "EMAIL": { "covered": 648, "total": 648 }, "PHONE": { "covered": 264, "total": 264 }, "ORGANIZATION": { "covered": 1631, "total": 1631 } }, "conservative_accuracy_gate_passed": true, "scope": "all source spans; incomplete processing counts as missing, not leaked output", "deterministic_replay_latency_ms": { "median": 0.20667949866037816, "max": 3.778295998927206 } }, "candidate_outcomes": { "complete_no_measured_leakage": 120, "candidate_partial_mask": 0, "candidate_fully_unmasked": 0, "blocked_conflicting_spans": 0, "blocked_model_runtime": 0, "other_incomplete": 0 }, "candidate_return_rate": 1.0, "safe_abstention_rate": 0.0, "incomplete_rate": 0.0, "no_measured_leakage_rate_among_returned": 1.0, "candidate_partial_canaries": { "PERSON": 0, "CITY": 0, "RELIGION": 0, "EMAIL": 0, "PHONE": 0, "ORGANIZATION": 0 }, "candidate_fully_unmasked_canaries": { "PERSON": 0, "CITY": 0, "RELIGION": 0, "EMAIL": 0, "PHONE": 0, "ORGANIZATION": 0 }, "returned_candidate_entity_metrics": { "PERSON": { "tp": 132, "fp": 12, "fn": 0, "precision": 0.9166666666666666, "recall": 1.0, "f1": 0.9565217391304348 }, "CITY": { "tp": 84, "fp": 12, "fn": 0, "precision": 0.875, "recall": 1.0, "f1": 0.9333333333333333 }, "RELIGION": { "tp": 36, "fp": 12, "fn": 0, "precision": 0.75, "recall": 1.0, "f1": 0.8571428571428571 }, "EMAIL": { "tp": 24, "fp": 0, "fn": 0, "precision": 1.0, "recall": 1.0, "f1": 1.0 }, "PHONE": { "tp": 24, "fp": 0, "fn": 0, "precision": 1.0, "recall": 1.0, "f1": 1.0 }, "ORGANIZATION": { "tp": 84, "fp": 12, "fn": 0, "precision": 0.875, "recall": 1.0, "f1": 0.9333333333333333 } }, "per_case": [ { "case_id": "holdout03c-000", "scenario": null, "predictions": [ { "start": 10, "end": 25, "label": "PERSON", "text": "Saskia Dornfeld", "score": 0.9995438456535339, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 39, "end": 63, "label": "ORGANIZATION", "text": "Aster Research Institute", "score": 0.9974908828735352, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 67, "end": 76, "label": "CITY", "text": "Osnabrück", "score": 0.999496579170227, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 10, "end": 25, "label": "PERSON", "text": "Saskia Dornfeld", "score": 0.9995438456535339, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 39, "end": 63, "label": "ORGANIZATION", "text": "Aster Research Institute", "score": 0.9974908828735352, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 67, "end": 76, "label": "CITY", "text": "Osnabrück", "score": 0.999496579170227, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 10, "end": 25, "label": "PERSON", "text": "Saskia Dornfeld", "uncovered_positions": [] }, { "start": 39, "end": 63, "label": "ORGANIZATION", "text": "Aster Research Institute", "uncovered_positions": [] }, { "start": 67, "end": 76, "label": "CITY", "text": "Osnabrück", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-001", "scenario": null, "predictions": [ { "start": 12, "end": 27, "label": "PERSON", "text": "Saskia Dornfeld", "score": 0.9990620017051697, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 39, "end": 63, "label": "ORGANIZATION", "text": "Aster Research Institute", "score": 0.997700035572052, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 69, "end": 78, "label": "CITY", "text": "Osnabrück", "score": 0.9995871186256409, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 12, "end": 27, "label": "PERSON", "text": "Saskia Dornfeld", "score": 0.9990620017051697, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 39, "end": 63, "label": "ORGANIZATION", "text": "Aster Research Institute", "score": 0.997700035572052, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 69, "end": 78, "label": "CITY", "text": "Osnabrück", "score": 0.9995871186256409, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 12, "end": 27, "label": "PERSON", "text": "Saskia Dornfeld", "uncovered_positions": [] }, { "start": 39, "end": 63, "label": "ORGANIZATION", "text": "Aster Research Institute", "uncovered_positions": [] }, { "start": 69, "end": 78, "label": "CITY", "text": "Osnabrück", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-002", "scenario": null, "predictions": [ { "start": 0, "end": 15, "label": "PERSON", "text": "Saskia Dornfeld", "score": 0.9997074007987976, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 22, "end": 33, "label": "RELIGION", "text": "sikhistisch", "score": 0.9321887493133545, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 61, "end": 91, "label": "EMAIL", "text": "case101.review@example.invalid", "score": 0.9007220268249512, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 102, "end": 116, "label": "PHONE", "text": "+49 541 730101", "score": 0.9946468472480774, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 0, "end": 15, "label": "PERSON", "text": "Saskia Dornfeld", "score": 0.9997074007987976, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 22, "end": 33, "label": "RELIGION", "text": "sikhistisch", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" }, { "start": 61, "end": 91, "label": "EMAIL", "text": "case101.review@example.invalid", "score": null, "source": "deterministic", "model_revision": "contact-patterns-v1", "normalization": "identity" }, { "start": 102, "end": 116, "label": "PHONE", "text": "+49 541 730101", "score": null, "source": "deterministic", "model_revision": "contact-patterns-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 0, "end": 15, "label": "PERSON", "text": "Saskia Dornfeld", "uncovered_positions": [] }, { "start": 22, "end": 33, "label": "RELIGION", "text": "sikhistisch", "uncovered_positions": [] }, { "start": 61, "end": 91, "label": "EMAIL", "text": "case101.review@example.invalid", "uncovered_positions": [] }, { "start": 102, "end": 116, "label": "PHONE", "text": "+49 541 730101", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-003", "scenario": null, "predictions": [ { "start": 0, "end": 15, "label": "PERSON", "text": "Saskia Dornfeld", "score": 0.9992882609367371, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 33, "end": 40, "label": "RELIGION", "text": "Sikhism", "score": 0.9966249465942383, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 66, "end": 96, "label": "EMAIL", "text": "case101.review@example.invalid", "score": 0.908982515335083, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 105, "end": 119, "label": "PHONE", "text": "+49 541 730101", "score": 0.9701294898986816, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 0, "end": 15, "label": "PERSON", "text": "Saskia Dornfeld", "score": 0.9992882609367371, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 33, "end": 40, "label": "RELIGION", "text": "Sikhism", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" }, { "start": 66, "end": 96, "label": "EMAIL", "text": "case101.review@example.invalid", "score": null, "source": "deterministic", "model_revision": "contact-patterns-v1", "normalization": "identity" }, { "start": 105, "end": 119, "label": "PHONE", "text": "+49 541 730101", "score": null, "source": "deterministic", "model_revision": "contact-patterns-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 0, "end": 15, "label": "PERSON", "text": "Saskia Dornfeld", "uncovered_positions": [] }, { "start": 33, "end": 40, "label": "RELIGION", "text": "Sikhism", "uncovered_positions": [] }, { "start": 66, "end": 96, "label": "EMAIL", "text": "case101.review@example.invalid", "uncovered_positions": [] }, { "start": 105, "end": 119, "label": "PHONE", "text": "+49 541 730101", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-004", "scenario": null, "predictions": [ { "start": 0, "end": 15, "label": "PERSON", "text": "Saskia Dornfeld", "score": 0.9996077418327332, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 24, "end": 33, "label": "CITY", "text": "Osnabrück", "score": 0.998096764087677, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 36, "end": 51, "label": "PERSON", "text": "Saskia Dornfeld", "score": 0.9998266100883484, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 77, "end": 101, "label": "ORGANIZATION", "text": "Aster Research Institute", "score": 0.9952711462974548, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 0, "end": 15, "label": "PERSON", "text": "Saskia Dornfeld", "score": 0.9996077418327332, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 24, "end": 33, "label": "CITY", "text": "Osnabrück", "score": 0.998096764087677, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 36, "end": 51, "label": "PERSON", "text": "Saskia Dornfeld", "score": 0.9998266100883484, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 77, "end": 101, "label": "ORGANIZATION", "text": "Aster Research Institute", "score": 0.9952711462974548, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 0, "end": 15, "label": "PERSON", "text": "Saskia Dornfeld", "uncovered_positions": [] }, { "start": 24, "end": 33, "label": "CITY", "text": "Osnabrück", "uncovered_positions": [] }, { "start": 36, "end": 51, "label": "PERSON", "text": "Saskia Dornfeld", "uncovered_positions": [] }, { "start": 77, "end": 101, "label": "ORGANIZATION", "text": "Aster Research Institute", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-005", "scenario": null, "predictions": [ { "start": 3, "end": 12, "label": "CITY", "text": "Osnabrück", "score": 0.9999159574508667, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 14, "end": 29, "label": "PERSON", "text": "Saskia Dornfeld", "score": 0.9998455047607422, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 41, "end": 65, "label": "ORGANIZATION", "text": "Aster Research Institute", "score": 0.9969690442085266, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 86, "end": 101, "label": "PERSON", "text": "Saskia Dornfeld", "score": 0.9999145269393921, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 114, "end": 123, "label": "CITY", "text": "Osnabrück", "score": 0.999828577041626, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 3, "end": 12, "label": "CITY", "text": "Osnabrück", "score": 0.9999159574508667, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 14, "end": 29, "label": "PERSON", "text": "Saskia Dornfeld", "score": 0.9998455047607422, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 41, "end": 65, "label": "ORGANIZATION", "text": "Aster Research Institute", "score": 0.9969690442085266, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 86, "end": 101, "label": "PERSON", "text": "Saskia Dornfeld", "score": 0.9999145269393921, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 114, "end": 123, "label": "CITY", "text": "Osnabrück", "score": 0.999828577041626, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 3, "end": 12, "label": "CITY", "text": "Osnabrück", "uncovered_positions": [] }, { "start": 14, "end": 29, "label": "PERSON", "text": "Saskia Dornfeld", "uncovered_positions": [] }, { "start": 41, "end": 65, "label": "ORGANIZATION", "text": "Aster Research Institute", "uncovered_positions": [] }, { "start": 86, "end": 101, "label": "PERSON", "text": "Saskia Dornfeld", "uncovered_positions": [] }, { "start": 114, "end": 123, "label": "CITY", "text": "Osnabrück", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-006", "scenario": null, "predictions": [ { "start": 6, "end": 22, "label": "PERSON", "text": "Saskia-\nDornfeld", "score": 0.9996308088302612, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "dehyphenation-v1" }, { "start": 28, "end": 39, "label": "CITY", "text": "Osna-\nbrück", "score": 0.9999524354934692, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "dehyphenation-v1" }, { "start": 53, "end": 77, "label": "ORGANIZATION", "text": "Aster Research Institute", "score": 0.9986594915390015, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "dehyphenation-v1" } ], "combined_predictions": [ { "start": 6, "end": 22, "label": "PERSON", "text": "Saskia-\nDornfeld", "score": 0.9996308088302612, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "dehyphenation-v1" }, { "start": 28, "end": 39, "label": "CITY", "text": "Osna-\nbrück", "score": 0.9999524354934692, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "dehyphenation-v1" }, { "start": 53, "end": 77, "label": "ORGANIZATION", "text": "Aster Research Institute", "score": 0.9986594915390015, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "dehyphenation-v1" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 6, "end": 22, "label": "PERSON", "text": "Saskia-\nDornfeld", "uncovered_positions": [] }, { "start": 28, "end": 39, "label": "CITY", "text": "Osna-\nbrück", "uncovered_positions": [] }, { "start": 53, "end": 77, "label": "ORGANIZATION", "text": "Aster Research Institute", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-007", "scenario": null, "predictions": [ { "start": 16, "end": 31, "label": "PERSON", "text": "Sask1a D0rnfeld", "score": 0.9964213371276855, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 43, "end": 52, "label": "CITY", "text": "Osn4brück", "score": 0.999607503414154, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 66, "end": 90, "label": "ORGANIZATION", "text": "Aster Research Institute", "score": 0.9983013868331909, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 16, "end": 31, "label": "PERSON", "text": "Sask1a D0rnfeld", "score": 0.9964213371276855, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 43, "end": 52, "label": "CITY", "text": "Osn4brück", "score": 0.999607503414154, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 66, "end": 90, "label": "ORGANIZATION", "text": "Aster Research Institute", "score": 0.9983013868331909, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 16, "end": 31, "label": "PERSON", "text": "Sask1a D0rnfeld", "uncovered_positions": [] }, { "start": 43, "end": 52, "label": "CITY", "text": "Osn4brück", "uncovered_positions": [] }, { "start": 66, "end": 90, "label": "ORGANIZATION", "text": "Aster Research Institute", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-008", "scenario": null, "predictions": [ { "start": 10, "end": 21, "label": "RELIGION", "text": "sikhistisch", "score": 0.820220947265625, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 48, "end": 72, "label": "ORGANIZATION", "text": "Aster Research Institute", "score": 0.983711302280426, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 86, "end": 101, "label": "PERSON", "text": "Saskia Dornfeld", "score": 0.9986269474029541, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 10, "end": 21, "label": "RELIGION", "text": "sikhistisch", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" }, { "start": 48, "end": 72, "label": "ORGANIZATION", "text": "Aster Research Institute", "score": 0.983711302280426, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 86, "end": 101, "label": "PERSON", "text": "Saskia Dornfeld", "score": 0.9986269474029541, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 10, "end": 21, "label": "RELIGION", "text": "sikhistisch", "uncovered_positions": [] }, { "start": 48, "end": 72, "label": "ORGANIZATION", "text": "Aster Research Institute", "uncovered_positions": [] }, { "start": 86, "end": 101, "label": "PERSON", "text": "Saskia Dornfeld", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-009", "scenario": null, "predictions": [ { "start": 101, "end": 107, "label": "PERSON", "text": "person", "score": 0.7040723562240601, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 109, "end": 114, "label": "CITY", "text": "place", "score": 0.7776683568954468, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 116, "end": 128, "label": "ORGANIZATION", "text": "organization", "score": 0.8319209218025208, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 40, "end": 64, "label": "RELIGION", "text": "no religious affiliation", "score": null, "source": "deterministic", "model_revision": "religious-status-v1", "normalization": "identity" }, { "start": 101, "end": 107, "label": "PERSON", "text": "person", "score": 0.7040723562240601, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 109, "end": 114, "label": "CITY", "text": "place", "score": 0.7776683568954468, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 116, "end": 128, "label": "ORGANIZATION", "text": "organization", "score": 0.8319209218025208, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [ [ 40, 64, "RELIGION" ], [ 101, 107, "PERSON" ], [ 109, 114, "CITY" ], [ 116, 128, "ORGANIZATION" ] ], "false_negatives": [], "exact_match": false, "error_code": null, "canaries": [], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-010", "scenario": null, "predictions": [ { "start": 10, "end": 21, "label": "PERSON", "text": "Emil Farber", "score": 0.9987824559211731, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 35, "end": 56, "label": "ORGANIZATION", "text": "Northbridge Analytics", "score": 0.9969447255134583, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 60, "end": 67, "label": "CITY", "text": "Potsdam", "score": 0.9994106292724609, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 10, "end": 21, "label": "PERSON", "text": "Emil Farber", "score": 0.9987824559211731, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 35, "end": 56, "label": "ORGANIZATION", "text": "Northbridge Analytics", "score": 0.9969447255134583, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 60, "end": 67, "label": "CITY", "text": "Potsdam", "score": 0.9994106292724609, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 10, "end": 21, "label": "PERSON", "text": "Emil Farber", "uncovered_positions": [] }, { "start": 35, "end": 56, "label": "ORGANIZATION", "text": "Northbridge Analytics", "uncovered_positions": [] }, { "start": 60, "end": 67, "label": "CITY", "text": "Potsdam", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-011", "scenario": null, "predictions": [ { "start": 12, "end": 23, "label": "PERSON", "text": "Emil Farber", "score": 0.9989939332008362, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 35, "end": 56, "label": "ORGANIZATION", "text": "Northbridge Analytics", "score": 0.9984185695648193, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 62, "end": 69, "label": "CITY", "text": "Potsdam", "score": 0.9994024038314819, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 12, "end": 23, "label": "PERSON", "text": "Emil Farber", "score": 0.9989939332008362, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 35, "end": 56, "label": "ORGANIZATION", "text": "Northbridge Analytics", "score": 0.9984185695648193, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 62, "end": 69, "label": "CITY", "text": "Potsdam", "score": 0.9994024038314819, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 12, "end": 23, "label": "PERSON", "text": "Emil Farber", "uncovered_positions": [] }, { "start": 35, "end": 56, "label": "ORGANIZATION", "text": "Northbridge Analytics", "uncovered_positions": [] }, { "start": 62, "end": 69, "label": "CITY", "text": "Potsdam", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-012", "scenario": null, "predictions": [ { "start": 0, "end": 11, "label": "PERSON", "text": "Emil Farber", "score": 0.9991351962089539, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 18, "end": 26, "label": "RELIGION", "text": "orthodox", "score": 0.9795414805412292, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 54, "end": 84, "label": "EMAIL", "text": "case102.review@example.invalid", "score": 0.9129301309585571, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 95, "end": 109, "label": "PHONE", "text": "+49 541 730102", "score": 0.9935047626495361, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 0, "end": 11, "label": "PERSON", "text": "Emil Farber", "score": 0.9991351962089539, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 18, "end": 26, "label": "RELIGION", "text": "orthodox", "score": 0.9795414805412292, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 54, "end": 84, "label": "EMAIL", "text": "case102.review@example.invalid", "score": null, "source": "deterministic", "model_revision": "contact-patterns-v1", "normalization": "identity" }, { "start": 95, "end": 109, "label": "PHONE", "text": "+49 541 730102", "score": null, "source": "deterministic", "model_revision": "contact-patterns-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 0, "end": 11, "label": "PERSON", "text": "Emil Farber", "uncovered_positions": [] }, { "start": 18, "end": 26, "label": "RELIGION", "text": "orthodox", "uncovered_positions": [] }, { "start": 54, "end": 84, "label": "EMAIL", "text": "case102.review@example.invalid", "uncovered_positions": [] }, { "start": 95, "end": 109, "label": "PHONE", "text": "+49 541 730102", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-013", "scenario": null, "predictions": [ { "start": 0, "end": 11, "label": "PERSON", "text": "Emil Farber", "score": 0.9992079138755798, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 29, "end": 50, "label": "RELIGION", "text": "Orthodox Christianity", "score": 0.9964632391929626, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 76, "end": 106, "label": "EMAIL", "text": "case102.review@example.invalid", "score": 0.9135122895240784, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 115, "end": 129, "label": "PHONE", "text": "+49 541 730102", "score": 0.9687923192977905, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 0, "end": 11, "label": "PERSON", "text": "Emil Farber", "score": 0.9992079138755798, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 29, "end": 50, "label": "RELIGION", "text": "Orthodox Christianity", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" }, { "start": 76, "end": 106, "label": "EMAIL", "text": "case102.review@example.invalid", "score": null, "source": "deterministic", "model_revision": "contact-patterns-v1", "normalization": "identity" }, { "start": 115, "end": 129, "label": "PHONE", "text": "+49 541 730102", "score": null, "source": "deterministic", "model_revision": "contact-patterns-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 0, "end": 11, "label": "PERSON", "text": "Emil Farber", "uncovered_positions": [] }, { "start": 29, "end": 50, "label": "RELIGION", "text": "Orthodox Christianity", "uncovered_positions": [] }, { "start": 76, "end": 106, "label": "EMAIL", "text": "case102.review@example.invalid", "uncovered_positions": [] }, { "start": 115, "end": 129, "label": "PHONE", "text": "+49 541 730102", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-014", "scenario": null, "predictions": [ { "start": 0, "end": 11, "label": "PERSON", "text": "Emil Farber", "score": 0.9997215867042542, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 20, "end": 27, "label": "CITY", "text": "Potsdam", "score": 0.999298095703125, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 30, "end": 41, "label": "PERSON", "text": "Emil Farber", "score": 0.9996708631515503, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 67, "end": 88, "label": "ORGANIZATION", "text": "Northbridge Analytics", "score": 0.9961026906967163, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 0, "end": 11, "label": "PERSON", "text": "Emil Farber", "score": 0.9997215867042542, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 20, "end": 27, "label": "CITY", "text": "Potsdam", "score": 0.999298095703125, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 30, "end": 41, "label": "PERSON", "text": "Emil Farber", "score": 0.9996708631515503, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 67, "end": 88, "label": "ORGANIZATION", "text": "Northbridge Analytics", "score": 0.9961026906967163, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 0, "end": 11, "label": "PERSON", "text": "Emil Farber", "uncovered_positions": [] }, { "start": 20, "end": 27, "label": "CITY", "text": "Potsdam", "uncovered_positions": [] }, { "start": 30, "end": 41, "label": "PERSON", "text": "Emil Farber", "uncovered_positions": [] }, { "start": 67, "end": 88, "label": "ORGANIZATION", "text": "Northbridge Analytics", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-015", "scenario": null, "predictions": [ { "start": 3, "end": 10, "label": "CITY", "text": "Potsdam", "score": 0.9998461008071899, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 12, "end": 23, "label": "PERSON", "text": "Emil Farber", "score": 0.9998921155929565, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 35, "end": 56, "label": "ORGANIZATION", "text": "Northbridge Analytics", "score": 0.9982470273971558, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 77, "end": 88, "label": "PERSON", "text": "Emil Farber", "score": 0.9998248219490051, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 101, "end": 108, "label": "CITY", "text": "Potsdam", "score": 0.9998860359191895, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 3, "end": 10, "label": "CITY", "text": "Potsdam", "score": 0.9998461008071899, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 12, "end": 23, "label": "PERSON", "text": "Emil Farber", "score": 0.9998921155929565, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 35, "end": 56, "label": "ORGANIZATION", "text": "Northbridge Analytics", "score": 0.9982470273971558, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 77, "end": 88, "label": "PERSON", "text": "Emil Farber", "score": 0.9998248219490051, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 101, "end": 108, "label": "CITY", "text": "Potsdam", "score": 0.9998860359191895, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 3, "end": 10, "label": "CITY", "text": "Potsdam", "uncovered_positions": [] }, { "start": 12, "end": 23, "label": "PERSON", "text": "Emil Farber", "uncovered_positions": [] }, { "start": 35, "end": 56, "label": "ORGANIZATION", "text": "Northbridge Analytics", "uncovered_positions": [] }, { "start": 77, "end": 88, "label": "PERSON", "text": "Emil Farber", "uncovered_positions": [] }, { "start": 101, "end": 108, "label": "CITY", "text": "Potsdam", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-016", "scenario": null, "predictions": [ { "start": 6, "end": 18, "label": "PERSON", "text": "Emil-\nFarber", "score": 0.9998793601989746, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "dehyphenation-v1" }, { "start": 24, "end": 33, "label": "CITY", "text": "Pot-\nsdam", "score": 0.9999624490737915, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "dehyphenation-v1" }, { "start": 47, "end": 68, "label": "ORGANIZATION", "text": "Northbridge Analytics", "score": 0.997786283493042, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "dehyphenation-v1" } ], "combined_predictions": [ { "start": 6, "end": 18, "label": "PERSON", "text": "Emil-\nFarber", "score": 0.9998793601989746, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "dehyphenation-v1" }, { "start": 24, "end": 33, "label": "CITY", "text": "Pot-\nsdam", "score": 0.9999624490737915, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "dehyphenation-v1" }, { "start": 47, "end": 68, "label": "ORGANIZATION", "text": "Northbridge Analytics", "score": 0.997786283493042, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "dehyphenation-v1" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 6, "end": 18, "label": "PERSON", "text": "Emil-\nFarber", "uncovered_positions": [] }, { "start": 24, "end": 33, "label": "CITY", "text": "Pot-\nsdam", "uncovered_positions": [] }, { "start": 47, "end": 68, "label": "ORGANIZATION", "text": "Northbridge Analytics", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-017", "scenario": null, "predictions": [ { "start": 16, "end": 27, "label": "PERSON", "text": "Em1l Farber", "score": 0.9948354959487915, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 39, "end": 46, "label": "CITY", "text": "P0tsd4m", "score": 0.9718403220176697, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 60, "end": 81, "label": "ORGANIZATION", "text": "Northbridge Analytics", "score": 0.9991218447685242, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 16, "end": 27, "label": "PERSON", "text": "Em1l Farber", "score": 0.9948354959487915, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 39, "end": 46, "label": "CITY", "text": "P0tsd4m", "score": 0.9718403220176697, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 60, "end": 81, "label": "ORGANIZATION", "text": "Northbridge Analytics", "score": 0.9991218447685242, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 16, "end": 27, "label": "PERSON", "text": "Em1l Farber", "uncovered_positions": [] }, { "start": 39, "end": 46, "label": "CITY", "text": "P0tsd4m", "uncovered_positions": [] }, { "start": 60, "end": 81, "label": "ORGANIZATION", "text": "Northbridge Analytics", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-018", "scenario": null, "predictions": [ { "start": 10, "end": 18, "label": "RELIGION", "text": "orthodox", "score": 0.8374165296554565, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 45, "end": 66, "label": "ORGANIZATION", "text": "Northbridge Analytics", "score": 0.9889349341392517, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 80, "end": 91, "label": "PERSON", "text": "Emil Farber", "score": 0.9984270334243774, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 10, "end": 18, "label": "RELIGION", "text": "orthodox", "score": 0.8374165296554565, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 45, "end": 66, "label": "ORGANIZATION", "text": "Northbridge Analytics", "score": 0.9889349341392517, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 80, "end": 91, "label": "PERSON", "text": "Emil Farber", "score": 0.9984270334243774, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 10, "end": 18, "label": "RELIGION", "text": "orthodox", "uncovered_positions": [] }, { "start": 45, "end": 66, "label": "ORGANIZATION", "text": "Northbridge Analytics", "uncovered_positions": [] }, { "start": 80, "end": 91, "label": "PERSON", "text": "Emil Farber", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-019", "scenario": null, "predictions": [ { "start": 101, "end": 107, "label": "PERSON", "text": "person", "score": 0.7027997374534607, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 109, "end": 114, "label": "CITY", "text": "place", "score": 0.7789004445075989, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 116, "end": 128, "label": "ORGANIZATION", "text": "organization", "score": 0.8365603685379028, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 40, "end": 64, "label": "RELIGION", "text": "no religious affiliation", "score": null, "source": "deterministic", "model_revision": "religious-status-v1", "normalization": "identity" }, { "start": 101, "end": 107, "label": "PERSON", "text": "person", "score": 0.7027997374534607, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 109, "end": 114, "label": "CITY", "text": "place", "score": 0.7789004445075989, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 116, "end": 128, "label": "ORGANIZATION", "text": "organization", "score": 0.8365603685379028, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [ [ 40, 64, "RELIGION" ], [ 101, 107, "PERSON" ], [ 109, 114, "CITY" ], [ 116, 128, "ORGANIZATION" ] ], "false_negatives": [], "exact_match": false, "error_code": null, "canaries": [], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-020", "scenario": null, "predictions": [ { "start": 10, "end": 25, "label": "PERSON", "text": "Cora Lindenfels", "score": 0.9973998069763184, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 39, "end": 62, "label": "ORGANIZATION", "text": "Amber Health Foundation", "score": 0.9973830580711365, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 66, "end": 72, "label": "CITY", "text": "Dundee", "score": 0.9992653727531433, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 10, "end": 25, "label": "PERSON", "text": "Cora Lindenfels", "score": 0.9973998069763184, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 39, "end": 62, "label": "ORGANIZATION", "text": "Amber Health Foundation", "score": 0.9973830580711365, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 66, "end": 72, "label": "CITY", "text": "Dundee", "score": 0.9992653727531433, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 10, "end": 25, "label": "PERSON", "text": "Cora Lindenfels", "uncovered_positions": [] }, { "start": 39, "end": 62, "label": "ORGANIZATION", "text": "Amber Health Foundation", "uncovered_positions": [] }, { "start": 66, "end": 72, "label": "CITY", "text": "Dundee", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-021", "scenario": null, "predictions": [ { "start": 12, "end": 27, "label": "PERSON", "text": "Cora Lindenfels", "score": 0.997750461101532, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 39, "end": 62, "label": "ORGANIZATION", "text": "Amber Health Foundation", "score": 0.9969223141670227, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 68, "end": 74, "label": "CITY", "text": "Dundee", "score": 0.9994328618049622, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 12, "end": 27, "label": "PERSON", "text": "Cora Lindenfels", "score": 0.997750461101532, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 39, "end": 62, "label": "ORGANIZATION", "text": "Amber Health Foundation", "score": 0.9969223141670227, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 68, "end": 74, "label": "CITY", "text": "Dundee", "score": 0.9994328618049622, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 12, "end": 27, "label": "PERSON", "text": "Cora Lindenfels", "uncovered_positions": [] }, { "start": 39, "end": 62, "label": "ORGANIZATION", "text": "Amber Health Foundation", "uncovered_positions": [] }, { "start": 68, "end": 74, "label": "CITY", "text": "Dundee", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-022", "scenario": null, "predictions": [ { "start": 0, "end": 15, "label": "PERSON", "text": "Cora Lindenfels", "score": 0.9937719702720642, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 22, "end": 33, "label": "RELIGION", "text": "jainistisch", "score": 0.9889543056488037, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 61, "end": 91, "label": "EMAIL", "text": "case103.review@example.invalid", "score": 0.9086615443229675, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 102, "end": 116, "label": "PHONE", "text": "+49 541 730103", "score": 0.9945864677429199, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 0, "end": 15, "label": "PERSON", "text": "Cora Lindenfels", "score": 0.9937719702720642, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 22, "end": 33, "label": "RELIGION", "text": "jainistisch", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" }, { "start": 61, "end": 91, "label": "EMAIL", "text": "case103.review@example.invalid", "score": null, "source": "deterministic", "model_revision": "contact-patterns-v1", "normalization": "identity" }, { "start": 102, "end": 116, "label": "PHONE", "text": "+49 541 730103", "score": null, "source": "deterministic", "model_revision": "contact-patterns-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 0, "end": 15, "label": "PERSON", "text": "Cora Lindenfels", "uncovered_positions": [] }, { "start": 22, "end": 33, "label": "RELIGION", "text": "jainistisch", "uncovered_positions": [] }, { "start": 61, "end": 91, "label": "EMAIL", "text": "case103.review@example.invalid", "uncovered_positions": [] }, { "start": 102, "end": 116, "label": "PHONE", "text": "+49 541 730103", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-023", "scenario": null, "predictions": [ { "start": 0, "end": 15, "label": "PERSON", "text": "Cora Lindenfels", "score": 0.9916738867759705, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 33, "end": 40, "label": "RELIGION", "text": "Jainism", "score": 0.9980966448783875, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 66, "end": 96, "label": "EMAIL", "text": "case103.review@example.invalid", "score": 0.9247252345085144, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 105, "end": 119, "label": "PHONE", "text": "+49 541 730103", "score": 0.9735237956047058, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 0, "end": 15, "label": "PERSON", "text": "Cora Lindenfels", "score": 0.9916738867759705, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 33, "end": 40, "label": "RELIGION", "text": "Jainism", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" }, { "start": 66, "end": 96, "label": "EMAIL", "text": "case103.review@example.invalid", "score": null, "source": "deterministic", "model_revision": "contact-patterns-v1", "normalization": "identity" }, { "start": 105, "end": 119, "label": "PHONE", "text": "+49 541 730103", "score": null, "source": "deterministic", "model_revision": "contact-patterns-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 0, "end": 15, "label": "PERSON", "text": "Cora Lindenfels", "uncovered_positions": [] }, { "start": 33, "end": 40, "label": "RELIGION", "text": "Jainism", "uncovered_positions": [] }, { "start": 66, "end": 96, "label": "EMAIL", "text": "case103.review@example.invalid", "uncovered_positions": [] }, { "start": 105, "end": 119, "label": "PHONE", "text": "+49 541 730103", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-024", "scenario": null, "predictions": [ { "start": 0, "end": 15, "label": "PERSON", "text": "Cora Lindenfels", "score": 0.9980031847953796, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 24, "end": 30, "label": "CITY", "text": "Dundee", "score": 0.9969285130500793, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 33, "end": 48, "label": "PERSON", "text": "Cora Lindenfels", "score": 0.9981155395507812, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 74, "end": 97, "label": "ORGANIZATION", "text": "Amber Health Foundation", "score": 0.9936736822128296, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 0, "end": 15, "label": "PERSON", "text": "Cora Lindenfels", "score": 0.9980031847953796, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 24, "end": 30, "label": "CITY", "text": "Dundee", "score": 0.9969285130500793, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 33, "end": 48, "label": "PERSON", "text": "Cora Lindenfels", "score": 0.9981155395507812, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 74, "end": 97, "label": "ORGANIZATION", "text": "Amber Health Foundation", "score": 0.9936736822128296, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 0, "end": 15, "label": "PERSON", "text": "Cora Lindenfels", "uncovered_positions": [] }, { "start": 24, "end": 30, "label": "CITY", "text": "Dundee", "uncovered_positions": [] }, { "start": 33, "end": 48, "label": "PERSON", "text": "Cora Lindenfels", "uncovered_positions": [] }, { "start": 74, "end": 97, "label": "ORGANIZATION", "text": "Amber Health Foundation", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-025", "scenario": null, "predictions": [ { "start": 3, "end": 9, "label": "CITY", "text": "Dundee", "score": 0.9998049139976501, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 11, "end": 26, "label": "PERSON", "text": "Cora Lindenfels", "score": 0.9993300437927246, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 38, "end": 61, "label": "ORGANIZATION", "text": "Amber Health Foundation", "score": 0.9960338473320007, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 82, "end": 97, "label": "PERSON", "text": "Cora Lindenfels", "score": 0.9993378520011902, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 110, "end": 116, "label": "CITY", "text": "Dundee", "score": 0.9997878670692444, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 3, "end": 9, "label": "CITY", "text": "Dundee", "score": 0.9998049139976501, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 11, "end": 26, "label": "PERSON", "text": "Cora Lindenfels", "score": 0.9993300437927246, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 38, "end": 61, "label": "ORGANIZATION", "text": "Amber Health Foundation", "score": 0.9960338473320007, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 82, "end": 97, "label": "PERSON", "text": "Cora Lindenfels", "score": 0.9993378520011902, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 110, "end": 116, "label": "CITY", "text": "Dundee", "score": 0.9997878670692444, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 3, "end": 9, "label": "CITY", "text": "Dundee", "uncovered_positions": [] }, { "start": 11, "end": 26, "label": "PERSON", "text": "Cora Lindenfels", "uncovered_positions": [] }, { "start": 38, "end": 61, "label": "ORGANIZATION", "text": "Amber Health Foundation", "uncovered_positions": [] }, { "start": 82, "end": 97, "label": "PERSON", "text": "Cora Lindenfels", "uncovered_positions": [] }, { "start": 110, "end": 116, "label": "CITY", "text": "Dundee", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-026", "scenario": null, "predictions": [ { "start": 6, "end": 22, "label": "PERSON", "text": "Cora-\nLindenfels", "score": 0.997488260269165, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "dehyphenation-v1" }, { "start": 28, "end": 36, "label": "CITY", "text": "Dun-\ndee", "score": 0.999882698059082, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "dehyphenation-v1" }, { "start": 50, "end": 73, "label": "ORGANIZATION", "text": "Amber Health Foundation", "score": 0.9973509311676025, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "dehyphenation-v1" } ], "combined_predictions": [ { "start": 6, "end": 22, "label": "PERSON", "text": "Cora-\nLindenfels", "score": 0.997488260269165, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "dehyphenation-v1" }, { "start": 28, "end": 36, "label": "CITY", "text": "Dun-\ndee", "score": 0.999882698059082, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "dehyphenation-v1" }, { "start": 50, "end": 73, "label": "ORGANIZATION", "text": "Amber Health Foundation", "score": 0.9973509311676025, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "dehyphenation-v1" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 6, "end": 22, "label": "PERSON", "text": "Cora-\nLindenfels", "uncovered_positions": [] }, { "start": 28, "end": 36, "label": "CITY", "text": "Dun-\ndee", "uncovered_positions": [] }, { "start": 50, "end": 73, "label": "ORGANIZATION", "text": "Amber Health Foundation", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-027", "scenario": null, "predictions": [ { "start": 16, "end": 31, "label": "PERSON", "text": "C0ra L1ndenfels", "score": 0.9803281426429749, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 43, "end": 49, "label": "CITY", "text": "Dundee", "score": 0.9996110796928406, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 63, "end": 86, "label": "ORGANIZATION", "text": "Amber Health Foundation", "score": 0.9959501028060913, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 16, "end": 31, "label": "PERSON", "text": "C0ra L1ndenfels", "score": 0.9803281426429749, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 43, "end": 49, "label": "CITY", "text": "Dundee", "score": 0.9996110796928406, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 63, "end": 86, "label": "ORGANIZATION", "text": "Amber Health Foundation", "score": 0.9959501028060913, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 16, "end": 31, "label": "PERSON", "text": "C0ra L1ndenfels", "uncovered_positions": [] }, { "start": 43, "end": 49, "label": "CITY", "text": "Dundee", "uncovered_positions": [] }, { "start": 63, "end": 86, "label": "ORGANIZATION", "text": "Amber Health Foundation", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-028", "scenario": null, "predictions": [ { "start": 10, "end": 21, "label": "RELIGION", "text": "jainistisch", "score": 0.7595014572143555, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 48, "end": 71, "label": "ORGANIZATION", "text": "Amber Health Foundation", "score": 0.9929428100585938, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 85, "end": 100, "label": "PERSON", "text": "Cora Lindenfels", "score": 0.9964152574539185, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 10, "end": 21, "label": "RELIGION", "text": "jainistisch", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" }, { "start": 48, "end": 71, "label": "ORGANIZATION", "text": "Amber Health Foundation", "score": 0.9929428100585938, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 85, "end": 100, "label": "PERSON", "text": "Cora Lindenfels", "score": 0.9964152574539185, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 10, "end": 21, "label": "RELIGION", "text": "jainistisch", "uncovered_positions": [] }, { "start": 48, "end": 71, "label": "ORGANIZATION", "text": "Amber Health Foundation", "uncovered_positions": [] }, { "start": 85, "end": 100, "label": "PERSON", "text": "Cora Lindenfels", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-029", "scenario": null, "predictions": [ { "start": 101, "end": 107, "label": "PERSON", "text": "person", "score": 0.7094536423683167, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 109, "end": 114, "label": "CITY", "text": "place", "score": 0.7874830365180969, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 116, "end": 128, "label": "ORGANIZATION", "text": "organization", "score": 0.8446553945541382, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 40, "end": 64, "label": "RELIGION", "text": "no religious affiliation", "score": null, "source": "deterministic", "model_revision": "religious-status-v1", "normalization": "identity" }, { "start": 101, "end": 107, "label": "PERSON", "text": "person", "score": 0.7094536423683167, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 109, "end": 114, "label": "CITY", "text": "place", "score": 0.7874830365180969, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 116, "end": 128, "label": "ORGANIZATION", "text": "organization", "score": 0.8446553945541382, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [ [ 40, 64, "RELIGION" ], [ 101, 107, "PERSON" ], [ 109, 114, "CITY" ], [ 116, 128, "ORGANIZATION" ] ], "false_negatives": [], "exact_match": false, "error_code": null, "canaries": [], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-030", "scenario": null, "predictions": [ { "start": 10, "end": 22, "label": "PERSON", "text": "Jasper Keene", "score": 0.997222900390625, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 36, "end": 57, "label": "ORGANIZATION", "text": "Silver Finch Robotics", "score": 0.9982925057411194, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 61, "end": 69, "label": "CITY", "text": "Aberdeen", "score": 0.9969948530197144, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 10, "end": 22, "label": "PERSON", "text": "Jasper Keene", "score": 0.997222900390625, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 36, "end": 57, "label": "ORGANIZATION", "text": "Silver Finch Robotics", "score": 0.9982925057411194, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 61, "end": 69, "label": "CITY", "text": "Aberdeen", "score": 0.9969948530197144, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 10, "end": 22, "label": "PERSON", "text": "Jasper Keene", "uncovered_positions": [] }, { "start": 36, "end": 57, "label": "ORGANIZATION", "text": "Silver Finch Robotics", "uncovered_positions": [] }, { "start": 61, "end": 69, "label": "CITY", "text": "Aberdeen", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-031", "scenario": null, "predictions": [ { "start": 12, "end": 24, "label": "PERSON", "text": "Jasper Keene", "score": 0.9982971549034119, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 36, "end": 57, "label": "ORGANIZATION", "text": "Silver Finch Robotics", "score": 0.9989097118377686, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 63, "end": 71, "label": "CITY", "text": "Aberdeen", "score": 0.9978512525558472, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 12, "end": 24, "label": "PERSON", "text": "Jasper Keene", "score": 0.9982971549034119, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 36, "end": 57, "label": "ORGANIZATION", "text": "Silver Finch Robotics", "score": 0.9989097118377686, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 63, "end": 71, "label": "CITY", "text": "Aberdeen", "score": 0.9978512525558472, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 12, "end": 24, "label": "PERSON", "text": "Jasper Keene", "uncovered_positions": [] }, { "start": 36, "end": 57, "label": "ORGANIZATION", "text": "Silver Finch Robotics", "uncovered_positions": [] }, { "start": 63, "end": 71, "label": "CITY", "text": "Aberdeen", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-032", "scenario": null, "predictions": [ { "start": 0, "end": 12, "label": "PERSON", "text": "Jasper Keene", "score": 0.9964504241943359, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 19, "end": 31, "label": "RELIGION", "text": "hinduistisch", "score": 0.9917290806770325, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 59, "end": 89, "label": "EMAIL", "text": "case104.review@example.invalid", "score": 0.9179330468177795, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 100, "end": 114, "label": "PHONE", "text": "+49 541 730104", "score": 0.9929746389389038, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 0, "end": 12, "label": "PERSON", "text": "Jasper Keene", "score": 0.9964504241943359, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 19, "end": 31, "label": "RELIGION", "text": "hinduistisch", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" }, { "start": 59, "end": 89, "label": "EMAIL", "text": "case104.review@example.invalid", "score": null, "source": "deterministic", "model_revision": "contact-patterns-v1", "normalization": "identity" }, { "start": 100, "end": 114, "label": "PHONE", "text": "+49 541 730104", "score": null, "source": "deterministic", "model_revision": "contact-patterns-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 0, "end": 12, "label": "PERSON", "text": "Jasper Keene", "uncovered_positions": [] }, { "start": 19, "end": 31, "label": "RELIGION", "text": "hinduistisch", "uncovered_positions": [] }, { "start": 59, "end": 89, "label": "EMAIL", "text": "case104.review@example.invalid", "uncovered_positions": [] }, { "start": 100, "end": 114, "label": "PHONE", "text": "+49 541 730104", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-033", "scenario": null, "predictions": [ { "start": 0, "end": 12, "label": "PERSON", "text": "Jasper Keene", "score": 0.9973819851875305, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 30, "end": 38, "label": "RELIGION", "text": "Hinduism", "score": 0.9998874664306641, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 64, "end": 94, "label": "EMAIL", "text": "case104.review@example.invalid", "score": 0.9190782308578491, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 103, "end": 117, "label": "PHONE", "text": "+49 541 730104", "score": 0.9675639271736145, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 0, "end": 12, "label": "PERSON", "text": "Jasper Keene", "score": 0.9973819851875305, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 30, "end": 38, "label": "RELIGION", "text": "Hinduism", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" }, { "start": 64, "end": 94, "label": "EMAIL", "text": "case104.review@example.invalid", "score": null, "source": "deterministic", "model_revision": "contact-patterns-v1", "normalization": "identity" }, { "start": 103, "end": 117, "label": "PHONE", "text": "+49 541 730104", "score": null, "source": "deterministic", "model_revision": "contact-patterns-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 0, "end": 12, "label": "PERSON", "text": "Jasper Keene", "uncovered_positions": [] }, { "start": 30, "end": 38, "label": "RELIGION", "text": "Hinduism", "uncovered_positions": [] }, { "start": 64, "end": 94, "label": "EMAIL", "text": "case104.review@example.invalid", "uncovered_positions": [] }, { "start": 103, "end": 117, "label": "PHONE", "text": "+49 541 730104", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-034", "scenario": null, "predictions": [ { "start": 0, "end": 12, "label": "PERSON", "text": "Jasper Keene", "score": 0.9992870688438416, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 21, "end": 29, "label": "CITY", "text": "Aberdeen", "score": 0.9939911961555481, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 32, "end": 44, "label": "PERSON", "text": "Jasper Keene", "score": 0.9992408752441406, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 70, "end": 91, "label": "ORGANIZATION", "text": "Silver Finch Robotics", "score": 0.9981598258018494, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 0, "end": 12, "label": "PERSON", "text": "Jasper Keene", "score": 0.9992870688438416, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 21, "end": 29, "label": "CITY", "text": "Aberdeen", "score": 0.9939911961555481, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 32, "end": 44, "label": "PERSON", "text": "Jasper Keene", "score": 0.9992408752441406, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 70, "end": 91, "label": "ORGANIZATION", "text": "Silver Finch Robotics", "score": 0.9981598258018494, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 0, "end": 12, "label": "PERSON", "text": "Jasper Keene", "uncovered_positions": [] }, { "start": 21, "end": 29, "label": "CITY", "text": "Aberdeen", "uncovered_positions": [] }, { "start": 32, "end": 44, "label": "PERSON", "text": "Jasper Keene", "uncovered_positions": [] }, { "start": 70, "end": 91, "label": "ORGANIZATION", "text": "Silver Finch Robotics", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-035", "scenario": null, "predictions": [ { "start": 3, "end": 11, "label": "CITY", "text": "Aberdeen", "score": 0.9991244673728943, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 13, "end": 25, "label": "PERSON", "text": "Jasper Keene", "score": 0.9997237324714661, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 37, "end": 58, "label": "ORGANIZATION", "text": "Silver Finch Robotics", "score": 0.9979164004325867, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 79, "end": 91, "label": "PERSON", "text": "Jasper Keene", "score": 0.9997982382774353, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 104, "end": 112, "label": "CITY", "text": "Aberdeen", "score": 0.9984569549560547, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 3, "end": 11, "label": "CITY", "text": "Aberdeen", "score": 0.9991244673728943, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 13, "end": 25, "label": "PERSON", "text": "Jasper Keene", "score": 0.9997237324714661, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 37, "end": 58, "label": "ORGANIZATION", "text": "Silver Finch Robotics", "score": 0.9979164004325867, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 79, "end": 91, "label": "PERSON", "text": "Jasper Keene", "score": 0.9997982382774353, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 104, "end": 112, "label": "CITY", "text": "Aberdeen", "score": 0.9984569549560547, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 3, "end": 11, "label": "CITY", "text": "Aberdeen", "uncovered_positions": [] }, { "start": 13, "end": 25, "label": "PERSON", "text": "Jasper Keene", "uncovered_positions": [] }, { "start": 37, "end": 58, "label": "ORGANIZATION", "text": "Silver Finch Robotics", "uncovered_positions": [] }, { "start": 79, "end": 91, "label": "PERSON", "text": "Jasper Keene", "uncovered_positions": [] }, { "start": 104, "end": 112, "label": "CITY", "text": "Aberdeen", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-036", "scenario": null, "predictions": [ { "start": 6, "end": 19, "label": "PERSON", "text": "Jasper-\nKeene", "score": 0.9998306035995483, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "dehyphenation-v1" }, { "start": 25, "end": 35, "label": "CITY", "text": "Aber-\ndeen", "score": 0.9998304843902588, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "dehyphenation-v1" }, { "start": 49, "end": 70, "label": "ORGANIZATION", "text": "Silver Finch Robotics", "score": 0.9984242916107178, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "dehyphenation-v1" } ], "combined_predictions": [ { "start": 6, "end": 19, "label": "PERSON", "text": "Jasper-\nKeene", "score": 0.9998306035995483, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "dehyphenation-v1" }, { "start": 25, "end": 35, "label": "CITY", "text": "Aber-\ndeen", "score": 0.9998304843902588, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "dehyphenation-v1" }, { "start": 49, "end": 70, "label": "ORGANIZATION", "text": "Silver Finch Robotics", "score": 0.9984242916107178, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "dehyphenation-v1" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 6, "end": 19, "label": "PERSON", "text": "Jasper-\nKeene", "uncovered_positions": [] }, { "start": 25, "end": 35, "label": "CITY", "text": "Aber-\ndeen", "uncovered_positions": [] }, { "start": 49, "end": 70, "label": "ORGANIZATION", "text": "Silver Finch Robotics", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-037", "scenario": null, "predictions": [ { "start": 16, "end": 28, "label": "PERSON", "text": "Jasper Keene", "score": 0.9981817007064819, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 40, "end": 48, "label": "CITY", "text": "Aberdeen", "score": 0.999523401260376, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 62, "end": 83, "label": "ORGANIZATION", "text": "Silver Finch Robotics", "score": 0.9983817338943481, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 16, "end": 28, "label": "PERSON", "text": "Jasper Keene", "score": 0.9981817007064819, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 40, "end": 48, "label": "CITY", "text": "Aberdeen", "score": 0.999523401260376, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 62, "end": 83, "label": "ORGANIZATION", "text": "Silver Finch Robotics", "score": 0.9983817338943481, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 16, "end": 28, "label": "PERSON", "text": "Jasper Keene", "uncovered_positions": [] }, { "start": 40, "end": 48, "label": "CITY", "text": "Aberdeen", "uncovered_positions": [] }, { "start": 62, "end": 83, "label": "ORGANIZATION", "text": "Silver Finch Robotics", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-038", "scenario": null, "predictions": [ { "start": 10, "end": 22, "label": "RELIGION", "text": "hinduistisch", "score": 0.8530606627464294, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 49, "end": 70, "label": "ORGANIZATION", "text": "Silver Finch Robotics", "score": 0.991775631904602, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 84, "end": 96, "label": "PERSON", "text": "Jasper Keene", "score": 0.9976971745491028, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 10, "end": 22, "label": "RELIGION", "text": "hinduistisch", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" }, { "start": 49, "end": 70, "label": "ORGANIZATION", "text": "Silver Finch Robotics", "score": 0.991775631904602, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 84, "end": 96, "label": "PERSON", "text": "Jasper Keene", "score": 0.9976971745491028, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 10, "end": 22, "label": "RELIGION", "text": "hinduistisch", "uncovered_positions": [] }, { "start": 49, "end": 70, "label": "ORGANIZATION", "text": "Silver Finch Robotics", "uncovered_positions": [] }, { "start": 84, "end": 96, "label": "PERSON", "text": "Jasper Keene", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-039", "scenario": null, "predictions": [ { "start": 101, "end": 107, "label": "PERSON", "text": "person", "score": 0.7161614894866943, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 109, "end": 114, "label": "CITY", "text": "place", "score": 0.7881067395210266, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 116, "end": 128, "label": "ORGANIZATION", "text": "organization", "score": 0.847603976726532, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 40, "end": 64, "label": "RELIGION", "text": "no religious affiliation", "score": null, "source": "deterministic", "model_revision": "religious-status-v1", "normalization": "identity" }, { "start": 101, "end": 107, "label": "PERSON", "text": "person", "score": 0.7161614894866943, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 109, "end": 114, "label": "CITY", "text": "place", "score": 0.7881067395210266, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 116, "end": 128, "label": "ORGANIZATION", "text": "organization", "score": 0.847603976726532, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [ [ 40, 64, "RELIGION" ], [ 101, 107, "PERSON" ], [ 109, 114, "CITY" ], [ 116, 128, "ORGANIZATION" ] ], "false_negatives": [], "exact_match": false, "error_code": null, "canaries": [], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-040", "scenario": null, "predictions": [ { "start": 10, "end": 23, "label": "PERSON", "text": "Frida Östberg", "score": 0.9995619654655457, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 37, "end": 59, "label": "ORGANIZATION", "text": "Blue Cedar Cooperative", "score": 0.9986785054206848, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 63, "end": 70, "label": "CITY", "text": "Uppsala", "score": 0.9995537400245667, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 10, "end": 23, "label": "PERSON", "text": "Frida Östberg", "score": 0.9995619654655457, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 37, "end": 59, "label": "ORGANIZATION", "text": "Blue Cedar Cooperative", "score": 0.9986785054206848, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 63, "end": 70, "label": "CITY", "text": "Uppsala", "score": 0.9995537400245667, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 10, "end": 23, "label": "PERSON", "text": "Frida Östberg", "uncovered_positions": [] }, { "start": 37, "end": 59, "label": "ORGANIZATION", "text": "Blue Cedar Cooperative", "uncovered_positions": [] }, { "start": 63, "end": 70, "label": "CITY", "text": "Uppsala", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-041", "scenario": null, "predictions": [ { "start": 12, "end": 25, "label": "PERSON", "text": "Frida Östberg", "score": 0.9993315935134888, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 37, "end": 59, "label": "ORGANIZATION", "text": "Blue Cedar Cooperative", "score": 0.9984630346298218, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 65, "end": 72, "label": "CITY", "text": "Uppsala", "score": 0.9996377229690552, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 12, "end": 25, "label": "PERSON", "text": "Frida Östberg", "score": 0.9993315935134888, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 37, "end": 59, "label": "ORGANIZATION", "text": "Blue Cedar Cooperative", "score": 0.9984630346298218, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 65, "end": 72, "label": "CITY", "text": "Uppsala", "score": 0.9996377229690552, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 12, "end": 25, "label": "PERSON", "text": "Frida Östberg", "uncovered_positions": [] }, { "start": 37, "end": 59, "label": "ORGANIZATION", "text": "Blue Cedar Cooperative", "uncovered_positions": [] }, { "start": 65, "end": 72, "label": "CITY", "text": "Uppsala", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-042", "scenario": null, "predictions": [ { "start": 0, "end": 13, "label": "PERSON", "text": "Frida Östberg", "score": 0.9994117021560669, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 20, "end": 25, "label": "RELIGION", "text": "bahai", "score": 0.9852348566055298, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 53, "end": 83, "label": "EMAIL", "text": "case105.review@example.invalid", "score": 0.8997676968574524, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 94, "end": 108, "label": "PHONE", "text": "+49 541 730105", "score": 0.9945896863937378, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 0, "end": 13, "label": "PERSON", "text": "Frida Östberg", "score": 0.9994117021560669, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 20, "end": 25, "label": "RELIGION", "text": "bahai", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" }, { "start": 53, "end": 83, "label": "EMAIL", "text": "case105.review@example.invalid", "score": null, "source": "deterministic", "model_revision": "contact-patterns-v1", "normalization": "identity" }, { "start": 94, "end": 108, "label": "PHONE", "text": "+49 541 730105", "score": null, "source": "deterministic", "model_revision": "contact-patterns-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 0, "end": 13, "label": "PERSON", "text": "Frida Östberg", "uncovered_positions": [] }, { "start": 20, "end": 25, "label": "RELIGION", "text": "bahai", "uncovered_positions": [] }, { "start": 53, "end": 83, "label": "EMAIL", "text": "case105.review@example.invalid", "uncovered_positions": [] }, { "start": 94, "end": 108, "label": "PHONE", "text": "+49 541 730105", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-043", "scenario": null, "predictions": [ { "start": 0, "end": 13, "label": "PERSON", "text": "Frida Östberg", "score": 0.9995088577270508, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 31, "end": 43, "label": "RELIGION", "text": "Baháʼí Faith", "score": 0.9641181826591492, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 69, "end": 99, "label": "EMAIL", "text": "case105.review@example.invalid", "score": 0.9057281017303467, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 108, "end": 122, "label": "PHONE", "text": "+49 541 730105", "score": 0.9755640625953674, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 0, "end": 13, "label": "PERSON", "text": "Frida Östberg", "score": 0.9995088577270508, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 31, "end": 43, "label": "RELIGION", "text": "Baháʼí Faith", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" }, { "start": 69, "end": 99, "label": "EMAIL", "text": "case105.review@example.invalid", "score": null, "source": "deterministic", "model_revision": "contact-patterns-v1", "normalization": "identity" }, { "start": 108, "end": 122, "label": "PHONE", "text": "+49 541 730105", "score": null, "source": "deterministic", "model_revision": "contact-patterns-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 0, "end": 13, "label": "PERSON", "text": "Frida Östberg", "uncovered_positions": [] }, { "start": 31, "end": 43, "label": "RELIGION", "text": "Baháʼí Faith", "uncovered_positions": [] }, { "start": 69, "end": 99, "label": "EMAIL", "text": "case105.review@example.invalid", "uncovered_positions": [] }, { "start": 108, "end": 122, "label": "PHONE", "text": "+49 541 730105", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-044", "scenario": null, "predictions": [ { "start": 0, "end": 13, "label": "PERSON", "text": "Frida Östberg", "score": 0.9998176693916321, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 22, "end": 29, "label": "CITY", "text": "Uppsala", "score": 0.9994066953659058, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 32, "end": 45, "label": "PERSON", "text": "Frida Östberg", "score": 0.999852180480957, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 71, "end": 93, "label": "ORGANIZATION", "text": "Blue Cedar Cooperative", "score": 0.9967365860939026, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 0, "end": 13, "label": "PERSON", "text": "Frida Östberg", "score": 0.9998176693916321, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 22, "end": 29, "label": "CITY", "text": "Uppsala", "score": 0.9994066953659058, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 32, "end": 45, "label": "PERSON", "text": "Frida Östberg", "score": 0.999852180480957, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 71, "end": 93, "label": "ORGANIZATION", "text": "Blue Cedar Cooperative", "score": 0.9967365860939026, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 0, "end": 13, "label": "PERSON", "text": "Frida Östberg", "uncovered_positions": [] }, { "start": 22, "end": 29, "label": "CITY", "text": "Uppsala", "uncovered_positions": [] }, { "start": 32, "end": 45, "label": "PERSON", "text": "Frida Östberg", "uncovered_positions": [] }, { "start": 71, "end": 93, "label": "ORGANIZATION", "text": "Blue Cedar Cooperative", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-045", "scenario": null, "predictions": [ { "start": 3, "end": 10, "label": "CITY", "text": "Uppsala", "score": 0.9998781681060791, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 12, "end": 25, "label": "PERSON", "text": "Frida Östberg", "score": 0.9998877048492432, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 37, "end": 59, "label": "ORGANIZATION", "text": "Blue Cedar Cooperative", "score": 0.9978963136672974, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 80, "end": 93, "label": "PERSON", "text": "Frida Östberg", "score": 0.9998670816421509, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 106, "end": 113, "label": "CITY", "text": "Uppsala", "score": 0.9998447895050049, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 3, "end": 10, "label": "CITY", "text": "Uppsala", "score": 0.9998781681060791, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 12, "end": 25, "label": "PERSON", "text": "Frida Östberg", "score": 0.9998877048492432, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 37, "end": 59, "label": "ORGANIZATION", "text": "Blue Cedar Cooperative", "score": 0.9978963136672974, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 80, "end": 93, "label": "PERSON", "text": "Frida Östberg", "score": 0.9998670816421509, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 106, "end": 113, "label": "CITY", "text": "Uppsala", "score": 0.9998447895050049, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 3, "end": 10, "label": "CITY", "text": "Uppsala", "uncovered_positions": [] }, { "start": 12, "end": 25, "label": "PERSON", "text": "Frida Östberg", "uncovered_positions": [] }, { "start": 37, "end": 59, "label": "ORGANIZATION", "text": "Blue Cedar Cooperative", "uncovered_positions": [] }, { "start": 80, "end": 93, "label": "PERSON", "text": "Frida Östberg", "uncovered_positions": [] }, { "start": 106, "end": 113, "label": "CITY", "text": "Uppsala", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-046", "scenario": null, "predictions": [ { "start": 6, "end": 20, "label": "PERSON", "text": "Frida-\nÖstberg", "score": 0.9996815919876099, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "dehyphenation-v1" }, { "start": 26, "end": 35, "label": "CITY", "text": "Upp-\nsala", "score": 0.9999419450759888, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "dehyphenation-v1" }, { "start": 49, "end": 71, "label": "ORGANIZATION", "text": "Blue Cedar Cooperative", "score": 0.9979609251022339, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "dehyphenation-v1" } ], "combined_predictions": [ { "start": 6, "end": 20, "label": "PERSON", "text": "Frida-\nÖstberg", "score": 0.9996815919876099, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "dehyphenation-v1" }, { "start": 26, "end": 35, "label": "CITY", "text": "Upp-\nsala", "score": 0.9999419450759888, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "dehyphenation-v1" }, { "start": 49, "end": 71, "label": "ORGANIZATION", "text": "Blue Cedar Cooperative", "score": 0.9979609251022339, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "dehyphenation-v1" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 6, "end": 20, "label": "PERSON", "text": "Frida-\nÖstberg", "uncovered_positions": [] }, { "start": 26, "end": 35, "label": "CITY", "text": "Upp-\nsala", "uncovered_positions": [] }, { "start": 49, "end": 71, "label": "ORGANIZATION", "text": "Blue Cedar Cooperative", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-047", "scenario": null, "predictions": [ { "start": 16, "end": 29, "label": "PERSON", "text": "Fr1da Östberg", "score": 0.9966031312942505, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 41, "end": 48, "label": "CITY", "text": "Upps4l4", "score": 0.990062415599823, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 62, "end": 84, "label": "ORGANIZATION", "text": "Blue Cedar Cooperative", "score": 0.9982581734657288, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 16, "end": 29, "label": "PERSON", "text": "Fr1da Östberg", "score": 0.9966031312942505, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 41, "end": 48, "label": "CITY", "text": "Upps4l4", "score": 0.990062415599823, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 62, "end": 84, "label": "ORGANIZATION", "text": "Blue Cedar Cooperative", "score": 0.9982581734657288, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 16, "end": 29, "label": "PERSON", "text": "Fr1da Östberg", "uncovered_positions": [] }, { "start": 41, "end": 48, "label": "CITY", "text": "Upps4l4", "uncovered_positions": [] }, { "start": 62, "end": 84, "label": "ORGANIZATION", "text": "Blue Cedar Cooperative", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-048", "scenario": null, "predictions": [ { "start": 10, "end": 15, "label": "RELIGION", "text": "bahai", "score": 0.8015578985214233, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 42, "end": 64, "label": "ORGANIZATION", "text": "Blue Cedar Cooperative", "score": 0.9946749210357666, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 78, "end": 91, "label": "PERSON", "text": "Frida Östberg", "score": 0.9991986155509949, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 10, "end": 15, "label": "RELIGION", "text": "bahai", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" }, { "start": 42, "end": 64, "label": "ORGANIZATION", "text": "Blue Cedar Cooperative", "score": 0.9946749210357666, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 78, "end": 91, "label": "PERSON", "text": "Frida Östberg", "score": 0.9991986155509949, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 10, "end": 15, "label": "RELIGION", "text": "bahai", "uncovered_positions": [] }, { "start": 42, "end": 64, "label": "ORGANIZATION", "text": "Blue Cedar Cooperative", "uncovered_positions": [] }, { "start": 78, "end": 91, "label": "PERSON", "text": "Frida Östberg", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-049", "scenario": null, "predictions": [ { "start": 101, "end": 107, "label": "PERSON", "text": "person", "score": 0.7069818377494812, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 109, "end": 114, "label": "CITY", "text": "place", "score": 0.785081684589386, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 116, "end": 128, "label": "ORGANIZATION", "text": "organization", "score": 0.8447502255439758, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 40, "end": 64, "label": "RELIGION", "text": "no religious affiliation", "score": null, "source": "deterministic", "model_revision": "religious-status-v1", "normalization": "identity" }, { "start": 101, "end": 107, "label": "PERSON", "text": "person", "score": 0.7069818377494812, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 109, "end": 114, "label": "CITY", "text": "place", "score": 0.785081684589386, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 116, "end": 128, "label": "ORGANIZATION", "text": "organization", "score": 0.8447502255439758, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [ [ 40, 64, "RELIGION" ], [ 101, 107, "PERSON" ], [ 109, 114, "CITY" ], [ 116, 128, "ORGANIZATION" ] ], "false_negatives": [], "exact_match": false, "error_code": null, "canaries": [], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-050", "scenario": null, "predictions": [ { "start": 10, "end": 24, "label": "PERSON", "text": "Owen Fairchild", "score": 0.9981813430786133, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 38, "end": 60, "label": "ORGANIZATION", "text": "Oakwell Public Library", "score": 0.9969713687896729, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 64, "end": 73, "label": "CITY", "text": "Leicester", "score": 0.999505877494812, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 10, "end": 24, "label": "PERSON", "text": "Owen Fairchild", "score": 0.9981813430786133, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 38, "end": 60, "label": "ORGANIZATION", "text": "Oakwell Public Library", "score": 0.9969713687896729, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 64, "end": 73, "label": "CITY", "text": "Leicester", "score": 0.999505877494812, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 10, "end": 24, "label": "PERSON", "text": "Owen Fairchild", "uncovered_positions": [] }, { "start": 38, "end": 60, "label": "ORGANIZATION", "text": "Oakwell Public Library", "uncovered_positions": [] }, { "start": 64, "end": 73, "label": "CITY", "text": "Leicester", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-051", "scenario": null, "predictions": [ { "start": 12, "end": 26, "label": "PERSON", "text": "Owen Fairchild", "score": 0.9984040856361389, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 38, "end": 60, "label": "ORGANIZATION", "text": "Oakwell Public Library", "score": 0.9967267513275146, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 66, "end": 75, "label": "CITY", "text": "Leicester", "score": 0.9997547268867493, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 12, "end": 26, "label": "PERSON", "text": "Owen Fairchild", "score": 0.9984040856361389, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 38, "end": 60, "label": "ORGANIZATION", "text": "Oakwell Public Library", "score": 0.9967267513275146, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 66, "end": 75, "label": "CITY", "text": "Leicester", "score": 0.9997547268867493, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 12, "end": 26, "label": "PERSON", "text": "Owen Fairchild", "uncovered_positions": [] }, { "start": 38, "end": 60, "label": "ORGANIZATION", "text": "Oakwell Public Library", "uncovered_positions": [] }, { "start": 66, "end": 75, "label": "CITY", "text": "Leicester", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-052", "scenario": null, "predictions": [ { "start": 0, "end": 14, "label": "PERSON", "text": "Owen Fairchild", "score": 0.9966481328010559, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 21, "end": 33, "label": "RELIGION", "text": "buddhistisch", "score": 0.9927167296409607, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 61, "end": 91, "label": "EMAIL", "text": "case106.review@example.invalid", "score": 0.9082765579223633, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 102, "end": 116, "label": "PHONE", "text": "+49 541 730106", "score": 0.9930872321128845, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 0, "end": 14, "label": "PERSON", "text": "Owen Fairchild", "score": 0.9966481328010559, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 21, "end": 33, "label": "RELIGION", "text": "buddhistisch", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" }, { "start": 61, "end": 91, "label": "EMAIL", "text": "case106.review@example.invalid", "score": null, "source": "deterministic", "model_revision": "contact-patterns-v1", "normalization": "identity" }, { "start": 102, "end": 116, "label": "PHONE", "text": "+49 541 730106", "score": null, "source": "deterministic", "model_revision": "contact-patterns-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 0, "end": 14, "label": "PERSON", "text": "Owen Fairchild", "uncovered_positions": [] }, { "start": 21, "end": 33, "label": "RELIGION", "text": "buddhistisch", "uncovered_positions": [] }, { "start": 61, "end": 91, "label": "EMAIL", "text": "case106.review@example.invalid", "uncovered_positions": [] }, { "start": 102, "end": 116, "label": "PHONE", "text": "+49 541 730106", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-053", "scenario": null, "predictions": [ { "start": 0, "end": 14, "label": "PERSON", "text": "Owen Fairchild", "score": 0.9962939620018005, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 32, "end": 40, "label": "RELIGION", "text": "Buddhism", "score": 0.9998692274093628, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 66, "end": 96, "label": "EMAIL", "text": "case106.review@example.invalid", "score": 0.9211816191673279, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 105, "end": 119, "label": "PHONE", "text": "+49 541 730106", "score": 0.9646264910697937, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 0, "end": 14, "label": "PERSON", "text": "Owen Fairchild", "score": 0.9962939620018005, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 32, "end": 40, "label": "RELIGION", "text": "Buddhism", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" }, { "start": 66, "end": 96, "label": "EMAIL", "text": "case106.review@example.invalid", "score": null, "source": "deterministic", "model_revision": "contact-patterns-v1", "normalization": "identity" }, { "start": 105, "end": 119, "label": "PHONE", "text": "+49 541 730106", "score": null, "source": "deterministic", "model_revision": "contact-patterns-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 0, "end": 14, "label": "PERSON", "text": "Owen Fairchild", "uncovered_positions": [] }, { "start": 32, "end": 40, "label": "RELIGION", "text": "Buddhism", "uncovered_positions": [] }, { "start": 66, "end": 96, "label": "EMAIL", "text": "case106.review@example.invalid", "uncovered_positions": [] }, { "start": 105, "end": 119, "label": "PHONE", "text": "+49 541 730106", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-054", "scenario": null, "predictions": [ { "start": 0, "end": 14, "label": "PERSON", "text": "Owen Fairchild", "score": 0.9986757636070251, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 23, "end": 32, "label": "CITY", "text": "Leicester", "score": 0.9993718266487122, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 35, "end": 49, "label": "PERSON", "text": "Owen Fairchild", "score": 0.9993817806243896, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 75, "end": 97, "label": "ORGANIZATION", "text": "Oakwell Public Library", "score": 0.9926812052726746, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 0, "end": 14, "label": "PERSON", "text": "Owen Fairchild", "score": 0.9986757636070251, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 23, "end": 32, "label": "CITY", "text": "Leicester", "score": 0.9993718266487122, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 35, "end": 49, "label": "PERSON", "text": "Owen Fairchild", "score": 0.9993817806243896, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 75, "end": 97, "label": "ORGANIZATION", "text": "Oakwell Public Library", "score": 0.9926812052726746, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 0, "end": 14, "label": "PERSON", "text": "Owen Fairchild", "uncovered_positions": [] }, { "start": 23, "end": 32, "label": "CITY", "text": "Leicester", "uncovered_positions": [] }, { "start": 35, "end": 49, "label": "PERSON", "text": "Owen Fairchild", "uncovered_positions": [] }, { "start": 75, "end": 97, "label": "ORGANIZATION", "text": "Oakwell Public Library", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-055", "scenario": null, "predictions": [ { "start": 3, "end": 12, "label": "CITY", "text": "Leicester", "score": 0.9998503923416138, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 14, "end": 28, "label": "PERSON", "text": "Owen Fairchild", "score": 0.999387264251709, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 40, "end": 62, "label": "ORGANIZATION", "text": "Oakwell Public Library", "score": 0.9959281086921692, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 83, "end": 97, "label": "PERSON", "text": "Owen Fairchild", "score": 0.9997162222862244, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 110, "end": 119, "label": "CITY", "text": "Leicester", "score": 0.9999486207962036, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 3, "end": 12, "label": "CITY", "text": "Leicester", "score": 0.9998503923416138, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 14, "end": 28, "label": "PERSON", "text": "Owen Fairchild", "score": 0.999387264251709, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 40, "end": 62, "label": "ORGANIZATION", "text": "Oakwell Public Library", "score": 0.9959281086921692, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 83, "end": 97, "label": "PERSON", "text": "Owen Fairchild", "score": 0.9997162222862244, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 110, "end": 119, "label": "CITY", "text": "Leicester", "score": 0.9999486207962036, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 3, "end": 12, "label": "CITY", "text": "Leicester", "uncovered_positions": [] }, { "start": 14, "end": 28, "label": "PERSON", "text": "Owen Fairchild", "uncovered_positions": [] }, { "start": 40, "end": 62, "label": "ORGANIZATION", "text": "Oakwell Public Library", "uncovered_positions": [] }, { "start": 83, "end": 97, "label": "PERSON", "text": "Owen Fairchild", "uncovered_positions": [] }, { "start": 110, "end": 119, "label": "CITY", "text": "Leicester", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-056", "scenario": null, "predictions": [ { "start": 6, "end": 21, "label": "PERSON", "text": "Owen-\nFairchild", "score": 0.9996999502182007, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "dehyphenation-v1" }, { "start": 27, "end": 38, "label": "CITY", "text": "Leic-\nester", "score": 0.9999775886535645, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "dehyphenation-v1" }, { "start": 52, "end": 74, "label": "ORGANIZATION", "text": "Oakwell Public Library", "score": 0.9970940351486206, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "dehyphenation-v1" } ], "combined_predictions": [ { "start": 6, "end": 21, "label": "PERSON", "text": "Owen-\nFairchild", "score": 0.9996999502182007, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "dehyphenation-v1" }, { "start": 27, "end": 38, "label": "CITY", "text": "Leic-\nester", "score": 0.9999775886535645, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "dehyphenation-v1" }, { "start": 52, "end": 74, "label": "ORGANIZATION", "text": "Oakwell Public Library", "score": 0.9970940351486206, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "dehyphenation-v1" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 6, "end": 21, "label": "PERSON", "text": "Owen-\nFairchild", "uncovered_positions": [] }, { "start": 27, "end": 38, "label": "CITY", "text": "Leic-\nester", "uncovered_positions": [] }, { "start": 52, "end": 74, "label": "ORGANIZATION", "text": "Oakwell Public Library", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-057", "scenario": null, "predictions": [ { "start": 16, "end": 30, "label": "PERSON", "text": "Owen Fa1rch1ld", "score": 0.9971356391906738, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 42, "end": 51, "label": "CITY", "text": "Leicester", "score": 0.9998725652694702, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 65, "end": 87, "label": "ORGANIZATION", "text": "Oakwell Public Library", "score": 0.9962441921234131, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 16, "end": 30, "label": "PERSON", "text": "Owen Fa1rch1ld", "score": 0.9971356391906738, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 42, "end": 51, "label": "CITY", "text": "Leicester", "score": 0.9998725652694702, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 65, "end": 87, "label": "ORGANIZATION", "text": "Oakwell Public Library", "score": 0.9962441921234131, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 16, "end": 30, "label": "PERSON", "text": "Owen Fa1rch1ld", "uncovered_positions": [] }, { "start": 42, "end": 51, "label": "CITY", "text": "Leicester", "uncovered_positions": [] }, { "start": 65, "end": 87, "label": "ORGANIZATION", "text": "Oakwell Public Library", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-058", "scenario": null, "predictions": [ { "start": 10, "end": 22, "label": "RELIGION", "text": "buddhistisch", "score": 0.8650351166725159, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 49, "end": 71, "label": "ORGANIZATION", "text": "Oakwell Public Library", "score": 0.9895272254943848, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 85, "end": 99, "label": "PERSON", "text": "Owen Fairchild", "score": 0.9970881342887878, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 10, "end": 22, "label": "RELIGION", "text": "buddhistisch", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" }, { "start": 49, "end": 71, "label": "ORGANIZATION", "text": "Oakwell Public Library", "score": 0.9895272254943848, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 85, "end": 99, "label": "PERSON", "text": "Owen Fairchild", "score": 0.9970881342887878, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 10, "end": 22, "label": "RELIGION", "text": "buddhistisch", "uncovered_positions": [] }, { "start": 49, "end": 71, "label": "ORGANIZATION", "text": "Oakwell Public Library", "uncovered_positions": [] }, { "start": 85, "end": 99, "label": "PERSON", "text": "Owen Fairchild", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-059", "scenario": null, "predictions": [ { "start": 101, "end": 107, "label": "PERSON", "text": "person", "score": 0.7122743725776672, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 109, "end": 114, "label": "CITY", "text": "place", "score": 0.7866309881210327, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 116, "end": 128, "label": "ORGANIZATION", "text": "organization", "score": 0.8471381068229675, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 40, "end": 64, "label": "RELIGION", "text": "no religious affiliation", "score": null, "source": "deterministic", "model_revision": "religious-status-v1", "normalization": "identity" }, { "start": 101, "end": 107, "label": "PERSON", "text": "person", "score": 0.7122743725776672, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 109, "end": 114, "label": "CITY", "text": "place", "score": 0.7866309881210327, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 116, "end": 128, "label": "ORGANIZATION", "text": "organization", "score": 0.8471381068229675, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [ [ 40, 64, "RELIGION" ], [ 101, 107, "PERSON" ], [ 109, 114, "CITY" ], [ 116, 128, "ORGANIZATION" ] ], "false_negatives": [], "exact_match": false, "error_code": null, "canaries": [], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-060", "scenario": null, "predictions": [ { "start": 10, "end": 23, "label": "PERSON", "text": "Nele Brückner", "score": 0.9995220899581909, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 37, "end": 62, "label": "ORGANIZATION", "text": "Westhaven Water Authority", "score": 0.9957793951034546, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 66, "end": 74, "label": "CITY", "text": "Würzburg", "score": 0.9995125532150269, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 10, "end": 23, "label": "PERSON", "text": "Nele Brückner", "score": 0.9995220899581909, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 37, "end": 62, "label": "ORGANIZATION", "text": "Westhaven Water Authority", "score": 0.9957793951034546, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 66, "end": 74, "label": "CITY", "text": "Würzburg", "score": 0.9995125532150269, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 10, "end": 23, "label": "PERSON", "text": "Nele Brückner", "uncovered_positions": [] }, { "start": 37, "end": 62, "label": "ORGANIZATION", "text": "Westhaven Water Authority", "uncovered_positions": [] }, { "start": 66, "end": 74, "label": "CITY", "text": "Würzburg", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-061", "scenario": null, "predictions": [ { "start": 12, "end": 25, "label": "PERSON", "text": "Nele Brückner", "score": 0.9993897676467896, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 37, "end": 62, "label": "ORGANIZATION", "text": "Westhaven Water Authority", "score": 0.9940574169158936, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 68, "end": 76, "label": "CITY", "text": "Würzburg", "score": 0.9994066953659058, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 12, "end": 25, "label": "PERSON", "text": "Nele Brückner", "score": 0.9993897676467896, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 37, "end": 62, "label": "ORGANIZATION", "text": "Westhaven Water Authority", "score": 0.9940574169158936, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 68, "end": 76, "label": "CITY", "text": "Würzburg", "score": 0.9994066953659058, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 12, "end": 25, "label": "PERSON", "text": "Nele Brückner", "uncovered_positions": [] }, { "start": 37, "end": 62, "label": "ORGANIZATION", "text": "Westhaven Water Authority", "uncovered_positions": [] }, { "start": 68, "end": 76, "label": "CITY", "text": "Würzburg", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-062", "scenario": null, "predictions": [ { "start": 0, "end": 13, "label": "PERSON", "text": "Nele Brückner", "score": 0.9983347058296204, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 20, "end": 30, "label": "RELIGION", "text": "katholisch", "score": 0.9748141169548035, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 73, "end": 88, "label": "CITY", "text": "example.invalid", "score": 0.9023637175559998, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 99, "end": 113, "label": "PHONE", "text": "+49 541 730107", "score": 0.9952542781829834, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 0, "end": 13, "label": "PERSON", "text": "Nele Brückner", "score": 0.9983347058296204, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 20, "end": 30, "label": "RELIGION", "text": "katholisch", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" }, { "start": 58, "end": 88, "label": "EMAIL", "text": "case107.review@example.invalid", "score": null, "source": "deterministic", "model_revision": "contact-patterns-v1", "normalization": "identity" }, { "start": 99, "end": 113, "label": "PHONE", "text": "+49 541 730107", "score": null, "source": "deterministic", "model_revision": "contact-patterns-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 0, "end": 13, "label": "PERSON", "text": "Nele Brückner", "uncovered_positions": [] }, { "start": 20, "end": 30, "label": "RELIGION", "text": "katholisch", "uncovered_positions": [] }, { "start": 58, "end": 88, "label": "EMAIL", "text": "case107.review@example.invalid", "uncovered_positions": [] }, { "start": 99, "end": 113, "label": "PHONE", "text": "+49 541 730107", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-063", "scenario": null, "predictions": [ { "start": 0, "end": 13, "label": "PERSON", "text": "Nele Brückner", "score": 0.9967458248138428, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 31, "end": 42, "label": "RELIGION", "text": "Catholicism", "score": 0.9980359673500061, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 68, "end": 98, "label": "EMAIL", "text": "case107.review@example.invalid", "score": 0.9019798040390015, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 107, "end": 121, "label": "PHONE", "text": "+49 541 730107", "score": 0.9717420339584351, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 0, "end": 13, "label": "PERSON", "text": "Nele Brückner", "score": 0.9967458248138428, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 31, "end": 42, "label": "RELIGION", "text": "Catholicism", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" }, { "start": 68, "end": 98, "label": "EMAIL", "text": "case107.review@example.invalid", "score": null, "source": "deterministic", "model_revision": "contact-patterns-v1", "normalization": "identity" }, { "start": 107, "end": 121, "label": "PHONE", "text": "+49 541 730107", "score": null, "source": "deterministic", "model_revision": "contact-patterns-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 0, "end": 13, "label": "PERSON", "text": "Nele Brückner", "uncovered_positions": [] }, { "start": 31, "end": 42, "label": "RELIGION", "text": "Catholicism", "uncovered_positions": [] }, { "start": 68, "end": 98, "label": "EMAIL", "text": "case107.review@example.invalid", "uncovered_positions": [] }, { "start": 107, "end": 121, "label": "PHONE", "text": "+49 541 730107", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-064", "scenario": null, "predictions": [ { "start": 0, "end": 13, "label": "PERSON", "text": "Nele Brückner", "score": 0.9998051524162292, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 22, "end": 30, "label": "CITY", "text": "Würzburg", "score": 0.9990678429603577, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 33, "end": 46, "label": "PERSON", "text": "Nele Brückner", "score": 0.9998308420181274, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 72, "end": 97, "label": "ORGANIZATION", "text": "Westhaven Water Authority", "score": 0.9937987923622131, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 0, "end": 13, "label": "PERSON", "text": "Nele Brückner", "score": 0.9998051524162292, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 22, "end": 30, "label": "CITY", "text": "Würzburg", "score": 0.9990678429603577, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 33, "end": 46, "label": "PERSON", "text": "Nele Brückner", "score": 0.9998308420181274, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 72, "end": 97, "label": "ORGANIZATION", "text": "Westhaven Water Authority", "score": 0.9937987923622131, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 0, "end": 13, "label": "PERSON", "text": "Nele Brückner", "uncovered_positions": [] }, { "start": 22, "end": 30, "label": "CITY", "text": "Würzburg", "uncovered_positions": [] }, { "start": 33, "end": 46, "label": "PERSON", "text": "Nele Brückner", "uncovered_positions": [] }, { "start": 72, "end": 97, "label": "ORGANIZATION", "text": "Westhaven Water Authority", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-065", "scenario": null, "predictions": [ { "start": 3, "end": 11, "label": "CITY", "text": "Würzburg", "score": 0.9997674822807312, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 13, "end": 26, "label": "PERSON", "text": "Nele Brückner", "score": 0.9999117851257324, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 38, "end": 63, "label": "ORGANIZATION", "text": "Westhaven Water Authority", "score": 0.9946510195732117, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 84, "end": 97, "label": "PERSON", "text": "Nele Brückner", "score": 0.9998905658721924, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 110, "end": 118, "label": "CITY", "text": "Würzburg", "score": 0.9998799562454224, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 3, "end": 11, "label": "CITY", "text": "Würzburg", "score": 0.9997674822807312, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 13, "end": 26, "label": "PERSON", "text": "Nele Brückner", "score": 0.9999117851257324, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 38, "end": 63, "label": "ORGANIZATION", "text": "Westhaven Water Authority", "score": 0.9946510195732117, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 84, "end": 97, "label": "PERSON", "text": "Nele Brückner", "score": 0.9998905658721924, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 110, "end": 118, "label": "CITY", "text": "Würzburg", "score": 0.9998799562454224, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 3, "end": 11, "label": "CITY", "text": "Würzburg", "uncovered_positions": [] }, { "start": 13, "end": 26, "label": "PERSON", "text": "Nele Brückner", "uncovered_positions": [] }, { "start": 38, "end": 63, "label": "ORGANIZATION", "text": "Westhaven Water Authority", "uncovered_positions": [] }, { "start": 84, "end": 97, "label": "PERSON", "text": "Nele Brückner", "uncovered_positions": [] }, { "start": 110, "end": 118, "label": "CITY", "text": "Würzburg", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-066", "scenario": null, "predictions": [ { "start": 6, "end": 20, "label": "PERSON", "text": "Nele-\nBrückner", "score": 0.9997313618659973, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "dehyphenation-v1" }, { "start": 26, "end": 36, "label": "CITY", "text": "Würz-\nburg", "score": 0.9999507665634155, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "dehyphenation-v1" }, { "start": 50, "end": 75, "label": "ORGANIZATION", "text": "Westhaven Water Authority", "score": 0.9986609220504761, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "dehyphenation-v1" } ], "combined_predictions": [ { "start": 6, "end": 20, "label": "PERSON", "text": "Nele-\nBrückner", "score": 0.9997313618659973, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "dehyphenation-v1" }, { "start": 26, "end": 36, "label": "CITY", "text": "Würz-\nburg", "score": 0.9999507665634155, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "dehyphenation-v1" }, { "start": 50, "end": 75, "label": "ORGANIZATION", "text": "Westhaven Water Authority", "score": 0.9986609220504761, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "dehyphenation-v1" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 6, "end": 20, "label": "PERSON", "text": "Nele-\nBrückner", "uncovered_positions": [] }, { "start": 26, "end": 36, "label": "CITY", "text": "Würz-\nburg", "uncovered_positions": [] }, { "start": 50, "end": 75, "label": "ORGANIZATION", "text": "Westhaven Water Authority", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-067", "scenario": null, "predictions": [ { "start": 16, "end": 29, "label": "PERSON", "text": "Nele Brückner", "score": 0.9996700286865234, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 41, "end": 49, "label": "CITY", "text": "Würzburg", "score": 0.9998364448547363, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 63, "end": 88, "label": "ORGANIZATION", "text": "Westhaven Water Authority", "score": 0.9963690042495728, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 16, "end": 29, "label": "PERSON", "text": "Nele Brückner", "score": 0.9996700286865234, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 41, "end": 49, "label": "CITY", "text": "Würzburg", "score": 0.9998364448547363, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 63, "end": 88, "label": "ORGANIZATION", "text": "Westhaven Water Authority", "score": 0.9963690042495728, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 16, "end": 29, "label": "PERSON", "text": "Nele Brückner", "uncovered_positions": [] }, { "start": 41, "end": 49, "label": "CITY", "text": "Würzburg", "uncovered_positions": [] }, { "start": 63, "end": 88, "label": "ORGANIZATION", "text": "Westhaven Water Authority", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-068", "scenario": null, "predictions": [ { "start": 10, "end": 20, "label": "RELIGION", "text": "katholisch", "score": 0.8750774264335632, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 47, "end": 72, "label": "ORGANIZATION", "text": "Westhaven Water Authority", "score": 0.9934499859809875, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 86, "end": 99, "label": "PERSON", "text": "Nele Brückner", "score": 0.9986261129379272, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 10, "end": 20, "label": "RELIGION", "text": "katholisch", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" }, { "start": 47, "end": 72, "label": "ORGANIZATION", "text": "Westhaven Water Authority", "score": 0.9934499859809875, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 86, "end": 99, "label": "PERSON", "text": "Nele Brückner", "score": 0.9986261129379272, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 10, "end": 20, "label": "RELIGION", "text": "katholisch", "uncovered_positions": [] }, { "start": 47, "end": 72, "label": "ORGANIZATION", "text": "Westhaven Water Authority", "uncovered_positions": [] }, { "start": 86, "end": 99, "label": "PERSON", "text": "Nele Brückner", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-069", "scenario": null, "predictions": [ { "start": 101, "end": 107, "label": "PERSON", "text": "person", "score": 0.7072301506996155, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 109, "end": 114, "label": "CITY", "text": "place", "score": 0.7869006395339966, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 116, "end": 128, "label": "ORGANIZATION", "text": "organization", "score": 0.844225287437439, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 40, "end": 64, "label": "RELIGION", "text": "no religious affiliation", "score": null, "source": "deterministic", "model_revision": "religious-status-v1", "normalization": "identity" }, { "start": 101, "end": 107, "label": "PERSON", "text": "person", "score": 0.7072301506996155, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 109, "end": 114, "label": "CITY", "text": "place", "score": 0.7869006395339966, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 116, "end": 128, "label": "ORGANIZATION", "text": "organization", "score": 0.844225287437439, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [ [ 40, 64, "RELIGION" ], [ 101, 107, "PERSON" ], [ 109, 114, "CITY" ], [ 116, 128, "ORGANIZATION" ] ], "false_negatives": [], "exact_match": false, "error_code": null, "canaries": [], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-070", "scenario": null, "predictions": [ { "start": 10, "end": 25, "label": "PERSON", "text": "Silas Hawthorne", "score": 0.9949835538864136, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 39, "end": 61, "label": "ORGANIZATION", "text": "Juniper Legal Services", "score": 0.9961141347885132, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 65, "end": 73, "label": "CITY", "text": "Portland", "score": 0.9993465542793274, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 10, "end": 25, "label": "PERSON", "text": "Silas Hawthorne", "score": 0.9949835538864136, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 39, "end": 61, "label": "ORGANIZATION", "text": "Juniper Legal Services", "score": 0.9961141347885132, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 65, "end": 73, "label": "CITY", "text": "Portland", "score": 0.9993465542793274, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 10, "end": 25, "label": "PERSON", "text": "Silas Hawthorne", "uncovered_positions": [] }, { "start": 39, "end": 61, "label": "ORGANIZATION", "text": "Juniper Legal Services", "uncovered_positions": [] }, { "start": 65, "end": 73, "label": "CITY", "text": "Portland", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-071", "scenario": null, "predictions": [ { "start": 12, "end": 27, "label": "PERSON", "text": "Silas Hawthorne", "score": 0.9974763989448547, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 39, "end": 61, "label": "ORGANIZATION", "text": "Juniper Legal Services", "score": 0.9980837106704712, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 67, "end": 75, "label": "CITY", "text": "Portland", "score": 0.9996472597122192, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 12, "end": 27, "label": "PERSON", "text": "Silas Hawthorne", "score": 0.9974763989448547, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 39, "end": 61, "label": "ORGANIZATION", "text": "Juniper Legal Services", "score": 0.9980837106704712, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 67, "end": 75, "label": "CITY", "text": "Portland", "score": 0.9996472597122192, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 12, "end": 27, "label": "PERSON", "text": "Silas Hawthorne", "uncovered_positions": [] }, { "start": 39, "end": 61, "label": "ORGANIZATION", "text": "Juniper Legal Services", "uncovered_positions": [] }, { "start": 67, "end": 75, "label": "CITY", "text": "Portland", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-072", "scenario": null, "predictions": [ { "start": 0, "end": 15, "label": "PERSON", "text": "Silas Hawthorne", "score": 0.9946703314781189, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 22, "end": 33, "label": "RELIGION", "text": "evangelisch", "score": 0.9349200129508972, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 61, "end": 91, "label": "EMAIL", "text": "case108.review@example.invalid", "score": 0.9081872701644897, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 102, "end": 116, "label": "PHONE", "text": "+49 541 730108", "score": 0.9933851361274719, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 0, "end": 15, "label": "PERSON", "text": "Silas Hawthorne", "score": 0.9946703314781189, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 22, "end": 33, "label": "RELIGION", "text": "evangelisch", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" }, { "start": 61, "end": 91, "label": "EMAIL", "text": "case108.review@example.invalid", "score": null, "source": "deterministic", "model_revision": "contact-patterns-v1", "normalization": "identity" }, { "start": 102, "end": 116, "label": "PHONE", "text": "+49 541 730108", "score": null, "source": "deterministic", "model_revision": "contact-patterns-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 0, "end": 15, "label": "PERSON", "text": "Silas Hawthorne", "uncovered_positions": [] }, { "start": 22, "end": 33, "label": "RELIGION", "text": "evangelisch", "uncovered_positions": [] }, { "start": 61, "end": 91, "label": "EMAIL", "text": "case108.review@example.invalid", "uncovered_positions": [] }, { "start": 102, "end": 116, "label": "PHONE", "text": "+49 541 730108", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-073", "scenario": null, "predictions": [ { "start": 0, "end": 15, "label": "PERSON", "text": "Silas Hawthorne", "score": 0.9950461387634277, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 33, "end": 46, "label": "RELIGION", "text": "Protestantism", "score": 0.9982935786247253, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 72, "end": 102, "label": "EMAIL", "text": "case108.review@example.invalid", "score": 0.9170582890510559, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 111, "end": 125, "label": "PHONE", "text": "+49 541 730108", "score": 0.9616659283638, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 0, "end": 15, "label": "PERSON", "text": "Silas Hawthorne", "score": 0.9950461387634277, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 33, "end": 46, "label": "RELIGION", "text": "Protestantism", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" }, { "start": 72, "end": 102, "label": "EMAIL", "text": "case108.review@example.invalid", "score": null, "source": "deterministic", "model_revision": "contact-patterns-v1", "normalization": "identity" }, { "start": 111, "end": 125, "label": "PHONE", "text": "+49 541 730108", "score": null, "source": "deterministic", "model_revision": "contact-patterns-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 0, "end": 15, "label": "PERSON", "text": "Silas Hawthorne", "uncovered_positions": [] }, { "start": 33, "end": 46, "label": "RELIGION", "text": "Protestantism", "uncovered_positions": [] }, { "start": 72, "end": 102, "label": "EMAIL", "text": "case108.review@example.invalid", "uncovered_positions": [] }, { "start": 111, "end": 125, "label": "PHONE", "text": "+49 541 730108", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-074", "scenario": null, "predictions": [ { "start": 0, "end": 15, "label": "PERSON", "text": "Silas Hawthorne", "score": 0.9976925849914551, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 24, "end": 32, "label": "CITY", "text": "Portland", "score": 0.9988584518432617, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 35, "end": 50, "label": "PERSON", "text": "Silas Hawthorne", "score": 0.9983211159706116, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 76, "end": 98, "label": "ORGANIZATION", "text": "Juniper Legal Services", "score": 0.9957102537155151, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 0, "end": 15, "label": "PERSON", "text": "Silas Hawthorne", "score": 0.9976925849914551, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 24, "end": 32, "label": "CITY", "text": "Portland", "score": 0.9988584518432617, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 35, "end": 50, "label": "PERSON", "text": "Silas Hawthorne", "score": 0.9983211159706116, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 76, "end": 98, "label": "ORGANIZATION", "text": "Juniper Legal Services", "score": 0.9957102537155151, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 0, "end": 15, "label": "PERSON", "text": "Silas Hawthorne", "uncovered_positions": [] }, { "start": 24, "end": 32, "label": "CITY", "text": "Portland", "uncovered_positions": [] }, { "start": 35, "end": 50, "label": "PERSON", "text": "Silas Hawthorne", "uncovered_positions": [] }, { "start": 76, "end": 98, "label": "ORGANIZATION", "text": "Juniper Legal Services", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-075", "scenario": null, "predictions": [ { "start": 3, "end": 11, "label": "CITY", "text": "Portland", "score": 0.9997417330741882, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 13, "end": 28, "label": "PERSON", "text": "Silas Hawthorne", "score": 0.9993771910667419, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 40, "end": 62, "label": "ORGANIZATION", "text": "Juniper Legal Services", "score": 0.9973406195640564, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 83, "end": 98, "label": "PERSON", "text": "Silas Hawthorne", "score": 0.9994988441467285, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 111, "end": 119, "label": "CITY", "text": "Portland", "score": 0.9996170997619629, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 3, "end": 11, "label": "CITY", "text": "Portland", "score": 0.9997417330741882, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 13, "end": 28, "label": "PERSON", "text": "Silas Hawthorne", "score": 0.9993771910667419, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 40, "end": 62, "label": "ORGANIZATION", "text": "Juniper Legal Services", "score": 0.9973406195640564, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 83, "end": 98, "label": "PERSON", "text": "Silas Hawthorne", "score": 0.9994988441467285, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 111, "end": 119, "label": "CITY", "text": "Portland", "score": 0.9996170997619629, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 3, "end": 11, "label": "CITY", "text": "Portland", "uncovered_positions": [] }, { "start": 13, "end": 28, "label": "PERSON", "text": "Silas Hawthorne", "uncovered_positions": [] }, { "start": 40, "end": 62, "label": "ORGANIZATION", "text": "Juniper Legal Services", "uncovered_positions": [] }, { "start": 83, "end": 98, "label": "PERSON", "text": "Silas Hawthorne", "uncovered_positions": [] }, { "start": 111, "end": 119, "label": "CITY", "text": "Portland", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-076", "scenario": null, "predictions": [ { "start": 6, "end": 22, "label": "PERSON", "text": "Silas-\nHawthorne", "score": 0.9994223117828369, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "dehyphenation-v1" }, { "start": 28, "end": 38, "label": "CITY", "text": "Port-\nland", "score": 0.9999568462371826, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "dehyphenation-v1" }, { "start": 52, "end": 74, "label": "ORGANIZATION", "text": "Juniper Legal Services", "score": 0.9987984895706177, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "dehyphenation-v1" } ], "combined_predictions": [ { "start": 6, "end": 22, "label": "PERSON", "text": "Silas-\nHawthorne", "score": 0.9994223117828369, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "dehyphenation-v1" }, { "start": 28, "end": 38, "label": "CITY", "text": "Port-\nland", "score": 0.9999568462371826, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "dehyphenation-v1" }, { "start": 52, "end": 74, "label": "ORGANIZATION", "text": "Juniper Legal Services", "score": 0.9987984895706177, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "dehyphenation-v1" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 6, "end": 22, "label": "PERSON", "text": "Silas-\nHawthorne", "uncovered_positions": [] }, { "start": 28, "end": 38, "label": "CITY", "text": "Port-\nland", "uncovered_positions": [] }, { "start": 52, "end": 74, "label": "ORGANIZATION", "text": "Juniper Legal Services", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-077", "scenario": null, "predictions": [ { "start": 16, "end": 31, "label": "PERSON", "text": "S1las Hawth0rne", "score": 0.9958475828170776, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 43, "end": 51, "label": "CITY", "text": "P0rtl4nd", "score": 0.9920943379402161, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 65, "end": 87, "label": "ORGANIZATION", "text": "Juniper Legal Services", "score": 0.9983524084091187, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 16, "end": 31, "label": "PERSON", "text": "S1las Hawth0rne", "score": 0.9958475828170776, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 43, "end": 51, "label": "CITY", "text": "P0rtl4nd", "score": 0.9920943379402161, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 65, "end": 87, "label": "ORGANIZATION", "text": "Juniper Legal Services", "score": 0.9983524084091187, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 16, "end": 31, "label": "PERSON", "text": "S1las Hawth0rne", "uncovered_positions": [] }, { "start": 43, "end": 51, "label": "CITY", "text": "P0rtl4nd", "uncovered_positions": [] }, { "start": 65, "end": 87, "label": "ORGANIZATION", "text": "Juniper Legal Services", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-078", "scenario": null, "predictions": [ { "start": 10, "end": 21, "label": "RELIGION", "text": "evangelisch", "score": 0.6795831918716431, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 48, "end": 70, "label": "ORGANIZATION", "text": "Juniper Legal Services", "score": 0.9773863554000854, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 84, "end": 99, "label": "PERSON", "text": "Silas Hawthorne", "score": 0.9956725835800171, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 10, "end": 21, "label": "RELIGION", "text": "evangelisch", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" }, { "start": 48, "end": 70, "label": "ORGANIZATION", "text": "Juniper Legal Services", "score": 0.9773863554000854, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 84, "end": 99, "label": "PERSON", "text": "Silas Hawthorne", "score": 0.9956725835800171, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 10, "end": 21, "label": "RELIGION", "text": "evangelisch", "uncovered_positions": [] }, { "start": 48, "end": 70, "label": "ORGANIZATION", "text": "Juniper Legal Services", "uncovered_positions": [] }, { "start": 84, "end": 99, "label": "PERSON", "text": "Silas Hawthorne", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-079", "scenario": null, "predictions": [ { "start": 101, "end": 107, "label": "PERSON", "text": "person", "score": 0.711972177028656, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 109, "end": 114, "label": "CITY", "text": "place", "score": 0.7886146903038025, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 116, "end": 128, "label": "ORGANIZATION", "text": "organization", "score": 0.8439294695854187, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 40, "end": 64, "label": "RELIGION", "text": "no religious affiliation", "score": null, "source": "deterministic", "model_revision": "religious-status-v1", "normalization": "identity" }, { "start": 101, "end": 107, "label": "PERSON", "text": "person", "score": 0.711972177028656, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 109, "end": 114, "label": "CITY", "text": "place", "score": 0.7886146903038025, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 116, "end": 128, "label": "ORGANIZATION", "text": "organization", "score": 0.8439294695854187, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [ [ 40, 64, "RELIGION" ], [ 101, 107, "PERSON" ], [ 109, 114, "CITY" ], [ 116, 128, "ORGANIZATION" ] ], "false_negatives": [], "exact_match": false, "error_code": null, "canaries": [], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-080", "scenario": null, "predictions": [ { "start": 10, "end": 25, "label": "PERSON", "text": "Alva Rosenqvist", "score": 0.998843789100647, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 39, "end": 61, "label": "ORGANIZATION", "text": "Maple Ridge University", "score": 0.9975813627243042, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 65, "end": 71, "label": "CITY", "text": "Bergen", "score": 0.9971677660942078, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 10, "end": 25, "label": "PERSON", "text": "Alva Rosenqvist", "score": 0.998843789100647, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 39, "end": 61, "label": "ORGANIZATION", "text": "Maple Ridge University", "score": 0.9975813627243042, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 65, "end": 71, "label": "CITY", "text": "Bergen", "score": 0.9971677660942078, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 10, "end": 25, "label": "PERSON", "text": "Alva Rosenqvist", "uncovered_positions": [] }, { "start": 39, "end": 61, "label": "ORGANIZATION", "text": "Maple Ridge University", "uncovered_positions": [] }, { "start": 65, "end": 71, "label": "CITY", "text": "Bergen", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-081", "scenario": null, "predictions": [ { "start": 12, "end": 27, "label": "PERSON", "text": "Alva Rosenqvist", "score": 0.9987525939941406, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 39, "end": 61, "label": "ORGANIZATION", "text": "Maple Ridge University", "score": 0.9985166192054749, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 67, "end": 73, "label": "CITY", "text": "Bergen", "score": 0.9985432624816895, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 12, "end": 27, "label": "PERSON", "text": "Alva Rosenqvist", "score": 0.9987525939941406, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 39, "end": 61, "label": "ORGANIZATION", "text": "Maple Ridge University", "score": 0.9985166192054749, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 67, "end": 73, "label": "CITY", "text": "Bergen", "score": 0.9985432624816895, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 12, "end": 27, "label": "PERSON", "text": "Alva Rosenqvist", "uncovered_positions": [] }, { "start": 39, "end": 61, "label": "ORGANIZATION", "text": "Maple Ridge University", "uncovered_positions": [] }, { "start": 67, "end": 73, "label": "CITY", "text": "Bergen", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-082", "scenario": null, "predictions": [ { "start": 0, "end": 15, "label": "PERSON", "text": "Alva Rosenqvist", "score": 0.9994316697120667, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 22, "end": 29, "label": "RELIGION", "text": "jüdisch", "score": 0.9905497431755066, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 57, "end": 87, "label": "EMAIL", "text": "case109.review@example.invalid", "score": 0.9014635682106018, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 98, "end": 112, "label": "PHONE", "text": "+49 541 730109", "score": 0.993009090423584, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 0, "end": 15, "label": "PERSON", "text": "Alva Rosenqvist", "score": 0.9994316697120667, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 22, "end": 29, "label": "RELIGION", "text": "jüdisch", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" }, { "start": 57, "end": 87, "label": "EMAIL", "text": "case109.review@example.invalid", "score": null, "source": "deterministic", "model_revision": "contact-patterns-v1", "normalization": "identity" }, { "start": 98, "end": 112, "label": "PHONE", "text": "+49 541 730109", "score": null, "source": "deterministic", "model_revision": "contact-patterns-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 0, "end": 15, "label": "PERSON", "text": "Alva Rosenqvist", "uncovered_positions": [] }, { "start": 22, "end": 29, "label": "RELIGION", "text": "jüdisch", "uncovered_positions": [] }, { "start": 57, "end": 87, "label": "EMAIL", "text": "case109.review@example.invalid", "uncovered_positions": [] }, { "start": 98, "end": 112, "label": "PHONE", "text": "+49 541 730109", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-083", "scenario": null, "predictions": [ { "start": 0, "end": 15, "label": "PERSON", "text": "Alva Rosenqvist", "score": 0.9990854263305664, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 33, "end": 40, "label": "RELIGION", "text": "Judaism", "score": 0.9997567534446716, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 66, "end": 96, "label": "EMAIL", "text": "case109.review@example.invalid", "score": 0.9070643186569214, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 105, "end": 119, "label": "PHONE", "text": "+49 541 730109", "score": 0.9656404256820679, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 0, "end": 15, "label": "PERSON", "text": "Alva Rosenqvist", "score": 0.9990854263305664, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 33, "end": 40, "label": "RELIGION", "text": "Judaism", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" }, { "start": 66, "end": 96, "label": "EMAIL", "text": "case109.review@example.invalid", "score": null, "source": "deterministic", "model_revision": "contact-patterns-v1", "normalization": "identity" }, { "start": 105, "end": 119, "label": "PHONE", "text": "+49 541 730109", "score": null, "source": "deterministic", "model_revision": "contact-patterns-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 0, "end": 15, "label": "PERSON", "text": "Alva Rosenqvist", "uncovered_positions": [] }, { "start": 33, "end": 40, "label": "RELIGION", "text": "Judaism", "uncovered_positions": [] }, { "start": 66, "end": 96, "label": "EMAIL", "text": "case109.review@example.invalid", "uncovered_positions": [] }, { "start": 105, "end": 119, "label": "PHONE", "text": "+49 541 730109", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-084", "scenario": null, "predictions": [ { "start": 0, "end": 15, "label": "PERSON", "text": "Alva Rosenqvist", "score": 0.99967360496521, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 24, "end": 30, "label": "CITY", "text": "Bergen", "score": 0.9969269633293152, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 33, "end": 48, "label": "PERSON", "text": "Alva Rosenqvist", "score": 0.9995853304862976, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 74, "end": 96, "label": "ORGANIZATION", "text": "Maple Ridge University", "score": 0.9907796382904053, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 0, "end": 15, "label": "PERSON", "text": "Alva Rosenqvist", "score": 0.99967360496521, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 24, "end": 30, "label": "CITY", "text": "Bergen", "score": 0.9969269633293152, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 33, "end": 48, "label": "PERSON", "text": "Alva Rosenqvist", "score": 0.9995853304862976, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 74, "end": 96, "label": "ORGANIZATION", "text": "Maple Ridge University", "score": 0.9907796382904053, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 0, "end": 15, "label": "PERSON", "text": "Alva Rosenqvist", "uncovered_positions": [] }, { "start": 24, "end": 30, "label": "CITY", "text": "Bergen", "uncovered_positions": [] }, { "start": 33, "end": 48, "label": "PERSON", "text": "Alva Rosenqvist", "uncovered_positions": [] }, { "start": 74, "end": 96, "label": "ORGANIZATION", "text": "Maple Ridge University", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-085", "scenario": null, "predictions": [ { "start": 3, "end": 9, "label": "CITY", "text": "Bergen", "score": 0.9995262622833252, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 11, "end": 26, "label": "PERSON", "text": "Alva Rosenqvist", "score": 0.9998084902763367, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 38, "end": 60, "label": "ORGANIZATION", "text": "Maple Ridge University", "score": 0.99769526720047, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 81, "end": 96, "label": "PERSON", "text": "Alva Rosenqvist", "score": 0.999796450138092, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 109, "end": 115, "label": "CITY", "text": "Bergen", "score": 0.9991483688354492, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 3, "end": 9, "label": "CITY", "text": "Bergen", "score": 0.9995262622833252, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 11, "end": 26, "label": "PERSON", "text": "Alva Rosenqvist", "score": 0.9998084902763367, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 38, "end": 60, "label": "ORGANIZATION", "text": "Maple Ridge University", "score": 0.99769526720047, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 81, "end": 96, "label": "PERSON", "text": "Alva Rosenqvist", "score": 0.999796450138092, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 109, "end": 115, "label": "CITY", "text": "Bergen", "score": 0.9991483688354492, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 3, "end": 9, "label": "CITY", "text": "Bergen", "uncovered_positions": [] }, { "start": 11, "end": 26, "label": "PERSON", "text": "Alva Rosenqvist", "uncovered_positions": [] }, { "start": 38, "end": 60, "label": "ORGANIZATION", "text": "Maple Ridge University", "uncovered_positions": [] }, { "start": 81, "end": 96, "label": "PERSON", "text": "Alva Rosenqvist", "uncovered_positions": [] }, { "start": 109, "end": 115, "label": "CITY", "text": "Bergen", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-086", "scenario": null, "predictions": [ { "start": 6, "end": 22, "label": "PERSON", "text": "Alva-\nRosenqvist", "score": 0.9996846914291382, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "dehyphenation-v1" }, { "start": 28, "end": 36, "label": "CITY", "text": "Ber-\ngen", "score": 0.9998655319213867, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "dehyphenation-v1" }, { "start": 50, "end": 72, "label": "ORGANIZATION", "text": "Maple Ridge University", "score": 0.9983516931533813, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "dehyphenation-v1" } ], "combined_predictions": [ { "start": 6, "end": 22, "label": "PERSON", "text": "Alva-\nRosenqvist", "score": 0.9996846914291382, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "dehyphenation-v1" }, { "start": 28, "end": 36, "label": "CITY", "text": "Ber-\ngen", "score": 0.9998655319213867, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "dehyphenation-v1" }, { "start": 50, "end": 72, "label": "ORGANIZATION", "text": "Maple Ridge University", "score": 0.9983516931533813, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "dehyphenation-v1" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 6, "end": 22, "label": "PERSON", "text": "Alva-\nRosenqvist", "uncovered_positions": [] }, { "start": 28, "end": 36, "label": "CITY", "text": "Ber-\ngen", "uncovered_positions": [] }, { "start": 50, "end": 72, "label": "ORGANIZATION", "text": "Maple Ridge University", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-087", "scenario": null, "predictions": [ { "start": 16, "end": 31, "label": "PERSON", "text": "Alva R0senqv1st", "score": 0.951907753944397, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 43, "end": 49, "label": "CITY", "text": "Bergen", "score": 0.9994907379150391, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 63, "end": 85, "label": "ORGANIZATION", "text": "Maple Ridge University", "score": 0.9973485469818115, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 16, "end": 31, "label": "PERSON", "text": "Alva R0senqv1st", "score": 0.951907753944397, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 43, "end": 49, "label": "CITY", "text": "Bergen", "score": 0.9994907379150391, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 63, "end": 85, "label": "ORGANIZATION", "text": "Maple Ridge University", "score": 0.9973485469818115, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 16, "end": 31, "label": "PERSON", "text": "Alva R0senqv1st", "uncovered_positions": [] }, { "start": 43, "end": 49, "label": "CITY", "text": "Bergen", "uncovered_positions": [] }, { "start": 63, "end": 85, "label": "ORGANIZATION", "text": "Maple Ridge University", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-088", "scenario": null, "predictions": [ { "start": 10, "end": 17, "label": "RELIGION", "text": "jüdisch", "score": 0.8295804262161255, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 44, "end": 66, "label": "ORGANIZATION", "text": "Maple Ridge University", "score": 0.9811441898345947, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 80, "end": 95, "label": "PERSON", "text": "Alva Rosenqvist", "score": 0.998078465461731, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 10, "end": 17, "label": "RELIGION", "text": "jüdisch", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" }, { "start": 44, "end": 66, "label": "ORGANIZATION", "text": "Maple Ridge University", "score": 0.9811441898345947, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 80, "end": 95, "label": "PERSON", "text": "Alva Rosenqvist", "score": 0.998078465461731, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 10, "end": 17, "label": "RELIGION", "text": "jüdisch", "uncovered_positions": [] }, { "start": 44, "end": 66, "label": "ORGANIZATION", "text": "Maple Ridge University", "uncovered_positions": [] }, { "start": 80, "end": 95, "label": "PERSON", "text": "Alva Rosenqvist", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-089", "scenario": null, "predictions": [ { "start": 101, "end": 107, "label": "PERSON", "text": "person", "score": 0.7157840132713318, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 109, "end": 114, "label": "CITY", "text": "place", "score": 0.7889829277992249, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 116, "end": 128, "label": "ORGANIZATION", "text": "organization", "score": 0.8459369540214539, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 40, "end": 64, "label": "RELIGION", "text": "no religious affiliation", "score": null, "source": "deterministic", "model_revision": "religious-status-v1", "normalization": "identity" }, { "start": 101, "end": 107, "label": "PERSON", "text": "person", "score": 0.7157840132713318, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 109, "end": 114, "label": "CITY", "text": "place", "score": 0.7889829277992249, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 116, "end": 128, "label": "ORGANIZATION", "text": "organization", "score": 0.8459369540214539, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [ [ 40, 64, "RELIGION" ], [ 101, 107, "PERSON" ], [ 109, 114, "CITY" ], [ 116, 128, "ORGANIZATION" ] ], "false_negatives": [], "exact_match": false, "error_code": null, "canaries": [], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-090", "scenario": null, "predictions": [ { "start": 10, "end": 27, "label": "PERSON", "text": "Theo Merriweather", "score": 0.9981582760810852, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 41, "end": 59, "label": "ORGANIZATION", "text": "Polar Systems Ltd.", "score": 0.9988139867782593, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 63, "end": 67, "label": "CITY", "text": "York", "score": 0.9996792078018188, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 10, "end": 27, "label": "PERSON", "text": "Theo Merriweather", "score": 0.9981582760810852, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 41, "end": 59, "label": "ORGANIZATION", "text": "Polar Systems Ltd.", "score": 0.9988139867782593, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 63, "end": 67, "label": "CITY", "text": "York", "score": 0.9996792078018188, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 10, "end": 27, "label": "PERSON", "text": "Theo Merriweather", "uncovered_positions": [] }, { "start": 41, "end": 59, "label": "ORGANIZATION", "text": "Polar Systems Ltd.", "uncovered_positions": [] }, { "start": 63, "end": 67, "label": "CITY", "text": "York", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-091", "scenario": null, "predictions": [ { "start": 12, "end": 29, "label": "PERSON", "text": "Theo Merriweather", "score": 0.9984655380249023, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 41, "end": 59, "label": "ORGANIZATION", "text": "Polar Systems Ltd.", "score": 0.998237133026123, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 65, "end": 69, "label": "CITY", "text": "York", "score": 0.9996891021728516, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 12, "end": 29, "label": "PERSON", "text": "Theo Merriweather", "score": 0.9984655380249023, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 41, "end": 59, "label": "ORGANIZATION", "text": "Polar Systems Ltd.", "score": 0.998237133026123, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 65, "end": 69, "label": "CITY", "text": "York", "score": 0.9996891021728516, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 12, "end": 29, "label": "PERSON", "text": "Theo Merriweather", "uncovered_positions": [] }, { "start": 41, "end": 59, "label": "ORGANIZATION", "text": "Polar Systems Ltd.", "uncovered_positions": [] }, { "start": 65, "end": 69, "label": "CITY", "text": "York", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-092", "scenario": null, "predictions": [ { "start": 0, "end": 17, "label": "PERSON", "text": "Theo Merriweather", "score": 0.9836243391036987, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 24, "end": 34, "label": "RELIGION", "text": "muslimisch", "score": 0.9906712770462036, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 62, "end": 92, "label": "EMAIL", "text": "case110.review@example.invalid", "score": 0.9017051458358765, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 103, "end": 117, "label": "PHONE", "text": "+49 541 730110", "score": 0.9932438731193542, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 0, "end": 17, "label": "PERSON", "text": "Theo Merriweather", "score": 0.9836243391036987, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 24, "end": 34, "label": "RELIGION", "text": "muslimisch", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" }, { "start": 62, "end": 92, "label": "EMAIL", "text": "case110.review@example.invalid", "score": null, "source": "deterministic", "model_revision": "contact-patterns-v1", "normalization": "identity" }, { "start": 103, "end": 117, "label": "PHONE", "text": "+49 541 730110", "score": null, "source": "deterministic", "model_revision": "contact-patterns-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 0, "end": 17, "label": "PERSON", "text": "Theo Merriweather", "uncovered_positions": [] }, { "start": 24, "end": 34, "label": "RELIGION", "text": "muslimisch", "uncovered_positions": [] }, { "start": 62, "end": 92, "label": "EMAIL", "text": "case110.review@example.invalid", "uncovered_positions": [] }, { "start": 103, "end": 117, "label": "PHONE", "text": "+49 541 730110", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-093", "scenario": null, "predictions": [ { "start": 0, "end": 17, "label": "PERSON", "text": "Theo Merriweather", "score": 0.9977192282676697, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 35, "end": 40, "label": "RELIGION", "text": "Islam", "score": 0.999936580657959, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 66, "end": 96, "label": "EMAIL", "text": "case110.review@example.invalid", "score": 0.8946349620819092, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 105, "end": 119, "label": "PHONE", "text": "+49 541 730110", "score": 0.9599586129188538, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 0, "end": 17, "label": "PERSON", "text": "Theo Merriweather", "score": 0.9977192282676697, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 35, "end": 40, "label": "RELIGION", "text": "Islam", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" }, { "start": 66, "end": 96, "label": "EMAIL", "text": "case110.review@example.invalid", "score": null, "source": "deterministic", "model_revision": "contact-patterns-v1", "normalization": "identity" }, { "start": 105, "end": 119, "label": "PHONE", "text": "+49 541 730110", "score": null, "source": "deterministic", "model_revision": "contact-patterns-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 0, "end": 17, "label": "PERSON", "text": "Theo Merriweather", "uncovered_positions": [] }, { "start": 35, "end": 40, "label": "RELIGION", "text": "Islam", "uncovered_positions": [] }, { "start": 66, "end": 96, "label": "EMAIL", "text": "case110.review@example.invalid", "uncovered_positions": [] }, { "start": 105, "end": 119, "label": "PHONE", "text": "+49 541 730110", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-094", "scenario": null, "predictions": [ { "start": 0, "end": 17, "label": "PERSON", "text": "Theo Merriweather", "score": 0.9987108707427979, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 26, "end": 30, "label": "CITY", "text": "York", "score": 0.9983842372894287, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 33, "end": 50, "label": "PERSON", "text": "Theo Merriweather", "score": 0.9984440207481384, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 76, "end": 94, "label": "ORGANIZATION", "text": "Polar Systems Ltd.", "score": 0.9534488320350647, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 0, "end": 17, "label": "PERSON", "text": "Theo Merriweather", "score": 0.9987108707427979, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 26, "end": 30, "label": "CITY", "text": "York", "score": 0.9983842372894287, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 33, "end": 50, "label": "PERSON", "text": "Theo Merriweather", "score": 0.9984440207481384, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 76, "end": 94, "label": "ORGANIZATION", "text": "Polar Systems Ltd.", "score": 0.9534488320350647, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 0, "end": 17, "label": "PERSON", "text": "Theo Merriweather", "uncovered_positions": [] }, { "start": 26, "end": 30, "label": "CITY", "text": "York", "uncovered_positions": [] }, { "start": 33, "end": 50, "label": "PERSON", "text": "Theo Merriweather", "uncovered_positions": [] }, { "start": 76, "end": 94, "label": "ORGANIZATION", "text": "Polar Systems Ltd.", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-095", "scenario": null, "predictions": [ { "start": 3, "end": 7, "label": "CITY", "text": "York", "score": 0.9998012185096741, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 9, "end": 26, "label": "PERSON", "text": "Theo Merriweather", "score": 0.9998037219047546, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 38, "end": 56, "label": "ORGANIZATION", "text": "Polar Systems Ltd.", "score": 0.9765823483467102, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 77, "end": 94, "label": "PERSON", "text": "Theo Merriweather", "score": 0.9997954964637756, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 107, "end": 111, "label": "CITY", "text": "York", "score": 0.9997724890708923, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 3, "end": 7, "label": "CITY", "text": "York", "score": 0.9998012185096741, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 9, "end": 26, "label": "PERSON", "text": "Theo Merriweather", "score": 0.9998037219047546, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 38, "end": 56, "label": "ORGANIZATION", "text": "Polar Systems Ltd.", "score": 0.9765823483467102, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 77, "end": 94, "label": "PERSON", "text": "Theo Merriweather", "score": 0.9997954964637756, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 107, "end": 111, "label": "CITY", "text": "York", "score": 0.9997724890708923, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 3, "end": 7, "label": "CITY", "text": "York", "uncovered_positions": [] }, { "start": 9, "end": 26, "label": "PERSON", "text": "Theo Merriweather", "uncovered_positions": [] }, { "start": 38, "end": 56, "label": "ORGANIZATION", "text": "Polar Systems Ltd.", "uncovered_positions": [] }, { "start": 77, "end": 94, "label": "PERSON", "text": "Theo Merriweather", "uncovered_positions": [] }, { "start": 107, "end": 111, "label": "CITY", "text": "York", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-096", "scenario": null, "predictions": [ { "start": 6, "end": 24, "label": "PERSON", "text": "Theo-\nMerriweather", "score": 0.9998553991317749, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "dehyphenation-v1" }, { "start": 30, "end": 36, "label": "CITY", "text": "Yo-\nrk", "score": 0.9999533891677856, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "dehyphenation-v1" }, { "start": 50, "end": 68, "label": "ORGANIZATION", "text": "Polar Systems Ltd.", "score": 0.9765849709510803, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "dehyphenation-v1" } ], "combined_predictions": [ { "start": 6, "end": 24, "label": "PERSON", "text": "Theo-\nMerriweather", "score": 0.9998553991317749, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "dehyphenation-v1" }, { "start": 30, "end": 36, "label": "CITY", "text": "Yo-\nrk", "score": 0.9999533891677856, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "dehyphenation-v1" }, { "start": 50, "end": 68, "label": "ORGANIZATION", "text": "Polar Systems Ltd.", "score": 0.9765849709510803, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "dehyphenation-v1" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 6, "end": 24, "label": "PERSON", "text": "Theo-\nMerriweather", "uncovered_positions": [] }, { "start": 30, "end": 36, "label": "CITY", "text": "Yo-\nrk", "uncovered_positions": [] }, { "start": 50, "end": 68, "label": "ORGANIZATION", "text": "Polar Systems Ltd.", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-097", "scenario": null, "predictions": [ { "start": 16, "end": 33, "label": "PERSON", "text": "The0 Merr1weather", "score": 0.9711146950721741, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 45, "end": 49, "label": "CITY", "text": "Y0rk", "score": 0.9964478015899658, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 63, "end": 81, "label": "ORGANIZATION", "text": "Polar Systems Ltd.", "score": 0.9427483677864075, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 16, "end": 33, "label": "PERSON", "text": "The0 Merr1weather", "score": 0.9711146950721741, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 45, "end": 49, "label": "CITY", "text": "Y0rk", "score": 0.9964478015899658, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 63, "end": 81, "label": "ORGANIZATION", "text": "Polar Systems Ltd.", "score": 0.9427483677864075, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 16, "end": 33, "label": "PERSON", "text": "The0 Merr1weather", "uncovered_positions": [] }, { "start": 45, "end": 49, "label": "CITY", "text": "Y0rk", "uncovered_positions": [] }, { "start": 63, "end": 81, "label": "ORGANIZATION", "text": "Polar Systems Ltd.", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-098", "scenario": null, "predictions": [ { "start": 10, "end": 20, "label": "RELIGION", "text": "muslimisch", "score": 0.8173997402191162, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 47, "end": 65, "label": "ORGANIZATION", "text": "Polar Systems Ltd.", "score": 0.990338146686554, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 79, "end": 96, "label": "PERSON", "text": "Theo Merriweather", "score": 0.9961032867431641, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 10, "end": 20, "label": "RELIGION", "text": "muslimisch", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" }, { "start": 47, "end": 65, "label": "ORGANIZATION", "text": "Polar Systems Ltd.", "score": 0.990338146686554, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 79, "end": 96, "label": "PERSON", "text": "Theo Merriweather", "score": 0.9961032867431641, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 10, "end": 20, "label": "RELIGION", "text": "muslimisch", "uncovered_positions": [] }, { "start": 47, "end": 65, "label": "ORGANIZATION", "text": "Polar Systems Ltd.", "uncovered_positions": [] }, { "start": 79, "end": 96, "label": "PERSON", "text": "Theo Merriweather", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-099", "scenario": null, "predictions": [ { "start": 101, "end": 107, "label": "PERSON", "text": "person", "score": 0.7109703421592712, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 109, "end": 114, "label": "CITY", "text": "place", "score": 0.7852903604507446, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 116, "end": 128, "label": "ORGANIZATION", "text": "organization", "score": 0.8414570093154907, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 40, "end": 64, "label": "RELIGION", "text": "no religious affiliation", "score": null, "source": "deterministic", "model_revision": "religious-status-v1", "normalization": "identity" }, { "start": 101, "end": 107, "label": "PERSON", "text": "person", "score": 0.7109703421592712, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 109, "end": 114, "label": "CITY", "text": "place", "score": 0.7852903604507446, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 116, "end": 128, "label": "ORGANIZATION", "text": "organization", "score": 0.8414570093154907, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [ [ 40, 64, "RELIGION" ], [ 101, 107, "PERSON" ], [ 109, 114, "CITY" ], [ 116, 128, "ORGANIZATION" ] ], "false_negatives": [], "exact_match": false, "error_code": null, "canaries": [], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-100", "scenario": null, "predictions": [ { "start": 10, "end": 25, "label": "PERSON", "text": "Maja Kirschbaum", "score": 0.9991471767425537, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 39, "end": 57, "label": "ORGANIZATION", "text": "Sonnenbogen Verein", "score": 0.9799805283546448, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 61, "end": 69, "label": "CITY", "text": "Lüneburg", "score": 0.9990435242652893, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 10, "end": 25, "label": "PERSON", "text": "Maja Kirschbaum", "score": 0.9991471767425537, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 39, "end": 57, "label": "ORGANIZATION", "text": "Sonnenbogen Verein", "score": 0.9799805283546448, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 61, "end": 69, "label": "CITY", "text": "Lüneburg", "score": 0.9990435242652893, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 10, "end": 25, "label": "PERSON", "text": "Maja Kirschbaum", "uncovered_positions": [] }, { "start": 39, "end": 57, "label": "ORGANIZATION", "text": "Sonnenbogen Verein", "uncovered_positions": [] }, { "start": 61, "end": 69, "label": "CITY", "text": "Lüneburg", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-101", "scenario": null, "predictions": [ { "start": 12, "end": 27, "label": "PERSON", "text": "Maja Kirschbaum", "score": 0.9968947172164917, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 39, "end": 57, "label": "ORGANIZATION", "text": "Sonnenbogen Verein", "score": 0.9929682612419128, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 63, "end": 71, "label": "CITY", "text": "Lüneburg", "score": 0.9971613883972168, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 12, "end": 27, "label": "PERSON", "text": "Maja Kirschbaum", "score": 0.9968947172164917, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 39, "end": 57, "label": "ORGANIZATION", "text": "Sonnenbogen Verein", "score": 0.9929682612419128, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 63, "end": 71, "label": "CITY", "text": "Lüneburg", "score": 0.9971613883972168, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 12, "end": 27, "label": "PERSON", "text": "Maja Kirschbaum", "uncovered_positions": [] }, { "start": 39, "end": 57, "label": "ORGANIZATION", "text": "Sonnenbogen Verein", "uncovered_positions": [] }, { "start": 63, "end": 71, "label": "CITY", "text": "Lüneburg", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-102", "scenario": null, "predictions": [ { "start": 0, "end": 15, "label": "PERSON", "text": "Maja Kirschbaum", "score": 0.998397171497345, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 64, "end": 94, "label": "EMAIL", "text": "case111.review@example.invalid", "score": 0.8911723494529724, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 105, "end": 119, "label": "PHONE", "text": "+49 541 730111", "score": 0.9943364262580872, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 0, "end": 15, "label": "PERSON", "text": "Maja Kirschbaum", "score": 0.998397171497345, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 22, "end": 36, "label": "RELIGION", "text": "konfessionslos", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" }, { "start": 64, "end": 94, "label": "EMAIL", "text": "case111.review@example.invalid", "score": null, "source": "deterministic", "model_revision": "contact-patterns-v1", "normalization": "identity" }, { "start": 105, "end": 119, "label": "PHONE", "text": "+49 541 730111", "score": null, "source": "deterministic", "model_revision": "contact-patterns-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 0, "end": 15, "label": "PERSON", "text": "Maja Kirschbaum", "uncovered_positions": [] }, { "start": 22, "end": 36, "label": "RELIGION", "text": "konfessionslos", "uncovered_positions": [] }, { "start": 64, "end": 94, "label": "EMAIL", "text": "case111.review@example.invalid", "uncovered_positions": [] }, { "start": 105, "end": 119, "label": "PHONE", "text": "+49 541 730111", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-103", "scenario": null, "predictions": [ { "start": 0, "end": 15, "label": "PERSON", "text": "Maja Kirschbaum", "score": 0.9988783001899719, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 33, "end": 40, "label": "RELIGION", "text": "atheism", "score": 0.99891197681427, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 66, "end": 96, "label": "EMAIL", "text": "case111.review@example.invalid", "score": 0.8873010277748108, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 105, "end": 119, "label": "PHONE", "text": "+49 541 730111", "score": 0.9670958518981934, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 0, "end": 15, "label": "PERSON", "text": "Maja Kirschbaum", "score": 0.9988783001899719, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 33, "end": 40, "label": "RELIGION", "text": "atheism", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" }, { "start": 66, "end": 96, "label": "EMAIL", "text": "case111.review@example.invalid", "score": null, "source": "deterministic", "model_revision": "contact-patterns-v1", "normalization": "identity" }, { "start": 105, "end": 119, "label": "PHONE", "text": "+49 541 730111", "score": null, "source": "deterministic", "model_revision": "contact-patterns-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 0, "end": 15, "label": "PERSON", "text": "Maja Kirschbaum", "uncovered_positions": [] }, { "start": 33, "end": 40, "label": "RELIGION", "text": "atheism", "uncovered_positions": [] }, { "start": 66, "end": 96, "label": "EMAIL", "text": "case111.review@example.invalid", "uncovered_positions": [] }, { "start": 105, "end": 119, "label": "PHONE", "text": "+49 541 730111", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-104", "scenario": null, "predictions": [ { "start": 0, "end": 15, "label": "PERSON", "text": "Maja Kirschbaum", "score": 0.9995399713516235, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 24, "end": 32, "label": "CITY", "text": "Lüneburg", "score": 0.9943020343780518, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 35, "end": 50, "label": "PERSON", "text": "Maja Kirschbaum", "score": 0.9996929168701172, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 76, "end": 94, "label": "ORGANIZATION", "text": "Sonnenbogen Verein", "score": 0.9872075915336609, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 0, "end": 15, "label": "PERSON", "text": "Maja Kirschbaum", "score": 0.9995399713516235, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 24, "end": 32, "label": "CITY", "text": "Lüneburg", "score": 0.9943020343780518, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 35, "end": 50, "label": "PERSON", "text": "Maja Kirschbaum", "score": 0.9996929168701172, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 76, "end": 94, "label": "ORGANIZATION", "text": "Sonnenbogen Verein", "score": 0.9872075915336609, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 0, "end": 15, "label": "PERSON", "text": "Maja Kirschbaum", "uncovered_positions": [] }, { "start": 24, "end": 32, "label": "CITY", "text": "Lüneburg", "uncovered_positions": [] }, { "start": 35, "end": 50, "label": "PERSON", "text": "Maja Kirschbaum", "uncovered_positions": [] }, { "start": 76, "end": 94, "label": "ORGANIZATION", "text": "Sonnenbogen Verein", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-105", "scenario": null, "predictions": [ { "start": 3, "end": 11, "label": "CITY", "text": "Lüneburg", "score": 0.9998090863227844, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 13, "end": 28, "label": "PERSON", "text": "Maja Kirschbaum", "score": 0.9998596906661987, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 40, "end": 58, "label": "ORGANIZATION", "text": "Sonnenbogen Verein", "score": 0.9954612851142883, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 79, "end": 94, "label": "PERSON", "text": "Maja Kirschbaum", "score": 0.9997907280921936, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 107, "end": 115, "label": "CITY", "text": "Lüneburg", "score": 0.9995126724243164, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 3, "end": 11, "label": "CITY", "text": "Lüneburg", "score": 0.9998090863227844, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 13, "end": 28, "label": "PERSON", "text": "Maja Kirschbaum", "score": 0.9998596906661987, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 40, "end": 58, "label": "ORGANIZATION", "text": "Sonnenbogen Verein", "score": 0.9954612851142883, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 79, "end": 94, "label": "PERSON", "text": "Maja Kirschbaum", "score": 0.9997907280921936, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 107, "end": 115, "label": "CITY", "text": "Lüneburg", "score": 0.9995126724243164, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 3, "end": 11, "label": "CITY", "text": "Lüneburg", "uncovered_positions": [] }, { "start": 13, "end": 28, "label": "PERSON", "text": "Maja Kirschbaum", "uncovered_positions": [] }, { "start": 40, "end": 58, "label": "ORGANIZATION", "text": "Sonnenbogen Verein", "uncovered_positions": [] }, { "start": 79, "end": 94, "label": "PERSON", "text": "Maja Kirschbaum", "uncovered_positions": [] }, { "start": 107, "end": 115, "label": "CITY", "text": "Lüneburg", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-106", "scenario": null, "predictions": [ { "start": 6, "end": 22, "label": "PERSON", "text": "Maja-\nKirschbaum", "score": 0.999875545501709, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "dehyphenation-v1" }, { "start": 28, "end": 38, "label": "CITY", "text": "Lüne-\nburg", "score": 0.9999223947525024, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "dehyphenation-v1" }, { "start": 52, "end": 70, "label": "ORGANIZATION", "text": "Sonnenbogen Verein", "score": 0.9918258190155029, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "dehyphenation-v1" } ], "combined_predictions": [ { "start": 6, "end": 22, "label": "PERSON", "text": "Maja-\nKirschbaum", "score": 0.999875545501709, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "dehyphenation-v1" }, { "start": 28, "end": 38, "label": "CITY", "text": "Lüne-\nburg", "score": 0.9999223947525024, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "dehyphenation-v1" }, { "start": 52, "end": 70, "label": "ORGANIZATION", "text": "Sonnenbogen Verein", "score": 0.9918258190155029, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "dehyphenation-v1" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 6, "end": 22, "label": "PERSON", "text": "Maja-\nKirschbaum", "uncovered_positions": [] }, { "start": 28, "end": 38, "label": "CITY", "text": "Lüne-\nburg", "uncovered_positions": [] }, { "start": 52, "end": 70, "label": "ORGANIZATION", "text": "Sonnenbogen Verein", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-107", "scenario": null, "predictions": [ { "start": 16, "end": 31, "label": "PERSON", "text": "Maja K1rschbaum", "score": 0.9977808594703674, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 43, "end": 51, "label": "CITY", "text": "Lüneburg", "score": 0.9994350075721741, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 65, "end": 83, "label": "ORGANIZATION", "text": "Sonnenbogen Verein", "score": 0.9930818676948547, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 16, "end": 31, "label": "PERSON", "text": "Maja K1rschbaum", "score": 0.9977808594703674, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 43, "end": 51, "label": "CITY", "text": "Lüneburg", "score": 0.9994350075721741, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 65, "end": 83, "label": "ORGANIZATION", "text": "Sonnenbogen Verein", "score": 0.9930818676948547, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 16, "end": 31, "label": "PERSON", "text": "Maja K1rschbaum", "uncovered_positions": [] }, { "start": 43, "end": 51, "label": "CITY", "text": "Lüneburg", "uncovered_positions": [] }, { "start": 65, "end": 83, "label": "ORGANIZATION", "text": "Sonnenbogen Verein", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-108", "scenario": null, "predictions": [ { "start": 51, "end": 69, "label": "ORGANIZATION", "text": "Sonnenbogen Verein", "score": 0.9550279378890991, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 83, "end": 98, "label": "PERSON", "text": "Maja Kirschbaum", "score": 0.9982492923736572, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 10, "end": 24, "label": "RELIGION", "text": "konfessionslos", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" }, { "start": 51, "end": 69, "label": "ORGANIZATION", "text": "Sonnenbogen Verein", "score": 0.9550279378890991, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 83, "end": 98, "label": "PERSON", "text": "Maja Kirschbaum", "score": 0.9982492923736572, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 10, "end": 24, "label": "RELIGION", "text": "konfessionslos", "uncovered_positions": [] }, { "start": 51, "end": 69, "label": "ORGANIZATION", "text": "Sonnenbogen Verein", "uncovered_positions": [] }, { "start": 83, "end": 98, "label": "PERSON", "text": "Maja Kirschbaum", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-109", "scenario": null, "predictions": [ { "start": 101, "end": 107, "label": "PERSON", "text": "person", "score": 0.6989420056343079, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 109, "end": 114, "label": "CITY", "text": "place", "score": 0.7754619121551514, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 116, "end": 128, "label": "ORGANIZATION", "text": "organization", "score": 0.8311129808425903, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 40, "end": 64, "label": "RELIGION", "text": "no religious affiliation", "score": null, "source": "deterministic", "model_revision": "religious-status-v1", "normalization": "identity" }, { "start": 101, "end": 107, "label": "PERSON", "text": "person", "score": 0.6989420056343079, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 109, "end": 114, "label": "CITY", "text": "place", "score": 0.7754619121551514, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 116, "end": 128, "label": "ORGANIZATION", "text": "organization", "score": 0.8311129808425903, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [ [ 40, 64, "RELIGION" ], [ 101, 107, "PERSON" ], [ 109, 114, "CITY" ], [ 116, 128, "ORGANIZATION" ] ], "false_negatives": [], "exact_match": false, "error_code": null, "canaries": [], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-110", "scenario": null, "predictions": [ { "start": 10, "end": 24, "label": "PERSON", "text": "Ronan Whitmore", "score": 0.9985572695732117, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 38, "end": 56, "label": "ORGANIZATION", "text": "Beacon Valley Labs", "score": 0.9976409673690796, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 60, "end": 64, "label": "CITY", "text": "Bath", "score": 0.9994964599609375, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 10, "end": 24, "label": "PERSON", "text": "Ronan Whitmore", "score": 0.9985572695732117, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 38, "end": 56, "label": "ORGANIZATION", "text": "Beacon Valley Labs", "score": 0.9976409673690796, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 60, "end": 64, "label": "CITY", "text": "Bath", "score": 0.9994964599609375, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 10, "end": 24, "label": "PERSON", "text": "Ronan Whitmore", "uncovered_positions": [] }, { "start": 38, "end": 56, "label": "ORGANIZATION", "text": "Beacon Valley Labs", "uncovered_positions": [] }, { "start": 60, "end": 64, "label": "CITY", "text": "Bath", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-111", "scenario": null, "predictions": [ { "start": 12, "end": 26, "label": "PERSON", "text": "Ronan Whitmore", "score": 0.998796820640564, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 38, "end": 56, "label": "ORGANIZATION", "text": "Beacon Valley Labs", "score": 0.9980693459510803, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 62, "end": 66, "label": "CITY", "text": "Bath", "score": 0.9995183944702148, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 12, "end": 26, "label": "PERSON", "text": "Ronan Whitmore", "score": 0.998796820640564, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 38, "end": 56, "label": "ORGANIZATION", "text": "Beacon Valley Labs", "score": 0.9980693459510803, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 62, "end": 66, "label": "CITY", "text": "Bath", "score": 0.9995183944702148, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 12, "end": 26, "label": "PERSON", "text": "Ronan Whitmore", "uncovered_positions": [] }, { "start": 38, "end": 56, "label": "ORGANIZATION", "text": "Beacon Valley Labs", "uncovered_positions": [] }, { "start": 62, "end": 66, "label": "CITY", "text": "Bath", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-112", "scenario": null, "predictions": [ { "start": 0, "end": 14, "label": "PERSON", "text": "Ronan Whitmore", "score": 0.99787437915802, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 21, "end": 31, "label": "RELIGION", "text": "christlich", "score": 0.9895966649055481, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 59, "end": 89, "label": "EMAIL", "text": "case112.review@example.invalid", "score": 0.9079257249832153, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 100, "end": 114, "label": "PHONE", "text": "+49 541 730112", "score": 0.9931344985961914, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 0, "end": 14, "label": "PERSON", "text": "Ronan Whitmore", "score": 0.99787437915802, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 21, "end": 31, "label": "RELIGION", "text": "christlich", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" }, { "start": 59, "end": 89, "label": "EMAIL", "text": "case112.review@example.invalid", "score": null, "source": "deterministic", "model_revision": "contact-patterns-v1", "normalization": "identity" }, { "start": 100, "end": 114, "label": "PHONE", "text": "+49 541 730112", "score": null, "source": "deterministic", "model_revision": "contact-patterns-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 0, "end": 14, "label": "PERSON", "text": "Ronan Whitmore", "uncovered_positions": [] }, { "start": 21, "end": 31, "label": "RELIGION", "text": "christlich", "uncovered_positions": [] }, { "start": 59, "end": 89, "label": "EMAIL", "text": "case112.review@example.invalid", "uncovered_positions": [] }, { "start": 100, "end": 114, "label": "PHONE", "text": "+49 541 730112", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-113", "scenario": null, "predictions": [ { "start": 0, "end": 14, "label": "PERSON", "text": "Ronan Whitmore", "score": 0.9984022974967957, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 32, "end": 44, "label": "RELIGION", "text": "Christianity", "score": 0.9992650151252747, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 70, "end": 100, "label": "EMAIL", "text": "case112.review@example.invalid", "score": 0.9013476371765137, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 109, "end": 123, "label": "PHONE", "text": "+49 541 730112", "score": 0.9611164331436157, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 0, "end": 14, "label": "PERSON", "text": "Ronan Whitmore", "score": 0.9984022974967957, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 32, "end": 44, "label": "RELIGION", "text": "Christianity", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" }, { "start": 70, "end": 100, "label": "EMAIL", "text": "case112.review@example.invalid", "score": null, "source": "deterministic", "model_revision": "contact-patterns-v1", "normalization": "identity" }, { "start": 109, "end": 123, "label": "PHONE", "text": "+49 541 730112", "score": null, "source": "deterministic", "model_revision": "contact-patterns-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 0, "end": 14, "label": "PERSON", "text": "Ronan Whitmore", "uncovered_positions": [] }, { "start": 32, "end": 44, "label": "RELIGION", "text": "Christianity", "uncovered_positions": [] }, { "start": 70, "end": 100, "label": "EMAIL", "text": "case112.review@example.invalid", "uncovered_positions": [] }, { "start": 109, "end": 123, "label": "PHONE", "text": "+49 541 730112", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-114", "scenario": null, "predictions": [ { "start": 0, "end": 14, "label": "PERSON", "text": "Ronan Whitmore", "score": 0.9992198944091797, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 23, "end": 27, "label": "CITY", "text": "Bath", "score": 0.9989506006240845, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 30, "end": 44, "label": "PERSON", "text": "Ronan Whitmore", "score": 0.9990702271461487, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 70, "end": 88, "label": "ORGANIZATION", "text": "Beacon Valley Labs", "score": 0.9951003193855286, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 0, "end": 14, "label": "PERSON", "text": "Ronan Whitmore", "score": 0.9992198944091797, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 23, "end": 27, "label": "CITY", "text": "Bath", "score": 0.9989506006240845, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 30, "end": 44, "label": "PERSON", "text": "Ronan Whitmore", "score": 0.9990702271461487, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 70, "end": 88, "label": "ORGANIZATION", "text": "Beacon Valley Labs", "score": 0.9951003193855286, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 0, "end": 14, "label": "PERSON", "text": "Ronan Whitmore", "uncovered_positions": [] }, { "start": 23, "end": 27, "label": "CITY", "text": "Bath", "uncovered_positions": [] }, { "start": 30, "end": 44, "label": "PERSON", "text": "Ronan Whitmore", "uncovered_positions": [] }, { "start": 70, "end": 88, "label": "ORGANIZATION", "text": "Beacon Valley Labs", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-115", "scenario": null, "predictions": [ { "start": 3, "end": 7, "label": "CITY", "text": "Bath", "score": 0.9997978806495667, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 9, "end": 23, "label": "PERSON", "text": "Ronan Whitmore", "score": 0.999768078327179, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 35, "end": 53, "label": "ORGANIZATION", "text": "Beacon Valley Labs", "score": 0.9973915815353394, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 74, "end": 88, "label": "PERSON", "text": "Ronan Whitmore", "score": 0.9997625946998596, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 101, "end": 105, "label": "CITY", "text": "Bath", "score": 0.9997105002403259, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 3, "end": 7, "label": "CITY", "text": "Bath", "score": 0.9997978806495667, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 9, "end": 23, "label": "PERSON", "text": "Ronan Whitmore", "score": 0.999768078327179, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 35, "end": 53, "label": "ORGANIZATION", "text": "Beacon Valley Labs", "score": 0.9973915815353394, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 74, "end": 88, "label": "PERSON", "text": "Ronan Whitmore", "score": 0.9997625946998596, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 101, "end": 105, "label": "CITY", "text": "Bath", "score": 0.9997105002403259, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 3, "end": 7, "label": "CITY", "text": "Bath", "uncovered_positions": [] }, { "start": 9, "end": 23, "label": "PERSON", "text": "Ronan Whitmore", "uncovered_positions": [] }, { "start": 35, "end": 53, "label": "ORGANIZATION", "text": "Beacon Valley Labs", "uncovered_positions": [] }, { "start": 74, "end": 88, "label": "PERSON", "text": "Ronan Whitmore", "uncovered_positions": [] }, { "start": 101, "end": 105, "label": "CITY", "text": "Bath", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-116", "scenario": null, "predictions": [ { "start": 6, "end": 21, "label": "PERSON", "text": "Ronan-\nWhitmore", "score": 0.9997969269752502, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "dehyphenation-v1" }, { "start": 27, "end": 33, "label": "CITY", "text": "Ba-\nth", "score": 0.9999775886535645, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "dehyphenation-v1" }, { "start": 47, "end": 65, "label": "ORGANIZATION", "text": "Beacon Valley Labs", "score": 0.9984259605407715, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "dehyphenation-v1" } ], "combined_predictions": [ { "start": 6, "end": 21, "label": "PERSON", "text": "Ronan-\nWhitmore", "score": 0.9997969269752502, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "dehyphenation-v1" }, { "start": 27, "end": 33, "label": "CITY", "text": "Ba-\nth", "score": 0.9999775886535645, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "dehyphenation-v1" }, { "start": 47, "end": 65, "label": "ORGANIZATION", "text": "Beacon Valley Labs", "score": 0.9984259605407715, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "dehyphenation-v1" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 6, "end": 21, "label": "PERSON", "text": "Ronan-\nWhitmore", "uncovered_positions": [] }, { "start": 27, "end": 33, "label": "CITY", "text": "Ba-\nth", "uncovered_positions": [] }, { "start": 47, "end": 65, "label": "ORGANIZATION", "text": "Beacon Valley Labs", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-117", "scenario": null, "predictions": [ { "start": 16, "end": 30, "label": "PERSON", "text": "R0nan Wh1tm0re", "score": 0.9801152944564819, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 42, "end": 46, "label": "CITY", "text": "B4th", "score": 0.9872779846191406, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 60, "end": 78, "label": "ORGANIZATION", "text": "Beacon Valley Labs", "score": 0.9977201819419861, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 16, "end": 30, "label": "PERSON", "text": "R0nan Wh1tm0re", "score": 0.9801152944564819, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 42, "end": 46, "label": "CITY", "text": "B4th", "score": 0.9872779846191406, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 60, "end": 78, "label": "ORGANIZATION", "text": "Beacon Valley Labs", "score": 0.9977201819419861, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 16, "end": 30, "label": "PERSON", "text": "R0nan Wh1tm0re", "uncovered_positions": [] }, { "start": 42, "end": 46, "label": "CITY", "text": "B4th", "uncovered_positions": [] }, { "start": 60, "end": 78, "label": "ORGANIZATION", "text": "Beacon Valley Labs", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-118", "scenario": null, "predictions": [ { "start": 10, "end": 20, "label": "RELIGION", "text": "christlich", "score": 0.7355935573577881, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 47, "end": 65, "label": "ORGANIZATION", "text": "Beacon Valley Labs", "score": 0.9870188236236572, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 79, "end": 93, "label": "PERSON", "text": "Ronan Whitmore", "score": 0.9990026354789734, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 10, "end": 20, "label": "RELIGION", "text": "christlich", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" }, { "start": 47, "end": 65, "label": "ORGANIZATION", "text": "Beacon Valley Labs", "score": 0.9870188236236572, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 79, "end": 93, "label": "PERSON", "text": "Ronan Whitmore", "score": 0.9990026354789734, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 10, "end": 20, "label": "RELIGION", "text": "christlich", "uncovered_positions": [] }, { "start": 47, "end": 65, "label": "ORGANIZATION", "text": "Beacon Valley Labs", "uncovered_positions": [] }, { "start": 79, "end": 93, "label": "PERSON", "text": "Ronan Whitmore", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03c-119", "scenario": null, "predictions": [ { "start": 101, "end": 107, "label": "PERSON", "text": "person", "score": 0.7034855484962463, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 109, "end": 114, "label": "CITY", "text": "place", "score": 0.7791854739189148, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 116, "end": 128, "label": "ORGANIZATION", "text": "organization", "score": 0.8351515531539917, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 40, "end": 64, "label": "RELIGION", "text": "no religious affiliation", "score": null, "source": "deterministic", "model_revision": "religious-status-v1", "normalization": "identity" }, { "start": 101, "end": 107, "label": "PERSON", "text": "person", "score": 0.7034855484962463, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 109, "end": 114, "label": "CITY", "text": "place", "score": 0.7791854739189148, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 116, "end": 128, "label": "ORGANIZATION", "text": "organization", "score": 0.8351515531539917, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [ [ 40, 64, "RELIGION" ], [ 101, 107, "PERSON" ], [ 109, 114, "CITY" ], [ 116, 128, "ORGANIZATION" ] ], "false_negatives": [], "exact_match": false, "error_code": null, "canaries": [], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" } ], "automated_release_eligible": false, "recognition_qualified": false }, "development03d": { "schema_version": "development-candidate-outcomes/v1", "accuracy": { "documents": 160, "ner_entity_metrics": { "PERSON": { "tp": 21, "fp": 9, "fn": 1, "precision": 0.7, "recall": 0.9545454545454546, "f1": 0.8076923076923077 }, "CITY": { "tp": 8, "fp": 3, "fn": 0, "precision": 0.7272727272727273, "recall": 1.0, "f1": 0.8421052631578948 }, "RELIGION": { "tp": 108, "fp": 6, "fn": 42, "precision": 0.9473684210526315, "recall": 0.72, "f1": 0.8181818181818181 }, "EMAIL": { "tp": 3, "fp": 0, "fn": 1, "precision": 1.0, "recall": 0.75, "f1": 0.8571428571428571 }, "PHONE": { "tp": 2, "fp": 0, "fn": 0, "precision": 1.0, "recall": 1.0, "f1": 1.0 }, "ORGANIZATION": { "tp": 5, "fp": 14, "fn": 1, "precision": 0.2631578947368421, "recall": 0.8333333333333334, "f1": 0.39999999999999997 } }, "combined_entity_metrics": { "PERSON": { "tp": 20, "fp": 5, "fn": 2, "precision": 0.8, "recall": 0.9090909090909091, "f1": 0.8510638297872342 }, "CITY": { "tp": 7, "fp": 2, "fn": 1, "precision": 0.7777777777777778, "recall": 0.875, "f1": 0.823529411764706 }, "RELIGION": { "tp": 140, "fp": 2, "fn": 10, "precision": 0.9859154929577465, "recall": 0.9333333333333333, "f1": 0.9589041095890412 }, "EMAIL": { "tp": 3, "fp": 0, "fn": 1, "precision": 1.0, "recall": 0.75, "f1": 0.8571428571428571 }, "PHONE": { "tp": 2, "fp": 0, "fn": 0, "precision": 1.0, "recall": 1.0, "f1": 1.0 }, "ORGANIZATION": { "tp": 5, "fp": 12, "fn": 1, "precision": 0.29411764705882354, "recall": 0.8333333333333334, "f1": 0.4347826086956522 } }, "document_exact_match_rate": 0.8375, "groups": { "de": { "documents": 80, "exact_matches": 62, "false_negatives": 12 }, "en": { "documents": 80, "exact_matches": 72, "false_negatives": 3 }, "ocr_noise": { "documents": 16, "exact_matches": 11, "false_negatives": 3 } }, "blocked_documents": 5, "failure_codes": { "ambiguous_entity_overlap": 5 }, "automated_release_eligible": false, "uncovered_source_canaries_including_incomplete": { "PERSON": 1, "CITY": 1, "RELIGION": 6, "EMAIL": 1, "PHONE": 0, "ORGANIZATION": 1 }, "literal_presence_or_incomplete": { "PERSON": 1, "CITY": 1, "RELIGION": 6, "EMAIL": 1, "PHONE": 0, "ORGANIZATION": 0 }, "partially_covered_source_canaries_including_incomplete": { "PERSON": 0, "CITY": 0, "RELIGION": 0, "EMAIL": 0, "PHONE": 0, "ORGANIZATION": 1 }, "fully_uncovered_source_canaries_including_incomplete": { "PERSON": 1, "CITY": 1, "RELIGION": 6, "EMAIL": 1, "PHONE": 0, "ORGANIZATION": 0 }, "all_document_source_coverage": { "PERSON": { "covered": 258, "total": 270 }, "CITY": { "covered": 38, "total": 42 }, "RELIGION": { "covered": 1455, "total": 1510 }, "EMAIL": { "covered": 58, "total": 76 }, "PHONE": { "covered": 23, "total": 23 }, "ORGANIZATION": { "covered": 101, "total": 108 } }, "conservative_accuracy_gate_passed": false, "scope": "all source spans; incomplete processing counts as missing, not leaked output", "deterministic_replay_latency_ms": { "median": 0.11009700028807856, "max": 3.662794995761942 } }, "candidate_outcomes": { "complete_no_measured_leakage": 154, "candidate_partial_mask": 1, "candidate_fully_unmasked": 0, "blocked_conflicting_spans": 5, "blocked_model_runtime": 0, "other_incomplete": 0 }, "candidate_return_rate": 0.96875, "safe_abstention_rate": 0.03125, "incomplete_rate": 0.03125, "no_measured_leakage_rate_among_returned": 0.9935483870967742, "candidate_partial_canaries": { "PERSON": 0, "CITY": 0, "RELIGION": 0, "EMAIL": 0, "PHONE": 0, "ORGANIZATION": 1 }, "candidate_fully_unmasked_canaries": { "PERSON": 0, "CITY": 0, "RELIGION": 0, "EMAIL": 0, "PHONE": 0, "ORGANIZATION": 0 }, "returned_candidate_entity_metrics": { "PERSON": { "tp": 20, "fp": 5, "fn": 1, "precision": 0.8, "recall": 0.9523809523809523, "f1": 0.8695652173913043 }, "CITY": { "tp": 7, "fp": 2, "fn": 0, "precision": 0.7777777777777778, "recall": 1.0, "f1": 0.8750000000000001 }, "RELIGION": { "tp": 140, "fp": 2, "fn": 4, "precision": 0.9859154929577465, "recall": 0.9722222222222222, "f1": 0.979020979020979 }, "EMAIL": { "tp": 3, "fp": 0, "fn": 0, "precision": 1.0, "recall": 1.0, "f1": 1.0 }, "PHONE": { "tp": 2, "fp": 0, "fn": 0, "precision": 1.0, "recall": 1.0, "f1": 1.0 }, "ORGANIZATION": { "tp": 5, "fp": 12, "fn": 1, "precision": 0.29411764705882354, "recall": 0.8333333333333334, "f1": 0.4347826086956522 } }, "per_case": [ { "case_id": "holdout03d-001", "scenario": "affiliation", "predictions": [ { "start": 24, "end": 34, "label": "PERSON", "text": "Bekenntnis", "score": 0.5389757752418518, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 35, "end": 53, "label": "RELIGION", "text": "römisch-katholisch", "score": 0.843971312046051, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 24, "end": 34, "label": "PERSON", "text": "Bekenntnis", "score": 0.5389757752418518, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 35, "end": 53, "label": "RELIGION", "text": "römisch-katholisch", "score": 0.843971312046051, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [ [ 24, 34, "PERSON" ] ], "false_negatives": [], "exact_match": false, "error_code": null, "canaries": [ { "start": 35, "end": 53, "label": "RELIGION", "text": "römisch-katholisch", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-002", "scenario": "affiliation", "predictions": [ { "start": 35, "end": 49, "label": "RELIGION", "text": "Roman Catholic", "score": 0.9411200881004333, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 35, "end": 49, "label": "RELIGION", "text": "Roman Catholic", "score": 0.9411200881004333, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 35, "end": 49, "label": "RELIGION", "text": "Roman Catholic", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-003", "scenario": "affiliation", "predictions": [ { "start": 0, "end": 14, "label": "ORGANIZATION", "text": "Tabea Hornfels", "score": 0.9000996351242065, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 45, "end": 55, "label": "RELIGION", "text": "sunnitisch", "score": 0.6417439579963684, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 0, "end": 14, "label": "ORGANIZATION", "text": "Tabea Hornfels", "score": 0.9000996351242065, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 45, "end": 55, "label": "RELIGION", "text": "sunnitisch", "score": 0.6417439579963684, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [ [ 0, 14, "ORGANIZATION" ] ], "false_negatives": [ [ 0, 14, "PERSON" ] ], "exact_match": false, "error_code": null, "canaries": [ { "start": 0, "end": 14, "label": "PERSON", "text": "Tabea Hornfels", "uncovered_positions": [] }, { "start": 45, "end": 55, "label": "RELIGION", "text": "sunnitisch", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-004", "scenario": "affiliation", "predictions": [ { "start": 0, "end": 14, "label": "PERSON", "text": "Ewan Fairbrook", "score": 0.9914917349815369, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 29, "end": 40, "label": "RELIGION", "text": "Shia Muslim", "score": 0.7597324848175049, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 0, "end": 14, "label": "PERSON", "text": "Ewan Fairbrook", "score": 0.9914917349815369, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 29, "end": 40, "label": "RELIGION", "text": "Shia Muslim", "score": 0.7597324848175049, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 0, "end": 14, "label": "PERSON", "text": "Ewan Fairbrook", "uncovered_positions": [] }, { "start": 29, "end": 40, "label": "RELIGION", "text": "Shia Muslim", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-005", "scenario": "affiliation", "predictions": [ { "start": 27, "end": 37, "label": "RELIGION", "text": "Jesidentum", "score": 0.7354034185409546, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 27, "end": 37, "label": "RELIGION", "text": "Jesidentum", "score": 0.7354034185409546, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 27, "end": 37, "label": "RELIGION", "text": "Jesidentum", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-006", "scenario": "affiliation", "predictions": [ { "start": 24, "end": 30, "label": "ORGANIZATION", "text": "Quaker", "score": 0.9522815346717834, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 24, "end": 30, "label": "ORGANIZATION", "text": "Quaker", "score": 0.9522815346717834, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [ [ 24, 30, "ORGANIZATION" ] ], "false_negatives": [ [ 24, 30, "RELIGION" ] ], "exact_match": false, "error_code": null, "canaries": [ { "start": 24, "end": 30, "label": "RELIGION", "text": "Quaker", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-007", "scenario": "affiliation", "predictions": [ { "start": 26, "end": 35, "label": "RELIGION", "text": "Sikhismus", "score": 0.9770961403846741, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 26, "end": 35, "label": "RELIGION", "text": "Sikhismus", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 26, "end": 35, "label": "RELIGION", "text": "Sikhismus", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-008", "scenario": "affiliation", "predictions": [ { "start": 27, "end": 35, "label": "RELIGION", "text": "Hinduism", "score": 0.9994035959243774, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 27, "end": 35, "label": "RELIGION", "text": "Hinduism", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 27, "end": 35, "label": "RELIGION", "text": "Hinduism", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-009", "scenario": "affiliation", "predictions": [ { "start": 44, "end": 52, "label": "RELIGION", "text": "Judentum", "score": 0.918600857257843, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 44, "end": 52, "label": "RELIGION", "text": "Judentum", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 44, "end": 52, "label": "RELIGION", "text": "Judentum", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-010", "scenario": "affiliation", "predictions": [ { "start": 38, "end": 46, "label": "RELIGION", "text": "Buddhism", "score": 0.9944268465042114, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 38, "end": 46, "label": "RELIGION", "text": "Buddhism", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 38, "end": 46, "label": "RELIGION", "text": "Buddhism", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-011", "scenario": "affiliation", "predictions": [ { "start": 46, "end": 51, "label": "RELIGION", "text": "Bahai", "score": 0.8567683100700378, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 46, "end": 51, "label": "RELIGION", "text": "Bahai", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 46, "end": 51, "label": "RELIGION", "text": "Bahai", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-012", "scenario": "affiliation", "predictions": [ { "start": 46, "end": 53, "label": "RELIGION", "text": "Jainism", "score": 0.9846257567405701, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 46, "end": 53, "label": "RELIGION", "text": "Jainism", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 46, "end": 53, "label": "RELIGION", "text": "Jainism", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-013", "scenario": "affiliation", "predictions": [ { "start": 31, "end": 53, "label": "RELIGION", "text": "evangelisch-lutherisch", "score": 0.7786831259727478, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 31, "end": 53, "label": "RELIGION", "text": "evangelisch-lutherisch", "score": 0.7786831259727478, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 31, "end": 53, "label": "RELIGION", "text": "evangelisch-lutherisch", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-014", "scenario": "affiliation", "predictions": [ { "start": 51, "end": 72, "label": "RELIGION", "text": "Seventh-day Adventist", "score": 0.7101534605026245, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 51, "end": 72, "label": "RELIGION", "text": "Seventh-day Adventist", "score": 0.7101534605026245, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 51, "end": 72, "label": "RELIGION", "text": "Seventh-day Adventist", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-015", "scenario": "affiliation", "predictions": [ { "start": 0, "end": 14, "label": "PERSON", "text": "Raban Moosfeld", "score": 0.9412690997123718, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 32, "end": 43, "label": "RELIGION", "text": "Christentum", "score": 0.9764376282691956, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 0, "end": 14, "label": "PERSON", "text": "Raban Moosfeld", "score": 0.9412690997123718, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 32, "end": 43, "label": "RELIGION", "text": "Christentum", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 0, "end": 14, "label": "PERSON", "text": "Raban Moosfeld", "uncovered_positions": [] }, { "start": 32, "end": 43, "label": "RELIGION", "text": "Christentum", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-016", "scenario": "affiliation", "predictions": [ { "start": 0, "end": 14, "label": "PERSON", "text": "Iona Crestwell", "score": 0.9930251836776733, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 41, "end": 55, "label": "RELIGION", "text": "Zoroastrianism", "score": 0.9812561869621277, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 0, "end": 14, "label": "PERSON", "text": "Iona Crestwell", "score": 0.9930251836776733, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 41, "end": 55, "label": "RELIGION", "text": "Zoroastrianism", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 0, "end": 14, "label": "PERSON", "text": "Iona Crestwell", "uncovered_positions": [] }, { "start": 41, "end": 55, "label": "RELIGION", "text": "Zoroastrianism", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-017", "scenario": "nonaffiliation", "predictions": [], "combined_predictions": [ { "start": 38, "end": 53, "label": "RELIGION", "text": "konfessionsfrei", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 38, "end": 53, "label": "RELIGION", "text": "konfessionsfrei", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-018", "scenario": "nonaffiliation", "predictions": [], "combined_predictions": [ { "start": 18, "end": 42, "label": "RELIGION", "text": "no religious affiliation", "score": null, "source": "deterministic", "model_revision": "religious-status-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 18, "end": 42, "label": "RELIGION", "text": "no religious affiliation", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-019", "scenario": "nonaffiliation", "predictions": [], "combined_predictions": [ { "start": 27, "end": 39, "label": "RELIGION", "text": "religionslos", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 27, "end": 39, "label": "RELIGION", "text": "religionslos", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-020", "scenario": "nonaffiliation", "predictions": [ { "start": 43, "end": 50, "label": "RELIGION", "text": "atheist", "score": 0.6892726421356201, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 43, "end": 50, "label": "RELIGION", "text": "atheist", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 43, "end": 50, "label": "RELIGION", "text": "atheist", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-021", "scenario": "nonaffiliation", "predictions": [], "combined_predictions": [ { "start": 47, "end": 57, "label": "RELIGION", "text": "agnostisch", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 47, "end": 57, "label": "RELIGION", "text": "agnostisch", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-022", "scenario": "nonaffiliation", "predictions": [], "combined_predictions": [ { "start": 36, "end": 44, "label": "RELIGION", "text": "agnostic", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 36, "end": 44, "label": "RELIGION", "text": "agnostic", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-023", "scenario": "nonaffiliation", "predictions": [ { "start": 34, "end": 44, "label": "RELIGION", "text": "Konfession", "score": 0.8056620955467224, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 29, "end": 44, "label": "RELIGION", "text": "ohne Konfession", "score": null, "source": "deterministic", "model_revision": "religious-status-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 29, "end": 44, "label": "RELIGION", "text": "ohne Konfession", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-024", "scenario": "nonaffiliation", "predictions": [ { "start": 24, "end": 36, "label": "RELIGION", "text": "nonreligious", "score": 0.6173449158668518, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 24, "end": 36, "label": "RELIGION", "text": "nonreligious", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 24, "end": 36, "label": "RELIGION", "text": "nonreligious", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-025", "scenario": "nonaffiliation", "predictions": [ { "start": 27, "end": 35, "label": "RELIGION", "text": "Religion", "score": 0.6249727010726929, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 21, "end": 35, "label": "RELIGION", "text": "keine Religion", "score": null, "source": "deterministic", "model_revision": "religious-status-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 21, "end": 35, "label": "RELIGION", "text": "keine Religion", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-026", "scenario": "nonaffiliation", "predictions": [ { "start": 0, "end": 11, "label": "ORGANIZATION", "text": "Affiliation", "score": 0.6462724208831787, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 0, "end": 11, "label": "ORGANIZATION", "text": "Affiliation", "score": 0.6462724208831787, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 27, "end": 51, "label": "RELIGION", "text": "religiously unaffiliated", "score": null, "source": "deterministic", "model_revision": "religious-status-v1", "normalization": "identity" } ], "false_positives": [ [ 0, 11, "ORGANIZATION" ] ], "false_negatives": [], "exact_match": false, "error_code": null, "canaries": [ { "start": 27, "end": 51, "label": "RELIGION", "text": "religiously unaffiliated", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-027", "scenario": "nonaffiliation", "predictions": [], "combined_predictions": [ { "start": 32, "end": 46, "label": "RELIGION", "text": "konfessionslos", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 32, "end": 46, "label": "RELIGION", "text": "konfessionslos", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-028", "scenario": "nonaffiliation", "predictions": [ { "start": 17, "end": 24, "label": "RELIGION", "text": "atheism", "score": 0.8736236095428467, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 17, "end": 24, "label": "RELIGION", "text": "atheism", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 17, "end": 24, "label": "RELIGION", "text": "atheism", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-029", "scenario": "nonaffiliation", "predictions": [ { "start": 21, "end": 30, "label": "RELIGION", "text": "Atheismus", "score": 0.7562140226364136, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 21, "end": 30, "label": "RELIGION", "text": "Atheismus", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 21, "end": 30, "label": "RELIGION", "text": "Atheismus", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-030", "scenario": "nonaffiliation", "predictions": [], "combined_predictions": [ { "start": 29, "end": 40, "label": "RELIGION", "text": "no religion", "score": null, "source": "deterministic", "model_revision": "religious-status-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 29, "end": 40, "label": "RELIGION", "text": "no religion", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-031", "scenario": "nonaffiliation", "predictions": [], "combined_predictions": [ { "start": 24, "end": 37, "label": "RELIGION", "text": "Agnostizismus", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 24, "end": 37, "label": "RELIGION", "text": "Agnostizismus", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-032", "scenario": "nonaffiliation", "predictions": [], "combined_predictions": [ { "start": 29, "end": 42, "label": "RELIGION", "text": "non-religious", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 29, "end": 42, "label": "RELIGION", "text": "non-religious", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-033", "scenario": "belief_names", "predictions": [ { "start": 4, "end": 11, "label": "ORGANIZATION", "text": "Glossar", "score": 0.5172879099845886, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 22, "end": 30, "label": "RELIGION", "text": "Daoismus", "score": 0.6094017624855042, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 4, "end": 11, "label": "ORGANIZATION", "text": "Glossar", "score": 0.5172879099845886, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 22, "end": 30, "label": "RELIGION", "text": "Daoismus", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [ [ 4, 11, "ORGANIZATION" ] ], "false_negatives": [], "exact_match": false, "error_code": null, "canaries": [ { "start": 22, "end": 30, "label": "RELIGION", "text": "Daoismus", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-034", "scenario": "belief_names", "predictions": [ { "start": 32, "end": 41, "label": "RELIGION", "text": "Shintoism", "score": 0.9690232872962952, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 32, "end": 41, "label": "RELIGION", "text": "Shintoism", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 32, "end": 41, "label": "RELIGION", "text": "Shintoism", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-035", "scenario": "belief_names", "predictions": [ { "start": 40, "end": 45, "label": "RELIGION", "text": "Islam", "score": 0.9760571718215942, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 40, "end": 45, "label": "RELIGION", "text": "Islam", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 40, "end": 45, "label": "RELIGION", "text": "Islam", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-036", "scenario": "belief_names", "predictions": [ { "start": 27, "end": 34, "label": "RELIGION", "text": "Judaism", "score": 0.9818722605705261, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 27, "end": 34, "label": "RELIGION", "text": "Judaism", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 27, "end": 34, "label": "RELIGION", "text": "Judaism", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-037", "scenario": "belief_names", "predictions": [ { "start": 29, "end": 39, "label": "RELIGION", "text": "Hinduismus", "score": 0.9524589776992798, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 44, "end": 54, "label": "RELIGION", "text": "Buddhismus", "score": 0.9651915431022644, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 29, "end": 39, "label": "RELIGION", "text": "Hinduismus", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" }, { "start": 44, "end": 54, "label": "RELIGION", "text": "Buddhismus", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 29, "end": 39, "label": "RELIGION", "text": "Hinduismus", "uncovered_positions": [] }, { "start": 44, "end": 54, "label": "RELIGION", "text": "Buddhismus", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-038", "scenario": "belief_names", "predictions": [ { "start": 22, "end": 34, "label": "RELIGION", "text": "Christianity", "score": 0.9961896538734436, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 40, "end": 46, "label": "RELIGION", "text": "Taoism", "score": 0.9618191719055176, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 22, "end": 34, "label": "RELIGION", "text": "Christianity", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" }, { "start": 40, "end": 46, "label": "RELIGION", "text": "Taoism", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 22, "end": 34, "label": "RELIGION", "text": "Christianity", "uncovered_positions": [] }, { "start": 40, "end": 46, "label": "RELIGION", "text": "Taoism", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-039", "scenario": "belief_names", "predictions": [ { "start": 0, "end": 10, "label": "ORGANIZATION", "text": "Das Museum", "score": 0.8367996215820312, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 43, "end": 52, "label": "RELIGION", "text": "Jainismus", "score": 0.907336413860321, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 0, "end": 10, "label": "ORGANIZATION", "text": "Das Museum", "score": 0.8367996215820312, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 43, "end": 52, "label": "RELIGION", "text": "Jainismus", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [ [ 0, 10, "ORGANIZATION" ] ], "false_negatives": [], "exact_match": false, "error_code": null, "canaries": [ { "start": 43, "end": 52, "label": "RELIGION", "text": "Jainismus", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-040", "scenario": "belief_names", "predictions": [ { "start": 44, "end": 51, "label": "RELIGION", "text": "Sikhism", "score": 0.984706461429596, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 44, "end": 51, "label": "RELIGION", "text": "Sikhism", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 44, "end": 51, "label": "RELIGION", "text": "Sikhism", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-041", "scenario": "belief_names", "predictions": [ { "start": 22, "end": 35, "label": "RELIGION", "text": "Zoroastrismus", "score": 0.8952449560165405, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 42, "end": 50, "label": "PERSON", "text": "Personen", "score": 0.7890533804893494, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 22, "end": 35, "label": "RELIGION", "text": "Zoroastrismus", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" }, { "start": 42, "end": 50, "label": "PERSON", "text": "Personen", "score": 0.7890533804893494, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [ [ 42, 50, "PERSON" ] ], "false_negatives": [], "exact_match": false, "error_code": null, "canaries": [ { "start": 22, "end": 35, "label": "RELIGION", "text": "Zoroastrismus", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-042", "scenario": "belief_names", "predictions": [ { "start": 24, "end": 30, "label": "CITY", "text": "Baháʼí", "score": 0.7278924584388733, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 31, "end": 36, "label": "RELIGION", "text": "Faith", "score": 0.5462192296981812, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 24, "end": 36, "label": "RELIGION", "text": "Baháʼí Faith", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 24, "end": 36, "label": "RELIGION", "text": "Baháʼí Faith", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-043", "scenario": "belief_names", "predictions": [ { "start": 4, "end": 13, "label": "ORGANIZATION", "text": "Übersicht", "score": 0.5883280038833618, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 24, "end": 37, "label": "RELIGION", "text": "Katholizismus", "score": 0.6876825094223022, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 4, "end": 13, "label": "ORGANIZATION", "text": "Übersicht", "score": 0.5883280038833618, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 24, "end": 37, "label": "RELIGION", "text": "Katholizismus", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [ [ 4, 13, "ORGANIZATION" ] ], "false_negatives": [], "exact_match": false, "error_code": null, "canaries": [ { "start": 24, "end": 37, "label": "RELIGION", "text": "Katholizismus", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-044", "scenario": "belief_names", "predictions": [ { "start": 41, "end": 54, "label": "RELIGION", "text": "Protestantism", "score": 0.9522858262062073, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 41, "end": 54, "label": "RELIGION", "text": "Protestantism", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 41, "end": 54, "label": "RELIGION", "text": "Protestantism", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-045", "scenario": "belief_names", "predictions": [ { "start": 24, "end": 34, "label": "RELIGION", "text": "Orthodoxie", "score": 0.8815929293632507, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 24, "end": 34, "label": "RELIGION", "text": "Orthodoxie", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 24, "end": 34, "label": "RELIGION", "text": "Orthodoxie", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-046", "scenario": "belief_names", "predictions": [ { "start": 30, "end": 51, "label": "RELIGION", "text": "Orthodox Christianity", "score": 0.9181906580924988, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 30, "end": 51, "label": "RELIGION", "text": "Orthodox Christianity", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 30, "end": 51, "label": "RELIGION", "text": "Orthodox Christianity", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-047", "scenario": "belief_names", "predictions": [ { "start": 27, "end": 36, "label": "RELIGION", "text": "Sikhismus", "score": 0.9480417370796204, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 41, "end": 50, "label": "RELIGION", "text": "Jainismus", "score": 0.9625227451324463, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 27, "end": 36, "label": "RELIGION", "text": "Sikhismus", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" }, { "start": 41, "end": 50, "label": "RELIGION", "text": "Jainismus", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 27, "end": 36, "label": "RELIGION", "text": "Sikhismus", "uncovered_positions": [] }, { "start": 41, "end": 50, "label": "RELIGION", "text": "Jainismus", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-048", "scenario": "belief_names", "predictions": [ { "start": 4, "end": 16, "label": "ORGANIZATION", "text": "encyclopedia", "score": 0.740946352481842, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 42, "end": 54, "label": "RELIGION", "text": "Confucianism", "score": 0.9735894203186035, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 4, "end": 16, "label": "ORGANIZATION", "text": "encyclopedia", "score": 0.740946352481842, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 42, "end": 54, "label": "RELIGION", "text": "Confucianism", "score": 0.9735894203186035, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [ [ 4, 16, "ORGANIZATION" ] ], "false_negatives": [], "exact_match": false, "error_code": null, "canaries": [ { "start": 42, "end": 54, "label": "RELIGION", "text": "Confucianism", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-049", "scenario": "general_topics", "predictions": [ { "start": 38, "end": 43, "label": "RELIGION", "text": "Islam", "score": 0.9990894794464111, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 38, "end": 43, "label": "RELIGION", "text": "Islam", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 38, "end": 43, "label": "RELIGION", "text": "Islam", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-050", "scenario": "general_topics", "predictions": [ { "start": 33, "end": 41, "label": "RELIGION", "text": "Hinduism", "score": 0.9996596574783325, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 33, "end": 41, "label": "RELIGION", "text": "Hinduism", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 33, "end": 41, "label": "RELIGION", "text": "Hinduism", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-051", "scenario": "general_topics", "predictions": [ { "start": 23, "end": 34, "label": "RELIGION", "text": "Christentum", "score": 0.9296461343765259, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 45, "end": 56, "label": "CITY", "text": "Mittelalter", "score": 0.600753903388977, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 23, "end": 34, "label": "RELIGION", "text": "Christentum", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" }, { "start": 45, "end": 56, "label": "CITY", "text": "Mittelalter", "score": 0.600753903388977, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [ [ 45, 56, "CITY" ] ], "false_negatives": [], "exact_match": false, "error_code": null, "canaries": [ { "start": 23, "end": 34, "label": "RELIGION", "text": "Christentum", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-052", "scenario": "general_topics", "predictions": [ { "start": 28, "end": 36, "label": "RELIGION", "text": "Buddhism", "score": 0.9992027878761292, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 28, "end": 36, "label": "RELIGION", "text": "Buddhism", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 28, "end": 36, "label": "RELIGION", "text": "Buddhism", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-053", "scenario": "general_topics", "predictions": [ { "start": 37, "end": 45, "label": "RELIGION", "text": "Judentum", "score": 0.9257993698120117, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 37, "end": 45, "label": "RELIGION", "text": "Judentum", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 37, "end": 45, "label": "RELIGION", "text": "Judentum", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-054", "scenario": "general_topics", "predictions": [ { "start": 20, "end": 27, "label": "RELIGION", "text": "Jainism", "score": 0.9963538646697998, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 20, "end": 27, "label": "RELIGION", "text": "Jainism", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 20, "end": 27, "label": "RELIGION", "text": "Jainism", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-055", "scenario": "general_topics", "predictions": [ { "start": 24, "end": 29, "label": "PERSON", "text": "Bahai", "score": 0.9998281002044678, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [], "false_positives": [], "false_negatives": [ [ 24, 29, "RELIGION" ] ], "exact_match": false, "error_code": "ambiguous_entity_overlap", "canaries": [ { "start": 24, "end": 29, "label": "RELIGION", "text": "Bahai", "uncovered_positions": [ 24, 25, 26, 27, 28 ] } ], "candidate_returned": false, "candidate_canaries": [], "outcome": "blocked_conflicting_spans" }, { "case_id": "holdout03d-056", "scenario": "general_topics", "predictions": [ { "start": 25, "end": 32, "label": "RELIGION", "text": "Sikhism", "score": 0.9963860511779785, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 25, "end": 32, "label": "RELIGION", "text": "Sikhism", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 25, "end": 32, "label": "RELIGION", "text": "Sikhism", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-057", "scenario": "general_topics", "predictions": [ { "start": 20, "end": 29, "label": "RELIGION", "text": "Atheismus", "score": 0.5075995326042175, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 20, "end": 29, "label": "RELIGION", "text": "Atheismus", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 20, "end": 29, "label": "RELIGION", "text": "Atheismus", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-058", "scenario": "general_topics", "predictions": [ { "start": 19, "end": 26, "label": "RELIGION", "text": "atheism", "score": 0.9609501957893372, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 31, "end": 42, "label": "RELIGION", "text": "agnosticism", "score": 0.9465921521186829, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 19, "end": 26, "label": "RELIGION", "text": "atheism", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" }, { "start": 31, "end": 42, "label": "RELIGION", "text": "agnosticism", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 19, "end": 26, "label": "RELIGION", "text": "atheism", "uncovered_positions": [] }, { "start": 31, "end": 42, "label": "RELIGION", "text": "agnosticism", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-059", "scenario": "general_topics", "predictions": [ { "start": 30, "end": 41, "label": "RELIGION", "text": "Shintoismus", "score": 0.9589027762413025, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 30, "end": 41, "label": "RELIGION", "text": "Shintoismus", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 30, "end": 41, "label": "RELIGION", "text": "Shintoismus", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-060", "scenario": "general_topics", "predictions": [ { "start": 23, "end": 29, "label": "RELIGION", "text": "Taoism", "score": 0.9697747826576233, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 23, "end": 29, "label": "RELIGION", "text": "Taoism", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 23, "end": 29, "label": "RELIGION", "text": "Taoism", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-061", "scenario": "general_topics", "predictions": [ { "start": 30, "end": 45, "label": "RELIGION", "text": "Protestantismus", "score": 0.6991519927978516, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 30, "end": 45, "label": "RELIGION", "text": "Protestantismus", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 30, "end": 45, "label": "RELIGION", "text": "Protestantismus", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-062", "scenario": "general_topics", "predictions": [ { "start": 25, "end": 36, "label": "RELIGION", "text": "Catholicism", "score": 0.9966902732849121, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 25, "end": 36, "label": "RELIGION", "text": "Catholicism", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 25, "end": 36, "label": "RELIGION", "text": "Catholicism", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-063", "scenario": "general_topics", "predictions": [ { "start": 30, "end": 44, "label": "PERSON", "text": "konfessionslos", "score": 0.5395556092262268, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [], "false_positives": [], "false_negatives": [ [ 30, 44, "RELIGION" ] ], "exact_match": false, "error_code": "ambiguous_entity_overlap", "canaries": [ { "start": 30, "end": 44, "label": "RELIGION", "text": "konfessionslos", "uncovered_positions": [ 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43 ] } ], "candidate_returned": false, "candidate_canaries": [], "outcome": "blocked_conflicting_spans" }, { "case_id": "holdout03d-064", "scenario": "general_topics", "predictions": [ { "start": 29, "end": 42, "label": "RELIGION", "text": "non-religious", "score": 0.5220276713371277, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 29, "end": 42, "label": "RELIGION", "text": "non-religious", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 29, "end": 42, "label": "RELIGION", "text": "non-religious", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-065", "scenario": "negation_quotation", "predictions": [ { "start": 21, "end": 24, "label": "PERSON", "text": "Ich", "score": 0.7314160466194153, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 35, "end": 45, "label": "RELIGION", "text": "muslimisch", "score": 0.8888826966285706, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 21, "end": 24, "label": "PERSON", "text": "Ich", "score": 0.7314160466194153, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 35, "end": 45, "label": "RELIGION", "text": "muslimisch", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [ [ 21, 24, "PERSON" ] ], "false_negatives": [], "exact_match": false, "error_code": null, "canaries": [ { "start": 35, "end": 45, "label": "RELIGION", "text": "muslimisch", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-066", "scenario": "negation_quotation", "predictions": [ { "start": 28, "end": 37, "label": "RELIGION", "text": "Christian", "score": 0.931776762008667, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 28, "end": 37, "label": "RELIGION", "text": "Christian", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 28, "end": 37, "label": "RELIGION", "text": "Christian", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-067", "scenario": "negation_quotation", "predictions": [ { "start": 29, "end": 39, "label": "RELIGION", "text": "katholisch", "score": 0.8747991323471069, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 29, "end": 39, "label": "RELIGION", "text": "katholisch", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 29, "end": 39, "label": "RELIGION", "text": "katholisch", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-068", "scenario": "negation_quotation", "predictions": [], "combined_predictions": [ { "start": 35, "end": 42, "label": "RELIGION", "text": "atheist", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 35, "end": 42, "label": "RELIGION", "text": "atheist", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-069", "scenario": "negation_quotation", "predictions": [ { "start": 10, "end": 22, "label": "RELIGION", "text": "buddhistisch", "score": 0.78475022315979, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 10, "end": 22, "label": "RELIGION", "text": "buddhistisch", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 10, "end": 22, "label": "RELIGION", "text": "buddhistisch", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-070", "scenario": "negation_quotation", "predictions": [ { "start": 1, "end": 7, "label": "RELIGION", "text": "Jewish", "score": 0.922519862651825, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 1, "end": 7, "label": "RELIGION", "text": "Jewish", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 1, "end": 7, "label": "RELIGION", "text": "Jewish", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-071", "scenario": "negation_quotation", "predictions": [ { "start": 40, "end": 52, "label": "RELIGION", "text": "hinduistisch", "score": 0.7200407981872559, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 40, "end": 52, "label": "RELIGION", "text": "hinduistisch", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 40, "end": 52, "label": "RELIGION", "text": "hinduistisch", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-072", "scenario": "negation_quotation", "predictions": [ { "start": 37, "end": 43, "label": "RELIGION", "text": "Muslim", "score": 0.995964527130127, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 37, "end": 43, "label": "RELIGION", "text": "Muslim", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 37, "end": 43, "label": "RELIGION", "text": "Muslim", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-073", "scenario": "negation_quotation", "predictions": [], "combined_predictions": [ { "start": 1, "end": 16, "label": "RELIGION", "text": "Ohne Konfession", "score": null, "source": "deterministic", "model_revision": "religious-status-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 1, "end": 16, "label": "RELIGION", "text": "Ohne Konfession", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-074", "scenario": "negation_quotation", "predictions": [ { "start": 30, "end": 38, "label": "RELIGION", "text": "religion", "score": 0.5185865163803101, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 27, "end": 38, "label": "RELIGION", "text": "no religion", "score": null, "source": "deterministic", "model_revision": "religious-status-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 27, "end": 38, "label": "RELIGION", "text": "no religion", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-075", "scenario": "negation_quotation", "predictions": [], "combined_predictions": [ { "start": 6, "end": 17, "label": "RELIGION", "text": "evangelisch", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" }, { "start": 27, "end": 37, "label": "RELIGION", "text": "agnostisch", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 6, "end": 17, "label": "RELIGION", "text": "evangelisch", "uncovered_positions": [] }, { "start": 27, "end": 37, "label": "RELIGION", "text": "agnostisch", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-076", "scenario": "negation_quotation", "predictions": [ { "start": 18, "end": 26, "label": "RELIGION", "text": "Catholic", "score": 0.9851877689361572, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 30, "end": 42, "label": "RELIGION", "text": "nonreligious", "score": 0.5536500811576843, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 18, "end": 26, "label": "RELIGION", "text": "Catholic", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" }, { "start": 30, "end": 42, "label": "RELIGION", "text": "nonreligious", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 18, "end": 26, "label": "RELIGION", "text": "Catholic", "uncovered_positions": [] }, { "start": 30, "end": 42, "label": "RELIGION", "text": "nonreligious", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-077", "scenario": "negation_quotation", "predictions": [ { "start": 22, "end": 30, "label": "RELIGION", "text": "Judentum", "score": 0.9432740807533264, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 22, "end": 30, "label": "RELIGION", "text": "Judentum", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 22, "end": 30, "label": "RELIGION", "text": "Judentum", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-078", "scenario": "negation_quotation", "predictions": [ { "start": 23, "end": 29, "label": "PERSON", "text": "Baha'i", "score": 0.8763203620910645, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [], "false_positives": [], "false_negatives": [ [ 23, 29, "RELIGION" ] ], "exact_match": false, "error_code": "ambiguous_entity_overlap", "canaries": [ { "start": 23, "end": 29, "label": "RELIGION", "text": "Baha'i", "uncovered_positions": [ 23, 24, 25, 26, 28 ] } ], "candidate_returned": false, "candidate_canaries": [], "outcome": "blocked_conflicting_spans" }, { "case_id": "holdout03d-079", "scenario": "negation_quotation", "predictions": [ { "start": 21, "end": 32, "label": "RELIGION", "text": "islamischen", "score": 0.6762732863426208, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 21, "end": 32, "label": "RELIGION", "text": "islamischen", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 21, "end": 32, "label": "RELIGION", "text": "islamischen", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-080", "scenario": "negation_quotation", "predictions": [ { "start": 15, "end": 23, "label": "RELIGION", "text": "Buddhist", "score": 0.964024007320404, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 15, "end": 23, "label": "RELIGION", "text": "Buddhist", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 15, "end": 23, "label": "RELIGION", "text": "Buddhist", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-081", "scenario": "ambiguity", "predictions": [], "combined_predictions": [], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-082", "scenario": "ambiguity", "predictions": [ { "start": 4, "end": 12, "label": "RELIGION", "text": "catholic", "score": 0.6429972648620605, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 4, "end": 12, "label": "RELIGION", "text": "catholic", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [ [ 4, 12, "RELIGION" ] ], "false_negatives": [], "exact_match": false, "error_code": null, "canaries": [], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-083", "scenario": "ambiguity", "predictions": [], "combined_predictions": [], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-084", "scenario": "ambiguity", "predictions": [ { "start": 3, "end": 14, "label": "RELIGION", "text": "evangelical", "score": 0.8391301035881042, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 3, "end": 14, "label": "RELIGION", "text": "evangelical", "score": 0.8391301035881042, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [ [ 3, 14, "RELIGION" ] ], "false_negatives": [], "exact_match": false, "error_code": null, "canaries": [], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-085", "scenario": "ambiguity", "predictions": [ { "start": 18, "end": 26, "label": "RELIGION", "text": "orthodox", "score": 0.8430137634277344, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 18, "end": 26, "label": "RELIGION", "text": "orthodox", "score": 0.8430137634277344, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 18, "end": 26, "label": "RELIGION", "text": "orthodox", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-086", "scenario": "ambiguity", "predictions": [ { "start": 52, "end": 60, "label": "RELIGION", "text": "Orthodox", "score": 0.8877473473548889, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 52, "end": 60, "label": "RELIGION", "text": "Orthodox", "score": 0.8877473473548889, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 52, "end": 60, "label": "RELIGION", "text": "Orthodox", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-087", "scenario": "ambiguity", "predictions": [ { "start": 12, "end": 23, "label": "RELIGION", "text": "evangelisch", "score": 0.7735995650291443, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 25, "end": 36, "label": "ORGANIZATION", "text": "Algorithmus", "score": 0.5926805734634399, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 12, "end": 23, "label": "RELIGION", "text": "evangelisch", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" }, { "start": 25, "end": 36, "label": "ORGANIZATION", "text": "Algorithmus", "score": 0.5926805734634399, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [ [ 25, 36, "ORGANIZATION" ] ], "false_negatives": [], "exact_match": false, "error_code": null, "canaries": [ { "start": 12, "end": 23, "label": "RELIGION", "text": "evangelisch", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-088", "scenario": "ambiguity", "predictions": [ { "start": 26, "end": 37, "label": "RELIGION", "text": "Evangelical", "score": 0.9244890809059143, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 26, "end": 37, "label": "RELIGION", "text": "Evangelical", "score": 0.9244890809059143, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 26, "end": 37, "label": "RELIGION", "text": "Evangelical", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-089", "scenario": "ambiguity", "predictions": [], "combined_predictions": [], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-090", "scenario": "ambiguity", "predictions": [], "combined_predictions": [], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-091", "scenario": "ambiguity", "predictions": [ { "start": 17, "end": 27, "label": "RELIGION", "text": "christlich", "score": 0.9146808981895447, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 48, "end": 60, "label": "PERSON", "text": "Personenname", "score": 0.6654636263847351, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 17, "end": 27, "label": "RELIGION", "text": "christlich", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" }, { "start": 48, "end": 60, "label": "PERSON", "text": "Personenname", "score": 0.6654636263847351, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [ [ 48, 60, "PERSON" ] ], "false_negatives": [], "exact_match": false, "error_code": null, "canaries": [ { "start": 17, "end": 27, "label": "RELIGION", "text": "christlich", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-092", "scenario": "ambiguity", "predictions": [ { "start": 22, "end": 31, "label": "RELIGION", "text": "Christian", "score": 0.9971898198127747, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 22, "end": 31, "label": "RELIGION", "text": "Christian", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 22, "end": 31, "label": "RELIGION", "text": "Christian", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-093", "scenario": "ambiguity", "predictions": [], "combined_predictions": [], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-094", "scenario": "ambiguity", "predictions": [], "combined_predictions": [], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-095", "scenario": "ambiguity", "predictions": [ { "start": 26, "end": 33, "label": "RELIGION", "text": "jüdisch", "score": 0.9290997982025146, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 26, "end": 33, "label": "RELIGION", "text": "jüdisch", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 26, "end": 33, "label": "RELIGION", "text": "jüdisch", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-096", "scenario": "ambiguity", "predictions": [ { "start": 44, "end": 52, "label": "RELIGION", "text": "Reformed", "score": 0.8552035689353943, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 44, "end": 52, "label": "RELIGION", "text": "Reformed", "score": 0.8552035689353943, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 44, "end": 52, "label": "RELIGION", "text": "Reformed", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-097", "scenario": "ocr_noise", "predictions": [ { "start": 9, "end": 13, "label": "CITY", "text": "laut", "score": 0.6887308955192566, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 20, "end": 30, "label": "RELIGION", "text": "Buddh1smus", "score": 0.8085448145866394, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 9, "end": 13, "label": "CITY", "text": "laut", "score": 0.6887308955192566, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 20, "end": 30, "label": "RELIGION", "text": "Buddh1smus", "score": 0.8085448145866394, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [ [ 9, 13, "CITY" ] ], "false_negatives": [], "exact_match": false, "error_code": null, "canaries": [ { "start": 20, "end": 30, "label": "RELIGION", "text": "Buddh1smus", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-098", "scenario": "ocr_noise", "predictions": [ { "start": 31, "end": 43, "label": "RELIGION", "text": "Christlanity", "score": 0.5787172317504883, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 31, "end": 43, "label": "RELIGION", "text": "Christlanity", "score": 0.5787172317504883, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 31, "end": 43, "label": "RELIGION", "text": "Christlanity", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-099", "scenario": "ocr_noise", "predictions": [], "combined_predictions": [ { "start": 24, "end": 40, "label": "RELIGION", "text": "konfessions-\nlos", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "dehyphenation-v1" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 24, "end": 40, "label": "RELIGION", "text": "konfessions-\nlos", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-100", "scenario": "ocr_noise", "predictions": [], "combined_predictions": [ { "start": 21, "end": 35, "label": "RELIGION", "text": "non-\nreligious", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "dehyphenation-v1" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 21, "end": 35, "label": "RELIGION", "text": "non-\nreligious", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-101", "scenario": "ocr_noise", "predictions": [ { "start": 11, "end": 21, "label": "RELIGION", "text": "katho1isch", "score": 0.7328035831451416, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 11, "end": 21, "label": "RELIGION", "text": "katho1isch", "score": 0.7328035831451416, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 11, "end": 21, "label": "RELIGION", "text": "katho1isch", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-102", "scenario": "ocr_noise", "predictions": [ { "start": 24, "end": 33, "label": "RELIGION", "text": "Hinduisrn", "score": 0.7377975583076477, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 24, "end": 33, "label": "RELIGION", "text": "Hinduisrn", "score": 0.7377975583076477, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 24, "end": 33, "label": "RELIGION", "text": "Hinduisrn", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-103", "scenario": "ocr_noise", "predictions": [], "combined_predictions": [ { "start": 18, "end": 30, "label": "RELIGION", "text": "agnos-\ntisch", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "dehyphenation-v1" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 18, "end": 30, "label": "RELIGION", "text": "agnos-\ntisch", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-104", "scenario": "ocr_noise", "predictions": [ { "start": 25, "end": 34, "label": "RELIGION", "text": "Juda-\nism", "score": 0.984998881816864, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "dehyphenation-v1" } ], "combined_predictions": [ { "start": 25, "end": 34, "label": "RELIGION", "text": "Juda-\nism", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "dehyphenation-v1" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 25, "end": 34, "label": "RELIGION", "text": "Juda-\nism", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-105", "scenario": "ocr_noise", "predictions": [ { "start": 19, "end": 24, "label": "PERSON", "text": "lslam", "score": 0.8412295579910278, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 19, "end": 24, "label": "PERSON", "text": "lslam", "score": 0.8412295579910278, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [ [ 19, 24, "PERSON" ] ], "false_negatives": [ [ 19, 24, "RELIGION" ] ], "exact_match": false, "error_code": null, "canaries": [ { "start": 19, "end": 24, "label": "RELIGION", "text": "lslam", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-106", "scenario": "ocr_noise", "predictions": [ { "start": 4, "end": 7, "label": "ORGANIZATION", "text": "OCR", "score": 0.993300199508667, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 18, "end": 25, "label": "RELIGION", "text": "S1khism", "score": 0.9311596751213074, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 4, "end": 7, "label": "ORGANIZATION", "text": "OCR", "score": 0.993300199508667, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 18, "end": 25, "label": "RELIGION", "text": "S1khism", "score": 0.9311596751213074, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [ [ 4, 7, "ORGANIZATION" ] ], "false_negatives": [], "exact_match": false, "error_code": null, "canaries": [ { "start": 18, "end": 25, "label": "RELIGION", "text": "S1khism", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-107", "scenario": "ocr_noise", "predictions": [ { "start": 35, "end": 49, "label": "RELIGION", "text": "buddhis-\ntisch", "score": 0.7703148126602173, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "dehyphenation-v1" } ], "combined_predictions": [ { "start": 35, "end": 49, "label": "RELIGION", "text": "buddhis-\ntisch", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "dehyphenation-v1" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 35, "end": 49, "label": "RELIGION", "text": "buddhis-\ntisch", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-108", "scenario": "ocr_noise", "predictions": [ { "start": 27, "end": 43, "label": "RELIGION", "text": "Zoroas-\ntrianism", "score": 0.9521776437759399, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "dehyphenation-v1" } ], "combined_predictions": [ { "start": 27, "end": 43, "label": "RELIGION", "text": "Zoroas-\ntrianism", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "dehyphenation-v1" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 27, "end": 43, "label": "RELIGION", "text": "Zoroas-\ntrianism", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-109", "scenario": "ocr_noise", "predictions": [ { "start": 14, "end": 27, "label": "PERSON", "text": "Fenja Torwald", "score": 0.9993094205856323, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 29, "end": 49, "label": "ORGANIZATION", "text": "Bekenntnis Judenturn", "score": 0.8337629437446594, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 14, "end": 27, "label": "PERSON", "text": "Fenja Torwald", "score": 0.9993094205856323, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 29, "end": 49, "label": "ORGANIZATION", "text": "Bekenntnis Judenturn", "score": 0.8337629437446594, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [ [ 29, 49, "ORGANIZATION" ] ], "false_negatives": [ [ 40, 49, "RELIGION" ] ], "exact_match": false, "error_code": null, "canaries": [ { "start": 14, "end": 27, "label": "PERSON", "text": "Fenja Torwald", "uncovered_positions": [] }, { "start": 40, "end": 49, "label": "RELIGION", "text": "Judenturn", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-110", "scenario": "ocr_noise", "predictions": [ { "start": 21, "end": 33, "label": "PERSON", "text": "Orin Wexford", "score": 0.9491236209869385, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 38, "end": 49, "label": "RELIGION", "text": "Cath0licism", "score": 0.8634129166603088, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 21, "end": 33, "label": "PERSON", "text": "Orin Wexford", "score": 0.9491236209869385, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 38, "end": 49, "label": "RELIGION", "text": "Cath0licism", "score": 0.8634129166603088, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 21, "end": 33, "label": "PERSON", "text": "Orin Wexford", "uncovered_positions": [] }, { "start": 38, "end": 49, "label": "RELIGION", "text": "Cath0licism", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-111", "scenario": "ocr_noise", "predictions": [], "combined_predictions": [ { "start": 13, "end": 28, "label": "RELIGION", "text": "konfesslonsfrei", "score": null, "source": "deterministic", "model_revision": "religious-status-v1", "normalization": "status-ocr-v1" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 13, "end": 28, "label": "RELIGION", "text": "konfesslonsfrei", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-112", "scenario": "ocr_noise", "predictions": [ { "start": 21, "end": 29, "label": "ORGANIZATION", "text": "agn0stic", "score": 0.8037384748458862, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 21, "end": 29, "label": "ORGANIZATION", "text": "agn0stic", "score": 0.8037384748458862, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [ [ 21, 29, "ORGANIZATION" ] ], "false_negatives": [ [ 21, 29, "RELIGION" ] ], "exact_match": false, "error_code": null, "canaries": [ { "start": 21, "end": 29, "label": "RELIGION", "text": "agn0stic", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-113", "scenario": "multiple_entities", "predictions": [ { "start": 0, "end": 14, "label": "PERSON", "text": "Silja Dornbach", "score": 0.9992048144340515, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 19, "end": 26, "label": "CITY", "text": "Rostock", "score": 0.999563992023468, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 34, "end": 42, "label": "RELIGION", "text": "Judentum", "score": 0.9391108751296997, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 48, "end": 66, "label": "ORGANIZATION", "text": "Morgenkai Institut", "score": 0.9742982983589172, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 0, "end": 14, "label": "PERSON", "text": "Silja Dornbach", "score": 0.9992048144340515, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 19, "end": 26, "label": "CITY", "text": "Rostock", "score": 0.999563992023468, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 34, "end": 42, "label": "RELIGION", "text": "Judentum", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" }, { "start": 48, "end": 66, "label": "ORGANIZATION", "text": "Morgenkai Institut", "score": 0.9742982983589172, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 0, "end": 14, "label": "PERSON", "text": "Silja Dornbach", "uncovered_positions": [] }, { "start": 19, "end": 26, "label": "CITY", "text": "Rostock", "uncovered_positions": [] }, { "start": 34, "end": 42, "label": "RELIGION", "text": "Judentum", "uncovered_positions": [] }, { "start": 48, "end": 66, "label": "ORGANIZATION", "text": "Morgenkai Institut", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-114", "scenario": "multiple_entities", "predictions": [ { "start": 0, "end": 15, "label": "PERSON", "text": "Nolan Briarwood", "score": 0.9830585718154907, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 19, "end": 23, "label": "CITY", "text": "York", "score": 0.9994452595710754, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 30, "end": 37, "label": "RELIGION", "text": "Jainism", "score": 0.9943238496780396, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 45, "end": 62, "label": "ORGANIZATION", "text": "Cobalt Quay Trust", "score": 0.9601506590843201, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 0, "end": 15, "label": "PERSON", "text": "Nolan Briarwood", "score": 0.9830585718154907, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 19, "end": 23, "label": "CITY", "text": "York", "score": 0.9994452595710754, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 30, "end": 37, "label": "RELIGION", "text": "Jainism", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" }, { "start": 45, "end": 62, "label": "ORGANIZATION", "text": "Cobalt Quay Trust", "score": 0.9601506590843201, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 0, "end": 15, "label": "PERSON", "text": "Nolan Briarwood", "uncovered_positions": [] }, { "start": 19, "end": 23, "label": "CITY", "text": "York", "uncovered_positions": [] }, { "start": 30, "end": 37, "label": "RELIGION", "text": "Jainism", "uncovered_positions": [] }, { "start": 45, "end": 62, "label": "ORGANIZATION", "text": "Cobalt Quay Trust", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-115", "scenario": "multiple_entities", "predictions": [ { "start": 0, "end": 22, "label": "ORGANIZATION", "text": "Bernsteinpfad Stiftung", "score": 0.9246055483818054, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 32, "end": 47, "label": "PERSON", "text": "Henrik Wiesenau", "score": 0.9986317753791809, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 0, "end": 22, "label": "ORGANIZATION", "text": "Bernsteinpfad Stiftung", "score": 0.9246055483818054, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 32, "end": 47, "label": "PERSON", "text": "Henrik Wiesenau", "score": 0.9986317753791809, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 52, "end": 66, "label": "RELIGION", "text": "konfessionslos", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 0, "end": 22, "label": "ORGANIZATION", "text": "Bernsteinpfad Stiftung", "uncovered_positions": [] }, { "start": 32, "end": 47, "label": "PERSON", "text": "Henrik Wiesenau", "uncovered_positions": [] }, { "start": 52, "end": 66, "label": "RELIGION", "text": "konfessionslos", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-116", "scenario": "multiple_entities", "predictions": [ { "start": 4, "end": 26, "label": "ORGANIZATION", "text": "Silver Estuary Council", "score": 0.8049675822257996, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 34, "end": 47, "label": "PERSON", "text": "Ada Merefield", "score": 0.9995853304862976, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 4, "end": 26, "label": "ORGANIZATION", "text": "Silver Estuary Council", "score": 0.8049675822257996, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 34, "end": 47, "label": "PERSON", "text": "Ada Merefield", "score": 0.9995853304862976, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 50, "end": 57, "label": "RELIGION", "text": "atheism", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 4, "end": 26, "label": "ORGANIZATION", "text": "Silver Estuary Council", "uncovered_positions": [] }, { "start": 34, "end": 47, "label": "PERSON", "text": "Ada Merefield", "uncovered_positions": [] }, { "start": 50, "end": 57, "label": "RELIGION", "text": "atheism", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-117", "scenario": "multiple_entities", "predictions": [ { "start": 0, "end": 12, "label": "PERSON", "text": "Ida Talbrück", "score": 0.9952776432037354, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 19, "end": 24, "label": "RELIGION", "text": "Islam", "score": 0.9999401569366455, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 35, "end": 58, "label": "EMAIL", "text": "ida.tal@example.invalid", "score": 0.8027673959732056, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 0, "end": 12, "label": "PERSON", "text": "Ida Talbrück", "score": 0.9952776432037354, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 19, "end": 24, "label": "RELIGION", "text": "Islam", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" }, { "start": 35, "end": 58, "label": "EMAIL", "text": "ida.tal@example.invalid", "score": null, "source": "deterministic", "model_revision": "contact-patterns-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 0, "end": 12, "label": "PERSON", "text": "Ida Talbrück", "uncovered_positions": [] }, { "start": 19, "end": 24, "label": "RELIGION", "text": "Islam", "uncovered_positions": [] }, { "start": 35, "end": 58, "label": "EMAIL", "text": "ida.tal@example.invalid", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-118", "scenario": "multiple_entities", "predictions": [ { "start": 0, "end": 13, "label": "PERSON", "text": "Lyle Ashcombe", "score": 0.9961810111999512, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 20, "end": 28, "label": "RELIGION", "text": "Buddhism", "score": 0.9979920387268066, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 0, "end": 13, "label": "PERSON", "text": "Lyle Ashcombe", "score": 0.9961810111999512, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 20, "end": 28, "label": "RELIGION", "text": "Buddhism", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" }, { "start": 38, "end": 60, "label": "EMAIL", "text": "lyle.a@example.invalid", "score": null, "source": "deterministic", "model_revision": "contact-patterns-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 0, "end": 13, "label": "PERSON", "text": "Lyle Ashcombe", "uncovered_positions": [] }, { "start": 20, "end": 28, "label": "RELIGION", "text": "Buddhism", "uncovered_positions": [] }, { "start": 38, "end": 60, "label": "EMAIL", "text": "lyle.a@example.invalid", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-119", "scenario": "multiple_entities", "predictions": [ { "start": 13, "end": 27, "label": "PERSON", "text": "Arvid Sandborn", "score": 0.998414158821106, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 29, "end": 43, "label": "PHONE", "text": "+49 381 482701", "score": 0.9627817869186401, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 13, "end": 27, "label": "PERSON", "text": "Arvid Sandborn", "score": 0.998414158821106, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 29, "end": 43, "label": "PHONE", "text": "+49 381 482701", "score": null, "source": "deterministic", "model_revision": "contact-patterns-v1", "normalization": "identity" }, { "start": 52, "end": 64, "label": "RELIGION", "text": "religionslos", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 13, "end": 27, "label": "PERSON", "text": "Arvid Sandborn", "uncovered_positions": [] }, { "start": 29, "end": 43, "label": "PHONE", "text": "+49 381 482701", "uncovered_positions": [] }, { "start": 52, "end": 64, "label": "RELIGION", "text": "religionslos", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-120", "scenario": "multiple_entities", "predictions": [ { "start": 5, "end": 17, "label": "PERSON", "text": "Mira Fenwick", "score": 0.9872230291366577, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 21, "end": 36, "label": "PHONE", "text": "+44 1904 582701", "score": 0.9610803127288818, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 45, "end": 53, "label": "RELIGION", "text": "Hinduism", "score": 0.9806439876556396, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 5, "end": 17, "label": "PERSON", "text": "Mira Fenwick", "score": 0.9872230291366577, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 21, "end": 36, "label": "PHONE", "text": "+44 1904 582701", "score": null, "source": "deterministic", "model_revision": "contact-patterns-v1", "normalization": "identity" }, { "start": 45, "end": 53, "label": "RELIGION", "text": "Hinduism", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 5, "end": 17, "label": "PERSON", "text": "Mira Fenwick", "uncovered_positions": [] }, { "start": 21, "end": 36, "label": "PHONE", "text": "+44 1904 582701", "uncovered_positions": [] }, { "start": 45, "end": 53, "label": "RELIGION", "text": "Hinduism", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-121", "scenario": "multiple_entities", "predictions": [ { "start": 0, "end": 6, "label": "CITY", "text": "Erfurt", "score": 0.9716014862060547, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 8, "end": 21, "label": "PERSON", "text": "Svea Morgenau", "score": 0.8673108220100403, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 33, "end": 43, "label": "RELIGION", "text": "katholisch", "score": 0.7634422779083252, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 47, "end": 57, "label": "RELIGION", "text": "agnostisch", "score": 0.5659558773040771, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 0, "end": 6, "label": "CITY", "text": "Erfurt", "score": 0.9716014862060547, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 8, "end": 21, "label": "PERSON", "text": "Svea Morgenau", "score": 0.8673108220100403, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 33, "end": 43, "label": "RELIGION", "text": "katholisch", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" }, { "start": 47, "end": 57, "label": "RELIGION", "text": "agnostisch", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 0, "end": 6, "label": "CITY", "text": "Erfurt", "uncovered_positions": [] }, { "start": 8, "end": 21, "label": "PERSON", "text": "Svea Morgenau", "uncovered_positions": [] }, { "start": 33, "end": 43, "label": "RELIGION", "text": "katholisch", "uncovered_positions": [] }, { "start": 47, "end": 57, "label": "RELIGION", "text": "agnostisch", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-122", "scenario": "multiple_entities", "predictions": [ { "start": 0, "end": 7, "label": "CITY", "text": "Bristol", "score": 0.9915996789932251, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 9, "end": 22, "label": "PERSON", "text": "Owen Thornley", "score": 0.9985686540603638, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 32, "end": 44, "label": "RELIGION", "text": "Christianity", "score": 0.9914804697036743, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 0, "end": 7, "label": "CITY", "text": "Bristol", "score": 0.9915996789932251, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 9, "end": 22, "label": "PERSON", "text": "Owen Thornley", "score": 0.9985686540603638, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 32, "end": 44, "label": "RELIGION", "text": "Christianity", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" }, { "start": 50, "end": 61, "label": "RELIGION", "text": "no religion", "score": null, "source": "deterministic", "model_revision": "religious-status-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 0, "end": 7, "label": "CITY", "text": "Bristol", "uncovered_positions": [] }, { "start": 9, "end": 22, "label": "PERSON", "text": "Owen Thornley", "uncovered_positions": [] }, { "start": 32, "end": 44, "label": "RELIGION", "text": "Christianity", "uncovered_positions": [] }, { "start": 50, "end": 61, "label": "RELIGION", "text": "no religion", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-123", "scenario": "multiple_entities", "predictions": [ { "start": 0, "end": 13, "label": "PERSON", "text": "Jona Falkenau", "score": 0.9965296387672424, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 18, "end": 31, "label": "PERSON", "text": "Jona Falkenau", "score": 0.9980782270431519, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 47, "end": 56, "label": "RELIGION", "text": "Sikhismus", "score": 0.9455693364143372, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 0, "end": 13, "label": "PERSON", "text": "Jona Falkenau", "score": 0.9965296387672424, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 18, "end": 31, "label": "PERSON", "text": "Jona Falkenau", "score": 0.9980782270431519, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 47, "end": 56, "label": "RELIGION", "text": "Sikhismus", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 0, "end": 13, "label": "PERSON", "text": "Jona Falkenau", "uncovered_positions": [] }, { "start": 18, "end": 31, "label": "PERSON", "text": "Jona Falkenau", "uncovered_positions": [] }, { "start": 47, "end": 56, "label": "RELIGION", "text": "Sikhismus", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-124", "scenario": "multiple_entities", "predictions": [ { "start": 0, "end": 12, "label": "PERSON", "text": "Esme Rowanby", "score": 0.9983696341514587, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 22, "end": 28, "label": "RELIGION", "text": "Taoism", "score": 0.9475066661834717, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 42, "end": 54, "label": "PERSON", "text": "Esme Rowanby", "score": 0.9983136653900146, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 0, "end": 12, "label": "PERSON", "text": "Esme Rowanby", "score": 0.9983696341514587, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 22, "end": 28, "label": "RELIGION", "text": "Taoism", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" }, { "start": 42, "end": 54, "label": "PERSON", "text": "Esme Rowanby", "score": 0.9983136653900146, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 0, "end": 12, "label": "PERSON", "text": "Esme Rowanby", "uncovered_positions": [] }, { "start": 22, "end": 28, "label": "RELIGION", "text": "Taoism", "uncovered_positions": [] }, { "start": 42, "end": 54, "label": "PERSON", "text": "Esme Rowanby", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-125", "scenario": "multiple_entities", "predictions": [ { "start": 23, "end": 33, "label": "ORGANIZATION", "text": "Hafenbogen", "score": 0.7308627963066101, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 43, "end": 47, "label": "CITY", "text": "Kiel", "score": 0.9980584979057312, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 49, "end": 60, "label": "RELIGION", "text": "evangelisch", "score": 0.7706614136695862, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 23, "end": 33, "label": "ORGANIZATION", "text": "Hafenbogen", "score": 0.7308627963066101, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 43, "end": 47, "label": "CITY", "text": "Kiel", "score": 0.9980584979057312, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 49, "end": 60, "label": "RELIGION", "text": "evangelisch", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [ [ 23, 33, "ORGANIZATION" ] ], "false_negatives": [ [ 23, 41, "ORGANIZATION" ] ], "exact_match": false, "error_code": null, "canaries": [ { "start": 23, "end": 41, "label": "ORGANIZATION", "text": "Hafenbogen Archivs", "uncovered_positions": [ 34, 35, 36, 37, 38, 39, 40 ] }, { "start": 43, "end": 47, "label": "CITY", "text": "Kiel", "uncovered_positions": [] }, { "start": 49, "end": 60, "label": "RELIGION", "text": "evangelisch", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [ { "label": "ORGANIZATION", "start": 23, "end": 41, "kind": "partial_mask" } ], "outcome": "candidate_partial_mask" }, { "case_id": "holdout03d-126", "scenario": "multiple_entities", "predictions": [ { "start": 2, "end": 22, "label": "ORGANIZATION", "text": "Maple Tide Institute", "score": 0.9682912230491638, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 35, "end": 41, "label": "CITY", "text": "Durham", "score": 0.99836665391922, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 51, "end": 58, "label": "RELIGION", "text": "Judaism", "score": 0.9933406710624695, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 2, "end": 22, "label": "ORGANIZATION", "text": "Maple Tide Institute", "score": 0.9682912230491638, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 35, "end": 41, "label": "CITY", "text": "Durham", "score": 0.99836665391922, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 51, "end": 58, "label": "RELIGION", "text": "Judaism", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 2, "end": 22, "label": "ORGANIZATION", "text": "Maple Tide Institute", "uncovered_positions": [] }, { "start": 35, "end": 41, "label": "CITY", "text": "Durham", "uncovered_positions": [] }, { "start": 51, "end": 58, "label": "RELIGION", "text": "Judaism", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-127", "scenario": "multiple_entities", "predictions": [ { "start": 0, "end": 13, "label": "PERSON", "text": "Liv Hohenrain", "score": 0.9968641400337219, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 20, "end": 24, "label": "CITY", "text": "Bonn", "score": 0.9716768860816956, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 29, "end": 34, "label": "PERSON", "text": "Bahai", "score": 0.8550848364830017, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 36, "end": 57, "label": "EMAIL", "text": "liv.h@example.invalid", "score": 0.7995248436927795, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [], "false_positives": [], "false_negatives": [ [ 0, 13, "PERSON" ], [ 20, 24, "CITY" ], [ 29, 34, "RELIGION" ], [ 36, 57, "EMAIL" ] ], "exact_match": false, "error_code": "ambiguous_entity_overlap", "canaries": [ { "start": 0, "end": 13, "label": "PERSON", "text": "Liv Hohenrain", "uncovered_positions": [ 0, 1, 2, 4, 5, 6, 7, 8, 9, 10, 11, 12 ] }, { "start": 20, "end": 24, "label": "CITY", "text": "Bonn", "uncovered_positions": [ 20, 21, 22, 23 ] }, { "start": 29, "end": 34, "label": "RELIGION", "text": "Bahai", "uncovered_positions": [ 29, 30, 31, 32, 33 ] }, { "start": 36, "end": 57, "label": "EMAIL", "text": "liv.h@example.invalid", "uncovered_positions": [ 36, 37, 38, 40, 42, 43, 44, 45, 46, 47, 48, 50, 51, 52, 53, 54, 55, 56 ] } ], "candidate_returned": false, "candidate_canaries": [], "outcome": "blocked_conflicting_spans" }, { "case_id": "holdout03d-128", "scenario": "multiple_entities", "predictions": [ { "start": 0, "end": 14, "label": "PERSON", "text": "Reed Alderwick", "score": 0.98525071144104, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 16, "end": 20, "label": "CITY", "text": "Bath", "score": 0.9997182488441467, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 22, "end": 33, "label": "RELIGION", "text": "agnosticism", "score": 0.9372888803482056, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 35, "end": 57, "label": "EMAIL", "text": "reed.a@example.invalid", "score": 0.644429087638855, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 0, "end": 14, "label": "PERSON", "text": "Reed Alderwick", "score": 0.98525071144104, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 16, "end": 20, "label": "CITY", "text": "Bath", "score": 0.9997182488441467, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 22, "end": 33, "label": "RELIGION", "text": "agnosticism", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" }, { "start": 35, "end": 57, "label": "EMAIL", "text": "reed.a@example.invalid", "score": null, "source": "deterministic", "model_revision": "contact-patterns-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 0, "end": 14, "label": "PERSON", "text": "Reed Alderwick", "uncovered_positions": [] }, { "start": 16, "end": 20, "label": "CITY", "text": "Bath", "uncovered_positions": [] }, { "start": 22, "end": 33, "label": "RELIGION", "text": "agnosticism", "uncovered_positions": [] }, { "start": 35, "end": 57, "label": "EMAIL", "text": "reed.a@example.invalid", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-129", "scenario": "formats_repetition", "predictions": [ { "start": 15, "end": 20, "label": "RELIGION", "text": "Islam", "score": 0.9793844223022461, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 15, "end": 20, "label": "RELIGION", "text": "Islam", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 15, "end": 20, "label": "RELIGION", "text": "Islam", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-130", "scenario": "formats_repetition", "predictions": [ { "start": 11, "end": 19, "label": "RELIGION", "text": "Buddhism", "score": 0.9502246379852295, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 11, "end": 19, "label": "RELIGION", "text": "Buddhism", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 11, "end": 19, "label": "RELIGION", "text": "Buddhism", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-131", "scenario": "formats_repetition", "predictions": [], "combined_predictions": [ { "start": 9, "end": 24, "label": "RELIGION", "text": "konfessionsfrei", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" }, { "start": 35, "end": 50, "label": "RELIGION", "text": "konfessionsfrei", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 9, "end": 24, "label": "RELIGION", "text": "konfessionsfrei", "uncovered_positions": [] }, { "start": 35, "end": 50, "label": "RELIGION", "text": "konfessionsfrei", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-132", "scenario": "formats_repetition", "predictions": [], "combined_predictions": [ { "start": 9, "end": 16, "label": "RELIGION", "text": "atheist", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" }, { "start": 27, "end": 34, "label": "RELIGION", "text": "atheist", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 9, "end": 16, "label": "RELIGION", "text": "atheist", "uncovered_positions": [] }, { "start": 27, "end": 34, "label": "RELIGION", "text": "atheist", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-133", "scenario": "formats_repetition", "predictions": [ { "start": 9, "end": 17, "label": "RELIGION", "text": "Judentum", "score": 0.9829030632972717, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 9, "end": 17, "label": "RELIGION", "text": "Judentum", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 9, "end": 17, "label": "RELIGION", "text": "Judentum", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-134", "scenario": "formats_repetition", "predictions": [ { "start": 7, "end": 13, "label": "RELIGION", "text": "Shinto", "score": 0.9690591096878052, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 7, "end": 13, "label": "RELIGION", "text": "Shinto", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 7, "end": 13, "label": "RELIGION", "text": "Shinto", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-135", "scenario": "formats_repetition", "predictions": [ { "start": 18, "end": 31, "label": "RELIGION", "text": "Agnostizismus", "score": 0.5336430072784424, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 18, "end": 31, "label": "RELIGION", "text": "Agnostizismus", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 18, "end": 31, "label": "RELIGION", "text": "Agnostizismus", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-136", "scenario": "formats_repetition", "predictions": [ { "start": 10, "end": 17, "label": "RELIGION", "text": "Sikhism", "score": 0.973417341709137, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 10, "end": 17, "label": "RELIGION", "text": "Sikhism", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 10, "end": 17, "label": "RELIGION", "text": "Sikhism", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-137", "scenario": "formats_repetition", "predictions": [ { "start": 11, "end": 24, "label": "ORGANIZATION", "text": "Katholizismus", "score": 0.6781677603721619, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 35, "end": 48, "label": "ORGANIZATION", "text": "Katholizismus", "score": 0.73296058177948, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [], "false_positives": [], "false_negatives": [ [ 11, 24, "RELIGION" ], [ 35, 48, "RELIGION" ] ], "exact_match": false, "error_code": "ambiguous_entity_overlap", "canaries": [ { "start": 11, "end": 24, "label": "RELIGION", "text": "Katholizismus", "uncovered_positions": [ 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23 ] }, { "start": 35, "end": 48, "label": "RELIGION", "text": "Katholizismus", "uncovered_positions": [ 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47 ] } ], "candidate_returned": false, "candidate_canaries": [], "outcome": "blocked_conflicting_spans" }, { "case_id": "holdout03d-138", "scenario": "formats_repetition", "predictions": [ { "start": 7, "end": 15, "label": "RELIGION", "text": "Hinduism", "score": 0.9410762786865234, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 26, "end": 34, "label": "RELIGION", "text": "Hinduism", "score": 0.9376835227012634, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 7, "end": 15, "label": "RELIGION", "text": "Hinduism", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" }, { "start": 26, "end": 34, "label": "RELIGION", "text": "Hinduism", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 7, "end": 15, "label": "RELIGION", "text": "Hinduism", "uncovered_positions": [] }, { "start": 26, "end": 34, "label": "RELIGION", "text": "Hinduism", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-139", "scenario": "formats_repetition", "predictions": [], "combined_predictions": [ { "start": 13, "end": 28, "label": "RELIGION", "text": "ohne Konfession", "score": null, "source": "deterministic", "model_revision": "religious-status-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 13, "end": 28, "label": "RELIGION", "text": "ohne Konfession", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-140", "scenario": "formats_repetition", "predictions": [], "combined_predictions": [ { "start": 11, "end": 24, "label": "RELIGION", "text": "non-religious", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 11, "end": 24, "label": "RELIGION", "text": "non-religious", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-141", "scenario": "formats_repetition", "predictions": [ { "start": 0, "end": 11, "label": "RELIGION", "text": "Christentum", "score": 0.8982365727424622, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 14, "end": 23, "label": "RELIGION", "text": "Atheismus", "score": 0.7216019034385681, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 0, "end": 11, "label": "RELIGION", "text": "Christentum", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" }, { "start": 14, "end": 23, "label": "RELIGION", "text": "Atheismus", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 0, "end": 11, "label": "RELIGION", "text": "Christentum", "uncovered_positions": [] }, { "start": 14, "end": 23, "label": "RELIGION", "text": "Atheismus", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-142", "scenario": "formats_repetition", "predictions": [ { "start": 0, "end": 12, "label": "RELIGION", "text": "Christianity", "score": 0.9415823221206665, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" }, { "start": 15, "end": 26, "label": "RELIGION", "text": "agnosticism", "score": 0.61921626329422, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 0, "end": 12, "label": "RELIGION", "text": "Christianity", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" }, { "start": 15, "end": 26, "label": "RELIGION", "text": "agnosticism", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 0, "end": 12, "label": "RELIGION", "text": "Christianity", "uncovered_positions": [] }, { "start": 15, "end": 26, "label": "RELIGION", "text": "agnosticism", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-143", "scenario": "formats_repetition", "predictions": [ { "start": 11, "end": 21, "label": "RELIGION", "text": "Buddhismus", "score": 0.7847886085510254, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 11, "end": 21, "label": "RELIGION", "text": "Buddhismus", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 11, "end": 21, "label": "RELIGION", "text": "Buddhismus", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-144", "scenario": "formats_repetition", "predictions": [ { "start": 9, "end": 16, "label": "RELIGION", "text": "Jainism", "score": 0.7793956995010376, "source": "multilingual_ner", "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "normalization": "identity" } ], "combined_predictions": [ { "start": 9, "end": 16, "label": "RELIGION", "text": "Jainism", "score": null, "source": "deterministic", "model_revision": "religious-terminology-v1", "normalization": "identity" } ], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [ { "start": 9, "end": 16, "label": "RELIGION", "text": "Jainism", "uncovered_positions": [] } ], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-145", "scenario": "negative_controls", "predictions": [], "combined_predictions": [], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-146", "scenario": "negative_controls", "predictions": [], "combined_predictions": [], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-147", "scenario": "negative_controls", "predictions": [], "combined_predictions": [], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-148", "scenario": "negative_controls", "predictions": [], "combined_predictions": [], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-149", "scenario": "negative_controls", "predictions": [], "combined_predictions": [], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-150", "scenario": "negative_controls", "predictions": [], "combined_predictions": [], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-151", "scenario": "negative_controls", "predictions": [], "combined_predictions": [], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-152", "scenario": "negative_controls", "predictions": [], "combined_predictions": [], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-153", "scenario": "negative_controls", "predictions": [], "combined_predictions": [], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-154", "scenario": "negative_controls", "predictions": [], "combined_predictions": [], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-155", "scenario": "negative_controls", "predictions": [], "combined_predictions": [], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-156", "scenario": "negative_controls", "predictions": [], "combined_predictions": [], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-157", "scenario": "negative_controls", "predictions": [], "combined_predictions": [], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-158", "scenario": "negative_controls", "predictions": [], "combined_predictions": [], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-159", "scenario": "negative_controls", "predictions": [], "combined_predictions": [], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" }, { "case_id": "holdout03d-160", "scenario": "negative_controls", "predictions": [], "combined_predictions": [], "false_positives": [], "false_negatives": [], "exact_match": true, "error_code": null, "canaries": [], "candidate_returned": true, "candidate_canaries": [], "outcome": "complete_no_measured_leakage" } ], "automated_release_eligible": false, "recognition_qualified": false } } }, "runtime_versions": { "gliner": "0.2.29", "torch": "2.8.0+cpu", "transformers": "4.57.1", "numpy": "2.4.6", "pydantic": "2.14.1" }, "actual_inference_latency_ms": { "qualification": { "median": 92.65962899735314, "max": 244.89703599829227 }, "validation": { "median": 141.5363085034187, "max": 273.8748889969429 }, "development03c": { "median": 119.45272150114761, "max": 545.1156909985002 }, "development03d": { "median": 95.93377900091582, "max": 392.20124499843223 } }, "automated_release_eligible": false, "independent_qualification": false, "selection_criteria": { "contextual_status": { "passed": true, "net_canary_improvements": 2 }, "status_scoped_ocr": { "passed": true, "net_canary_improvements": 3 } }, "peak_rss_mib": 3018.828125, "denied_network_calls": [ "socket.bind" ] }