Span file and count file for LiteRT's GPU, per-chip NPU span files, GPU priority results
e845a2b verified | { | |
| "about": "Every call type against gliner2 2.0.0 with the original weights (runtime/verify_parity.py): LiteRT 2.2.0 on the CPU, wfp16 encoder and the heads file.", | |
| "source": "fastino/GLiNER2.5-Decide@bbe10ff77ebb238777c17d3a8ac9260e30929057", | |
| "corpus": { | |
| "file": "runtime/parity_corpus.json", | |
| "requests": 296, | |
| "identical": 296, | |
| "byCallType": { | |
| "entities": { | |
| "calls": 40, | |
| "nativeError": 0, | |
| "refused": 0, | |
| "matched": 40, | |
| "items": 146, | |
| "minSpanF1": 1.0, | |
| "maxConfidenceDifference": 0.000214 | |
| }, | |
| "entities_single": { | |
| "calls": 8, | |
| "nativeError": 0, | |
| "refused": 0, | |
| "matched": 8, | |
| "items": 16, | |
| "minSpanF1": 1.0, | |
| "maxConfidenceDifference": 0.000236 | |
| }, | |
| "relations_work": { | |
| "calls": 24, | |
| "nativeError": 0, | |
| "refused": 0, | |
| "matched": 24, | |
| "items": 26, | |
| "minSpanF1": 1.0, | |
| "maxConfidenceDifference": 0.000351 | |
| }, | |
| "record_natural": { | |
| "calls": 16, | |
| "nativeError": 0, | |
| "refused": 0, | |
| "matched": 16, | |
| "items": 31, | |
| "minSpanF1": 1.0, | |
| "maxConfidenceDifference": 0.000303 | |
| }, | |
| "record_latent": { | |
| "calls": 24, | |
| "nativeError": 0, | |
| "refused": 0, | |
| "matched": 24, | |
| "items": 30, | |
| "minSpanF1": 1.0, | |
| "maxConfidenceDifference": 0.000489 | |
| }, | |
| "attributes": { | |
| "calls": 16, | |
| "nativeError": 0, | |
| "refused": 0, | |
| "matched": 16, | |
| "items": 26, | |
| "minSpanF1": 1.0, | |
| "maxConfidenceDifference": 0.00045 | |
| }, | |
| "classification_topic": { | |
| "calls": 40, | |
| "nativeError": 0, | |
| "refused": 0, | |
| "matched": 40, | |
| "items": 40, | |
| "minSpanF1": 1.0, | |
| "maxConfidenceDifference": 0.000204 | |
| }, | |
| "mixed": { | |
| "calls": 16, | |
| "nativeError": 0, | |
| "refused": 0, | |
| "matched": 16, | |
| "items": 55, | |
| "minSpanF1": 1.0, | |
| "maxConfidenceDifference": 0.000207 | |
| }, | |
| "relations_family": { | |
| "calls": 8, | |
| "nativeError": 0, | |
| "refused": 0, | |
| "matched": 8, | |
| "items": 8, | |
| "minSpanF1": 1.0, | |
| "maxConfidenceDifference": 0.000234 | |
| }, | |
| "classification_described": { | |
| "calls": 24, | |
| "nativeError": 0, | |
| "refused": 0, | |
| "matched": 24, | |
| "items": 24, | |
| "minSpanF1": 1.0, | |
| "maxConfidenceDifference": 9.3e-05 | |
| }, | |
| "record_anchorless": { | |
| "calls": 8, | |
| "nativeError": 0, | |
| "refused": 0, | |
| "matched": 8, | |
| "items": 12, | |
| "minSpanF1": 1.0, | |
| "maxConfidenceDifference": 0.000269 | |
| }, | |
| "structure_event": { | |
| "calls": 24, | |
| "nativeError": 0, | |
| "refused": 0, | |
| "matched": 24, | |
| "items": 38, | |
| "minSpanF1": 1.0, | |
| "maxConfidenceDifference": 0.001315 | |
| }, | |
| "classification_multi": { | |
| "calls": 16, | |
| "nativeError": 0, | |
| "refused": 0, | |
| "matched": 16, | |
| "items": 33, | |
| "minSpanF1": 1.0, | |
| "maxConfidenceDifference": 0.000334 | |
| }, | |
| "entities_described": { | |
| "calls": 16, | |
| "nativeError": 0, | |
| "refused": 0, | |
| "matched": 16, | |
| "items": 27, | |
| "minSpanF1": 1.0, | |
| "maxConfidenceDifference": 0.000182 | |
| }, | |
| "structure_order": { | |
| "calls": 8, | |
| "nativeError": 0, | |
| "refused": 0, | |
| "matched": 8, | |
| "items": 20, | |
| "minSpanF1": 1.0, | |
| "maxConfidenceDifference": 0.000579 | |
| }, | |
| "relations_event": { | |
| "calls": 8, | |
| "nativeError": 0, | |
| "refused": 0, | |
| "matched": 8, | |
| "items": 3, | |
| "minSpanF1": 1.0, | |
| "maxConfidenceDifference": 0.000118 | |
| } | |
| } | |
| }, | |
| "fixtures": { | |
| "about": "FluidInference's eleven verify-full-extraction fixtures", | |
| "identical": 11, | |
| "total": 11, | |
| "rows": [ | |
| { | |
| "case": "entities", | |
| "structureMatch": true, | |
| "maxConfidenceDifference": 1e-06 | |
| }, | |
| { | |
| "case": "entities_multi", | |
| "structureMatch": true, | |
| "maxConfidenceDifference": 0.0 | |
| }, | |
| { | |
| "case": "relations", | |
| "structureMatch": true, | |
| "maxConfidenceDifference": 1e-06 | |
| }, | |
| { | |
| "case": "relations_two", | |
| "structureMatch": true, | |
| "maxConfidenceDifference": 2e-06 | |
| }, | |
| { | |
| "case": "record_natural", | |
| "structureMatch": true, | |
| "maxConfidenceDifference": 4e-06 | |
| }, | |
| { | |
| "case": "record_latent", | |
| "structureMatch": true, | |
| "maxConfidenceDifference": 4e-06 | |
| }, | |
| { | |
| "case": "record_anchorless", | |
| "structureMatch": true, | |
| "maxConfidenceDifference": 4e-06 | |
| }, | |
| { | |
| "case": "attributes", | |
| "structureMatch": true, | |
| "maxConfidenceDifference": 0.00014 | |
| }, | |
| { | |
| "case": "mixed_classification", | |
| "structureMatch": true, | |
| "maxConfidenceDifference": 5e-06 | |
| }, | |
| { | |
| "case": "classification_only", | |
| "structureMatch": true, | |
| "maxConfidenceDifference": 3e-05 | |
| }, | |
| { | |
| "case": "enum_choice", | |
| "structureMatch": true, | |
| "maxConfidenceDifference": 1e-06 | |
| } | |
| ] | |
| }, | |
| "spanFile": { | |
| "about": "gliner25_decide_span_wfp16.tflite and gliner25_decide_count_wfp16.tflite in place of the heads file (runtime/verify_parity.py --span --count), LiteRT 2.2.0 CPU", | |
| "corpus": { | |
| "requests": 296, | |
| "identical": 296, | |
| "byCallType": { | |
| "entities": { | |
| "calls": 40, | |
| "nativeError": 0, | |
| "refused": 0, | |
| "matched": 40, | |
| "items": 146, | |
| "minSpanF1": 1.0, | |
| "maxConfidenceDifference": 0.000214 | |
| }, | |
| "entities_single": { | |
| "calls": 8, | |
| "nativeError": 0, | |
| "refused": 0, | |
| "matched": 8, | |
| "items": 16, | |
| "minSpanF1": 1.0, | |
| "maxConfidenceDifference": 0.000236 | |
| }, | |
| "relations_work": { | |
| "calls": 24, | |
| "nativeError": 0, | |
| "refused": 0, | |
| "matched": 24, | |
| "items": 26, | |
| "minSpanF1": 1.0, | |
| "maxConfidenceDifference": 0.000351 | |
| }, | |
| "record_natural": { | |
| "calls": 16, | |
| "nativeError": 0, | |
| "refused": 0, | |
| "matched": 16, | |
| "items": 31, | |
| "minSpanF1": 1.0, | |
| "maxConfidenceDifference": 0.000303 | |
| }, | |
| "record_latent": { | |
| "calls": 24, | |
| "nativeError": 0, | |
| "refused": 0, | |
| "matched": 24, | |
| "items": 30, | |
| "minSpanF1": 1.0, | |
| "maxConfidenceDifference": 0.000489 | |
| }, | |
| "attributes": { | |
| "calls": 16, | |
| "nativeError": 0, | |
| "refused": 0, | |
| "matched": 16, | |
| "items": 26, | |
| "minSpanF1": 1.0, | |
| "maxConfidenceDifference": 0.00045 | |
| }, | |
| "classification_topic": { | |
| "calls": 40, | |
| "nativeError": 0, | |
| "refused": 0, | |
| "matched": 40, | |
| "items": 40, | |
| "minSpanF1": 1.0, | |
| "maxConfidenceDifference": 0.000204 | |
| }, | |
| "mixed": { | |
| "calls": 16, | |
| "nativeError": 0, | |
| "refused": 0, | |
| "matched": 16, | |
| "items": 55, | |
| "minSpanF1": 1.0, | |
| "maxConfidenceDifference": 0.000207 | |
| }, | |
| "relations_family": { | |
| "calls": 8, | |
| "nativeError": 0, | |
| "refused": 0, | |
| "matched": 8, | |
| "items": 8, | |
| "minSpanF1": 1.0, | |
| "maxConfidenceDifference": 0.000234 | |
| }, | |
| "classification_described": { | |
| "calls": 24, | |
| "nativeError": 0, | |
| "refused": 0, | |
| "matched": 24, | |
| "items": 24, | |
| "minSpanF1": 1.0, | |
| "maxConfidenceDifference": 9.3e-05 | |
| }, | |
| "record_anchorless": { | |
| "calls": 8, | |
| "nativeError": 0, | |
| "refused": 0, | |
| "matched": 8, | |
| "items": 12, | |
| "minSpanF1": 1.0, | |
| "maxConfidenceDifference": 0.000269 | |
| }, | |
| "structure_event": { | |
| "calls": 24, | |
| "nativeError": 0, | |
| "refused": 0, | |
| "matched": 24, | |
| "items": 38, | |
| "minSpanF1": 1.0, | |
| "maxConfidenceDifference": 0.001315 | |
| }, | |
| "classification_multi": { | |
| "calls": 16, | |
| "nativeError": 0, | |
| "refused": 0, | |
| "matched": 16, | |
| "items": 33, | |
| "minSpanF1": 1.0, | |
| "maxConfidenceDifference": 0.000334 | |
| }, | |
| "entities_described": { | |
| "calls": 16, | |
| "nativeError": 0, | |
| "refused": 0, | |
| "matched": 16, | |
| "items": 27, | |
| "minSpanF1": 1.0, | |
| "maxConfidenceDifference": 0.000182 | |
| }, | |
| "structure_order": { | |
| "calls": 8, | |
| "nativeError": 0, | |
| "refused": 0, | |
| "matched": 8, | |
| "items": 20, | |
| "minSpanF1": 1.0, | |
| "maxConfidenceDifference": 0.000579 | |
| }, | |
| "relations_event": { | |
| "calls": 8, | |
| "nativeError": 0, | |
| "refused": 0, | |
| "matched": 8, | |
| "items": 3, | |
| "minSpanF1": 1.0, | |
| "maxConfidenceDifference": 0.000118 | |
| } | |
| } | |
| }, | |
| "fixtures": { | |
| "identical": 11, | |
| "total": 11 | |
| } | |
| } | |
| } | |