| { |
| "mirage": { |
| "engine": "Cactus runtime 4.0.0", |
| "library": "libcactus.so", |
| "library_sha256": "89ad23bdcfa5b8e62137175eb297664933d75a3095fc44a24043e0bf9bc574c7", |
| "library_bytes": 1933336, |
| "weights": "mirage.cact", |
| "weights_sha256": "4249a0f7515272ff76b9414bec17d7e7daf84b15b66c7fb4f17fce1e66f9f237", |
| "weights_bytes": 21611043 |
| }, |
| "oasis": { |
| "engine": "Oasis engine", |
| "library": "liboasis.so", |
| "library_bytes": 470496, |
| "weights": "oasis.oasis", |
| "weights_sha256": "0ea335c906c6d50bcbe73200024d1770ee9ba231e331b88702bc2c1a26b09c90", |
| "weights_bytes": 18773064 |
| }, |
| "latency": { |
| "describe": { |
| "mirage": { |
| "images": 24, |
| "median_ms": 911.6, |
| "mean_ms": 928.7, |
| "invalid_json": 7, |
| "invalid_json_with_nan": 7 |
| }, |
| "oasis": { |
| "images": 24, |
| "median_ms": 251.1, |
| "mean_ms": 253.0, |
| "invalid_json": 0, |
| "invalid_json_with_nan": 0 |
| }, |
| "oasis_one_thread": { |
| "images": 24, |
| "median_ms": 520.0, |
| "mean_ms": 524.3, |
| "invalid_json": 0, |
| "invalid_json_with_nan": 0 |
| }, |
| "mirage_weights_on_oasis_engine": { |
| "images": 24, |
| "median_ms": 232.2, |
| "mean_ms": 242.4, |
| "invalid_json": 0, |
| "invalid_json_with_nan": 0 |
| }, |
| "options": { |
| "conf": 0.3, |
| "text_conf": 0.35, |
| "max_objects": 20, |
| "max_text_lines": 20, |
| "max_tokens": 512, |
| "mask_format": "largest" |
| } |
| }, |
| "text_only": { |
| "mirage": { |
| "images": 24, |
| "median_ms": 847.8, |
| "mean_ms": 890.1, |
| "invalid_json": 4, |
| "invalid_json_with_nan": 4 |
| }, |
| "oasis": { |
| "images": 24, |
| "median_ms": 68.0, |
| "mean_ms": 70.6, |
| "invalid_json": 0, |
| "invalid_json_with_nan": 0 |
| }, |
| "options": { |
| "sections": [ |
| "text" |
| ], |
| "conf": 0.3, |
| "text_conf": 0.35, |
| "max_text_lines": 20 |
| } |
| }, |
| "form": { |
| "mirage": { |
| "images": 24, |
| "median_ms": 650.4, |
| "mean_ms": 652.1, |
| "invalid_json": 0, |
| "invalid_json_with_nan": 0 |
| }, |
| "oasis": { |
| "images": 24, |
| "median_ms": 109.3, |
| "mean_ms": 110.0, |
| "invalid_json": 0, |
| "invalid_json_with_nan": 0 |
| }, |
| "options": { |
| "form": { |
| "setting": { |
| "enum": [ |
| "indoor", |
| "outdoor" |
| ] |
| }, |
| "people": { |
| "type": "integer", |
| "minimum": 0, |
| "maximum": 5 |
| }, |
| "main": { |
| "enum": [ |
| "person", |
| "vehicle", |
| "animal", |
| "food", |
| "furniture", |
| "sign" |
| ] |
| }, |
| "where": { |
| "x-source": "bbox", |
| "x-of": "main" |
| } |
| } |
| } |
| } |
| }, |
| "latency_arm64": { |
| "measured": "the two engines alternately, photo by photo, each with its own thread count", |
| "mirage_library": "libcactus.so", |
| "mirage_library_sha256": "7e1a128eaf221d98168d786b2be1c4e6314eda45d746b71ce125581148b5c613", |
| "mirage_library_bytes": 1782024, |
| "oasis_library_bytes": 445488, |
| "describe": { |
| "mirage": { |
| "images": 24, |
| "median_ms": 813.2, |
| "mean_ms": 857.0, |
| "invalid_json": 0 |
| }, |
| "oasis": { |
| "images": 24, |
| "median_ms": 1012.4, |
| "mean_ms": 1039.9, |
| "invalid_json": 0 |
| }, |
| "options": { |
| "conf": 0.3, |
| "text_conf": 0.35, |
| "max_objects": 20, |
| "max_text_lines": 20, |
| "max_tokens": 512, |
| "mask_format": "largest" |
| } |
| }, |
| "text_only": { |
| "mirage": { |
| "images": 24, |
| "median_ms": 683.2, |
| "mean_ms": 719.8, |
| "invalid_json": 0 |
| }, |
| "oasis": { |
| "images": 24, |
| "median_ms": 275.9, |
| "mean_ms": 280.9, |
| "invalid_json": 0 |
| }, |
| "options": { |
| "sections": [ |
| "text" |
| ], |
| "conf": 0.3, |
| "text_conf": 0.35, |
| "max_text_lines": 20 |
| } |
| }, |
| "form": { |
| "mirage": { |
| "images": 24, |
| "median_ms": 334.2, |
| "mean_ms": 333.8, |
| "invalid_json": 0 |
| }, |
| "oasis": { |
| "images": 24, |
| "median_ms": 488.4, |
| "mean_ms": 488.0, |
| "invalid_json": 0 |
| }, |
| "options": { |
| "form": { |
| "setting": { |
| "enum": [ |
| "indoor", |
| "outdoor" |
| ] |
| }, |
| "people": { |
| "type": "integer", |
| "minimum": 0, |
| "maximum": 5 |
| }, |
| "main": { |
| "enum": [ |
| "person", |
| "vehicle", |
| "animal", |
| "food", |
| "furniture", |
| "sign" |
| ] |
| }, |
| "where": { |
| "x-source": "bbox", |
| "x-of": "main" |
| } |
| } |
| } |
| } |
| }, |
| "records_coco_val2017": { |
| "options": { |
| "masks": false |
| }, |
| "mirage": { |
| "images": 300, |
| "invalid_json": 58, |
| "empty_records": 0 |
| }, |
| "oasis": { |
| "images": 300, |
| "invalid_json": 0, |
| "empty_records": 0 |
| } |
| }, |
| "records_coco_val2017_conf_0.5": { |
| "options": { |
| "masks": false, |
| "conf": 0.5 |
| }, |
| "mirage": { |
| "images": 300, |
| "invalid_json": 54, |
| "empty_records": 2 |
| }, |
| "oasis": { |
| "images": 300, |
| "invalid_json": 0, |
| "empty_records": 0 |
| } |
| }, |
| "ocr_end_to_end": { |
| "icdar2013_test": { |
| "mirage": { |
| "n": 1015, |
| "word_acc": 0.8089, |
| "exact": 0.7547, |
| "cer": 0.1454, |
| "invalid_json": 0 |
| }, |
| "oasis": { |
| "n": 1015, |
| "word_acc": 0.8167, |
| "exact": 0.7626, |
| "cer": 0.1318, |
| "invalid_json": 0 |
| }, |
| "mirage_weights_on_oasis_engine": { |
| "n": 1015, |
| "word_acc": 0.8108, |
| "exact": 0.7586, |
| "cer": 0.1257, |
| "invalid_json": 0 |
| } |
| }, |
| "iiit5k_test": { |
| "mirage": { |
| "n": 3000, |
| "word_acc": 0.7763, |
| "exact": 0.387, |
| "cer": 0.5196, |
| "invalid_json": 0 |
| }, |
| "oasis": { |
| "n": 3000, |
| "word_acc": 0.7893, |
| "exact": 0.3827, |
| "cer": 0.5182, |
| "invalid_json": 0 |
| }, |
| "mirage_weights_on_oasis_engine": { |
| "n": 3000, |
| "word_acc": 0.779, |
| "exact": 0.3897, |
| "cer": 0.5125, |
| "invalid_json": 0 |
| } |
| } |
| }, |
| "accented_text": { |
| "set": "500 synthetic lines in the description languages, cut as tightly as a text detector's box, in fonts unused in training", |
| "mirage": { |
| "n": 500, |
| "word_acc": 0.354, |
| "exact": 0.276, |
| "cer": 0.4248, |
| "invalid_json": 0, |
| "accent_recall": 0.0065, |
| "dead_recall": 0.0 |
| }, |
| "oasis": { |
| "n": 500, |
| "word_acc": 0.52, |
| "exact": 0.406, |
| "cer": 0.2887, |
| "invalid_json": 0, |
| "accent_recall": 0.2972, |
| "dead_recall": 0.4452 |
| } |
| }, |
| "untrained_characters": { |
| "count": 67, |
| "mirage_output_weights": "zero, so Mirage's reader never outputs them" |
| }, |
| "objects_coco_val2017": { |
| "images": 100, |
| "iou": 0.5, |
| "mirage": { |
| "images": 100, |
| "returned": 1070, |
| "ground_truth": 880, |
| "precision": 0.2486, |
| "recall": 0.3023, |
| "f1": 0.2728, |
| "invalid_json": 10 |
| }, |
| "oasis": { |
| "images": 100, |
| "returned": 841, |
| "ground_truth": 880, |
| "precision": 0.3092, |
| "recall": 0.2955, |
| "f1": 0.3021, |
| "invalid_json": 0 |
| }, |
| "mirage_weights_on_oasis_engine": { |
| "images": 100, |
| "returned": 792, |
| "ground_truth": 880, |
| "precision": 0.3056, |
| "recall": 0.275, |
| "f1": 0.2895, |
| "invalid_json": 0 |
| } |
| }, |
| "masks_coco_val2017": { |
| "images": 100, |
| "format": "bitmap", |
| "mirage": { |
| "images": 100, |
| "matched": 266, |
| "mean_mask_iou": 0.6699, |
| "mask_iou_50": 0.8496, |
| "mask_iou_75": 0.3985, |
| "invalid_json": 10 |
| }, |
| "oasis": { |
| "images": 100, |
| "matched": 260, |
| "mean_mask_iou": 0.6813, |
| "mask_iou_50": 0.85, |
| "mask_iou_75": 0.4231, |
| "invalid_json": 0 |
| }, |
| "mirage_weights_on_oasis_engine": { |
| "images": 100, |
| "matched": 242, |
| "mean_mask_iou": 0.6863, |
| "mask_iou_50": 0.8595, |
| "mask_iou_75": 0.4256, |
| "invalid_json": 0 |
| } |
| }, |
| "outlines_coco_val2017": { |
| "images": 100, |
| "format": "polygons, filled", |
| "mirage": { |
| "images": 100, |
| "matched": 266, |
| "mean_mask_iou": 0.6734, |
| "mask_iou_50": 0.8496, |
| "mask_iou_75": 0.4023, |
| "invalid_json": 10 |
| }, |
| "oasis": { |
| "images": 100, |
| "matched": 260, |
| "mean_mask_iou": 0.6773, |
| "mask_iou_50": 0.8538, |
| "mask_iou_75": 0.4115, |
| "invalid_json": 0 |
| } |
| }, |
| "accuracy_arm64": { |
| "objects_coco_val2017": { |
| "images": 100, |
| "precision": 0.3137, |
| "recall": 0.2852, |
| "f1": 0.2988, |
| "invalid_json": 0 |
| }, |
| "icdar2013_test": { |
| "n": 1015, |
| "word_acc": 0.8177, |
| "exact": 0.7635, |
| "cer": 0.1317, |
| "invalid_json": 0 |
| } |
| } |
| } |
