Private review: measured five-language release, Hindi sentence mode and custom GGUFs
5e783af verified | { | |
| "languages": { | |
| "hi": { | |
| "ours": { | |
| "n": 9991, | |
| "top1": 70.04, | |
| "top1_ci95": [ | |
| 69.16, | |
| 70.9 | |
| ], | |
| "top3": 88.31, | |
| "top8": 93.63 | |
| }, | |
| "indicxlit_full": { | |
| "top1": 69.33, | |
| "top1_ci95": [ | |
| 68.44, | |
| 70.23 | |
| ], | |
| "top3": 85.4, | |
| "top8": 88.29 | |
| }, | |
| "paired_ours_minus_indicxlit": { | |
| "top1": [ | |
| 0.71, | |
| -0.04, | |
| 1.4 | |
| ] | |
| }, | |
| "download_mb": 7.905041 | |
| }, | |
| "bn": { | |
| "ours": { | |
| "n": 13367, | |
| "top1": 64.64, | |
| "top1_ci95": [ | |
| 63.83, | |
| 65.44 | |
| ], | |
| "top3": 75.66, | |
| "top3_ci95": [ | |
| 74.9, | |
| 76.38 | |
| ], | |
| "top8": 77.97, | |
| "top8_ci95": [ | |
| 77.23, | |
| 78.66 | |
| ] | |
| }, | |
| "indicxlit_full": { | |
| "n": 13367, | |
| "top1": 70.52, | |
| "top1_ci95": [ | |
| 69.76, | |
| 71.34 | |
| ], | |
| "top3": 82.25, | |
| "top3_ci95": [ | |
| 81.61, | |
| 82.89 | |
| ], | |
| "top8": 85.44, | |
| "top8_ci95": [ | |
| 84.86, | |
| 86.06 | |
| ] | |
| }, | |
| "paired_ours_minus_indicxlit": { | |
| "top1": [ | |
| -5.88, | |
| -6.52, | |
| -5.22 | |
| ], | |
| "top3": [ | |
| -6.59, | |
| -7.18, | |
| -5.97 | |
| ], | |
| "top8": [ | |
| -7.47, | |
| -8.05, | |
| -6.89 | |
| ] | |
| }, | |
| "frozen_sha256": { | |
| "xlit.q4": "9f099f0b9524f446debf1d3f0ae1dc7ed1e88625c33f6dbb7b45e0348bf76376", | |
| "dict.tsv.gz": "edb2ad0bd7de4e87bd4d49ac406a9ef01011451bf23ac82d4c063bb9cce1dc2b", | |
| "params.json": "e0bd564ad61f6dcef8eca6bf3d64f8b231146c9885aa4c5b282a2b2420606c4f" | |
| }, | |
| "test_rows_sha256": "de2dc6a45db144a32074c586b2b92f6af07e9b3b4fef495150d62c59c31c9dfd", | |
| "baseline_predictions_sha256": "e3e72158035ae04c46691f75eb85df3b59959e9d801cd9099af26148cf7a5c9e", | |
| "baseline_missing_empty_inputs": 1, | |
| "baseline_note": "Existing IndicXlit PyTorch port beam-8 + full dictionary predictions, rescored on exactly the same rows; no new baseline inference.", | |
| "size": { | |
| "parameters": 989056, | |
| "model_raw": 589040, | |
| "model_gz9": 540107, | |
| "dict_raw": 14068340, | |
| "dict_gz9": 2508610, | |
| "dict_words": 500000 | |
| }, | |
| "download_mb": 3.049313 | |
| }, | |
| "te": { | |
| "ours": { | |
| "n": 9936, | |
| "top1": 78.74, | |
| "top1_ci95": [ | |
| 77.93, | |
| 79.53 | |
| ], | |
| "top3": 86.87, | |
| "top3_ci95": [ | |
| 86.2, | |
| 87.52 | |
| ], | |
| "top8": 88.63, | |
| "top8_ci95": [ | |
| 88.0, | |
| 89.25 | |
| ] | |
| }, | |
| "indicxlit_full": { | |
| "n": 9936, | |
| "top1": 82.42, | |
| "top1_ci95": [ | |
| 81.65, | |
| 83.14 | |
| ], | |
| "top3": 90.63, | |
| "top3_ci95": [ | |
| 90.04, | |
| 91.18 | |
| ], | |
| "top8": 92.75, | |
| "top8_ci95": [ | |
| 92.21, | |
| 93.26 | |
| ] | |
| }, | |
| "paired_ours_minus_indicxlit": { | |
| "top1": [ | |
| -3.67, | |
| -4.27, | |
| -3.07 | |
| ], | |
| "top3": [ | |
| -3.76, | |
| -4.27, | |
| -3.21 | |
| ], | |
| "top8": [ | |
| -4.13, | |
| -4.59, | |
| -3.62 | |
| ] | |
| }, | |
| "frozen_sha256": { | |
| "xlit.q4": "aea094e46a3c61eb9487d1aaba3824d3d9dc60f3f1cf6b6f99cadf04fd4318f9", | |
| "dict.tsv.gz": "20612a52ccf39c816d3068e78e9e9a3aa9aed9156f10de45ebbd543363943640", | |
| "params.json": "653f897efe1af78bb52149f2a18dc804b49ecac9c93add867871e64f56e07470" | |
| }, | |
| "test_rows_sha256": "698de748186eb92a43baed29e1e0aa9a15acbd5b20a79b3a4fd7db4b336ceda1", | |
| "baseline_predictions_sha256": "6c05dfc219ebc012fd96bac5f5b9586fd5ac35d0856a36af28c7362b04a31c82", | |
| "baseline_note": "Existing IndicXlit PyTorch port beam-8 + full dictionary predictions, rescored on exactly the same rows; no new baseline inference.", | |
| "size": { | |
| "parameters": 988928, | |
| "model_raw": 588904, | |
| "model_gz9": 541242, | |
| "dict_raw": 15989177, | |
| "dict_gz9": 2680527, | |
| "dict_words": 500000 | |
| }, | |
| "download_mb": 3.222365 | |
| }, | |
| "kn": { | |
| "ours": { | |
| "n": 11165, | |
| "top1": 77.37, | |
| "top1_ci95": [ | |
| 76.61, | |
| 78.13 | |
| ], | |
| "top3": 85.8, | |
| "top3_ci95": [ | |
| 85.16, | |
| 86.46 | |
| ], | |
| "top8": 87.68, | |
| "top8_ci95": [ | |
| 87.08, | |
| 88.32 | |
| ] | |
| }, | |
| "indicxlit_full": { | |
| "n": 11165, | |
| "top1": 82.61, | |
| "top1_ci95": [ | |
| 81.92, | |
| 83.29 | |
| ], | |
| "top3": 90.68, | |
| "top3_ci95": [ | |
| 90.16, | |
| 91.2 | |
| ], | |
| "top8": 92.41, | |
| "top8_ci95": [ | |
| 91.94, | |
| 92.9 | |
| ] | |
| }, | |
| "paired_ours_minus_indicxlit": { | |
| "top1": [ | |
| -5.24, | |
| -5.83, | |
| -4.67 | |
| ], | |
| "top3": [ | |
| -4.87, | |
| -5.39, | |
| -4.39 | |
| ], | |
| "top8": [ | |
| -4.73, | |
| -5.19, | |
| -4.26 | |
| ] | |
| }, | |
| "frozen_sha256": { | |
| "xlit.q4": "3788fd545c297febb7dbbdbdd780e831042395b4ec99260b95c963607afc8664", | |
| "dict.tsv.gz": "97035264d30eddb5810467dc5bbf088614221ea6ba2c3fff349c6bbd196bb644", | |
| "params.json": "6076d9126ffce66630b634e47ac5b38d03cda3bce7ab4d07162aeb0cbc925356" | |
| }, | |
| "test_rows_sha256": "5bfe000c741316d700f4caae046ed05c4cdcaf485ef31707f6254bcc4c9abbd9", | |
| "baseline_predictions_sha256": "928c3375ffdbca7811a36284ac4afb70b8b5192d28aa35f88a6291f941373407", | |
| "baseline_note": "Existing IndicXlit PyTorch port beam-8 + full dictionary predictions, rescored on exactly the same rows; no new baseline inference.", | |
| "size": { | |
| "parameters": 988928, | |
| "model_raw": 588904, | |
| "model_gz9": 539543, | |
| "dict_raw": 16786145, | |
| "dict_gz9": 2684083, | |
| "dict_words": 500000 | |
| }, | |
| "download_mb": 3.224222 | |
| }, | |
| "ta": { | |
| "ours": { | |
| "n": 11405, | |
| "top1": 71.78, | |
| "top1_ci95": [ | |
| 71.01, | |
| 72.61 | |
| ], | |
| "top3": 80.05, | |
| "top3_ci95": [ | |
| 79.37, | |
| 80.78 | |
| ], | |
| "top8": 82.61, | |
| "top8_ci95": [ | |
| 81.95, | |
| 83.31 | |
| ] | |
| }, | |
| "indicxlit_full": { | |
| "n": 11405, | |
| "top1": 75.95, | |
| "top1_ci95": [ | |
| 75.2, | |
| 76.73 | |
| ], | |
| "top3": 84.21, | |
| "top3_ci95": [ | |
| 83.56, | |
| 84.89 | |
| ], | |
| "top8": 87.26, | |
| "top8_ci95": [ | |
| 86.66, | |
| 87.87 | |
| ] | |
| }, | |
| "paired_ours_minus_indicxlit": { | |
| "top1": [ | |
| -4.16, | |
| -4.75, | |
| -3.59 | |
| ], | |
| "top3": [ | |
| -4.16, | |
| -4.66, | |
| -3.63 | |
| ], | |
| "top8": [ | |
| -4.65, | |
| -5.12, | |
| -4.16 | |
| ] | |
| }, | |
| "frozen_sha256": { | |
| "xlit.q4": "5dd185a033db79835fede66ab41a25f5897f24ad5d7c8ecc9864ca6ec167fdc0", | |
| "dict.tsv.gz": "4d14f514020e7737144c22991e9324a29734b5ab203bef3b68ec6d9f51e7cddb", | |
| "params.json": "f4520a8320f02573e6b8b6530a372c980903fbf757999540698957e8eec58385" | |
| }, | |
| "test_rows_sha256": "464c11fec7cfa71680a01c57899aeb691d88f1a871b1c7976184b0a21e2d75a6", | |
| "baseline_predictions_sha256": "0c2272e9423eb22e3fa04037a9973a777d50070cb9f3bcd03d4037865fd71bc3", | |
| "baseline_note": "Existing IndicXlit PyTorch port beam-8 + full dictionary predictions, rescored on exactly the same rows; no new baseline inference.", | |
| "size": { | |
| "parameters": 988928, | |
| "model_raw": 588904, | |
| "model_gz9": 540818, | |
| "dict_raw": 18105290, | |
| "dict_gz9": 2588634, | |
| "dict_words": 500000 | |
| }, | |
| "download_mb": 3.130045 | |
| } | |
| } | |
| } | |