Buckets:
| int main() { | |
| bool success = true; | |
| const std::vector<std::pair<std::vector<uint32_t>, std::vector<uint32_t>>> normalization = { | |
| {{}, {}}, | |
| {{0x65, 0x301}, {0xE9}}, | |
| {{0xE9}, {0xE9}}, | |
| {{0x212B}, {0xC5}}, | |
| {{0x61, 0x315, 0x300}, {0xE0, 0x315}}, | |
| {{0x61, 0x305, 0x301}, {0x61, 0x305, 0x301}}, | |
| {{0x301, 0x327}, {0x327, 0x301}}, | |
| {{0x1100, 0x1161, 0x11A8}, {0xAC01}}, | |
| {{0xAC01}, {0xAC01}}, | |
| {{0xFB01, 0xFF21}, {0xFB01, 0xFF21}}, | |
| {{0x958}, {0x915, 0x93C}}, | |
| {{0x1D15E}, {0x1D157, 0x1D165}}, | |
| }; | |
| for (const auto & test : normalization) { | |
| if (unicode_cpts_normalize_nfc(test.first) != test.second || unicode_cpts_normalize_nfc(test.second) != test.second) { | |
| fprintf(stderr, "unexpected NFC normalization\n"); | |
| success = false; | |
| break; | |
| } | |
| } | |
| const struct { | |
| std::string text; | |
| std::vector<uint32_t> expected; | |
| } utf8_tests[] = { | |
| { "\x7f", { 0x7f } }, | |
| { "\xc2\x80", { 0x80 } }, | |
| { "\xdf\xbf", { 0x7ff } }, | |
| { "\xe0\xa0\x80", { 0x800 } }, | |
| { "\xef\xbf\xbf", { 0xffff } }, | |
| { "\xf0\x90\x80\x80", { 0x10000 } }, | |
| { "\xf4\x8f\xbf\xbf", { 0x10ffff } }, | |
| // Invalid-input cases preserve the existing recovery behavior: one U+FFFD per byte. | |
| { "\xc0\xa0", { 0xfffd, 0xfffd } }, | |
| { "\xc1\xbf", { 0xfffd, 0xfffd } }, | |
| { "\xe0\x80\xa0", { 0xfffd, 0xfffd, 0xfffd } }, | |
| { "\xe0\x9f\xbf", { 0xfffd, 0xfffd, 0xfffd } }, | |
| { "\xf0\x80\x80\xa0", { 0xfffd, 0xfffd, 0xfffd, 0xfffd } }, | |
| { "\xf0\x8f\xbf\xbf", { 0xfffd, 0xfffd, 0xfffd, 0xfffd } }, | |
| { "\xf4\x90\x80\x80", { 0xfffd, 0xfffd, 0xfffd, 0xfffd } }, | |
| { "\xf5\x80\x80\x80", { 0xfffd, 0xfffd, 0xfffd, 0xfffd } }, | |
| { "\xf7\xbf\xbf\xbf", { 0xfffd, 0xfffd, 0xfffd, 0xfffd } }, | |
| { "\xe2\x80", { 0xfffd, 0xfffd } }, | |
| }; | |
| for (const auto & test : utf8_tests) { | |
| auto expected = test.expected; | |
| expected.insert(expected.begin(), 'a'); | |
| expected.push_back('b'); | |
| if (unicode_cpts_from_utf8("a" + test.text + "b") != expected) { | |
| fprintf(stderr, "unexpected UTF-8 decoding for bytes:"); | |
| for (const unsigned char byte : test.text) { | |
| fprintf(stderr, " %02x", byte); | |
| } | |
| fprintf(stderr, "\n"); | |
| success = false; | |
| } | |
| } | |
| const std::vector<std::string> regex_exprs = { | |
| "[~][A-Za-z]+| ?[\\p{S}]+|\\s+", | |
| }; | |
| const std::vector<std::string> expected = { " ~", "foo" }; | |
| const auto actual = unicode_regex_split(" ~foo", regex_exprs, false); | |
| if (actual != expected) { | |
| fprintf(stderr, "unexpected split:"); | |
| for (const auto & piece : actual) { | |
| fprintf(stderr, " [%s]", piece.c_str()); | |
| } | |
| fprintf(stderr, "\n"); | |
| success = false; | |
| } | |
| return success ? 0 : 1; | |
| } | |
Xet Storage Details
- Size:
- 3.21 kB
- Xet hash:
- 96472380c857970df472699a2e1f62c0159f2362950df44daa8327a3393cdf58
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.