Spaces:
Running
Running
Download 731ai/node_modules/pdf-lib/src/utils/unicode.ts from u11310019/interactive-multi-web: direct link, hf CLI and curl.
- Browser
- Download file 14.1 kB
-
https://huggingface.co/spaces/u11310019/interactive-multi-web/resolve/main/731ai/node_modules/pdf-lib/src/utils/unicode.ts
- Command line
-
hf download hf://spaces/u11310019/interactive-multi-web/731ai/node_modules/pdf-lib/src/utils/unicode.ts
-
curl -L -o unicode.ts https://huggingface.co/spaces/u11310019/interactive-multi-web/resolve/main/731ai/node_modules/pdf-lib/src/utils/unicode.ts
14.1 kB
| import { toHexString } from 'src/utils/strings'; | |
| /** | |
| * Encodes a string to UTF-8. | |
| * | |
| * @param input The string to be encoded. | |
| * @param byteOrderMark Whether or not a byte order marker (BOM) should be added | |
| * to the start of the encoding. (default `true`) | |
| * @returns A Uint8Array containing the UTF-8 encoding of the input string. | |
| * | |
| * ----------------------------------------------------------------------------- | |
| * | |
| * JavaScript strings are composed of Unicode code points. Code points are | |
| * integers in the range 0 to 1,114,111 (0x10FFFF). When serializing a string, | |
| * it must be encoded as a sequence of words. A word is typically 8, 16, or 32 | |
| * bytes in size. As such, Unicode defines three encoding forms: UTF-8, UTF-16, | |
| * and UTF-32. These encoding forms are described in the Unicode standard [1]. | |
| * This function implements the UTF-8 encoding form. | |
| * | |
| * ----------------------------------------------------------------------------- | |
| * | |
| * In UTF-8, each code point is mapped to a sequence of 1, 2, 3, or 4 bytes. | |
| * Note that the logic which defines this mapping is slightly convoluted, and | |
| * not as straightforward as the mapping logic for UTF-16 or UTF-32. The UTF-8 | |
| * mapping logic is as follows [2]: | |
| * | |
| * • If a code point is in the range U+0000..U+007F, then view it as a 7-bit | |
| * integer: 0bxxxxxxx. Map the code point to 1 byte with the first high order | |
| * bit set to 0: | |
| * | |
| * b1=0b0xxxxxxx | |
| * | |
| * • If a code point is in the range U+0080..U+07FF, then view it as an 11-bit | |
| * integer: 0byyyyyxxxxxx. Map the code point to 2 bytes with the first 5 bits | |
| * of the code point stored in the first byte, and the last 6 bits stored in | |
| * the second byte: | |
| * | |
| * b1=0b110yyyyy b2=0b10xxxxxx | |
| * | |
| * • If a code point is in the range U+0800..U+FFFF, then view it as a 16-bit | |
| * integer, 0bzzzzyyyyyyxxxxxx. Map the code point to 3 bytes with the first | |
| * 4 bits stored in the first byte, the next 6 bits stored in the second byte, | |
| * and the last 6 bits in the third byte: | |
| * | |
| * b1=0b1110zzzz b2=0b10yyyyyy b3=0b10xxxxxx | |
| * | |
| * • If a code point is in the range U+10000...U+10FFFF, then view it as a | |
| * 21-bit integer, 0bvvvzzzzzzyyyyyyxxxxxx. Map the code point to 4 bytes with | |
| * the first 3 bits stored in the first byte, the next 6 bits stored in the | |
| * second byte, the next 6 bits stored in the third byte, and the last 6 bits | |
| * stored in the fourth byte: | |
| * | |
| * b1=0b11110xxx b2=0b10zzzzzz b3=0b10yyyyyy b4=0b10xxxxxx | |
| * | |
| * ----------------------------------------------------------------------------- | |
| * | |
| * It is important to note, when iterating through the code points of a string | |
| * in JavaScript, that if a character is encoded as a surrogate pair it will | |
| * increase the string's length by 2 instead of 1 [4]. For example: | |
| * | |
| * ``` | |
| * > 'a'.length | |
| * 1 | |
| * > '💩'.length | |
| * 2 | |
| * > '語'.length | |
| * 1 | |
| * > 'a💩語'.length | |
| * 4 | |
| * ``` | |
| * | |
| * The results of the above example are explained by the fact that the | |
| * characters 'a' and '語' are not represented by surrogate pairs, but '💩' is. | |
| * | |
| * Because of this idiosyncrasy in JavaScript's string implementation and APIs, | |
| * we must "jump" an extra index after encoding a character as a surrogate | |
| * pair. In practice, this means we must increment the index of our for loop by | |
| * 2 if we encode a surrogate pair, and 1 in all other cases. | |
| * | |
| * ----------------------------------------------------------------------------- | |
| * | |
| * References: | |
| * - [1] https://www.unicode.org/versions/Unicode12.0.0/UnicodeStandard-12.0.pdf | |
| * 3.9 Unicode Encoding Forms - UTF-8 | |
| * - [2] http://www.herongyang.com/Unicode/UTF-8-UTF-8-Encoding.html | |
| * - [3] http://www.herongyang.com/Unicode/UTF-8-UTF-8-Encoding-Algorithm.html | |
| * - [4] https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/String/length#Description | |
| * | |
| */ | |
| export const utf8Encode = (input: string, byteOrderMark = true): Uint8Array => { | |
| const encoded = []; | |
| if (byteOrderMark) encoded.push(0xef, 0xbb, 0xbf); | |
| for (let idx = 0, len = input.length; idx < len; ) { | |
| const codePoint = input.codePointAt(idx)!; | |
| // One byte encoding | |
| if (codePoint < 0x80) { | |
| const byte1 = codePoint & 0x7f; | |
| encoded.push(byte1); | |
| idx += 1; | |
| } | |
| // Two byte encoding | |
| else if (codePoint < 0x0800) { | |
| const byte1 = ((codePoint >> 6) & 0x1f) | 0xc0; | |
| const byte2 = (codePoint & 0x3f) | 0x80; | |
| encoded.push(byte1, byte2); | |
| idx += 1; | |
| } | |
| // Three byte encoding | |
| else if (codePoint < 0x010000) { | |
| const byte1 = ((codePoint >> 12) & 0x0f) | 0xe0; | |
| const byte2 = ((codePoint >> 6) & 0x3f) | 0x80; | |
| const byte3 = (codePoint & 0x3f) | 0x80; | |
| encoded.push(byte1, byte2, byte3); | |
| idx += 1; | |
| } | |
| // Four byte encoding (surrogate pair) | |
| else if (codePoint < 0x110000) { | |
| const byte1 = ((codePoint >> 18) & 0x07) | 0xf0; | |
| const byte2 = ((codePoint >> 12) & 0x3f) | 0x80; | |
| const byte3 = ((codePoint >> 6) & 0x3f) | 0x80; | |
| const byte4 = ((codePoint >> 0) & 0x3f) | 0x80; | |
| encoded.push(byte1, byte2, byte3, byte4); | |
| idx += 2; | |
| } | |
| // Should never reach this case | |
| else throw new Error(`Invalid code point: 0x${toHexString(codePoint)}`); | |
| } | |
| return new Uint8Array(encoded); | |
| }; | |
| /** | |
| * Encodes a string to UTF-16. | |
| * | |
| * @param input The string to be encoded. | |
| * @param byteOrderMark Whether or not a byte order marker (BOM) should be added | |
| * to the start of the encoding. (default `true`) | |
| * @returns A Uint16Array containing the UTF-16 encoding of the input string. | |
| * | |
| * ----------------------------------------------------------------------------- | |
| * | |
| * JavaScript strings are composed of Unicode code points. Code points are | |
| * integers in the range 0 to 1,114,111 (0x10FFFF). When serializing a string, | |
| * it must be encoded as a sequence of words. A word is typically 8, 16, or 32 | |
| * bytes in size. As such, Unicode defines three encoding forms: UTF-8, UTF-16, | |
| * and UTF-32. These encoding forms are described in the Unicode standard [1]. | |
| * This function implements the UTF-16 encoding form. | |
| * | |
| * ----------------------------------------------------------------------------- | |
| * | |
| * In UTF-16, each code point is mapped to one or two 16-bit integers. The | |
| * UTF-16 mapping logic is as follows [2]: | |
| * | |
| * • If a code point is in the range U+0000..U+FFFF, then map the code point to | |
| * a 16-bit integer with the most significant byte first. | |
| * | |
| * • If a code point is in the range U+10000..U+10000, then map the code point | |
| * to two 16-bit integers. The first integer should contain the high surrogate | |
| * and the second integer should contain the low surrogate. Both surrogates | |
| * should be written with the most significant byte first. | |
| * | |
| * ----------------------------------------------------------------------------- | |
| * | |
| * It is important to note, when iterating through the code points of a string | |
| * in JavaScript, that if a character is encoded as a surrogate pair it will | |
| * increase the string's length by 2 instead of 1 [4]. For example: | |
| * | |
| * ``` | |
| * > 'a'.length | |
| * 1 | |
| * > '💩'.length | |
| * 2 | |
| * > '語'.length | |
| * 1 | |
| * > 'a💩語'.length | |
| * 4 | |
| * ``` | |
| * | |
| * The results of the above example are explained by the fact that the | |
| * characters 'a' and '語' are not represented by surrogate pairs, but '💩' is. | |
| * | |
| * Because of this idiosyncrasy in JavaScript's string implementation and APIs, | |
| * we must "jump" an extra index after encoding a character as a surrogate | |
| * pair. In practice, this means we must increment the index of our for loop by | |
| * 2 if we encode a surrogate pair, and 1 in all other cases. | |
| * | |
| * ----------------------------------------------------------------------------- | |
| * | |
| * References: | |
| * - [1] https://www.unicode.org/versions/Unicode12.0.0/UnicodeStandard-12.0.pdf | |
| * 3.9 Unicode Encoding Forms - UTF-8 | |
| * - [2] http://www.herongyang.com/Unicode/UTF-16-UTF-16-Encoding.html | |
| * - [3] https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/String/length#Description | |
| * | |
| */ | |
| export const utf16Encode = ( | |
| input: string, | |
| byteOrderMark = true, | |
| ): Uint16Array => { | |
| const encoded = []; | |
| if (byteOrderMark) encoded.push(0xfeff); | |
| for (let idx = 0, len = input.length; idx < len; ) { | |
| const codePoint = input.codePointAt(idx)!; | |
| // Two byte encoding | |
| if (codePoint < 0x010000) { | |
| encoded.push(codePoint); | |
| idx += 1; | |
| } | |
| // Four byte encoding (surrogate pair) | |
| else if (codePoint < 0x110000) { | |
| encoded.push(highSurrogate(codePoint), lowSurrogate(codePoint)); | |
| idx += 2; | |
| } | |
| // Should never reach this case | |
| else throw new Error(`Invalid code point: 0x${toHexString(codePoint)}`); | |
| } | |
| return new Uint16Array(encoded); | |
| }; | |
| /** | |
| * Returns `true` if the `codePoint` is within the | |
| * Basic Multilingual Plane (BMP). Code points inside the BMP are not encoded | |
| * with surrogate pairs. | |
| * @param codePoint The code point to be evaluated. | |
| * | |
| * Reference: https://en.wikipedia.org/wiki/UTF-16#Description | |
| */ | |
| export const isWithinBMP = (codePoint: number) => | |
| codePoint >= 0 && codePoint <= 0xffff; | |
| /** | |
| * Returns `true` if the given `codePoint` is valid and must be represented | |
| * with a surrogate pair when encoded. | |
| * @param codePoint The code point to be evaluated. | |
| * | |
| * Reference: https://en.wikipedia.org/wiki/UTF-16#Description | |
| */ | |
| export const hasSurrogates = (codePoint: number) => | |
| codePoint >= 0x010000 && codePoint <= 0x10ffff; | |
| // From Unicode 3.0 spec, section 3.7: | |
| // http://unicode.org/versions/Unicode3.0.0/ch03.pdf | |
| export const highSurrogate = (codePoint: number) => | |
| Math.floor((codePoint - 0x10000) / 0x400) + 0xd800; | |
| // From Unicode 3.0 spec, section 3.7: | |
| // http://unicode.org/versions/Unicode3.0.0/ch03.pdf | |
| export const lowSurrogate = (codePoint: number) => | |
| ((codePoint - 0x10000) % 0x400) + 0xdc00; | |
| enum ByteOrder { | |
| BigEndian = 'BigEndian', | |
| LittleEndian = 'LittleEndian', | |
| } | |
| const REPLACEMENT = '�'.codePointAt(0)!; | |
| /** | |
| * Decodes a Uint8Array of data to a string using UTF-16. | |
| * | |
| * Note that this function attempts to recover from erronous input by | |
| * inserting the replacement character (�) to mark invalid code points | |
| * and surrogate pairs. | |
| * | |
| * @param input A Uint8Array containing UTF-16 encoded data | |
| * @param byteOrderMark Whether or not a byte order marker (BOM) should be read | |
| * at the start of the encoding. (default `true`) | |
| * @returns The decoded string. | |
| */ | |
| export const utf16Decode = ( | |
| input: Uint8Array, | |
| byteOrderMark = true, | |
| ): string => { | |
| // Need at least 2 bytes of data in UTF-16 encodings | |
| if (input.length <= 1) return String.fromCodePoint(REPLACEMENT); | |
| const byteOrder = byteOrderMark ? readBOM(input) : ByteOrder.BigEndian; | |
| // Skip byte order mark if needed | |
| let idx = byteOrderMark ? 2 : 0; | |
| const codePoints: number[] = []; | |
| while (input.length - idx >= 2) { | |
| const first = decodeValues(input[idx++], input[idx++], byteOrder); | |
| if (isHighSurrogate(first)) { | |
| if (input.length - idx < 2) { | |
| // Need at least 2 bytes left for the low surrogate that is required | |
| codePoints.push(REPLACEMENT); | |
| } else { | |
| const second = decodeValues(input[idx++], input[idx++], byteOrder); | |
| if (isLowSurrogate(second)) { | |
| codePoints.push(first, second); | |
| } else { | |
| // Low surrogates should always follow high surrogates | |
| codePoints.push(REPLACEMENT); | |
| } | |
| } | |
| } else if (isLowSurrogate(first)) { | |
| // High surrogates should always come first since `decodeValues()` | |
| // accounts for the byte ordering | |
| idx += 2; | |
| codePoints.push(REPLACEMENT); | |
| } else { | |
| codePoints.push(first); | |
| } | |
| } | |
| // There shouldn't be extra byte(s) left over | |
| if (idx < input.length) codePoints.push(REPLACEMENT); | |
| return String.fromCodePoint(...codePoints); | |
| }; | |
| /** | |
| * Returns `true` if the given `codePoint` is a high surrogate. | |
| * @param codePoint The code point to be evaluated. | |
| * | |
| * Reference: https://en.wikipedia.org/wiki/UTF-16#Description | |
| */ | |
| const isHighSurrogate = (codePoint: number) => | |
| codePoint >= 0xd800 && codePoint <= 0xdbff; | |
| /** | |
| * Returns `true` if the given `codePoint` is a low surrogate. | |
| * @param codePoint The code point to be evaluated. | |
| * | |
| * Reference: https://en.wikipedia.org/wiki/UTF-16#Description | |
| */ | |
| const isLowSurrogate = (codePoint: number) => | |
| codePoint >= 0xdc00 && codePoint <= 0xdfff; | |
| /** | |
| * Decodes the given utf-16 values first and second using the specified | |
| * byte order. | |
| * @param first The first byte of the encoding. | |
| * @param second The second byte of the encoding. | |
| * @param byteOrder The byte order of the encoding. | |
| * Reference: https://en.wikipedia.org/wiki/UTF-16#Examples | |
| */ | |
| const decodeValues = (first: number, second: number, byteOrder: ByteOrder) => { | |
| // Append the binary representation of the preceding byte by shifting the | |
| // first one 8 to the left and than applying a bitwise or-operator to append | |
| // the second one. | |
| if (byteOrder === ByteOrder.LittleEndian) return (second << 8) | first; | |
| if (byteOrder === ByteOrder.BigEndian) return (first << 8) | second; | |
| throw new Error(`Invalid byteOrder: ${byteOrder}`); | |
| }; | |
| /** | |
| * Returns whether the given array contains a byte order mark for the | |
| * UTF-16BE or UTF-16LE encoding. If it has neither, BigEndian is assumed. | |
| * | |
| * Reference: https://en.wikipedia.org/wiki/Byte_order_mark#UTF-16 | |
| * | |
| * @param bytes The byte array to be evaluated. | |
| */ | |
| // prettier-ignore | |
| const readBOM = (bytes: Uint8Array): ByteOrder => ( | |
| hasUtf16BigEndianBOM(bytes) ? ByteOrder.BigEndian | |
| : hasUtf16LittleEndianBOM(bytes) ? ByteOrder.LittleEndian | |
| : ByteOrder.BigEndian | |
| ); | |
| const hasUtf16BigEndianBOM = (bytes: Uint8Array) => | |
| bytes[0] === 0xfe && bytes[1] === 0xff; | |
| const hasUtf16LittleEndianBOM = (bytes: Uint8Array) => | |
| bytes[0] === 0xff && bytes[1] === 0xfe; | |
| export const hasUtf16BOM = (bytes: Uint8Array) => | |
| hasUtf16BigEndianBOM(bytes) || hasUtf16LittleEndianBOM(bytes); | |