/** Shared UTF-8 encoder, for strings that are well-formed. */
const utf8Encoder = new TextEncoder();
/** Helper for {@link encodeWtf8}: Is the given code unit a low surrogate? */
function isLowSurrogate(unit: number): boolean {
return (unit >= 0xdc00) && (unit <= 0xdfff);
}
/**
* Encodes a string as WTF-8 (). For a
* well-formed string, which is to say one with no lone surrogates, the result
* is the same as its UTF-8 encoding. A lone surrogate is encoded as the
* three-byte sequence UTF-8 would use for a code point of the same value, where
* `TextEncoder` substitutes the replacement character U+FFFD. So, unlike UTF-8,
* this encoding is one-to-one: two strings encode to the same bytes if and only
* if they are equal (`===`). The byte order of two encodings is the order of
* the strings by code point, the same as `utf8Compare()`.
*
* The returned array is exact-sized and unshared -- the caller may mutate it
* freely.
*/
export function encodeWtf8(value: string): Uint8Array {
if (value.isWellFormed()) {
return utf8Encoder.encode(value);
}
// No UTF-16 code unit takes more than three bytes: a character outside the
// Basic Multilingual Plane takes four, but it is two code units long.
const bytes = new Uint8Array(value.length * 3);
let at = 0;
let runStart = 0;
for (let i = 0; i < value.length; i++) {
const unit = value.charCodeAt(i);
if ((unit < 0xd800) || (unit > 0xdfff)) {
continue;
} else if ((unit <= 0xdbff) && isLowSurrogate(value.charCodeAt(i + 1))) {
i++; // A pair, which `TextEncoder` encodes along with the run.
continue;
}
// A lone surrogate. The run before it is well-formed, so `TextEncoder`
// encodes it as it would on its own. The surrogate takes the three-byte
// form of a code point of the same value, which starts `0xED` for every
// surrogate.
at += utf8Encoder.encodeInto(
value.slice(runStart, i),
bytes.subarray(at),
).written;
bytes[at++] = 0xed;
bytes[at++] = 0x80 | ((unit >> 6) & 0x3f);
bytes[at++] = 0x80 | (unit & 0x3f);
runStart = i + 1;
}
at += utf8Encoder.encodeInto(value.slice(runStart), bytes.subarray(at))
.written;
return (at === bytes.length) ? bytes : bytes.slice(0, at);
}