1 /*
2 * Copyright (c) Facebook, Inc. and its affiliates.
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17 #include "utf8.h"
18
19 #include "Log.h"
20
21 namespace facebook {
22 namespace jni {
23
24 namespace {
25
26 const uint16_t kUtf8OneByteBoundary = 0x80;
27 const uint16_t kUtf8TwoBytesBoundary = 0x800;
28 const uint16_t kUtf16HighSubLowBoundary = 0xD800;
29 const uint16_t kUtf16HighSubHighBoundary = 0xDC00;
30 const uint16_t kUtf16LowSubHighBoundary = 0xE000;
31
encode3ByteUTF8(char32_t code,uint8_t * out)32 inline void encode3ByteUTF8(char32_t code, uint8_t* out) {
33 if ((code & 0xffff0000) != 0) {
34 FBJNI_LOGF("3 byte utf-8 encodings only valid for up to 16 bits");
35 }
36
37 out[0] = 0xE0 | (code >> 12);
38 out[1] = 0x80 | ((code >> 6) & 0x3F);
39 out[2] = 0x80 | (code & 0x3F);
40 }
41
decode3ByteUTF8(const uint8_t * in)42 inline char32_t decode3ByteUTF8(const uint8_t* in) {
43 return (((in[0] & 0x0f) << 12) | ((in[1] & 0x3f) << 6) | (in[2] & 0x3f));
44 }
45
encode4ByteUTF8(char32_t code,std::string & out,size_t offset)46 inline void encode4ByteUTF8(char32_t code, std::string& out, size_t offset) {
47 if ((code & 0xfff80000) != 0) {
48 FBJNI_LOGF("4 byte utf-8 encodings only valid for up to 21 bits");
49 }
50
51 out[offset] = (char)(0xF0 | (code >> 18));
52 out[offset + 1] = (char)(0x80 | ((code >> 12) & 0x3F));
53 out[offset + 2] = (char)(0x80 | ((code >> 6) & 0x3F));
54 out[offset + 3] = (char)(0x80 | (code & 0x3F));
55 }
56
57 template <typename T>
isFourByteUTF8Encoding(const T * utf8)58 inline bool isFourByteUTF8Encoding(const T* utf8) {
59 return ((*utf8 & 0xF8) == 0xF0);
60 }
61
62 } // namespace
63
64 namespace detail {
65
modifiedLength(const std::string & str)66 size_t modifiedLength(const std::string& str) {
67 // Scan for supplementary characters
68 size_t j = 0;
69 for (size_t i = 0; i < str.size();) {
70 if (str[i] == 0) {
71 i += 1;
72 j += 2;
73 } else if (i + 4 > str.size() || !isFourByteUTF8Encoding(&(str[i]))) {
74 // See the code in utf8ToModifiedUTF8 for what's happening here.
75 i += 1;
76 j += 1;
77 } else {
78 i += 4;
79 j += 6;
80 }
81 }
82
83 return j;
84 }
85
86 // returns modified utf8 length; *length is set to strlen(str)
modifiedLength(const uint8_t * str,size_t * length)87 size_t modifiedLength(const uint8_t* str, size_t* length) {
88 // NUL-terminated: Scan for length and supplementary characters
89 size_t i = 0;
90 size_t j = 0;
91 while (str[i] != 0) {
92 if (str[i + 1] == 0 || str[i + 2] == 0 || str[i + 3] == 0 ||
93 !isFourByteUTF8Encoding(&(str[i]))) {
94 i += 1;
95 j += 1;
96 } else {
97 i += 4;
98 j += 6;
99 }
100 }
101
102 *length = i;
103 return j;
104 }
105
utf8ToModifiedUTF8(const uint8_t * utf8,size_t len,uint8_t * modified,size_t modifiedBufLen)106 void utf8ToModifiedUTF8(
107 const uint8_t* utf8,
108 size_t len,
109 uint8_t* modified,
110 size_t modifiedBufLen) {
111 size_t j = 0;
112 for (size_t i = 0; i < len;) {
113 if (j >= modifiedBufLen) {
114 FBJNI_LOGF("output buffer is too short");
115 }
116 if (utf8[i] == 0) {
117 if (j + 1 >= modifiedBufLen) {
118 FBJNI_LOGF("output buffer is too short");
119 }
120 modified[j] = 0xc0;
121 modified[j + 1] = 0x80;
122 i += 1;
123 j += 2;
124 continue;
125 }
126
127 if (i + 4 > len || !isFourByteUTF8Encoding(utf8 + i)) {
128 // If the input is too short for this to be a four-byte
129 // encoding, or it isn't one for real, just copy it on through.
130 modified[j] = utf8[i];
131 i++;
132 j++;
133 continue;
134 }
135
136 // Convert 4 bytes of input to 2 * 3 bytes of output
137 char32_t code =
138 (((utf8[i] & 0x07) << 18) | ((utf8[i + 1] & 0x3f) << 12) |
139 ((utf8[i + 2] & 0x3f) << 6) | (utf8[i + 3] & 0x3f));
140 char32_t first;
141 char32_t second;
142
143 if (code > 0x10ffff) {
144 // These could be valid utf-8, but cannot be represented as modified
145 // UTF-8, due to the 20-bit limit on that representation. Encode two
146 // replacement characters, so the expected output length lines up.
147 const char32_t kUnicodeReplacementChar = 0xfffd;
148 first = kUnicodeReplacementChar;
149 second = kUnicodeReplacementChar;
150 } else {
151 // split into surrogate pair
152 first = ((code - 0x010000) >> 10) | 0xd800;
153 second = ((code - 0x010000) & 0x3ff) | 0xdc00;
154 }
155
156 // encode each as a 3 byte surrogate value
157 if (j + 5 >= modifiedBufLen) {
158 FBJNI_LOGF("output buffer is too short");
159 }
160 encode3ByteUTF8(first, modified + j);
161 encode3ByteUTF8(second, modified + j + 3);
162 i += 4;
163 j += 6;
164 }
165
166 if (j >= modifiedBufLen) {
167 FBJNI_LOGF("output buffer is too short");
168 }
169 modified[j++] = '\0';
170 }
171
modifiedUTF8ToUTF8(const uint8_t * modified,size_t len)172 std::string modifiedUTF8ToUTF8(const uint8_t* modified, size_t len) noexcept {
173 // Converting from modified utf8 to utf8 will always shrink, so this will
174 // always be sufficient
175 std::string utf8(len, 0);
176 size_t j = 0;
177 for (size_t i = 0; i < len;) {
178 // surrogate pair: 1101 10xx xxxx xxxx 1101 11xx xxxx xxxx
179 // encoded pair: 1110 1101 1010 xxxx 10xx xxxx 1110 1101 1011 xxxx 10xx
180 // xxxx
181
182 if (len >= i + 6 && modified[i] == 0xed &&
183 (modified[i + 1] & 0xf0) == 0xa0 && modified[i + 3] == 0xed &&
184 (modified[i + 4] & 0xf0) == 0xb0) {
185 // Valid surrogate pair
186 char32_t pair1 = decode3ByteUTF8(modified + i);
187 char32_t pair2 = decode3ByteUTF8(modified + i + 3);
188 char32_t ch = 0x10000 + (((pair1 & 0x3ff) << 10) | (pair2 & 0x3ff));
189 encode4ByteUTF8(ch, utf8, j);
190 i += 6;
191 j += 4;
192 continue;
193 } else if (len >= i + 2 && modified[i] == 0xc0 && modified[i + 1] == 0x80) {
194 utf8[j] = 0;
195 i += 2;
196 j += 1;
197 continue;
198 }
199
200 // copy one byte. This might be a one, two, or three-byte encoding. It
201 // might be an invalid encoding of some sort, but garbage in garbage out is
202 // ok.
203
204 utf8[j] = (char)modified[i];
205 i++;
206 j++;
207 }
208
209 utf8.resize(j);
210
211 return utf8;
212 }
213
214 // Calculate how many bytes are needed to convert an UTF16 string into UTF8
215 // UTF16 string
utf16toUTF8Length(const uint16_t * utf16String,size_t utf16StringLen)216 size_t utf16toUTF8Length(const uint16_t* utf16String, size_t utf16StringLen) {
217 if (!utf16String || utf16StringLen == 0) {
218 return 0;
219 }
220
221 uint32_t utf8StringLen = 0;
222 auto utf16StringEnd = utf16String + utf16StringLen;
223 auto idx16 = utf16String;
224 while (idx16 < utf16StringEnd) {
225 auto ch = *idx16++;
226 if (ch < kUtf8OneByteBoundary) {
227 utf8StringLen++;
228 } else if (ch < kUtf8TwoBytesBoundary) {
229 utf8StringLen += 2;
230 } else if (
231 (ch >= kUtf16HighSubLowBoundary) && (ch < kUtf16HighSubHighBoundary) &&
232 (idx16 < utf16StringEnd) && (*idx16 >= kUtf16HighSubHighBoundary) &&
233 (*idx16 < kUtf16LowSubHighBoundary)) {
234 utf8StringLen += 4;
235 idx16++;
236 } else {
237 utf8StringLen += 3;
238 }
239 }
240
241 return utf8StringLen;
242 }
243
utf16toUTF8(const uint16_t * utf16String,size_t utf16StringLen)244 std::string utf16toUTF8(
245 const uint16_t* utf16String,
246 size_t utf16StringLen) noexcept {
247 if (!utf16String || utf16StringLen <= 0) {
248 return "";
249 }
250
251 std::string utf8String(utf16toUTF8Length(utf16String, utf16StringLen), '\0');
252 auto idx8 = utf8String.begin();
253 auto idx16 = utf16String;
254 auto utf16StringEnd = utf16String + utf16StringLen;
255 while (idx16 < utf16StringEnd) {
256 auto ch = *idx16++;
257 if (ch < kUtf8OneByteBoundary) {
258 *idx8++ = (ch & 0x7F);
259 } else if (ch < kUtf8TwoBytesBoundary) {
260 *idx8++ = 0b11000000 | (ch >> 6);
261 *idx8++ = 0b10000000 | (ch & 0x3F);
262 } else if (
263 (ch >= kUtf16HighSubLowBoundary) && (ch < kUtf16HighSubHighBoundary) &&
264 (idx16 < utf16StringEnd) && (*idx16 >= kUtf16HighSubHighBoundary) &&
265 (*idx16 < kUtf16LowSubHighBoundary)) {
266 auto ch2 = *idx16++;
267 uint8_t trunc_byte = (((ch >> 6) & 0x0F) + 1);
268 *idx8++ = 0b11110000 | (trunc_byte >> 2);
269 *idx8++ = 0b10000000 | ((trunc_byte & 0x03) << 4) | ((ch >> 2) & 0x0F);
270 *idx8++ = 0b10000000 | ((ch & 0x03) << 4) | ((ch2 >> 6) & 0x0F);
271 *idx8++ = 0b10000000 | (ch2 & 0x3F);
272 } else {
273 *idx8++ = 0b11100000 | (ch >> 12);
274 *idx8++ = 0b10000000 | ((ch >> 6) & 0x3F);
275 *idx8++ = 0b10000000 | (ch & 0x3F);
276 }
277 }
278
279 return utf8String;
280 }
281
282 } // namespace detail
283 } // namespace jni
284 } // namespace facebook
285