1 // © 2016 and later: Unicode, Inc. and others. 2 // License & terms of use: http://www.unicode.org/copyright.html 3 /* 4 ******************************************************************************* 5 * 6 * Copyright (C) 1999-2011, International Business Machines 7 * Corporation and others. All Rights Reserved. 8 * 9 ******************************************************************************* 10 * file name: utf.h 11 * encoding: UTF-8 12 * tab size: 8 (not used) 13 * indentation:4 14 * 15 * created on: 1999sep09 16 * created by: Markus W. Scherer 17 */ 18 19 /** 20 * @addtogroup icu4c ICU4C 21 * @{ 22 * \file 23 * \brief C API: Code point macros 24 * 25 * This file defines macros for checking whether a code point is 26 * a surrogate or a non-character etc. 27 * 28 * If U_NO_DEFAULT_INCLUDE_UTF_HEADERS is 0 then utf.h is included by utypes.h 29 * and itself includes utf8.h and utf16.h after some 30 * common definitions. 31 * If U_NO_DEFAULT_INCLUDE_UTF_HEADERS is 1 then each of these headers must be 32 * included explicitly if their definitions are used. 33 * 34 * utf8.h and utf16.h define macros for efficiently getting code points 35 * in and out of UTF-8/16 strings. 36 * utf16.h macros have "U16_" prefixes. 37 * utf8.h defines similar macros with "U8_" prefixes for UTF-8 string handling. 38 * 39 * ICU mostly processes 16-bit Unicode strings. 40 * Most of the time, such strings are well-formed UTF-16. 41 * Single, unpaired surrogates must be handled as well, and are treated in ICU 42 * like regular code points where possible. 43 * (Pairs of surrogate code points are indistinguishable from supplementary 44 * code points encoded as pairs of supplementary code units.) 45 * 46 * In fact, almost all Unicode code points in normal text (>99%) 47 * are on the BMP (<=U+ffff) and even <=U+d7ff. 48 * ICU functions handle supplementary code points (U+10000..U+10ffff) 49 * but are optimized for the much more frequently occurring BMP code points. 50 * 51 * umachine.h defines UChar to be an unsigned 16-bit integer. 52 * Since ICU 59, ICU uses char16_t in C++, UChar only in C, 53 * and defines UChar=char16_t by default. See the UChar API docs for details. 54 * 55 * UChar32 is defined to be a signed 32-bit integer (int32_t), large enough for a 21-bit 56 * Unicode code point (Unicode scalar value, 0..0x10ffff) and U_SENTINEL (-1). 57 * Before ICU 2.4, the definition of UChar32 was similarly platform-dependent as 58 * the definition of UChar. For details see the documentation for UChar32 itself. 59 * 60 * utf.h defines a small number of C macros for single Unicode code points. 61 * These are simple checks for surrogates and non-characters. 62 * For actual Unicode character properties see uchar.h. 63 * 64 * By default, string operations must be done with error checking in case 65 * a string is not well-formed UTF-16 or UTF-8. 66 * 67 * The U16_ macros detect if a surrogate code unit is unpaired 68 * (lead unit without trail unit or vice versa) and just return the unit itself 69 * as the code point. 70 * 71 * The U8_ macros detect illegal byte sequences and return a negative value. 72 * Starting with ICU 60, the observable length of a single illegal byte sequence 73 * skipped by one of these macros follows the Unicode 6+ recommendation 74 * which is consistent with the W3C Encoding Standard. 75 * 76 * There are ..._OR_FFFD versions of both U16_ and U8_ macros 77 * that return U+FFFD for illegal code unit sequences. 78 * 79 * The regular "safe" macros require that the initial, passed-in string index 80 * is within bounds. They only check the index when they read more than one 81 * code unit. This is usually done with code similar to the following loop: 82 * <pre>while(i<length) { 83 * U16_NEXT(s, i, length, c); 84 * // use c 85 * }</pre> 86 * 87 * When it is safe to assume that text is well-formed UTF-16 88 * (does not contain single, unpaired surrogates), then one can use 89 * U16_..._UNSAFE macros. 90 * These do not check for proper code unit sequences or truncated text and may 91 * yield wrong results or even cause a crash if they are used with "malformed" 92 * text. 93 * In practice, U16_..._UNSAFE macros will produce slightly less code but 94 * should not be faster because the processing is only different when a 95 * surrogate code unit is detected, which will be rare. 96 * 97 * Similarly for UTF-8, there are "safe" macros without a suffix, 98 * and U8_..._UNSAFE versions. 99 * The performance differences are much larger here because UTF-8 provides so 100 * many opportunities for malformed sequences. 101 * The unsafe UTF-8 macros are entirely implemented inside the macro definitions 102 * and are fast, while the safe UTF-8 macros call functions for some complicated cases. 103 * 104 * Unlike with UTF-16, malformed sequences cannot be expressed with distinct 105 * code point values (0..U+10ffff). They are indicated with negative values instead. 106 * 107 * For more information see the ICU User Guide Strings chapter 108 * (https://unicode-org.github.io/icu/userguide/strings). 109 * 110 * <em>Usage:</em> 111 * ICU coding guidelines for if() statements should be followed when using these macros. 112 * Compound statements (curly braces {}) must be used for if-else-while... 113 * bodies and all macro statements should be terminated with semicolon. 114 * 115 * \xrefitem stable "Stable" "Stable List" ICU 2.4 116 */ 117 118 #ifndef __UTF_H__ 119 #define __UTF_H__ 120 121 #include "unicode/umachine.h" 122 /* include the utfXX.h after the following definitions */ 123 124 /* single-code point definitions -------------------------------------------- */ 125 126 /** 127 * Is this code point a Unicode noncharacter? 128 * @param c 32-bit code point 129 * @return true or false 130 * \xrefitem stable "Stable" "Stable List" ICU 2.4 131 */ 132 #define U_IS_UNICODE_NONCHAR(c) \ 133 ((c)>=0xfdd0 && \ 134 ((c)<=0xfdef || ((c)&0xfffe)==0xfffe) && (c)<=0x10ffff) 135 136 /** 137 * Is c a Unicode code point value (0..U+10ffff) 138 * that can be assigned a character? 139 * 140 * Code points that are not characters include: 141 * - single surrogate code points (U+d800..U+dfff, 2048 code points) 142 * - the last two code points on each plane (U+__fffe and U+__ffff, 34 code points) 143 * - U+fdd0..U+fdef (new with Unicode 3.1, 32 code points) 144 * - the highest Unicode code point value is U+10ffff 145 * 146 * This means that all code points below U+d800 are character code points, 147 * and that boundary is tested first for performance. 148 * 149 * @param c 32-bit code point 150 * @return true or false 151 * \xrefitem stable "Stable" "Stable List" ICU 2.4 152 */ 153 #define U_IS_UNICODE_CHAR(c) \ 154 ((uint32_t)(c)<0xd800 || \ 155 (0xdfff<(c) && (c)<=0x10ffff && !U_IS_UNICODE_NONCHAR(c))) 156 157 /** 158 * Is this code point a BMP code point (U+0000..U+ffff)? 159 * @param c 32-bit code point 160 * @return true or false 161 * \xrefitem stable "Stable" "Stable List" ICU 2.8 162 */ 163 #define U_IS_BMP(c) ((uint32_t)(c)<=0xffff) 164 165 /** 166 * Is this code point a supplementary code point (U+10000..U+10ffff)? 167 * @param c 32-bit code point 168 * @return true or false 169 * \xrefitem stable "Stable" "Stable List" ICU 2.8 170 */ 171 #define U_IS_SUPPLEMENTARY(c) ((uint32_t)((c)-0x10000)<=0xfffff) 172 173 /** 174 * Is this code point a lead surrogate (U+d800..U+dbff)? 175 * @param c 32-bit code point 176 * @return true or false 177 * \xrefitem stable "Stable" "Stable List" ICU 2.4 178 */ 179 #define U_IS_LEAD(c) (((c)&0xfffffc00)==0xd800) 180 181 /** 182 * Is this code point a trail surrogate (U+dc00..U+dfff)? 183 * @param c 32-bit code point 184 * @return true or false 185 * \xrefitem stable "Stable" "Stable List" ICU 2.4 186 */ 187 #define U_IS_TRAIL(c) (((c)&0xfffffc00)==0xdc00) 188 189 /** 190 * Is this code point a surrogate (U+d800..U+dfff)? 191 * @param c 32-bit code point 192 * @return true or false 193 * \xrefitem stable "Stable" "Stable List" ICU 2.4 194 */ 195 #define U_IS_SURROGATE(c) (((c)&0xfffff800)==0xd800) 196 197 /** 198 * Assuming c is a surrogate code point (U_IS_SURROGATE(c)), 199 * is it a lead surrogate? 200 * @param c 32-bit code point 201 * @return true or false 202 * \xrefitem stable "Stable" "Stable List" ICU 2.4 203 */ 204 #define U_IS_SURROGATE_LEAD(c) (((c)&0x400)==0) 205 206 /** 207 * Assuming c is a surrogate code point (U_IS_SURROGATE(c)), 208 * is it a trail surrogate? 209 * @param c 32-bit code point 210 * @return true or false 211 * \xrefitem stable "Stable" "Stable List" ICU 4.2 212 */ 213 #define U_IS_SURROGATE_TRAIL(c) (((c)&0x400)!=0) 214 215 /* include the utfXX.h ------------------------------------------------------ */ 216 217 #if !U_NO_DEFAULT_INCLUDE_UTF_HEADERS 218 219 #include "unicode/utf8.h" 220 #include "unicode/utf16.h" 221 222 /* utf_old.h contains deprecated, pre-ICU 2.4 definitions */ 223 #include "unicode/utf_old.h" 224 225 #endif /* !U_NO_DEFAULT_INCLUDE_UTF_HEADERS */ 226 227 #endif /* __UTF_H__ */ 228 229 /** @} */ // addtogroup 230