1 // © 2016 and later: Unicode, Inc. and others.
2 // License & terms of use: http://www.unicode.org/copyright.html
3 /*
4 ************************************************************************************
5 * Copyright (C) 2006-2016, International Business Machines Corporation
6 * and others. All Rights Reserved.
7 ************************************************************************************
8 */
9
10 #include "unicode/utypes.h"
11
12 #if !UCONFIG_NO_BREAK_ITERATION
13
14 #include "unicode/uchar.h"
15 #include "unicode/uniset.h"
16 #include "unicode/chariter.h"
17 #include "unicode/ures.h"
18 #include "unicode/udata.h"
19 #include "unicode/putil.h"
20 #include "unicode/ustring.h"
21 #include "unicode/uscript.h"
22 #include "unicode/ucharstrie.h"
23 #include "unicode/bytestrie.h"
24 #include "unicode/rbbi.h"
25
26 #include "brkeng.h"
27 #include "cmemory.h"
28 #include "dictbe.h"
29 #include "lstmbe.h"
30 #include "charstr.h"
31 #include "dictionarydata.h"
32 #include "mutex.h"
33 #include "uvector.h"
34 #include "umutex.h"
35 #include "uresimp.h"
36 #include "ubrkimpl.h"
37
38 U_NAMESPACE_BEGIN
39
40 /*
41 ******************************************************************
42 */
43
LanguageBreakEngine()44 LanguageBreakEngine::LanguageBreakEngine() {
45 }
46
~LanguageBreakEngine()47 LanguageBreakEngine::~LanguageBreakEngine() {
48 }
49
50 /*
51 ******************************************************************
52 */
53
LanguageBreakFactory()54 LanguageBreakFactory::LanguageBreakFactory() {
55 }
56
~LanguageBreakFactory()57 LanguageBreakFactory::~LanguageBreakFactory() {
58 }
59
60 /*
61 ******************************************************************
62 */
63
UnhandledEngine(UErrorCode & status)64 UnhandledEngine::UnhandledEngine(UErrorCode &status) : fHandled(nullptr) {
65 (void)status;
66 }
67
~UnhandledEngine()68 UnhandledEngine::~UnhandledEngine() {
69 delete fHandled;
70 fHandled = nullptr;
71 }
72
73 UBool
handles(UChar32 c,const char * locale) const74 UnhandledEngine::handles(UChar32 c, const char* locale) const {
75 (void)locale; // Unused
76 return fHandled && fHandled->contains(c);
77 }
78
79 int32_t
findBreaks(UText * text,int32_t startPos,int32_t endPos,UVector32 &,UBool,UErrorCode & status) const80 UnhandledEngine::findBreaks( UText *text,
81 int32_t startPos,
82 int32_t endPos,
83 UVector32 &/*foundBreaks*/,
84 UBool /* isPhraseBreaking */,
85 UErrorCode &status) const {
86 if (U_FAILURE(status)) return 0;
87 utext_setNativeIndex(text, startPos);
88 UChar32 c = utext_current32(text);
89 while((int32_t)utext_getNativeIndex(text) < endPos && fHandled->contains(c)) {
90 utext_next32(text); // TODO: recast loop to work with post-increment operations.
91 c = utext_current32(text);
92 }
93 return 0;
94 }
95
96 void
handleCharacter(UChar32 c)97 UnhandledEngine::handleCharacter(UChar32 c) {
98 if (fHandled == nullptr) {
99 fHandled = new UnicodeSet();
100 if (fHandled == nullptr) {
101 return;
102 }
103 }
104 if (!fHandled->contains(c)) {
105 UErrorCode status = U_ZERO_ERROR;
106 // Apply the entire script of the character.
107 int32_t script = u_getIntPropertyValue(c, UCHAR_SCRIPT);
108 fHandled->applyIntPropertyValue(UCHAR_SCRIPT, script, status);
109 }
110 }
111
112 /*
113 ******************************************************************
114 */
115
ICULanguageBreakFactory(UErrorCode &)116 ICULanguageBreakFactory::ICULanguageBreakFactory(UErrorCode &/*status*/) {
117 fEngines = 0;
118 }
119
~ICULanguageBreakFactory()120 ICULanguageBreakFactory::~ICULanguageBreakFactory() {
121 if (fEngines != 0) {
122 delete fEngines;
123 }
124 }
125
ensureEngines(UErrorCode & status)126 void ICULanguageBreakFactory::ensureEngines(UErrorCode& status) {
127 static UMutex gBreakEngineMutex;
128 Mutex m(&gBreakEngineMutex);
129 if (fEngines == nullptr) {
130 LocalPointer<UStack> engines(new UStack(uprv_deleteUObject, nullptr, status), status);
131 if (U_SUCCESS(status)) {
132 fEngines = engines.orphan();
133 }
134 }
135 }
136
137 const LanguageBreakEngine *
getEngineFor(UChar32 c,const char * locale)138 ICULanguageBreakFactory::getEngineFor(UChar32 c, const char* locale) {
139 const LanguageBreakEngine *lbe = nullptr;
140 UErrorCode status = U_ZERO_ERROR;
141 ensureEngines(status);
142 if (U_FAILURE(status) ) {
143 // Note: no way to return error code to caller.
144 return nullptr;
145 }
146
147 static UMutex gBreakEngineMutex;
148 Mutex m(&gBreakEngineMutex);
149 int32_t i = fEngines->size();
150 while (--i >= 0) {
151 lbe = (const LanguageBreakEngine *)(fEngines->elementAt(i));
152 if (lbe != nullptr && lbe->handles(c, locale)) {
153 return lbe;
154 }
155 }
156
157 // We didn't find an engine. Create one.
158 lbe = loadEngineFor(c, locale);
159 if (lbe != nullptr) {
160 fEngines->push((void *)lbe, status);
161 }
162 return U_SUCCESS(status) ? lbe : nullptr;
163 }
164
165 const LanguageBreakEngine *
loadEngineFor(UChar32 c,const char *)166 ICULanguageBreakFactory::loadEngineFor(UChar32 c, const char*) {
167 UErrorCode status = U_ZERO_ERROR;
168 UScriptCode code = uscript_getScript(c, &status);
169 if (U_SUCCESS(status)) {
170 const LanguageBreakEngine *engine = nullptr;
171 // Try to use LSTM first
172 const LSTMData *data = CreateLSTMDataForScript(code, status);
173 if (U_SUCCESS(status)) {
174 if (data != nullptr) {
175 engine = CreateLSTMBreakEngine(code, data, status);
176 if (U_SUCCESS(status) && engine != nullptr) {
177 return engine;
178 }
179 if (engine != nullptr) {
180 delete engine;
181 engine = nullptr;
182 } else {
183 DeleteLSTMData(data);
184 }
185 }
186 }
187 status = U_ZERO_ERROR; // fallback to dictionary based
188 DictionaryMatcher *m = loadDictionaryMatcherFor(code);
189 if (m != nullptr) {
190 switch(code) {
191 case USCRIPT_THAI:
192 engine = new ThaiBreakEngine(m, status);
193 break;
194 case USCRIPT_LAO:
195 engine = new LaoBreakEngine(m, status);
196 break;
197 case USCRIPT_MYANMAR:
198 engine = new BurmeseBreakEngine(m, status);
199 break;
200 case USCRIPT_KHMER:
201 engine = new KhmerBreakEngine(m, status);
202 break;
203
204 #if !UCONFIG_NO_NORMALIZATION
205 // CJK not available w/o normalization
206 case USCRIPT_HANGUL:
207 engine = new CjkBreakEngine(m, kKorean, status);
208 break;
209
210 // use same BreakEngine and dictionary for both Chinese and Japanese
211 case USCRIPT_HIRAGANA:
212 case USCRIPT_KATAKANA:
213 case USCRIPT_HAN:
214 engine = new CjkBreakEngine(m, kChineseJapanese, status);
215 break;
216 #if 0
217 // TODO: Have to get some characters with script=common handled
218 // by CjkBreakEngine (e.g. U+309B). Simply subjecting
219 // them to CjkBreakEngine does not work. The engine has to
220 // special-case them.
221 case USCRIPT_COMMON:
222 {
223 UBlockCode block = ublock_getCode(code);
224 if (block == UBLOCK_HIRAGANA || block == UBLOCK_KATAKANA)
225 engine = new CjkBreakEngine(dict, kChineseJapanese, status);
226 break;
227 }
228 #endif
229 #endif
230
231 default:
232 break;
233 }
234 if (engine == nullptr) {
235 delete m;
236 }
237 else if (U_FAILURE(status)) {
238 delete engine;
239 engine = nullptr;
240 }
241 return engine;
242 }
243 }
244 return nullptr;
245 }
246
247 DictionaryMatcher *
loadDictionaryMatcherFor(UScriptCode script)248 ICULanguageBreakFactory::loadDictionaryMatcherFor(UScriptCode script) {
249 UErrorCode status = U_ZERO_ERROR;
250 // open root from brkitr tree.
251 UResourceBundle *b = ures_open(U_ICUDATA_BRKITR, "", &status);
252 b = ures_getByKeyWithFallback(b, "dictionaries", b, &status);
253 int32_t dictnlength = 0;
254 const char16_t *dictfname =
255 ures_getStringByKeyWithFallback(b, uscript_getShortName(script), &dictnlength, &status);
256 if (U_FAILURE(status)) {
257 ures_close(b);
258 return nullptr;
259 }
260 CharString dictnbuf;
261 CharString ext;
262 const char16_t *extStart = u_memrchr(dictfname, 0x002e, dictnlength); // last dot
263 if (extStart != nullptr) {
264 int32_t len = (int32_t)(extStart - dictfname);
265 ext.appendInvariantChars(UnicodeString(false, extStart + 1, dictnlength - len - 1), status);
266 dictnlength = len;
267 }
268 dictnbuf.appendInvariantChars(UnicodeString(false, dictfname, dictnlength), status);
269 ures_close(b);
270
271 UDataMemory *file = udata_open(U_ICUDATA_BRKITR, ext.data(), dictnbuf.data(), &status);
272 if (U_SUCCESS(status)) {
273 // build trie
274 const uint8_t *data = (const uint8_t *)udata_getMemory(file);
275 const int32_t *indexes = (const int32_t *)data;
276 const int32_t offset = indexes[DictionaryData::IX_STRING_TRIE_OFFSET];
277 const int32_t trieType = indexes[DictionaryData::IX_TRIE_TYPE] & DictionaryData::TRIE_TYPE_MASK;
278 DictionaryMatcher *m = nullptr;
279 if (trieType == DictionaryData::TRIE_TYPE_BYTES) {
280 const int32_t transform = indexes[DictionaryData::IX_TRANSFORM];
281 const char *characters = (const char *)(data + offset);
282 m = new BytesDictionaryMatcher(characters, transform, file);
283 }
284 else if (trieType == DictionaryData::TRIE_TYPE_UCHARS) {
285 const char16_t *characters = (const char16_t *)(data + offset);
286 m = new UCharsDictionaryMatcher(characters, file);
287 }
288 if (m == nullptr) {
289 // no matcher exists to take ownership - either we are an invalid
290 // type or memory allocation failed
291 udata_close(file);
292 }
293 return m;
294 } else if (dictfname != nullptr) {
295 // we don't have a dictionary matcher.
296 // returning nullptr here will cause us to fail to find a dictionary break engine, as expected
297 status = U_ZERO_ERROR;
298 return nullptr;
299 }
300 return nullptr;
301 }
302
303
addExternalEngine(ExternalBreakEngine * external,UErrorCode & status)304 void ICULanguageBreakFactory::addExternalEngine(
305 ExternalBreakEngine* external, UErrorCode& status) {
306 LocalPointer<ExternalBreakEngine> engine(external, status);
307 ensureEngines(status);
308 LocalPointer<BreakEngineWrapper> wrapper(
309 new BreakEngineWrapper(engine.orphan(), status), status);
310 static UMutex gBreakEngineMutex;
311 Mutex m(&gBreakEngineMutex);
312 fEngines->push(wrapper.getAlias(), status);
313 wrapper.orphan();
314 }
315
BreakEngineWrapper(ExternalBreakEngine * engine,UErrorCode & status)316 BreakEngineWrapper::BreakEngineWrapper(
317 ExternalBreakEngine* engine, UErrorCode &status) : delegate(engine, status) {
318 }
319
~BreakEngineWrapper()320 BreakEngineWrapper::~BreakEngineWrapper() {
321 }
322
handles(UChar32 c,const char * locale) const323 UBool BreakEngineWrapper::handles(UChar32 c, const char* locale) const {
324 return delegate->isFor(c, locale);
325 }
326
findBreaks(UText * text,int32_t startPos,int32_t endPos,UVector32 & foundBreaks,UBool,UErrorCode & status) const327 int32_t BreakEngineWrapper::findBreaks(
328 UText *text,
329 int32_t startPos,
330 int32_t endPos,
331 UVector32 &foundBreaks,
332 UBool /* isPhraseBreaking */,
333 UErrorCode &status) const {
334 if (U_FAILURE(status)) return 0;
335 int32_t result = 0;
336
337 // Find the span of characters included in the set.
338 // The span to break begins at the current position in the text, and
339 // extends towards the start or end of the text, depending on 'reverse'.
340
341 utext_setNativeIndex(text, startPos);
342 int32_t start = (int32_t)utext_getNativeIndex(text);
343 int32_t current;
344 int32_t rangeStart;
345 int32_t rangeEnd;
346 UChar32 c = utext_current32(text);
347 while((current = (int32_t)utext_getNativeIndex(text)) < endPos && delegate->handles(c)) {
348 utext_next32(text); // TODO: recast loop for postincrement
349 c = utext_current32(text);
350 }
351 rangeStart = start;
352 rangeEnd = current;
353 int32_t beforeSize = foundBreaks.size();
354 int32_t additionalCapacity = rangeEnd - rangeStart + 1;
355 // enlarge to contains (rangeEnd-rangeStart+1) more items
356 foundBreaks.ensureCapacity(beforeSize+additionalCapacity, status);
357 if (U_FAILURE(status)) return 0;
358 foundBreaks.setSize(beforeSize + beforeSize+additionalCapacity);
359 result = delegate->fillBreaks(text, rangeStart, rangeEnd, foundBreaks.getBuffer()+beforeSize,
360 additionalCapacity, status);
361 if (U_FAILURE(status)) return 0;
362 foundBreaks.setSize(beforeSize + result);
363 utext_setNativeIndex(text, current);
364 return result;
365 }
366
367 U_NAMESPACE_END
368
369 #endif /* #if !UCONFIG_NO_BREAK_ITERATION */
370