Searched defs:BertTokenizerOptions (Results 1 – 2 of 2) sorted by relevance
59 struct BertTokenizerOptions { struct60 int max_bytes_per_token = kDefaultMaxBytesPerToken;61 int max_chars_per_subtoken = kDefaultMaxCharsPerSubToken;62 std::string suffix_indicator = kDefaultSuffixIndicator;63 bool use_unknown_token = kDefaultUseUnknownToken;64 std::string unknown_token = kDefaultUnknownToken;65 bool split_unknown_chars = kDefaultSplitUnknownChars;66 std::string delim_str = kDefaultDelimRe;67 std::string include_delim_str = kDefaultIncludeDelimRe;
58 struct BertTokenizerOptions { struct59 int max_bytes_per_token = kDefaultMaxBytesPerToken;60 int max_chars_per_subtoken = kDefaultMaxCharsPerSubToken;61 std::string suffix_indicator = kDefaultSuffixIndicator;62 bool use_unknown_token = kDefaultUseUnknownToken;63 std::string unknown_token = kDefaultUnknownToken;64 bool split_unknown_chars = kDefaultSplitUnknownChars;