1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
|
#include "cru/common/StringUtil.hpp"
namespace cru {
namespace {
template <typename UInt, int number_of_bit, typename ReturnType>
inline std::enable_if_t<std::is_unsigned_v<UInt>, ReturnType> ExtractBits(
UInt n) {
return static_cast<ReturnType>(n & ((1u << number_of_bit) - 1));
}
} // namespace
CodePoint Utf8NextCodePoint(std::string_view str, Index current,
Index* next_position) {
CodePoint result;
if (current >= static_cast<Index>(str.length())) {
result = k_invalid_code_point;
} else {
const auto cu0 = static_cast<std::uint8_t>(str[current++]);
auto read_next_folowing_code = [&str, ¤t]() -> CodePoint {
if (current == static_cast<Index>(str.length()))
throw TextEncodeException(
"Unexpected end when read continuing byte of multi-byte code "
"point.");
const auto u = static_cast<std::uint8_t>(str[current]);
if (!(u & (1u << 7)) || (u & (1u << 6))) {
throw TextEncodeException(
"Unexpected bad-format (not 0b10xxxxxx) continuing byte of "
"multi-byte code point.");
}
return ExtractBits<std::uint8_t, 6, CodePoint>(str[current++]);
};
if ((1u << 7) & cu0) {
if ((1u << 6) & cu0) { // 2~4-length code point
if ((1u << 5) & cu0) { // 3~4-length code point
if ((1u << 4) & cu0) { // 4-length code point
if (cu0 & (1u << 3)) {
throw TextEncodeException(
"Unexpected bad-format begin byte (not 0b11110xxx) of 4-byte"
"code point.");
}
const CodePoint s0 = ExtractBits<std::uint8_t, 3, CodePoint>(cu0)
<< (6 * 3);
const CodePoint s1 = read_next_folowing_code() << (6 * 2);
const CodePoint s2 = read_next_folowing_code() << 6;
const CodePoint s3 = read_next_folowing_code();
result = s0 + s1 + s2 + s3;
} else { // 3-length code point
const CodePoint s0 = ExtractBits<std::uint8_t, 4, CodePoint>(cu0)
<< (6 * 2);
const CodePoint s1 = read_next_folowing_code() << 6;
const CodePoint s2 = read_next_folowing_code();
result = s0 + s1 + s2;
}
} else { // 2-length code point
const CodePoint s0 = ExtractBits<std::uint8_t, 5, CodePoint>(cu0)
<< 6;
const CodePoint s1 = read_next_folowing_code();
result = s0 + s1;
}
} else {
throw TextEncodeException(
"Unexpected bad-format (0b10xxxxxx) begin byte of a code point.");
}
} else {
result = static_cast<CodePoint>(cu0);
}
}
if (next_position != nullptr) *next_position = current;
return result;
}
CodePoint Utf16NextCodePoint(std::u16string_view str, Index current,
Index* next_position) {
CodePoint result;
if (current >= static_cast<Index>(str.length())) {
result = k_invalid_code_point;
} else {
const auto cu0 = str[current++];
if (!IsUtf16SurrogatePairCodeUnit(cu0)) { // 1-length code point
result = static_cast<CodePoint>(cu0);
} else if (IsUtf16SurrogatePairLeading(cu0)) { // 2-length code point
if (current >= static_cast<Index>(str.length())) {
throw TextEncodeException(
"Unexpected end when reading second code unit of surrogate pair.");
}
const auto cu1 = str[current++];
if (!IsUtf16SurrogatePairTrailing(cu1)) {
throw TextEncodeException(
"Unexpected bad-range second code unit of surrogate pair.");
}
const auto s0 = ExtractBits<std::uint16_t, 10, CodePoint>(cu0) << 10;
const auto s1 = ExtractBits<std::uint16_t, 10, CodePoint>(cu1);
result = s0 + s1 + 0x10000;
} else {
throw TextEncodeException(
"Unexpected bad-range first code unit of surrogate pair.");
}
}
if (next_position != nullptr) *next_position = current;
return result;
}
CodePoint Utf16PreviousCodePoint(std::u16string_view str, Index current,
Index* previous_position) {
CodePoint result;
if (current <= 0) {
result = k_invalid_code_point;
} else {
const auto cu0 = str[--current];
if (!IsUtf16SurrogatePairCodeUnit(cu0)) { // 1-length code point
result = static_cast<CodePoint>(cu0);
} else if (IsUtf16SurrogatePairTrailing(cu0)) { // 2-length code point
if (current <= 0) {
throw TextEncodeException(
"Unexpected end when reading first code unit of surrogate pair.");
}
const auto cu1 = str[--current];
if (!IsUtf16SurrogatePairLeading(cu1)) {
throw TextEncodeException(
"Unexpected bad-range first code unit of surrogate pair.");
}
const auto s0 = ExtractBits<std::uint16_t, 10, CodePoint>(cu1) << 10;
const auto s1 = ExtractBits<std::uint16_t, 10, CodePoint>(cu0);
result = s0 + s1 + 0x10000;
} else {
throw TextEncodeException(
"Unexpected bad-range second code unit of surrogate pair.");
}
}
if (previous_position != nullptr) *previous_position = current;
return result;
}
void Utf8EncodeCodePointAppend(CodePoint code_point, std::string& str) {
auto write_continue_byte = [&str](std::uint8_t byte6) {
str.push_back((1u << 7) + (((1u << 6) - 1) & byte6));
};
if (code_point >= 0 && code_point <= 0x007F) {
str.push_back(static_cast<char>(code_point));
} else if (code_point >= 0x0080 && code_point <= 0x07FF) {
std::uint32_t unsigned_code_point = code_point;
str.push_back(static_cast<char>(ExtractBits<std::uint32_t, 5, std::uint8_t>(
(unsigned_code_point >> 6)) +
0b11000000));
write_continue_byte(
ExtractBits<std::uint32_t, 6, std::uint8_t>(unsigned_code_point));
} else if (code_point >= 0x0800 && code_point <= 0xFFFF) {
std::uint32_t unsigned_code_point = code_point;
str.push_back(static_cast<char>(ExtractBits<std::uint32_t, 4, std::uint8_t>(
(unsigned_code_point >> (6 * 2))) +
0b11100000));
write_continue_byte(
ExtractBits<std::uint32_t, 6, std::uint8_t>(unsigned_code_point >> 6));
write_continue_byte(
ExtractBits<std::uint32_t, 6, std::uint8_t>(unsigned_code_point));
} else if (code_point >= 0x10000 && code_point <= 0x10FFFF) {
std::uint32_t unsigned_code_point = code_point;
str.push_back(static_cast<char>(ExtractBits<std::uint32_t, 3, std::uint8_t>(
(unsigned_code_point >> (6 * 3))) +
0b11110000));
write_continue_byte(ExtractBits<std::uint32_t, 6, std::uint8_t>(
unsigned_code_point >> (6 * 2)));
write_continue_byte(
ExtractBits<std::uint32_t, 6, std::uint8_t>(unsigned_code_point >> 6));
write_continue_byte(
ExtractBits<std::uint32_t, 6, std::uint8_t>(unsigned_code_point));
} else {
throw TextEncodeException("Code point out of range.");
}
}
void Utf16EncodeCodePointAppend(CodePoint code_point, std::u16string& str) {
if (code_point >= 0 && code_point <= 0xD7FF ||
code_point >= 0xE000 && code_point <= 0xFFFF) {
str.push_back(static_cast<char16_t>(code_point));
} else if (code_point >= 0x10000 && code_point <= 0x10FFFF) {
std::uint32_t u = code_point - 0x10000;
str.push_back(static_cast<char16_t>(
ExtractBits<std::uint32_t, 10, std::uint32_t>(u >> 10) + 0xD800u));
str.push_back(static_cast<char16_t>(
ExtractBits<std::uint32_t, 10, std::uint32_t>(u) + 0xDC00u));
} else {
throw TextEncodeException("Code point out of range.");
}
}
std::string ToUtf8(std::u16string_view s) {
std::string result;
for (CodePoint cp : Utf16CodePointIterator{s}) {
Utf8EncodeCodePointAppend(cp, result);
}
return result;
}
std::u16string ToUtf16(std::string_view s) {
std::u16string result;
for (CodePoint cp : Utf8CodePointIterator{s}) {
Utf16EncodeCodePointAppend(cp, result);
}
return result;
}
} // namespace cru
|
