Rev 4199 | Rev 4205 | Go to most recent revision | Show entire file | Ignore whitespace | Details | Blame | Last modification | View Log | RSS feed
Rev 4199 | Rev 4200 | ||
---|---|---|---|
Line 54... | Line 54... | ||
54 | #define HI_MASK_8(n) (~LO_MASK_8(8 - (n))) |
54 | #define HI_MASK_8(n) (~LO_MASK_8(8 - (n))) |
55 | 55 | ||
56 | /** Number of data bits in a UTF-8 continuation byte. */ |
56 | /** Number of data bits in a UTF-8 continuation byte. */ |
57 | #define CONT_BITS 6 |
57 | #define CONT_BITS 6 |
58 | 58 | ||
59 | /** Decode a single UTF-8 character from a NULL-terminated string. |
59 | /** Decode a single character from a substring. |
60 | * |
60 | * |
61 | * Decode a single UTF-8 character from a plain char NULL-terminated |
61 | * Decode a single character from a substring of size @a sz. Decoding starts |
62 | * string. Decoding starts at @index and this index is moved to the |
62 | * at @a offset and this offset is moved to the beginning of the next |
63 | * beginning of the next character. In case of decoding error, |
63 | * character. In case of decoding error, offset generally advances at least |
64 | * index advances. However, index is never moved beyond (str+limit). |
64 | * by one. However, offset is never moved beyond (str + sz). |
65 | * |
65 | * |
66 | * @param str Plain character NULL-terminated string. |
66 | * @param str String (not necessarily NULL-terminated). |
67 | * @param index Index (counted in plain characters) where to start |
67 | * @param index Index (counted in plain characters) where to start |
68 | * the decoding. |
68 | * the decoding. |
69 | * @param limit Maximal allowed value of index. |
69 | * @param limit Size of the substring. |
70 | * |
70 | * |
71 | * @return Decoded character in UTF-32 or '?' if the encoding is wrong. |
71 | * @return Value of decoded character or '?' on decoding error. |
72 | * |
72 | * |
73 | */ |
73 | */ |
74 | wchar_t utf8_decode(const char *str, index_t *index, index_t limit) |
74 | wchar_t chr_decode(const char *str, size_t *offset, size_t sz) |
75 | { |
75 | { |
76 | uint8_t b0, b; /* Bytes read from str. */ |
76 | uint8_t b0, b; /* Bytes read from str. */ |
77 | wchar_t ch; |
77 | wchar_t ch; |
78 | 78 | ||
79 | int b0_bits; /* Data bits in first byte. */ |
79 | int b0_bits; /* Data bits in first byte. */ |
80 | int cbytes; /* Number of continuation bytes. */ |
80 | int cbytes; /* Number of continuation bytes. */ |
81 | 81 | ||
82 | if (*index + 1 > limit) |
82 | if (*offset + 1 > sz) |
83 | return invalch; |
83 | return invalch; |
84 | 84 | ||
85 | b0 = (uint8_t) str[(*index)++]; |
85 | b0 = (uint8_t) str[(*offset)++]; |
86 | 86 | ||
87 | /* Determine code length. */ |
87 | /* Determine code length. */ |
88 | 88 | ||
89 | if ((b0 & 0x80) == 0) { |
89 | if ((b0 & 0x80) == 0) { |
90 | /* 0xxxxxxx (Plain ASCII) */ |
90 | /* 0xxxxxxx (Plain ASCII) */ |
Line 105... | Line 105... | ||
105 | } else { |
105 | } else { |
106 | /* 10xxxxxx -- unexpected continuation byte. */ |
106 | /* 10xxxxxx -- unexpected continuation byte. */ |
107 | return invalch; |
107 | return invalch; |
108 | } |
108 | } |
109 | 109 | ||
110 | if (*index + cbytes > limit) { |
110 | if (*offset + cbytes > sz) { |
111 | return invalch; |
111 | return invalch; |
112 | } |
112 | } |
113 | 113 | ||
114 | ch = b0 & LO_MASK_8(b0_bits); |
114 | ch = b0 & LO_MASK_8(b0_bits); |
115 | 115 | ||
116 | /* Decode continuation bytes. */ |
116 | /* Decode continuation bytes. */ |
117 | while (cbytes > 0) { |
117 | while (cbytes > 0) { |
118 | b = (uint8_t) str[(*index)++]; |
118 | b = (uint8_t) str[(*offset)++]; |
119 | 119 | ||
120 | /* Must be 10xxxxxx. */ |
120 | /* Must be 10xxxxxx. */ |
121 | if ((b & 0xc0) != 0x80) { |
121 | if ((b & 0xc0) != 0x80) { |
122 | return invalch; |
122 | return invalch; |
123 | } |
123 | } |
Line 128... | Line 128... | ||
128 | } |
128 | } |
129 | 129 | ||
130 | return ch; |
130 | return ch; |
131 | } |
131 | } |
132 | 132 | ||
133 | /** Encode a single UTF-32 character as UTF-8 |
- | |
134 | * |
- | |
135 | * Encode a single UTF-32 character as UTF-8 and store it into |
133 | /** Encode a single character to string representation. |
136 | * the given buffer at @index. Encoding starts at @index and |
- | |
137 | * this index is moved at the position where the next character |
- | |
138 | * can be written to. |
- | |
139 | * |
- | |
140 | * @param ch Input UTF-32 character. |
- | |
141 | * @param str Output buffer. |
- | |
142 | * @param index Index (counted in plain characters) where to start |
- | |
143 | * the encoding |
- | |
144 | * @param limit Maximal allowed value of index. |
- | |
145 | * |
- | |
146 | * @return True if the character was encoded or false if there is not |
- | |
147 | * enought space in the output buffer or the character is invalid |
- | |
148 | * Unicode code point. |
- | |
149 | * |
134 | * |
- | 135 | * Encode a single character to string representation (i.e. UTF-8) and store |
|
- | 136 | * it into a buffer at @a offset. Encoding starts at @a offset and this offset |
|
- | 137 | * is moved to the position where the next character can be written to. |
|
- | 138 | * |
|
- | 139 | * @param ch Input character. |
|
- | 140 | * @param str Output buffer. |
|
- | 141 | * @param offset Offset (in bytes) where to start writing. |
|
- | 142 | * @param sz Size of the output buffer. |
|
- | 143 | * |
|
- | 144 | * @return True if the character was encoded successfully or false if there |
|
- | 145 | * was not enough space in the output buffer or the character code |
|
- | 146 | * was invalid. |
|
150 | */ |
147 | */ |
151 | bool utf8_encode(const wchar_t ch, char *str, index_t *index, index_t limit) |
148 | bool chr_encode(const wchar_t ch, char *str, size_t *offset, size_t sz) |
152 | { |
149 | { |
153 | uint32_t cc; /* Unsigned version of ch. */ |
150 | uint32_t cc; /* Unsigned version of ch. */ |
154 | 151 | ||
155 | int cbytes; /* Number of continuation bytes. */ |
152 | int cbytes; /* Number of continuation bytes. */ |
156 | int b0_bits; /* Number of data bits in first byte. */ |
153 | int b0_bits; /* Number of data bits in first byte. */ |
157 | int i; |
154 | int i; |
158 | 155 | ||
159 | if (*index >= limit) |
156 | if (*offset >= sz) |
160 | return false; |
157 | return false; |
161 | 158 | ||
162 | if (ch < 0) |
159 | if (ch < 0) |
163 | return false; |
160 | return false; |
164 | 161 | ||
Line 182... | Line 179... | ||
182 | /* Codes longer than 21 bits are not supported. */ |
179 | /* Codes longer than 21 bits are not supported. */ |
183 | return false; |
180 | return false; |
184 | } |
181 | } |
185 | 182 | ||
186 | /* Check for available space in buffer. */ |
183 | /* Check for available space in buffer. */ |
187 | if (*index + cbytes >= limit) |
184 | if (*offset + cbytes >= sz) |
188 | return false; |
185 | return false; |
189 | 186 | ||
190 | /* Encode continuation bytes. */ |
187 | /* Encode continuation bytes. */ |
191 | for (i = cbytes; i > 0; --i) { |
188 | for (i = cbytes; i > 0; --i) { |
192 | str[*index + i] = 0x80 | (cc & LO_MASK_32(CONT_BITS)); |
189 | str[*offset + i] = 0x80 | (cc & LO_MASK_32(CONT_BITS)); |
193 | cc = cc >> CONT_BITS; |
190 | cc = cc >> CONT_BITS; |
194 | } |
191 | } |
195 | 192 | ||
196 | /* Encode first byte. */ |
193 | /* Encode first byte. */ |
197 | str[*index] = (cc & LO_MASK_32(b0_bits)) | HI_MASK_8(8 - b0_bits - 1); |
194 | str[*offset] = (cc & LO_MASK_32(b0_bits)) | HI_MASK_8(8 - b0_bits - 1); |
198 | 195 | ||
199 | /* Advance index. */ |
196 | /* Advance offset. */ |
200 | *index += (1 + cbytes); |
197 | *offset += (1 + cbytes); |
201 | 198 | ||
202 | return true; |
199 | return true; |
203 | } |
200 | } |
204 | 201 | ||
205 | /** Get bytes used by UTF-8 characters. |
202 | /** Get bytes used by UTF-8 characters. |
Line 224... | Line 221... | ||
224 | 221 | ||
225 | while (true) { |
222 | while (true) { |
226 | iprev = index; |
223 | iprev = index; |
227 | if (size >= count) |
224 | if (size >= count) |
228 | break; |
225 | break; |
229 | ch = utf8_decode(str, &index, UTF8_NO_LIMIT); |
226 | ch = chr_decode(str, &index, UTF8_NO_LIMIT); |
230 | if (ch == '\0') break; |
227 | if (ch == '\0') break; |
231 | 228 | ||
232 | size++; |
229 | size++; |
233 | } |
230 | } |
234 | 231 | ||
Line 286... | Line 283... | ||
286 | size_t strlen_utf8(const char *str) |
283 | size_t strlen_utf8(const char *str) |
287 | { |
284 | { |
288 | size_t size = 0; |
285 | size_t size = 0; |
289 | index_t index = 0; |
286 | index_t index = 0; |
290 | 287 | ||
291 | while (utf8_decode(str, &index, UTF8_NO_LIMIT) != 0) { |
288 | while (chr_decode(str, &index, UTF8_NO_LIMIT) != 0) { |
292 | size++; |
289 | size++; |
293 | } |
290 | } |
294 | 291 | ||
295 | return size; |
292 | return size; |
296 | } |
293 | } |