Rev 4199 | Rev 4205 | Go to most recent revision | Show entire file | Ignore whitespace | Details | Blame | Last modification | View Log | RSS feed
| Rev 4199 | Rev 4200 | ||
|---|---|---|---|
| Line 54... | Line 54... | ||
| 54 | #define HI_MASK_8(n) (~LO_MASK_8(8 - (n))) |
54 | #define HI_MASK_8(n) (~LO_MASK_8(8 - (n))) |
| 55 | 55 | ||
| 56 | /** Number of data bits in a UTF-8 continuation byte. */ |
56 | /** Number of data bits in a UTF-8 continuation byte. */ |
| 57 | #define CONT_BITS 6 |
57 | #define CONT_BITS 6 |
| 58 | 58 | ||
| 59 | /** Decode a single UTF-8 character from a NULL-terminated string. |
59 | /** Decode a single character from a substring. |
| 60 | * |
60 | * |
| 61 | * Decode a single UTF-8 character from a plain char NULL-terminated |
61 | * Decode a single character from a substring of size @a sz. Decoding starts |
| 62 | * string. Decoding starts at @index and this index is moved to the |
62 | * at @a offset and this offset is moved to the beginning of the next |
| 63 | * beginning of the next character. In case of decoding error, |
63 | * character. In case of decoding error, offset generally advances at least |
| 64 | * index advances. However, index is never moved beyond (str+limit). |
64 | * by one. However, offset is never moved beyond (str + sz). |
| 65 | * |
65 | * |
| 66 | * @param str Plain character NULL-terminated string. |
66 | * @param str String (not necessarily NULL-terminated). |
| 67 | * @param index Index (counted in plain characters) where to start |
67 | * @param index Index (counted in plain characters) where to start |
| 68 | * the decoding. |
68 | * the decoding. |
| 69 | * @param limit Maximal allowed value of index. |
69 | * @param limit Size of the substring. |
| 70 | * |
70 | * |
| 71 | * @return Decoded character in UTF-32 or '?' if the encoding is wrong. |
71 | * @return Value of decoded character or '?' on decoding error. |
| 72 | * |
72 | * |
| 73 | */ |
73 | */ |
| 74 | wchar_t utf8_decode(const char *str, index_t *index, index_t limit) |
74 | wchar_t chr_decode(const char *str, size_t *offset, size_t sz) |
| 75 | { |
75 | { |
| 76 | uint8_t b0, b; /* Bytes read from str. */ |
76 | uint8_t b0, b; /* Bytes read from str. */ |
| 77 | wchar_t ch; |
77 | wchar_t ch; |
| 78 | 78 | ||
| 79 | int b0_bits; /* Data bits in first byte. */ |
79 | int b0_bits; /* Data bits in first byte. */ |
| 80 | int cbytes; /* Number of continuation bytes. */ |
80 | int cbytes; /* Number of continuation bytes. */ |
| 81 | 81 | ||
| 82 | if (*index + 1 > limit) |
82 | if (*offset + 1 > sz) |
| 83 | return invalch; |
83 | return invalch; |
| 84 | 84 | ||
| 85 | b0 = (uint8_t) str[(*index)++]; |
85 | b0 = (uint8_t) str[(*offset)++]; |
| 86 | 86 | ||
| 87 | /* Determine code length. */ |
87 | /* Determine code length. */ |
| 88 | 88 | ||
| 89 | if ((b0 & 0x80) == 0) { |
89 | if ((b0 & 0x80) == 0) { |
| 90 | /* 0xxxxxxx (Plain ASCII) */ |
90 | /* 0xxxxxxx (Plain ASCII) */ |
| Line 105... | Line 105... | ||
| 105 | } else { |
105 | } else { |
| 106 | /* 10xxxxxx -- unexpected continuation byte. */ |
106 | /* 10xxxxxx -- unexpected continuation byte. */ |
| 107 | return invalch; |
107 | return invalch; |
| 108 | } |
108 | } |
| 109 | 109 | ||
| 110 | if (*index + cbytes > limit) { |
110 | if (*offset + cbytes > sz) { |
| 111 | return invalch; |
111 | return invalch; |
| 112 | } |
112 | } |
| 113 | 113 | ||
| 114 | ch = b0 & LO_MASK_8(b0_bits); |
114 | ch = b0 & LO_MASK_8(b0_bits); |
| 115 | 115 | ||
| 116 | /* Decode continuation bytes. */ |
116 | /* Decode continuation bytes. */ |
| 117 | while (cbytes > 0) { |
117 | while (cbytes > 0) { |
| 118 | b = (uint8_t) str[(*index)++]; |
118 | b = (uint8_t) str[(*offset)++]; |
| 119 | 119 | ||
| 120 | /* Must be 10xxxxxx. */ |
120 | /* Must be 10xxxxxx. */ |
| 121 | if ((b & 0xc0) != 0x80) { |
121 | if ((b & 0xc0) != 0x80) { |
| 122 | return invalch; |
122 | return invalch; |
| 123 | } |
123 | } |
| Line 128... | Line 128... | ||
| 128 | } |
128 | } |
| 129 | 129 | ||
| 130 | return ch; |
130 | return ch; |
| 131 | } |
131 | } |
| 132 | 132 | ||
| 133 | /** Encode a single UTF-32 character as UTF-8 |
- | |
| 134 | * |
- | |
| 135 | * Encode a single UTF-32 character as UTF-8 and store it into |
133 | /** Encode a single character to string representation. |
| 136 | * the given buffer at @index. Encoding starts at @index and |
- | |
| 137 | * this index is moved at the position where the next character |
- | |
| 138 | * can be written to. |
- | |
| 139 | * |
- | |
| 140 | * @param ch Input UTF-32 character. |
- | |
| 141 | * @param str Output buffer. |
- | |
| 142 | * @param index Index (counted in plain characters) where to start |
- | |
| 143 | * the encoding |
- | |
| 144 | * @param limit Maximal allowed value of index. |
- | |
| 145 | * |
- | |
| 146 | * @return True if the character was encoded or false if there is not |
- | |
| 147 | * enought space in the output buffer or the character is invalid |
- | |
| 148 | * Unicode code point. |
- | |
| 149 | * |
134 | * |
| - | 135 | * Encode a single character to string representation (i.e. UTF-8) and store |
|
| - | 136 | * it into a buffer at @a offset. Encoding starts at @a offset and this offset |
|
| - | 137 | * is moved to the position where the next character can be written to. |
|
| - | 138 | * |
|
| - | 139 | * @param ch Input character. |
|
| - | 140 | * @param str Output buffer. |
|
| - | 141 | * @param offset Offset (in bytes) where to start writing. |
|
| - | 142 | * @param sz Size of the output buffer. |
|
| - | 143 | * |
|
| - | 144 | * @return True if the character was encoded successfully or false if there |
|
| - | 145 | * was not enough space in the output buffer or the character code |
|
| - | 146 | * was invalid. |
|
| 150 | */ |
147 | */ |
| 151 | bool utf8_encode(const wchar_t ch, char *str, index_t *index, index_t limit) |
148 | bool chr_encode(const wchar_t ch, char *str, size_t *offset, size_t sz) |
| 152 | { |
149 | { |
| 153 | uint32_t cc; /* Unsigned version of ch. */ |
150 | uint32_t cc; /* Unsigned version of ch. */ |
| 154 | 151 | ||
| 155 | int cbytes; /* Number of continuation bytes. */ |
152 | int cbytes; /* Number of continuation bytes. */ |
| 156 | int b0_bits; /* Number of data bits in first byte. */ |
153 | int b0_bits; /* Number of data bits in first byte. */ |
| 157 | int i; |
154 | int i; |
| 158 | 155 | ||
| 159 | if (*index >= limit) |
156 | if (*offset >= sz) |
| 160 | return false; |
157 | return false; |
| 161 | 158 | ||
| 162 | if (ch < 0) |
159 | if (ch < 0) |
| 163 | return false; |
160 | return false; |
| 164 | 161 | ||
| Line 182... | Line 179... | ||
| 182 | /* Codes longer than 21 bits are not supported. */ |
179 | /* Codes longer than 21 bits are not supported. */ |
| 183 | return false; |
180 | return false; |
| 184 | } |
181 | } |
| 185 | 182 | ||
| 186 | /* Check for available space in buffer. */ |
183 | /* Check for available space in buffer. */ |
| 187 | if (*index + cbytes >= limit) |
184 | if (*offset + cbytes >= sz) |
| 188 | return false; |
185 | return false; |
| 189 | 186 | ||
| 190 | /* Encode continuation bytes. */ |
187 | /* Encode continuation bytes. */ |
| 191 | for (i = cbytes; i > 0; --i) { |
188 | for (i = cbytes; i > 0; --i) { |
| 192 | str[*index + i] = 0x80 | (cc & LO_MASK_32(CONT_BITS)); |
189 | str[*offset + i] = 0x80 | (cc & LO_MASK_32(CONT_BITS)); |
| 193 | cc = cc >> CONT_BITS; |
190 | cc = cc >> CONT_BITS; |
| 194 | } |
191 | } |
| 195 | 192 | ||
| 196 | /* Encode first byte. */ |
193 | /* Encode first byte. */ |
| 197 | str[*index] = (cc & LO_MASK_32(b0_bits)) | HI_MASK_8(8 - b0_bits - 1); |
194 | str[*offset] = (cc & LO_MASK_32(b0_bits)) | HI_MASK_8(8 - b0_bits - 1); |
| 198 | 195 | ||
| 199 | /* Advance index. */ |
196 | /* Advance offset. */ |
| 200 | *index += (1 + cbytes); |
197 | *offset += (1 + cbytes); |
| 201 | 198 | ||
| 202 | return true; |
199 | return true; |
| 203 | } |
200 | } |
| 204 | 201 | ||
| 205 | /** Get bytes used by UTF-8 characters. |
202 | /** Get bytes used by UTF-8 characters. |
| Line 224... | Line 221... | ||
| 224 | 221 | ||
| 225 | while (true) { |
222 | while (true) { |
| 226 | iprev = index; |
223 | iprev = index; |
| 227 | if (size >= count) |
224 | if (size >= count) |
| 228 | break; |
225 | break; |
| 229 | ch = utf8_decode(str, &index, UTF8_NO_LIMIT); |
226 | ch = chr_decode(str, &index, UTF8_NO_LIMIT); |
| 230 | if (ch == '\0') break; |
227 | if (ch == '\0') break; |
| 231 | 228 | ||
| 232 | size++; |
229 | size++; |
| 233 | } |
230 | } |
| 234 | 231 | ||
| Line 286... | Line 283... | ||
| 286 | size_t strlen_utf8(const char *str) |
283 | size_t strlen_utf8(const char *str) |
| 287 | { |
284 | { |
| 288 | size_t size = 0; |
285 | size_t size = 0; |
| 289 | index_t index = 0; |
286 | index_t index = 0; |
| 290 | 287 | ||
| 291 | while (utf8_decode(str, &index, UTF8_NO_LIMIT) != 0) { |
288 | while (chr_decode(str, &index, UTF8_NO_LIMIT) != 0) { |
| 292 | size++; |
289 | size++; |
| 293 | } |
290 | } |
| 294 | 291 | ||
| 295 | return size; |
292 | return size; |
| 296 | } |
293 | } |