Subversion Repositories HelenOS

Rev

Rev 4199 | Rev 4205 | Go to most recent revision | Show entire file | Ignore whitespace | Details | Blame | Last modification | View Log | RSS feed

Rev 4199 Rev 4200
Line 54... Line 54...
54
#define HI_MASK_8(n) (~LO_MASK_8(8 - (n)))
54
#define HI_MASK_8(n) (~LO_MASK_8(8 - (n)))
55
 
55
 
56
/** Number of data bits in a UTF-8 continuation byte. */
56
/** Number of data bits in a UTF-8 continuation byte. */
57
#define CONT_BITS 6
57
#define CONT_BITS 6
58
 
58
 
59
/** Decode a single UTF-8 character from a NULL-terminated string.
59
/** Decode a single character from a substring.
60
 *
60
 *
61
 * Decode a single UTF-8 character from a plain char NULL-terminated
61
 * Decode a single character from a substring of size @a sz. Decoding starts
62
 * string. Decoding starts at @index and this index is moved to the
62
 * at @a offset and this offset is moved to the beginning of the next
63
 * beginning of the next character. In case of decoding error,
63
 * character. In case of decoding error, offset generally advances at least
64
 * index advances. However, index is never moved beyond (str+limit).
64
 * by one. However, offset is never moved beyond (str + sz).
65
 *
65
 *
66
 * @param str   Plain character NULL-terminated string.
66
 * @param str   String (not necessarily NULL-terminated).
67
 * @param index Index (counted in plain characters) where to start
67
 * @param index Index (counted in plain characters) where to start
68
 *              the decoding.
68
 *              the decoding.
69
 * @param limit Maximal allowed value of index.
69
 * @param limit Size of the substring.
70
 *
70
 *
71
 * @return Decoded character in UTF-32 or '?' if the encoding is wrong.
71
 * @return  Value of decoded character or '?' on decoding error.
72
 *
72
 *
73
 */
73
 */
74
wchar_t utf8_decode(const char *str, index_t *index, index_t limit)
74
wchar_t chr_decode(const char *str, size_t *offset, size_t sz)
75
{
75
{
76
    uint8_t b0, b;          /* Bytes read from str. */
76
    uint8_t b0, b;          /* Bytes read from str. */
77
    wchar_t ch;
77
    wchar_t ch;
78
 
78
 
79
    int b0_bits;        /* Data bits in first byte. */
79
    int b0_bits;        /* Data bits in first byte. */
80
    int cbytes;     /* Number of continuation bytes. */
80
    int cbytes;     /* Number of continuation bytes. */
81
 
81
 
82
    if (*index + 1 > limit)
82
    if (*offset + 1 > sz)
83
        return invalch;
83
        return invalch;
84
 
84
 
85
    b0 = (uint8_t) str[(*index)++];
85
    b0 = (uint8_t) str[(*offset)++];
86
 
86
 
87
    /* Determine code length. */
87
    /* Determine code length. */
88
 
88
 
89
    if ((b0 & 0x80) == 0) {
89
    if ((b0 & 0x80) == 0) {
90
        /* 0xxxxxxx (Plain ASCII) */
90
        /* 0xxxxxxx (Plain ASCII) */
Line 105... Line 105...
105
    } else {
105
    } else {
106
        /* 10xxxxxx -- unexpected continuation byte. */
106
        /* 10xxxxxx -- unexpected continuation byte. */
107
        return invalch;
107
        return invalch;
108
    }
108
    }
109
 
109
 
110
    if (*index + cbytes > limit) {
110
    if (*offset + cbytes > sz) {
111
        return invalch;
111
        return invalch;
112
    }
112
    }
113
 
113
 
114
    ch = b0 & LO_MASK_8(b0_bits);
114
    ch = b0 & LO_MASK_8(b0_bits);
115
 
115
 
116
    /* Decode continuation bytes. */
116
    /* Decode continuation bytes. */
117
    while (cbytes > 0) {
117
    while (cbytes > 0) {
118
        b = (uint8_t) str[(*index)++];
118
        b = (uint8_t) str[(*offset)++];
119
 
119
 
120
        /* Must be 10xxxxxx. */
120
        /* Must be 10xxxxxx. */
121
        if ((b & 0xc0) != 0x80) {
121
        if ((b & 0xc0) != 0x80) {
122
            return invalch;
122
            return invalch;
123
        }
123
        }
Line 128... Line 128...
128
    }
128
    }
129
 
129
 
130
    return ch;
130
    return ch;
131
}
131
}
132
 
132
 
133
/** Encode a single UTF-32 character as UTF-8
-
 
134
 *
-
 
135
 * Encode a single UTF-32 character as UTF-8 and store it into
133
/** Encode a single character to string representation.
136
 * the given buffer at @index. Encoding starts at @index and
-
 
137
 * this index is moved at the position where the next character
-
 
138
 * can be written to.
-
 
139
 *
-
 
140
 * @param ch    Input UTF-32 character.
-
 
141
 * @param str   Output buffer.
-
 
142
 * @param index Index (counted in plain characters) where to start
-
 
143
 *              the encoding
-
 
144
 * @param limit Maximal allowed value of index.
-
 
145
 *
-
 
146
 * @return True if the character was encoded or false if there is not
-
 
147
 *         enought space in the output buffer or the character is invalid
-
 
148
 *         Unicode code point.
-
 
149
 *
134
 *
-
 
135
 * Encode a single character to string representation (i.e. UTF-8) and store
-
 
136
 * it into a buffer at @a offset. Encoding starts at @a offset and this offset
-
 
137
 * is moved to the position where the next character can be written to.
-
 
138
 *
-
 
139
 * @param ch        Input character.
-
 
140
 * @param str       Output buffer.
-
 
141
 * @param offset    Offset (in bytes) where to start writing.
-
 
142
 * @param sz        Size of the output buffer.
-
 
143
 *
-
 
144
 * @return True if the character was encoded successfully or false if there
-
 
145
 *     was not enough space in the output buffer or the character code
-
 
146
 *     was invalid.
150
 */
147
 */
151
bool utf8_encode(const wchar_t ch, char *str, index_t *index, index_t limit)
148
bool chr_encode(const wchar_t ch, char *str, size_t *offset, size_t sz)
152
{
149
{
153
    uint32_t cc;        /* Unsigned version of ch. */
150
    uint32_t cc;        /* Unsigned version of ch. */
154
 
151
 
155
    int cbytes;     /* Number of continuation bytes. */
152
    int cbytes;     /* Number of continuation bytes. */
156
    int b0_bits;        /* Number of data bits in first byte. */
153
    int b0_bits;        /* Number of data bits in first byte. */
157
    int i;
154
    int i;
158
 
155
 
159
    if (*index >= limit)
156
    if (*offset >= sz)
160
        return false;
157
        return false;
161
 
158
 
162
    if (ch < 0)
159
    if (ch < 0)
163
        return false;
160
        return false;
164
 
161
 
Line 182... Line 179...
182
        /* Codes longer than 21 bits are not supported. */
179
        /* Codes longer than 21 bits are not supported. */
183
        return false;
180
        return false;
184
    }
181
    }
185
 
182
 
186
    /* Check for available space in buffer. */
183
    /* Check for available space in buffer. */
187
    if (*index + cbytes >= limit)
184
    if (*offset + cbytes >= sz)
188
        return false;
185
        return false;
189
 
186
 
190
    /* Encode continuation bytes. */
187
    /* Encode continuation bytes. */
191
    for (i = cbytes; i > 0; --i) {
188
    for (i = cbytes; i > 0; --i) {
192
        str[*index + i] = 0x80 | (cc & LO_MASK_32(CONT_BITS));
189
        str[*offset + i] = 0x80 | (cc & LO_MASK_32(CONT_BITS));
193
        cc = cc >> CONT_BITS;
190
        cc = cc >> CONT_BITS;
194
    }
191
    }
195
 
192
 
196
    /* Encode first byte. */
193
    /* Encode first byte. */
197
    str[*index] = (cc & LO_MASK_32(b0_bits)) | HI_MASK_8(8 - b0_bits - 1);
194
    str[*offset] = (cc & LO_MASK_32(b0_bits)) | HI_MASK_8(8 - b0_bits - 1);
198
 
195
 
199
    /* Advance index. */
196
    /* Advance offset. */
200
    *index += (1 + cbytes);
197
    *offset += (1 + cbytes);
201
   
198
   
202
    return true;
199
    return true;
203
}
200
}
204
 
201
 
205
/** Get bytes used by UTF-8 characters.
202
/** Get bytes used by UTF-8 characters.
Line 224... Line 221...
224
   
221
   
225
    while (true) {
222
    while (true) {
226
        iprev = index;
223
        iprev = index;
227
        if (size >= count)
224
        if (size >= count)
228
            break;
225
            break;
229
        ch = utf8_decode(str, &index, UTF8_NO_LIMIT);
226
        ch = chr_decode(str, &index, UTF8_NO_LIMIT);
230
        if (ch == '\0') break;
227
        if (ch == '\0') break;
231
 
228
 
232
        size++;
229
        size++;
233
    }
230
    }
234
   
231
   
Line 286... Line 283...
286
size_t strlen_utf8(const char *str)
283
size_t strlen_utf8(const char *str)
287
{
284
{
288
    size_t size = 0;
285
    size_t size = 0;
289
    index_t index = 0;
286
    index_t index = 0;
290
   
287
   
291
    while (utf8_decode(str, &index, UTF8_NO_LIMIT) != 0) {
288
    while (chr_decode(str, &index, UTF8_NO_LIMIT) != 0) {
292
        size++;
289
        size++;
293
    }
290
    }
294
   
291
   
295
    return size;
292
    return size;
296
}
293
}