From: Dale Weiler Date: Fri, 28 Dec 2012 12:57:41 +0000 (+0000) Subject: s/\t/ /g - utf8.c X-Git-Tag: before-library~450 X-Git-Url: https://de.git.xonotic.org/?p=xonotic%2Fgmqcc.git;a=commitdiff_plain;h=d6e7d8ca758e3ce16383b1f79a081e12ab130efc;hp=cb12460b959ecb662f0fac610f0eaea0d9dbfe18;ds=sidebyside s/\t/ /g - utf8.c --- diff --git a/utf8.c b/utf8.c index 14510bc..391fc09 100644 --- a/utf8.c +++ b/utf8.c @@ -23,35 +23,35 @@ #include "gmqcc.h" static unsigned char utf8_lengths[256] = { - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* ascii characters */ - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 0x80 - 0xBF are within multibyte sequences */ - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* they could be interpreted as 2-byte starts but */ - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* the codepoint would be < 127 */ - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* */ - 0, 0, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, /* C0 and C1 would also result in overlong encodings */ - 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, /* */ - 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, - 4, 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 - /* with F5 the codepoint is above 0x10FFFF, - * F8-FB would start 5-byte sequences - * FC-FD would start 6-byte sequences - * ... - */ + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* ascii characters */ + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 0x80 - 0xBF are within multibyte sequences */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* they could be interpreted as 2-byte starts but */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* the codepoint would be < 127 */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* */ + 0, 0, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, /* C0 and C1 would also result in overlong encodings */ + 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, /* */ + 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, + 4, 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 + /* with F5 the codepoint is above 0x10FFFF, + * F8-FB would start 5-byte sequences + * FC-FD would start 6-byte sequences + * ... + */ }; static uchar_t utf8_range[5] = { - 1, /* invalid - let's not allow the creation of 0-bytes :P */ - 1, /* ascii minimum */ - 0x80, /* 2-byte minimum */ - 0x800, /* 3-byte minimum */ - 0x10000, /* 4-byte minimum */ + 1, /* invalid - let's not allow the creation of 0-bytes :P */ + 1, /* ascii minimum */ + 0x80, /* 2-byte minimum */ + 0x800, /* 3-byte minimum */ + 0x10000, /* 4-byte minimum */ }; /** Analyze the next character and return various information if requested. @@ -64,220 +64,220 @@ static uchar_t utf8_range[5] = { */ bool u8_analyze(const char *_s, size_t *_start, size_t *_len, uchar_t *_ch, size_t _maxlen) { - const unsigned char *s = (const unsigned char*)_s; - size_t i, j; - size_t bits = 0; - uchar_t ch; + const unsigned char *s = (const unsigned char*)_s; + size_t i, j; + size_t bits = 0; + uchar_t ch; - i = 0; + i = 0; /* findchar: */ - while (i < _maxlen && s[i] && (bits = utf8_lengths[s[i]]) == 0) - ++i; + while (i < _maxlen && s[i] && (bits = utf8_lengths[s[i]]) == 0) + ++i; - if (i >= _maxlen || !s[i]) { - if (_start) *_start = i; - if (_len) *_len = 0; - return false; - } + if (i >= _maxlen || !s[i]) { + if (_start) *_start = i; + if (_len) *_len = 0; + return false; + } - if (bits == 1) { /* ascii */ - if (_start) *_start = i; - if (_len) *_len = 1; - if (_ch) *_ch = (uchar_t)s[i]; - return true; - } + if (bits == 1) { /* ascii */ + if (_start) *_start = i; + if (_len) *_len = 1; + if (_ch) *_ch = (uchar_t)s[i]; + return true; + } - ch = (s[i] & (0xFF >> bits)); - for (j = 1; j < bits; ++j) - { - if ( (s[i+j] & 0xC0) != 0x80 ) - { - i += j; - /* in gmqcc, invalid / overlong encodings are considered an error - * goto findchar; - */ - if (!s[i]) goto done; - return false; - } - ch = (ch << 6) | (s[i+j] & 0x3F); - } - if (ch < utf8_range[bits] || ch >= 0x10FFFF) - { - /* same: error - * i += bits; - * goto findchar; - */ - return false; - } + ch = (s[i] & (0xFF >> bits)); + for (j = 1; j < bits; ++j) + { + if ( (s[i+j] & 0xC0) != 0x80 ) + { + i += j; + /* in gmqcc, invalid / overlong encodings are considered an error + * goto findchar; + */ + if (!s[i]) goto done; + return false; + } + ch = (ch << 6) | (s[i+j] & 0x3F); + } + if (ch < utf8_range[bits] || ch >= 0x10FFFF) + { + /* same: error + * i += bits; + * goto findchar; + */ + return false; + } done: - if (_start) - *_start = i; - if (_len) - *_len = bits; - if (_ch) - *_ch = ch; - return true; + if (_start) + *_start = i; + if (_len) + *_len = bits; + if (_ch) + *_ch = ch; + return true; } /* might come in handy */ size_t u8_strlen(const char *_s) { - size_t st, ln; - size_t len = 0; - const unsigned char *s = (const unsigned char*)_s; + size_t st, ln; + size_t len = 0; + const unsigned char *s = (const unsigned char*)_s; - while (*s) - { - /* ascii char, skip u8_analyze */ - if (*s < 0x80) - { - ++len; - ++s; - continue; - } + while (*s) + { + /* ascii char, skip u8_analyze */ + if (*s < 0x80) + { + ++len; + ++s; + continue; + } - /* invalid, skip u8_analyze */ - if (*s < 0xC2) - { - ++s; - continue; - } + /* invalid, skip u8_analyze */ + if (*s < 0xC2) + { + ++s; + continue; + } - if (!u8_analyze((const char*)s, &st, &ln, NULL, 0x10)) - break; - /* valid character, skip after it */ - s += st + ln; - ++len; - } - return len; + if (!u8_analyze((const char*)s, &st, &ln, NULL, 0x10)) + break; + /* valid character, skip after it */ + s += st + ln; + ++len; + } + return len; } size_t u8_strnlen(const char *_s, size_t n) { - size_t st, ln; - size_t len = 0; - const unsigned char *s = (const unsigned char*)_s; + size_t st, ln; + size_t len = 0; + const unsigned char *s = (const unsigned char*)_s; - while (*s && n) - { - /* ascii char, skip u8_analyze */ - if (*s < 0x80) - { - ++len; - ++s; - --n; - continue; - } + while (*s && n) + { + /* ascii char, skip u8_analyze */ + if (*s < 0x80) + { + ++len; + ++s; + --n; + continue; + } - /* invalid, skip u8_analyze */ - if (*s < 0xC2) - { - ++s; - --n; - continue; - } + /* invalid, skip u8_analyze */ + if (*s < 0xC2) + { + ++s; + --n; + continue; + } - if (!u8_analyze((const char*)s, &st, &ln, NULL, n)) - break; - /* valid character, see if it's still inside the range specified by n: */ - if (n < st + ln) - return len; - ++len; - n -= st + ln; - s += st + ln; - } - return len; + if (!u8_analyze((const char*)s, &st, &ln, NULL, n)) + break; + /* valid character, see if it's still inside the range specified by n: */ + if (n < st + ln) + return len; + ++len; + n -= st + ln; + s += st + ln; + } + return len; } /* Required for character constants */ uchar_t u8_getchar(const char *_s, const char **_end) { - size_t st, ln; - uchar_t ch; + size_t st, ln; + uchar_t ch; - if (!u8_analyze(_s, &st, &ln, &ch, 0x10)) - ch = 0; - if (_end) - *_end = _s + st + ln; - return ch; + if (!u8_analyze(_s, &st, &ln, &ch, 0x10)) + ch = 0; + if (_end) + *_end = _s + st + ln; + return ch; } uchar_t u8_getnchar(const char *_s, const char **_end, size_t _maxlen) { - size_t st, ln; - uchar_t ch; + size_t st, ln; + uchar_t ch; - if (!u8_analyze(_s, &st, &ln, &ch, _maxlen)) - ch = 0; - if (_end) - *_end = _s + st + ln; - return ch; + if (!u8_analyze(_s, &st, &ln, &ch, _maxlen)) + ch = 0; + if (_end) + *_end = _s + st + ln; + return ch; } /* required for \x{asdf}-like string escape sequences */ int u8_fromchar(uchar_t w, char *to, size_t maxlen) { - if (maxlen < 1) - return 0; + if (maxlen < 1) + return 0; - if (!w) - return 0; + if (!w) + return 0; /* We may want an -f flag for this behaviour... - if (w >= 0xE000) - w -= 0xE000; + if (w >= 0xE000) + w -= 0xE000; */ - if (w < 0x80) - { - to[0] = (char)w; - if (maxlen < 2) - return -1; - to[1] = 0; - return 1; - } - /* for a little speedup */ - if (w < 0x800) - { - if (maxlen < 3) - { - to[0] = 0; - return -1; - } - to[2] = 0; - to[1] = 0x80 | (w & 0x3F); w >>= 6; - to[0] = 0xC0 | w; - return 2; - } - if (w < 0x10000) - { - if (maxlen < 4) - { - to[0] = 0; - return -1; - } - to[3] = 0; - to[2] = 0x80 | (w & 0x3F); w >>= 6; - to[1] = 0x80 | (w & 0x3F); w >>= 6; - to[0] = 0xE0 | w; - return 3; - } + if (w < 0x80) + { + to[0] = (char)w; + if (maxlen < 2) + return -1; + to[1] = 0; + return 1; + } + /* for a little speedup */ + if (w < 0x800) + { + if (maxlen < 3) + { + to[0] = 0; + return -1; + } + to[2] = 0; + to[1] = 0x80 | (w & 0x3F); w >>= 6; + to[0] = 0xC0 | w; + return 2; + } + if (w < 0x10000) + { + if (maxlen < 4) + { + to[0] = 0; + return -1; + } + to[3] = 0; + to[2] = 0x80 | (w & 0x3F); w >>= 6; + to[1] = 0x80 | (w & 0x3F); w >>= 6; + to[0] = 0xE0 | w; + return 3; + } - /* RFC 3629 */ - if (w <= 0x10FFFF) - { - if (maxlen < 5) - { - to[0] = 0; - return -1; - } - to[4] = 0; - to[3] = 0x80 | (w & 0x3F); w >>= 6; - to[2] = 0x80 | (w & 0x3F); w >>= 6; - to[1] = 0x80 | (w & 0x3F); w >>= 6; - to[0] = 0xF0 | w; - return 4; - } - return 0; + /* RFC 3629 */ + if (w <= 0x10FFFF) + { + if (maxlen < 5) + { + to[0] = 0; + return -1; + } + to[4] = 0; + to[3] = 0x80 | (w & 0x3F); w >>= 6; + to[2] = 0x80 | (w & 0x3F); w >>= 6; + to[1] = 0x80 | (w & 0x3F); w >>= 6; + to[0] = 0xF0 | w; + return 4; + } + return 0; }