s/\t/ /g - utf8.c

[xonotic/gmqcc.git] / lexer.c

diff --git a/lexer.c b/lexer.c
index 8fb7ef0547109fff9eb32e4d8537047a11a5559d..43182c335ae93ef94bf262ce82c065375a9b1423 100644 (file)
--- a/lexer.c
+++ b/lexer.c
@@ -74,20 +74,17 @@ void lexerror(lex_file *lex, const char *fmt, ...)
 
 bool lexwarn(lex_file *lex, int warntype, const char *fmt, ...)
 {
+    bool    r;
+    lex_ctx ctx;
     va_list ap;
-    int lvl = LVL_WARNING;
 
-    if (!OPTS_WARN(warntype))
-        return false;
-
-    if (opts_werror)
-        lvl = LVL_ERROR;
+    ctx.file = lex->name;
+    ctx.line = lex->sline;
 
     va_start(ap, fmt);
-    con_vprintmsg(lvl, lex->name, lex->sline, (opts_werror ? "error" : "warning"), fmt, ap);
+    r = vcompile_warning(ctx, warntype, fmt, ap);
     va_end(ap);
-
-    return opts_werror;
+    return r;
 }
 
 
@@ -187,7 +184,7 @@ static void lex_token_new(lex_file *lex)
 lex_file* lex_open(const char *file)
 {
     lex_file *lex;
-    FILE *in = util_fopen(file, "rb");
+    FILE *in = file_open(file, "rb");
 
     if (!in) {
         lexerror(NULL, "open failed: '%s'\n", file);
@@ -196,7 +193,7 @@ lex_file* lex_open(const char *file)
 
     lex = (lex_file*)mem_a(sizeof(*lex));
     if (!lex) {
-        fclose(in);
+        file_close(in);
         lexerror(NULL, "out of memory\n");
         return NULL;
     }
@@ -261,7 +258,7 @@ void lex_close(lex_file *lex)
         vec_free(lex->modelname);
 
     if (lex->file)
-        fclose(lex->file);
+        file_close(lex->file);
 #if 0
     if (lex->tok)
         token_delete(lex->tok);
@@ -739,12 +736,17 @@ static bool lex_finish_frames(lex_file *lex)
         vec_shrinkto(lex->tok.value, 0);
         vec_push(lex->frames, m);
     } while (true);
+
+    return false;
 }
 
 static int GMQCC_WARN lex_finish_string(lex_file *lex, int quote)
 {
     int ch = 0;
     int nextch;
+    bool hex;
+    char u8buf[8]; /* way more than enough */
+    int  u8len, uc;
 
     while (ch != EOF)
     {
@@ -826,13 +828,51 @@ static int GMQCC_WARN lex_finish_string(lex_file *lex, int quote)
             case ']':  ch = 17; break;
             case '{':
                 ch = 0;
+                nextch = lex_getch(lex);
+                hex = (nextch == 'x');
+                if (!hex)
+                    lex_ungetch(lex, nextch);
                 for (nextch = lex_getch(lex); nextch != '}'; nextch = lex_getch(lex)) {
-                    ch = ch * 10 + nextch - '0';
-                    if (nextch < '0' || nextch > '9' || ch > 255) {
-                        lexerror(lex, "bad character code");
+                    if (!hex) {
+                        if (nextch >= '0' && nextch <= '9')
+                            ch = ch * 10 + nextch - '0';
+                        else {
+                            lexerror(lex, "bad character code");
+                            return (lex->tok.ttype = TOKEN_ERROR);
+                        }
+                    } else {
+                        if (nextch >= '0' || nextch <= '9')
+                            ch = ch * 16 + nextch - '0';
+                        else if (nextch >= 'a' && nextch <= 'f')
+                            ch = ch * 16 + nextch - 'a' + 10;
+                        else if (nextch >= 'A' && nextch <= 'F')
+                            ch = ch * 16 + nextch - 'A' + 10;
+                        else {
+                            lexerror(lex, "bad character code");
+                            return (lex->tok.ttype = TOKEN_ERROR);
+                        }
+                    }
+                    if ( (!OPTS_FLAG(UTF8) && ch > 255) ||
+                         ( OPTS_FLAG(UTF8) && ch > 0x10FFFF) )
+                    {
+                        lexerror(lex, "character code out of range");
                         return (lex->tok.ttype = TOKEN_ERROR);
                     }
                 }
+                if (OPTS_FLAG(UTF8) && ch >= 128) {
+                    u8len = u8_fromchar((uchar_t)ch, u8buf, sizeof(u8buf));
+                    if (!u8len)
+                        ch = 0;
+                    else {
+                        --u8len;
+                        for (uc = 0; uc < u8len; ++uc)
+                            lex_tokench(lex, u8buf[uc]);
+                        /* the last character will be inserted with the tokench() call
+                         * below the switch
+                         */
+                        ch = u8buf[uc];
+                    }
+                }
                 break;
             case '\n':  ch = '\n'; break;
 
@@ -1064,7 +1104,6 @@ int lex_do(lex_file *lex)
             if (rc < 0)
                 return (lex->tok.ttype = TOKEN_FATAL);
 
-            v = lex->tok.value;
             if (lex->modelname) {
                 frame_macro m;
                 m.value = lex->framevalue;
@@ -1119,6 +1158,15 @@ int lex_do(lex_file *lex)
     switch (ch)
     {
         case '[':
+            nextch = lex_getch(lex);
+            if (nextch == '[') {
+                lex_tokench(lex, ch);
+                lex_tokench(lex, nextch);
+                lex_endtoken(lex);
+                return (lex->tok.ttype = TOKEN_ATTRIBUTE_OPEN);
+            }
+            lex_ungetch(lex, nextch);
+            /* FALL THROUGH */
         case '(':
         case ':':
         case '?':
@@ -1128,11 +1176,23 @@ int lex_do(lex_file *lex)
                 return (lex->tok.ttype = ch);
             else
                 return (lex->tok.ttype = TOKEN_OPERATOR);
+
+        case ']':
+            if (lex->flags.noops) {
+                nextch = lex_getch(lex);
+                if (nextch == ']') {
+                    lex_tokench(lex, ch);
+                    lex_tokench(lex, nextch);
+                    lex_endtoken(lex);
+                    return (lex->tok.ttype = TOKEN_ATTRIBUTE_CLOSE);
+                }
+                lex_ungetch(lex, nextch);
+            }
+            /* FALL THROUGH */
         case ')':
         case ';':
         case '{':
         case '}':
-        case ']':
 
         case '#':
             lex_tokench(lex, ch);
@@ -1304,7 +1364,7 @@ int lex_do(lex_file *lex)
                 if (!strcmp(v, keywords_qc[kw]))
                     return (lex->tok.ttype = TOKEN_KEYWORD);
             }
-            if (opts_standard != COMPILER_QCC) {
+            if (opts.standard != COMPILER_QCC) {
                 for (kw = 0; kw < num_keywords_fg; ++kw) {
                     if (!strcmp(v, keywords_fg[kw]))
                         return (lex->tok.ttype = TOKEN_KEYWORD);
@@ -1355,7 +1415,7 @@ int lex_do(lex_file *lex)
 
         lex->tok.ttype = TOKEN_CHARCONST;
          /* It's a vector if we can successfully scan 3 floats */
-#ifdef WIN32
+#ifdef _MSC_VER
         if (sscanf_s(lex->tok.value, " %f %f %f ",
                    &lex->tok.constval.v.x, &lex->tok.constval.v.y, &lex->tok.constval.v.z) == 3)
 #else
@@ -1369,10 +1429,20 @@ int lex_do(lex_file *lex)
         else
         {
             if (!lex->flags.preprocessing && strlen(lex->tok.value) > 1) {
-                if (lexwarn(lex, WARN_MULTIBYTE_CHARACTER, "multibyte character: `%s`", lex->tok.value))
-                    return (lex->tok.ttype = TOKEN_ERROR);
+                uchar_t u8char;
+                /* check for a valid utf8 character */
+                if (!OPTS_FLAG(UTF8) || !u8_analyze(lex->tok.value, NULL, NULL, &u8char, 8)) {
+                    if (lexwarn(lex, WARN_MULTIBYTE_CHARACTER,
+                                ( OPTS_FLAG(UTF8) ? "invalid multibyte character sequence `%s`"
+                                                  : "multibyte character: `%s`" ),
+                                lex->tok.value))
+                        return (lex->tok.ttype = TOKEN_ERROR);
+                }
+                else
+                    lex->tok.constval.i = u8char;
             }
-            lex->tok.constval.i = lex->tok.value[0];
+            else
+                lex->tok.constval.i = lex->tok.value[0];
         }
 
         return lex->tok.ttype;