X-Git-Url: http://de.git.xonotic.org/?p=xonotic%2Fgmqcc.git;a=blobdiff_plain;f=test.c;h=107140e5cc7697f4a7e5e346e7334f12dd7a53cb;hp=f9696715a868cfb4546a7dbd9617c91eb79388fd;hb=8c08897749366bd83a9a623727aca2927257d4cf;hpb=467a4740dae1c0fa9d50c64e07b03ee82904e8e5

diff --git a/test.c b/test.c
index f969671..107140e 100644
--- a/test.c
+++ b/test.c
@@ -26,7 +26,7 @@
 
 opts_cmd_t opts;
 
-char *task_bins[] = {
+const char *task_bins[] = {
     "./gmqcc",
     "./qcvm"
 };
@@ -66,7 +66,7 @@ FILE ** task_popen(const char *command, const char *mode) {
     int     errhandle [2];
     int     trypipe;
 
-    popen_t *data = mem_a(sizeof(popen_t));
+    popen_t *data = (popen_t*)mem_a(sizeof(popen_t));
 
     /*
      * Parse the command now into a list for execv, this is a pain
@@ -152,17 +152,10 @@ int task_pclose(FILE **handles) {
     return status;
 }
 #else
-#    define _WIN32_LEAN_AND_MEAN
-#    define popen  _popen
-#    define pclose _pclose
-#    include <windows.h>
-#    include <io.h>
-#    include <fcntl.h>
     /*
      * Bidirectional piping implementation for windows using CreatePipe and DuplicateHandle +
      * other hacks.
      */
-
     typedef struct {
         int __dummy;
         /* TODO: implement */
@@ -181,96 +174,10 @@ int task_pclose(FILE **handles) {
         (void)files;
         return;
     }
+#endif /*! _WIN32 */
 
-#    ifdef __MINGW32__
-        /* mingw32 has dirent.h */
-#        include <dirent.h>
-#    elif defined (_WIN32)
-        /* 
-         * visual studio lacks dirent.h it's a posix thing
-         * so we emulate it with the WinAPI.
-         */
-
-        struct dirent {
-            long           d_ino;
-            unsigned short d_reclen;
-            unsigned short d_namlen;
-            char           d_name[FILENAME_MAX];
-        };
-
-        typedef struct {
-            struct _finddata_t dd_dta;
-            struct dirent      dd_dir;
-            long               dd_handle;
-            int                dd_stat;
-            char               dd_name[1];
-        } DIR;
-
-        DIR *opendir(const char *name) {
-            DIR *dir = (DIR*)mem_a(sizeof(DIR) + strlen(name));
-            if (!dir)
-                return NULL;
-
-            strcpy(dir->dd_name, name);
-            return dir;
-        }
-            
-        int closedir(DIR *dir) {
-            FindClose((HANDLE)dir->dd_handle);
-            mem_d ((void*)dir);
-            return 0;
-        }
-
-        struct dirent *readdir(DIR *dir) {
-            WIN32_FIND_DATA info;
-            struct dirent  *data;
-            int             rets;
-
-            if (!dir->dd_handle) {
-                char *dirname;
-                if (*dir->dd_name) {
-                    size_t n = strlen(dir->dd_name);
-                    if ((dirname  = (char*)mem_a(n + 5) /* 4 + 1 */)) {
-                        strcpy(dirname,     dir->dd_name);
-                        strcpy(dirname + n, "\\*.*");   /* 4 + 1 */
-                    }
-                } else {
-                    if (!(dirname = util_strdup("\\*.*")))
-                        return NULL;
-                }
-
-                dir->dd_handle = (long)FindFirstFile(dirname, &info);
-                mem_d(dirname);
-                rets = !(!dir->dd_handle);
-            } else if (dir->dd_handle != -11) {
-                rets = FindNextFile ((HANDLE)dir->dd_handle, &info);
-            } else {
-                rets = 0;
-            }
-
-            if (!rets)
-                return NULL;
-            
-            if ((data = (struct dirent*)mem_a(sizeof(struct dirent)))) {
-                strncpy(data->d_name, info.cFileName, FILENAME_MAX - 1);
-                data->d_name[FILENAME_MAX - 1] = '\0'; /* terminate */
-                data->d_namlen                 = strlen(data->d_name);
-            }
-            return data;
-        }
-
-        /*
-         * Visual studio also lacks S_ISDIR for sys/stat.h, so we emulate this as well
-         * which is not hard at all.
-         */
-#        undef  S_ISDIR /* undef just incase */
-#        define S_ISDIR(X) ((X)&_S_IFDIR)
-#    endif
-#endif
-
-#define TASK_COMPILE 0
-#define TASK_EXECUTE 1
-
+#define TASK_COMPILE    0
+#define TASK_EXECUTE    1
 /*
  * Task template system:
  *  templates are rules for a specific test, used to create a "task" that
@@ -294,14 +201,6 @@ int task_pclose(FILE **handles) {
  *          Used to set a description of the current test, this must be
  *          provided, this tag is NOT optional.
  *
- *      F:
- *          Used to set a failure message, this message will be displayed
- *          if the test fails, this tag is optional
- *
- *      S:
- *          Used to set a success message, this message will be displayed
- *          if the test succeeds, this tag is optional.
- *
  *      T:
  *          Used to set the procedure for the given task, there are two
  *          options for this:
@@ -319,6 +218,9 @@ int task_pclose(FILE **handles) {
  *          Used to set the compilation flags for the given task, this
  *          must be provided, this tag is NOT optional.
  *
+ *      F:  Used to set some test suite flags, currently the only option
+ *          is -no-defs (to including of defs.qh)
+ *
  *      E:
  *          Used to set the execution flags for the given task. This tag
  *          must be provided if T == -execute, otherwise it's erroneous
@@ -351,34 +253,35 @@ int task_pclose(FILE **handles) {
  */
 typedef struct {
     char  *description;
-    char  *failuremessage;
-    char  *successmessage;
     char  *compileflags;
     char  *executeflags;
     char  *proceduretype;
     char  *sourcefile;
     char  *tempfilename;
     char **comparematch;
+    char  *rulesfile;
+    char  *testflags;
 } task_template_t;
 
 /*
  * This is very much like a compiler code generator :-).  This generates
  * a value from some data observed from the compiler.
  */
-bool task_template_generate(task_template_t *template, char tag, const char *file, size_t line, const char *value) {
+bool task_template_generate(task_template_t *tmpl, char tag, const char *file, size_t line, char *value, size_t *pad) {
+    size_t desclen = 0;
+    size_t filelen = 0;
     char **destval = NULL;
 
-    if (!template)
+    if (!tmpl)
         return false;
 
     switch(tag) {
-        case 'D': destval = &template->description;    break;
-        case 'F': destval = &template->failuremessage; break;
-        case 'S': destval = &template->successmessage; break;
-        case 'T': destval = &template->proceduretype;  break;
-        case 'C': destval = &template->compileflags;   break;
-        case 'E': destval = &template->executeflags;   break;
-        case 'I': destval = &template->sourcefile;     break;
+        case 'D': destval = &tmpl->description;    break;
+        case 'T': destval = &tmpl->proceduretype;  break;
+        case 'C': destval = &tmpl->compileflags;   break;
+        case 'E': destval = &tmpl->executeflags;   break;
+        case 'I': destval = &tmpl->sourcefile;     break;
+        case 'F': destval = &tmpl->testflags;      break;
         default:
             con_printmsg(LVL_ERROR, __FILE__, __LINE__, "internal error",
                 "invalid tag `%c:` during code generation\n",
@@ -413,7 +316,7 @@ bool task_template_generate(task_template_t *template, char tag, const char *fil
     if (strchr(value, '\n'))
         *strrchr(value, '\n')='\0';
     else /* cppcheck: possible nullpointer dereference */
-        abort();
+        exit(EXIT_FAILURE);
 
     /*
      * Now allocate and set the actual value for the specific tag. Which
@@ -421,20 +324,33 @@ bool task_template_generate(task_template_t *template, char tag, const char *fil
      */
     *destval = util_strdup(value);
 
+
+    if (*destval == tmpl->description) {
+        /*
+         * Create some padding for the description to align the
+         * printing of the rules file.
+         */  
+        if ((desclen = strlen(tmpl->description)) > pad[0])
+            pad[0] = desclen;
+    }
+
+    if ((filelen = strlen(file)) > pad[2])
+        pad[2] = filelen;
+
     return true;
 }
 
-bool task_template_parse(const char *file, task_template_t *template, FILE *fp) {
+bool task_template_parse(const char *file, task_template_t *tmpl, FILE *fp, size_t *pad) {
     char  *data = NULL;
     char  *back = NULL;
     size_t size = 0;
     size_t line = 1;
 
-    if (!template)
+    if (!tmpl)
         return false;
 
     /* top down parsing */
-    while (file_getline(&back, &size, fp) != EOF) {
+    while (fs_file_getline(&back, &size, fp) != EOF) {
         /* skip whitespace */
         data = back;
         if (*data && (*data == ' ' || *data == '\t'))
@@ -442,12 +358,12 @@ bool task_template_parse(const char *file, task_template_t *template, FILE *fp)
 
         switch (*data) {
             /*
-             * Handle comments inside task template files.  We're strict
+             * Handle comments inside task tmpl files.  We're strict
              * about the language for fun :-)
              */
             case '/':
                 if (data[1] != '/') {
-                    con_printmsg(LVL_ERROR, file, line, "template parse error",
+                    con_printmsg(LVL_ERROR, file, line, "tmpl parse error",
                         "invalid character `/`, perhaps you meant `//` ?");
 
                     mem_d(back);
@@ -471,21 +387,20 @@ bool task_template_parse(const char *file, task_template_t *template, FILE *fp)
              * it to.
              */
             case 'D':
-            case 'F':
-            case 'S':
             case 'T':
             case 'C':
             case 'E':
             case 'I':
+            case 'F':
                 if (data[1] != ':') {
-                    con_printmsg(LVL_ERROR, file, line, "template parse error",
+                    con_printmsg(LVL_ERROR, file, line, "tmpl parse error",
                         "expected `:` after `%c`",
                         *data
                     );
                     goto failure;
                 }
-                if (!task_template_generate(template, *data, file, line, &data[3])) {
-                    con_printmsg(LVL_ERROR, file, line, "template compile error",
+                if (!task_template_generate(tmpl, *data, file, line, &data[3], pad)) {
+                    con_printmsg(LVL_ERROR, file, line, "tmpl compile error",
                         "failed to generate for given task\n"
                     );
                     goto failure;
@@ -500,7 +415,7 @@ bool task_template_parse(const char *file, task_template_t *template, FILE *fp)
             {
                 char *value = &data[3];
                 if (data[1] != ':') {
-                    con_printmsg(LVL_ERROR, file, line, "template parse error",
+                    con_printmsg(LVL_ERROR, file, line, "tmpl parse error",
                         "expected `:` after `%c`",
                         *data
                     );
@@ -517,15 +432,15 @@ bool task_template_parse(const char *file, task_template_t *template, FILE *fp)
                 if (strrchr(value, '\n'))
                     *strrchr(value, '\n')='\0';
                 else /* cppcheck: possible null pointer dereference */
-                    abort();
+                    exit(EXIT_FAILURE);
 
-                vec_push(template->comparematch, util_strdup(value));
+                vec_push(tmpl->comparematch, util_strdup(value));
 
                 break;
             }
 
             default:
-                con_printmsg(LVL_ERROR, file, line, "template parse error",
+                con_printmsg(LVL_ERROR, file, line, "tmpl parse error",
                     "invalid tag `%c`", *data
                 );
                 goto failure;
@@ -551,33 +466,42 @@ failure:
  * Nullifies the template data: used during initialization of a new
  * template and free.
  */
-void task_template_nullify(task_template_t *template) {
-    if (!template)
+void task_template_nullify(task_template_t *tmpl) {
+    if (!tmpl)
         return;
 
-    template->description    = NULL;
-    template->failuremessage = NULL;
-    template->successmessage = NULL;
-    template->proceduretype  = NULL;
-    template->compileflags   = NULL;
-    template->executeflags   = NULL;
-    template->comparematch   = NULL;
-    template->sourcefile     = NULL;
-    template->tempfilename   = NULL;
+    tmpl->description    = NULL;
+    tmpl->proceduretype  = NULL;
+    tmpl->compileflags   = NULL;
+    tmpl->executeflags   = NULL;
+    tmpl->comparematch   = NULL;
+    tmpl->sourcefile     = NULL;
+    tmpl->tempfilename   = NULL;
+    tmpl->rulesfile      = NULL;
+    tmpl->testflags      = NULL;
 }
 
-task_template_t *task_template_compile(const char *file, const char *dir) {
+task_template_t *task_template_compile(const char *file, const char *dir, size_t *pad) {
     /* a page should be enough */
     char             fullfile[4096];
+    size_t           filepadd = 0;
     FILE            *tempfile = NULL;
-    task_template_t *template = NULL;
+    task_template_t *tmpl     = NULL;
 
-    memset  (fullfile, 0, sizeof(fullfile));
     snprintf(fullfile,    sizeof(fullfile), "%s/%s", dir, file);
 
-    tempfile = file_open(fullfile, "r");
-    template = mem_a(sizeof(task_template_t));
-    task_template_nullify(template);
+    tempfile = fs_file_open(fullfile, "r");
+    tmpl     = (task_template_t*)mem_a(sizeof(task_template_t));
+    task_template_nullify(tmpl);
+
+    /*
+     * Create some padding for the printing to align the
+     * printing of the rules file to the console.
+     */  
+    if ((filepadd = strlen(fullfile)) > pad[1])
+        pad[1] = filepadd;
+
+    tmpl->rulesfile = util_strdup(fullfile);
 
     /*
      * Esnure the file even exists for the task, this is pretty useless
@@ -590,7 +514,7 @@ task_template_t *task_template_compile(const char *file, const char *dir) {
         goto failure;
     }
 
-    if (!task_template_parse(file, template, tempfile)) {
+    if (!task_template_parse(file, tmpl, tempfile, pad)) {
         con_err("template parse error: error during parsing\n");
         goto failure;
     }
@@ -602,19 +526,19 @@ task_template_t *task_template_compile(const char *file, const char *dir) {
      *  C
      *  I
      */
-    if (!template->description) {
+    if (!tmpl->description) {
         con_err("template compile error: %s missing `D:` tag\n", file);
         goto failure;
     }
-    if (!template->proceduretype) {
+    if (!tmpl->proceduretype) {
         con_err("template compile error: %s missing `T:` tag\n", file);
         goto failure;
     }
-    if (!template->compileflags) {
+    if (!tmpl->compileflags) {
         con_err("template compile error: %s missing `C:` tag\n", file);
         goto failure;
     }
-    if (!template->sourcefile) {
+    if (!tmpl->sourcefile) {
         con_err("template compile error: %s missing `I:` tag\n", file);
         goto failure;
     }
@@ -623,35 +547,41 @@ task_template_t *task_template_compile(const char *file, const char *dir) {
      * Now lets compile the template, compilation is really just
      * the process of validating the input.
      */
-    if (!strcmp(template->proceduretype, "-compile")) {
-        if (template->executeflags)
+    if (!strcmp(tmpl->proceduretype, "-compile")) {
+        if (tmpl->executeflags)
             con_err("template compile warning: %s erroneous tag `E:` when only compiling\n", file);
-        if (template->comparematch)
+        if (tmpl->comparematch)
             con_err("template compile warning: %s erroneous tag `M:` when only compiling\n", file);
         goto success;
-    } else if (!strcmp(template->proceduretype, "-execute")) {
-        if (!template->executeflags) {
+    } else if (!strcmp(tmpl->proceduretype, "-execute")) {
+        if (!tmpl->executeflags) {
             /* default to $null */
-            template->executeflags = util_strdup("$null");
+            tmpl->executeflags = util_strdup("$null");
         }
-        if (!template->comparematch) {
+        if (!tmpl->comparematch) {
             con_err("template compile error: %s missing `M:` tag (use `$null` for exclude)\n", file);
             goto failure;
         }
-    } else if (!strcmp(template->proceduretype, "-fail")) {
-        if (template->executeflags)
+    } else if (!strcmp(tmpl->proceduretype, "-fail")) {
+        if (tmpl->executeflags)
             con_err("template compile warning: %s erroneous tag `E:` when only failing\n", file);
-        if (template->comparematch)
+        if (tmpl->comparematch)
             con_err("template compile warning: %s erroneous tag `M:` when only failing\n", file);
-        goto success;
+    } else if (!strcmp(tmpl->proceduretype, "-pp")) {
+        if (tmpl->executeflags)
+            con_err("template compile warning: %s erroneous tag `E:` when only preprocessing\n", file);
+        if (!tmpl->comparematch) {
+            con_err("template compile error: %s missing `M:` tag (use `$null` for exclude)\n", file);
+            goto failure;
+        }
     } else {
-        con_err("template compile error: %s invalid procedure type: %s\n", file, template->proceduretype);
+        con_err("template compile error: %s invalid procedure type: %s\n", file, tmpl->proceduretype);
         goto failure;
     }
 
 success:
-    file_close(tempfile);
-    return template;
+    fs_file_close(tempfile);
+    return tmpl;
 
 failure:
     /*
@@ -659,41 +589,41 @@ failure:
      * so the check to see if it's not null here is required.
      */
     if (tempfile)
-        file_close(tempfile);
-    mem_d (template);
+        fs_file_close(tempfile);
+    mem_d (tmpl);
 
     return NULL;
 }
 
-void task_template_destroy(task_template_t **template) {
-    if (!template)
+void task_template_destroy(task_template_t **tmpl) {
+    if (!tmpl)
         return;
 
-    if ((*template)->description)    mem_d((*template)->description);
-    if ((*template)->failuremessage) mem_d((*template)->failuremessage);
-    if ((*template)->successmessage) mem_d((*template)->successmessage);
-    if ((*template)->proceduretype)  mem_d((*template)->proceduretype);
-    if ((*template)->compileflags)   mem_d((*template)->compileflags);
-    if ((*template)->executeflags)   mem_d((*template)->executeflags);
-    if ((*template)->sourcefile)     mem_d((*template)->sourcefile);
+    if ((*tmpl)->description)    mem_d((*tmpl)->description);
+    if ((*tmpl)->proceduretype)  mem_d((*tmpl)->proceduretype);
+    if ((*tmpl)->compileflags)   mem_d((*tmpl)->compileflags);
+    if ((*tmpl)->executeflags)   mem_d((*tmpl)->executeflags);
+    if ((*tmpl)->sourcefile)     mem_d((*tmpl)->sourcefile);
+    if ((*tmpl)->rulesfile)      mem_d((*tmpl)->rulesfile);
+    if ((*tmpl)->testflags)      mem_d((*tmpl)->testflags);
 
     /*
-     * Delete all allocated string for task template then destroy the
+     * Delete all allocated string for task tmpl then destroy the
      * main vector.
      */
     {
         size_t i = 0;
-        for (; i < vec_size((*template)->comparematch); i++)
-            mem_d((*template)->comparematch[i]);
+        for (; i < vec_size((*tmpl)->comparematch); i++)
+            mem_d((*tmpl)->comparematch[i]);
 
-        vec_free((*template)->comparematch);
+        vec_free((*tmpl)->comparematch);
     }
 
     /*
      * Nullify all the template members otherwise NULL comparision
-     * checks will fail if template pointer is reused.
+     * checks will fail if tmpl pointer is reused.
      */
-    mem_d(*template);
+    mem_d(*tmpl);
 }
 
 /*
@@ -701,7 +631,7 @@ void task_template_destroy(task_template_t **template) {
  * of a task list.  This is the executor of the tasks essentially as well.
  */
 typedef struct {
-    task_template_t *template;
+    task_template_t *tmpl;
     FILE           **runhandles;
     FILE            *stderrlog;
     FILE            *stdoutlog;
@@ -716,7 +646,7 @@ task_t *task_tasks = NULL;
  * Read a directory and searches for all template files in it
  * which is later used to run all tests.
  */
-bool task_propagate(const char *curdir) {
+bool task_propagate(const char *curdir, size_t *pad, const char *defs) {
     bool             success = true;
     DIR             *dir;
     struct dirent   *files;
@@ -724,11 +654,10 @@ bool task_propagate(const char *curdir) {
     char             buffer[4096];
     size_t           found = 0;
 
-    dir = opendir(curdir);
+    dir = fs_dir_open(curdir);
 
-    while ((files = readdir(dir))) {
-        memset  (buffer, 0,sizeof(buffer));
-        snprintf(buffer,   sizeof(buffer), "%s/%s", curdir, files->d_name);
+    while ((files = fs_dir_read(dir))) {
+        snprintf(buffer, sizeof(buffer), "%s/%s", curdir, files->d_name);
 
         if (stat(buffer, &directory) == -1) {
             con_err("internal error: stat failed, aborting\n");
@@ -744,13 +673,14 @@ bool task_propagate(const char *curdir) {
          * actually a directory, so it must be a file :)
          */
         if (strcmp(files->d_name + strlen(files->d_name) - 5, ".tmpl") == 0) {
-            task_template_t *template = task_template_compile(files->d_name, curdir);
+            task_template_t *tmpl = task_template_compile(files->d_name, curdir, pad);
             char             buf[4096]; /* one page should be enough */
+            char            *qcflags = NULL;
             task_t           task;
 
             util_debug("TEST", "compiling task template: %s/%s\n", curdir, files->d_name);
             found ++;
-            if (!template) {
+            if (!tmpl) {
                 con_err("error compiling task template: %s\n", files->d_name);
                 success = false;
                 continue;
@@ -759,51 +689,113 @@ bool task_propagate(const char *curdir) {
              * Generate a temportary file name for the output binary
              * so we don't trample over an existing one.
              */
-            template->tempfilename = tempnam(curdir, "TMPDAT");
+            tmpl->tempfilename = NULL;
+            util_asprintf(&tmpl->tempfilename, "%s/TMPDAT.%s", curdir, files->d_name);
+
+            /*
+             * Additional QCFLAGS enviroment variable may be used
+             * to test compile flags for all tests.  This needs to be
+             * BEFORE other flags (so that the .tmpl can override them)
+             */
+            qcflags = getenv("QCFLAGS");
 
             /*
              * Generate the command required to open a pipe to a process
              * which will be refered to with a handle in the task for
              * reading the data from the pipe.
              */
-            memset  (buf,0,sizeof(buf));
-            snprintf(buf,  sizeof(buf), "%s %s/%s %s -o %s",
-                task_bins[TASK_COMPILE],
-                curdir,
-                template->sourcefile,
-                template->compileflags,
-                template->tempfilename
-            );
+            if (strcmp(tmpl->proceduretype, "-pp")) {
+                if (qcflags) {
+                    if (tmpl->testflags && !strcmp(tmpl->testflags, "-no-defs")) {
+                        snprintf(buf, sizeof(buf), "%s %s/%s %s %s -o %s",
+                            task_bins[TASK_COMPILE],
+                            curdir,
+                            tmpl->sourcefile,
+                            qcflags,
+                            tmpl->compileflags,
+                            tmpl->tempfilename
+                        );
+                    } else {
+                        snprintf(buf, sizeof(buf), "%s %s/%s %s/%s %s %s -o %s",
+                            task_bins[TASK_COMPILE],
+                            curdir,
+                            defs,
+                            curdir,
+                            tmpl->sourcefile,
+                            qcflags,
+                            tmpl->compileflags,
+                            tmpl->tempfilename
+                        );
+                    }
+                } else {
+                    if (tmpl->testflags && !strcmp(tmpl->testflags, "-no-defs")) {
+                        snprintf(buf, sizeof(buf), "%s %s/%s %s -o %s",
+                            task_bins[TASK_COMPILE],
+                            curdir,
+                            tmpl->sourcefile,
+                            tmpl->compileflags,
+                            tmpl->tempfilename
+                        );
+                    } else {
+                        snprintf(buf, sizeof(buf), "%s %s/%s %s/%s %s -o %s",
+                            task_bins[TASK_COMPILE],
+                            curdir,
+                            defs,
+                            curdir,
+                            tmpl->sourcefile,
+                            tmpl->compileflags,
+                            tmpl->tempfilename
+                        );
+                    }
+                }
+            } else {
+                /* Preprocessing (qcflags mean shit all here we don't allow them) */
+                if (tmpl->testflags && !strcmp(tmpl->testflags, "-no-defs")) {
+                    snprintf(buf, sizeof(buf), "%s -E %s/%s -o %s",
+                        task_bins[TASK_COMPILE],
+                        curdir,
+                        tmpl->sourcefile,
+                        tmpl->tempfilename
+                    );
+                } else {
+                    snprintf(buf, sizeof(buf), "%s -E %s/%s %s/%s -o %s",
+                        task_bins[TASK_COMPILE],
+                        curdir,
+                        defs,
+                        curdir,
+                        tmpl->sourcefile,
+                        tmpl->tempfilename
+                    );
+                }
+            }
 
             /*
              * The task template was compiled, now lets create a task from
              * the template data which has now been propagated.
              */
-            task.template = template;
+            task.tmpl = tmpl;
             if (!(task.runhandles = task_popen(buf, "r"))) {
-                con_err("error opening pipe to process for test: %s\n", template->description);
+                con_err("error opening pipe to process for test: %s\n", tmpl->description);
                 success = false;
                 continue;
             }
 
-            util_debug("TEST", "executing test: `%s` [%s]\n", template->description, buf);
+            util_debug("TEST", "executing test: `%s` [%s]\n", tmpl->description, buf);
 
             /*
              * Open up some file desciptors for logging the stdout/stderr
              * to our own.
              */
-            memset  (buf,0,sizeof(buf));
-            snprintf(buf,  sizeof(buf), "%s.stdout", template->tempfilename);
+            snprintf(buf,  sizeof(buf), "%s.stdout", tmpl->tempfilename);
             task.stdoutlogfile = util_strdup(buf);
-            if (!(task.stdoutlog     = file_open(buf, "w"))) {
+            if (!(task.stdoutlog     = fs_file_open(buf, "w"))) {
                 con_err("error opening %s for stdout\n", buf);
                 continue;
             }
 
-            memset  (buf,0,sizeof(buf));
-            snprintf(buf,  sizeof(buf), "%s.stderr", template->tempfilename);
+            snprintf(buf,  sizeof(buf), "%s.stderr", tmpl->tempfilename);
             task.stderrlogfile = util_strdup(buf);
-            if (!(task.stderrlog     = file_open(buf, "w"))) {
+            if (!(task.stderrlog = fs_file_open(buf, "w"))) {
                 con_err("error opening %s for stderr\n", buf);
                 continue;
             }
@@ -817,35 +809,10 @@ bool task_propagate(const char *curdir) {
         found
     );
 
-    closedir(dir);
+    fs_dir_close(dir);
     return success;
 }
 
-/*
- * Removes all temporary 'progs.dat' files created during compilation
- * of all tests'
- */
-void task_cleanup(const char *curdir) {
-    DIR             *dir;
-    struct dirent   *files;
-    char             buffer[4096];
-
-    dir = opendir(curdir);
-
-    while ((files = readdir(dir))) {
-        memset(buffer, 0, sizeof(buffer));
-        if (strstr(files->d_name, "TMP")) {
-            snprintf(buffer, sizeof(buffer), "%s/%s", curdir, files->d_name);
-            if (remove(buffer))
-                con_err("error removing temporary file: %s\n", buffer);
-            else
-                util_debug("TEST", "removed temporary file: %s\n", buffer);
-        }
-    }
-
-    closedir(dir);
-}
-
 /*
  * Task precleanup removes any existing temporary files or log files
  * left behind from a previous invoke of the test-suite.
@@ -855,10 +822,9 @@ void task_precleanup(const char *curdir) {
     struct dirent   *files;
     char             buffer[4096];
 
-    dir = opendir(curdir);
+    dir = fs_dir_open(curdir);
 
-    while ((files = readdir(dir))) {
-        memset(buffer, 0, sizeof(buffer));
+    while ((files = fs_dir_read(dir))) {
         if (strstr(files->d_name, "TMP")     ||
             strstr(files->d_name, ".stdout") ||
             strstr(files->d_name, ".stderr"))
@@ -871,10 +837,10 @@ void task_precleanup(const char *curdir) {
         }
     }
 
-    closedir(dir);
+    fs_dir_close(dir);
 }
 
-void task_destroy(const char *curdir) {
+void task_destroy(void) {
     /*
      * Free all the data in the task list and finally the list itself
      * then proceed to cleanup anything else outside the program like
@@ -887,77 +853,83 @@ void task_destroy(const char *curdir) {
          * annoying to have to do all this cleanup work.
          */
         if (task_tasks[i].runhandles) task_pclose(task_tasks[i].runhandles);
-        if (task_tasks[i].stdoutlog)  file_close (task_tasks[i].stdoutlog);
-        if (task_tasks[i].stderrlog)  file_close (task_tasks[i].stderrlog);
+        if (task_tasks[i].stdoutlog)  fs_file_close (task_tasks[i].stdoutlog);
+        if (task_tasks[i].stderrlog)  fs_file_close (task_tasks[i].stderrlog);
 
         /*
          * Only remove the log files if the test actually compiled otherwise
          * forget about it (or if it didn't compile, and the procedure type
          * was set to -fail (meaning it shouldn't compile) .. stil remove) 
          */
-        if (task_tasks[i].compiled || !strcmp(task_tasks[i].template->proceduretype, "-fail")) {
+        if (task_tasks[i].compiled || !strcmp(task_tasks[i].tmpl->proceduretype, "-fail")) {
             if (remove(task_tasks[i].stdoutlogfile))
                 con_err("error removing stdout log file: %s\n", task_tasks[i].stdoutlogfile);
             else
                 util_debug("TEST", "removed stdout log file: %s\n", task_tasks[i].stdoutlogfile);
-
             if (remove(task_tasks[i].stderrlogfile))
                 con_err("error removing stderr log file: %s\n", task_tasks[i].stderrlogfile);
             else
                 util_debug("TEST", "removed stderr log file: %s\n", task_tasks[i].stderrlogfile);
+
+            remove(task_tasks[i].tmpl->tempfilename);
         }
 
         /* free util_strdup data for log files */
         mem_d(task_tasks[i].stdoutlogfile);
         mem_d(task_tasks[i].stderrlogfile);
 
-        task_template_destroy(&task_tasks[i].template);
+        task_template_destroy(&task_tasks[i].tmpl);
     }
     vec_free(task_tasks);
-
-    /*
-     * Cleanup outside stuff like temporary files.
-     */
-    task_cleanup(curdir);
 }
 
 /*
  * This executes the QCVM task for a specificly compiled progs.dat
  * using the template passed into it for call-flags and user defined
- * messages.
+ * messages IF the procedure type is -execute, otherwise it matches
+ * the preprocessor output. 
  */
-bool task_execute(task_template_t *template, char ***line) {
+bool task_trymatch(task_template_t *tmpl, char ***line) {
     bool     success = true;
     FILE    *execute;
     char     buffer[4096];
     memset  (buffer,0,sizeof(buffer));
 
-    /*
-     * Drop the execution flags for the QCVM if none where
-     * actually specified.
-     */
-    if (!strcmp(template->executeflags, "$null")) {
-        snprintf(buffer,  sizeof(buffer), "%s %s",
-            task_bins[TASK_EXECUTE],
-            template->tempfilename
+    if (strcmp(tmpl->proceduretype, "-pp")) {
+        /*
+         * Drop the execution flags for the QCVM if none where
+         * actually specified.
+         */
+        if (!strcmp(tmpl->executeflags, "$null")) {
+            snprintf(buffer,  sizeof(buffer), "%s %s",
+                task_bins[TASK_EXECUTE],
+                tmpl->tempfilename
+            );
+        } else {
+            snprintf(buffer,  sizeof(buffer), "%s %s %s",
+                task_bins[TASK_EXECUTE],
+                tmpl->executeflags,
+                tmpl->tempfilename
+            );
+        }
+
+        util_debug("TEST", "executing qcvm: `%s` [%s]\n",
+            tmpl->description,
+            buffer
         );
+
+        execute = popen(buffer, "r");
+        if (!execute)
+            return false;
     } else {
-        snprintf(buffer,  sizeof(buffer), "%s %s %s",
-            task_bins[TASK_EXECUTE],
-            template->executeflags,
-            template->tempfilename
-        );
+        /*
+         * we're preprocessing, which means we need to read int
+         * the produced file and do some really weird shit.
+         */
+        if (!(execute = fs_file_open(tmpl->tempfilename, "r")))
+            return false;
     }
 
-    util_debug("TEST", "executing qcvm: `%s` [%s]\n",
-        template->description,
-        buffer
-    );
-
-    execute = popen(buffer, "r");
-    if (!execute)
-        return false;
-
     /*
      * Now lets read the lines and compare them to the matches we expect
      * and handle accordingly.
@@ -966,12 +938,11 @@ bool task_execute(task_template_t *template, char ***line) {
         char  *data    = NULL;
         size_t size    = 0;
         size_t compare = 0;
-        while (file_getline(&data, &size, execute) != EOF) {
+        while (fs_file_getline(&data, &size, execute) != EOF) {
             if (!strcmp(data, "No main function found\n")) {
-                con_err("test failure: `%s` [%s] (No main function found)\n",
-                    template->description,
-                    (template->failuremessage) ?
-                    template->failuremessage : "unknown"
+                con_err("test failure: `%s` (No main function found) [%s]\n",
+                    tmpl->description,
+                    tmpl->rulesfile
                 );
                 pclose(execute);
                 return false;
@@ -984,8 +955,15 @@ bool task_execute(task_template_t *template, char ***line) {
             if  (strrchr(data, '\n'))
                 *strrchr(data, '\n') = '\0';
 
-            if (vec_size(template->comparematch) > compare) {
-                if (strcmp(data, template->comparematch[compare++]))
+            /*
+             * If data is just null now, that means the line was an empty
+             * one and for that, we just ignore it.
+             */
+            if (!*data)
+                continue;
+
+            if (vec_size(tmpl->comparematch) > compare) {
+                if (strcmp(data, tmpl->comparematch[compare++]))
                     success = false;
             } else {
                     success = false;
@@ -1004,33 +982,56 @@ bool task_execute(task_template_t *template, char ***line) {
         mem_d(data);
         data = NULL;
     }
-    pclose(execute);
+
+    if (strcmp(tmpl->proceduretype, "-pp"))
+        pclose(execute);
+    else
+        fs_file_close(execute);
+
     return success;
 }
 
+const char *task_type(task_template_t *tmpl) {
+    if (!strcmp(tmpl->proceduretype, "-pp"))
+        return "type: preprocessor";
+    if (!strcmp(tmpl->proceduretype, "-execute"))
+        return "type: execution";
+    if (!strcmp(tmpl->proceduretype, "-compile"))
+        return "type: compile";
+    return "type: fail";
+}
+
 /*
  * This schedualizes all tasks and actually runs them individually
  * this is generally easy for just -compile variants.  For compile and
  * execution this takes more work since a task needs to be generated
  * from thin air and executed INLINE.
  */
-void task_schedualize() {
+#include <math.h>
+void task_schedualize(size_t *pad) {
+    char   space[2][64];
     bool   execute  = false;
     char  *data     = NULL;
     char **match    = NULL;
     size_t size     = 0;
-    size_t i;
-    size_t j;
+    size_t i        = 0;
+    size_t j        = 0;
 
-    util_debug("TEST", "found %d tasks, preparing to execute\n", vec_size(task_tasks));
+    snprintf(space[0], sizeof(space[0]), "%d", (int)vec_size(task_tasks));
 
-    for (i = 0; i < vec_size(task_tasks); i++) {
-        util_debug("TEST", "executing task: %d: %s\n", i, task_tasks[i].template->description);
+    for (; i < vec_size(task_tasks); i++) {
+        memset(space[1], 0, sizeof(space[1]));
+        snprintf(space[1], sizeof(space[1]), "%d", (int)(i + 1));
+        
+        con_out("test #%u %*s", i + 1, strlen(space[0]) - strlen(space[1]), "");
+
+        util_debug("TEST", "executing task: %d: %s\n", i, task_tasks[i].tmpl->description);
         /*
          * Generate a task from thin air if it requires execution in
          * the QCVM.
          */
-        execute = !!(!strcmp(task_tasks[i].template->proceduretype, "-execute"));
+        execute = !! (!strcmp(task_tasks[i].tmpl->proceduretype, "-execute")) ||
+                     (!strcmp(task_tasks[i].tmpl->proceduretype, "-pp"));
 
         /*
          * We assume it compiled before we actually compiled :).  On error
@@ -1042,17 +1043,15 @@ void task_schedualize() {
          * Read data from stdout first and pipe that stuff into a log file
          * then we do the same for stderr.
          */
-        while (file_getline(&data, &size, task_tasks[i].runhandles[1]) != EOF) {
-            file_puts(task_tasks[i].stdoutlog, data);
+        while (fs_file_getline(&data, &size, task_tasks[i].runhandles[1]) != EOF) {
+            fs_file_puts(task_tasks[i].stdoutlog, data);
 
             if (strstr(data, "failed to open file")) {
                 task_tasks[i].compiled = false;
                 execute                = false;
             }
-
-            fflush(task_tasks[i].stdoutlog);
         }
-        while (file_getline(&data, &size, task_tasks[i].runhandles[2]) != EOF) {
+        while (fs_file_getline(&data, &size, task_tasks[i].runhandles[2]) != EOF) {
             /*
              * If a string contains an error we just dissalow execution
              * of it in the vm.
@@ -1066,41 +1065,52 @@ void task_schedualize() {
                 task_tasks[i].compiled = false;
             }
 
-            file_puts(task_tasks[i].stderrlog, data);
-            fflush(task_tasks[i].stdoutlog);
+            fs_file_puts (task_tasks[i].stderrlog, data);
         }
 
-        if (!task_tasks[i].compiled && strcmp(task_tasks[i].template->proceduretype, "-fail")) {
-            con_err("test failure: `%s` [%s] (failed to compile) see %s.stdout and %s.stderr\n",
-                task_tasks[i].template->description,
-                (task_tasks[i].template->failuremessage) ?
-                task_tasks[i].template->failuremessage : "unknown",
-                task_tasks[i].template->tempfilename,
-                task_tasks[i].template->tempfilename
+        if (!task_tasks[i].compiled && strcmp(task_tasks[i].tmpl->proceduretype, "-fail")) {
+            con_out("failure:   `%s` %*s %*s\n",
+                task_tasks[i].tmpl->description,
+                (pad[0] + pad[1] - strlen(task_tasks[i].tmpl->description)) + (strlen(task_tasks[i].tmpl->rulesfile) - pad[1]),
+                task_tasks[i].tmpl->rulesfile,
+                (pad[1] + pad[2] - strlen(task_tasks[i].tmpl->rulesfile)) + (strlen("(failed to compile)") - pad[2]),
+                "(failed to compile)"
             );
             continue;
         }
 
         if (!execute) {
-            con_out("test succeeded: `%s` [%s]\n",
-                 task_tasks[i].template->description,
-                (task_tasks[i].template->successmessage) ?
-                 task_tasks[i].template->successmessage  : "unknown"
+            con_out("succeeded: `%s` %*s %*s\n",
+                task_tasks[i].tmpl->description,
+                (pad[0] + pad[1] - strlen(task_tasks[i].tmpl->description)) + (strlen(task_tasks[i].tmpl->rulesfile) - pad[1]),
+                task_tasks[i].tmpl->rulesfile,
+                (pad[1] + pad[2] - strlen(task_tasks[i].tmpl->rulesfile)) + (strlen(task_type(task_tasks[i].tmpl)) - pad[2]),
+                task_type(task_tasks[i].tmpl)
+                
             );
             continue;
         }
 
         /*
          * If we made it here that concludes the task is to be executed
-         * in the virtual machine.
+         * in the virtual machine (or the preprocessor output needs to
+         * be matched). 
          */
-        if (!task_execute(task_tasks[i].template, &match)) {
+        if (!task_trymatch(task_tasks[i].tmpl, &match)) {
             size_t d = 0;
 
-            con_err("test failure: `%s` [%s] (invalid results from execution)\n",
-                 task_tasks[i].template->description,
-                (task_tasks[i].template->failuremessage) ?
-                 task_tasks[i].template->failuremessage : "unknown"
+            con_out("failure:   `%s` %*s %*s\n",
+                task_tasks[i].tmpl->description,
+                (pad[0] + pad[1] - strlen(task_tasks[i].tmpl->description)) + (strlen(task_tasks[i].tmpl->rulesfile) - pad[1]),
+                task_tasks[i].tmpl->rulesfile,
+                (pad[1] + pad[2] - strlen(task_tasks[i].tmpl->rulesfile)) + (strlen(
+                    (strcmp(task_tasks[i].tmpl->proceduretype, "-pp"))
+                        ? "(invalid results from execution)"
+                        : "(invalid results from preprocessing)"
+                ) - pad[2]),
+                (strcmp(task_tasks[i].tmpl->proceduretype, "-pp"))
+                    ? "(invalid results from execution)"
+                    : "(invalid results from preprocessing)"
             );
 
             /*
@@ -1108,18 +1118,18 @@ void task_schedualize() {
              * handler for the all the given matches in the template file and
              * what was actually returned from executing.
              */
-            con_err("    Expected From %u Matches: (got %u Matches)\n",
-                vec_size(task_tasks[i].template->comparematch),
+            con_out("    Expected From %u Matches: (got %u Matches)\n",
+                vec_size(task_tasks[i].tmpl->comparematch),
                 vec_size(match)
             );
-            for (; d < vec_size(task_tasks[i].template->comparematch); d++) {
-                char  *select = task_tasks[i].template->comparematch[d];
+            for (; d < vec_size(task_tasks[i].tmpl->comparematch); d++) {
+                char  *select = task_tasks[i].tmpl->comparematch[d];
                 size_t length = 40 - strlen(select);
 
-                con_err("        Expected: \"%s\"", select);
+                con_out("        Expected: \"%s\"", select);
                 while (length --)
-                    con_err(" ");
-                con_err("| Got: \"%s\"\n", (d >= vec_size(match)) ? "<<nothing else to compare>>" : match[d]);
+                    con_out(" ");
+                con_out("| Got: \"%s\"\n", (d >= vec_size(match)) ? "<<nothing else to compare>>" : match[d]);
             }
 
             /*
@@ -1127,10 +1137,10 @@ void task_schedualize() {
              * This will help track down bugs in template files that fail to match
              * something.
              */  
-            if (vec_size(match) > vec_size(task_tasks[i].template->comparematch)) {
-                for (d = 0; d < vec_size(match) - vec_size(task_tasks[i].template->comparematch); d++) {
-                    con_err("        Expected: Nothing                                   | Got: \"%s\"\n",
-                        match[d + vec_size(task_tasks[i].template->comparematch)]
+            if (vec_size(match) > vec_size(task_tasks[i].tmpl->comparematch)) {
+                for (d = 0; d < vec_size(match) - vec_size(task_tasks[i].tmpl->comparematch); d++) {
+                    con_out("        Expected: Nothing                                   | Got: \"%s\"\n",
+                        match[d + vec_size(task_tasks[i].tmpl->comparematch)]
                     );
                 }
             }
@@ -1145,10 +1155,13 @@ void task_schedualize() {
             mem_d(match[j]);
         vec_free(match);
 
-        con_out("test succeeded: `%s` [%s]\n",
-             task_tasks[i].template->description,
-            (task_tasks[i].template->successmessage) ?
-             task_tasks[i].template->successmessage  : "unknown"
+        con_out("succeeded: `%s` %*s %*s\n",
+            task_tasks[i].tmpl->description,
+            (pad[0] + pad[1] - strlen(task_tasks[i].tmpl->description)) + (strlen(task_tasks[i].tmpl->rulesfile) - pad[1]),
+            task_tasks[i].tmpl->rulesfile,
+            (pad[1] + pad[2] - strlen(task_tasks[i].tmpl->rulesfile)) + (strlen(task_type(task_tasks[i].tmpl))- pad[2]),
+            task_type(task_tasks[i].tmpl)
+            
         );
     }
     mem_d(data);
@@ -1168,11 +1181,27 @@ void task_schedualize() {
  *
  * It expects con_init() was called before hand.
  */
-bool test_perform(const char *curdir) {
+GMQCC_WARN bool test_perform(const char *curdir, const char *defs) {
+    static const char *default_defs = "defs.qh";
+
+    size_t pad[] = {
+        /* test ### [succeed/fail]: `description`      [tests/template.tmpl]     [type] */
+                    0,                                 0,                        0
+    };
+
+    /*
+     * If the default definition file isn't set to anything.  We will
+     * use the default_defs here, which is "defs.qc"
+     */   
+    if (!defs) {
+        defs = default_defs;
+    }
+        
+
     task_precleanup(curdir);
-    if (!task_propagate(curdir)) {
+    if (!task_propagate(curdir, pad, defs)) {
         con_err("error: failed to propagate tasks\n");
-        task_destroy(curdir);
+        task_destroy();
         return false;
     }
     /*
@@ -1182,8 +1211,8 @@ bool test_perform(const char *curdir) {
      * it's designed to prevent lock contention, and possible syncronization
      * issues.
      */
-    task_schedualize();
-    task_destroy(curdir);
+    task_schedualize(pad);
+    task_destroy();
 
     return true;
 }
@@ -1219,8 +1248,10 @@ static bool parsecmd(const char *optname, int *argc_, char ***argv_, char **out,
 }
 
 int main(int argc, char **argv) {
+    bool          succeed  = false;
     char         *redirout = (char*)stdout;
     char         *redirerr = (char*)stderr;
+    char         *defs     = NULL;
 
     con_init();
 
@@ -1237,15 +1268,17 @@ int main(int argc, char **argv) {
                 continue;
             if (parsecmd("redirerr", &argc, &argv, &redirerr, 1, false))
                 continue;
+            if (parsecmd("defs",     &argc, &argv, &defs,     1, false))
+                continue;
 
             con_change(redirout, redirerr);
 
             if (!strcmp(argv[0]+1, "debug")) {
-                opts.debug = true;
+                OPTS_OPTION_BOOL(OPTION_DEBUG) = true;
                 continue;
             }
             if (!strcmp(argv[0]+1, "memchk")) {
-                opts.memchk = true;
+                OPTS_OPTION_BOOL(OPTION_MEMCHK) = true;
                 continue;
             }
             if (!strcmp(argv[0]+1, "nocolor")) {
@@ -1258,7 +1291,9 @@ int main(int argc, char **argv) {
         }
     }
     con_change(redirout, redirerr);
-    test_perform("tests");
+    succeed = test_perform("tests", defs);
     util_meminfo();
-    return 0;
+
+
+    return (succeed) ? EXIT_SUCCESS : EXIT_FAILURE;
 }