]> de.git.xonotic.org Git - xonotic/netradiant.git/blob - tools/quake3/q3data/compress.c
transfer from internal tree r5311 branches/1.4-gpl
[xonotic/netradiant.git] / tools / quake3 / q3data / compress.c
1 #include "q3data.h"\r
2 \r
3 #if 0\r
4 /*\r
5 ==================\r
6 MTF\r
7 ==================\r
8 */\r
9 cblock_t MTF (cblock_t in)\r
10 {\r
11         int                     i, j, b, code;\r
12         byte            *out_p;\r
13         int                     index[256];\r
14         cblock_t        out;\r
15 \r
16         out_p = out.data = malloc(in.count + 4);\r
17 \r
18         // write count\r
19         *out_p++ = in.count&255;\r
20         *out_p++ = (in.count>>8)&255;\r
21         *out_p++ = (in.count>>16)&255;\r
22         *out_p++ = (in.count>>24)&255;\r
23 \r
24         for (i=0 ; i<256 ; i++)\r
25                 index[i] = i;\r
26 \r
27         for (i=0 ; i<in.count ; i++)\r
28         {\r
29                 b = in.data[i];\r
30                 code = index[b];\r
31                 *out_p++ = code;\r
32                 \r
33                 // shuffle b indexes to 0\r
34                 for (j=0 ; j<256 ; j++)\r
35                         if (index[j] < code)\r
36                                 index[j]++;\r
37                 index[b] = 0;\r
38         }\r
39 \r
40         out.count = out_p - out.data;\r
41 \r
42         return out;\r
43 }\r
44 \r
45 \r
46 //==========================================================================\r
47 \r
48 int             bwt_size;\r
49 byte    *bwt_data;\r
50 \r
51 int bwtCompare (const void *elem1, const void *elem2)\r
52 {\r
53         int             i;\r
54         int             i1, i2;\r
55         int             b1, b2;\r
56 \r
57         i1 = *(int *)elem1;\r
58         i2 = *(int *)elem2;\r
59 \r
60         for (i=0 ; i<bwt_size ; i++)\r
61         {\r
62                 b1 = bwt_data[i1];\r
63                 b2 = bwt_data[i2];\r
64                 if (b1 < b2)\r
65                         return -1;\r
66                 if (b1 > b2)\r
67                         return 1;\r
68                 if (++i1 == bwt_size)\r
69                         i1 = 0;\r
70                 if (++i2 == bwt_size)\r
71                         i2 = 0;\r
72         }\r
73 \r
74         return 0;\r
75 }\r
76 \r
77 /*\r
78 ==================\r
79 BWT\r
80 ==================\r
81 */\r
82 cblock_t BWT (cblock_t in)\r
83 {\r
84         int             *sorted;\r
85         int             i;\r
86         byte    *out_p;\r
87         cblock_t        out;\r
88 \r
89         bwt_size = in.count;\r
90         bwt_data = in.data;\r
91 \r
92         sorted = malloc(in.count*sizeof(*sorted));\r
93         for (i=0 ; i<in.count ; i++)\r
94                 sorted[i] = i;\r
95         qsort (sorted, in.count, sizeof(*sorted), bwtCompare);\r
96 \r
97         out_p = out.data = malloc(in.count + 8);\r
98 \r
99         // write count\r
100         *out_p++ = in.count&255;\r
101         *out_p++ = (in.count>>8)&255;\r
102         *out_p++ = (in.count>>16)&255;\r
103         *out_p++ = (in.count>>24)&255;\r
104 \r
105         // write head index\r
106         for (i=0 ; i<in.count ; i++)\r
107                 if (sorted[i] == 0)\r
108                         break;\r
109         *out_p++ = i&255;\r
110         *out_p++ = (i>>8)&255;\r
111         *out_p++ = (i>>16)&255;\r
112         *out_p++ = (i>>24)&255;\r
113 \r
114         // write the L column\r
115         for (i=0 ; i<in.count ; i++)\r
116                 *out_p++ = in.data[(sorted[i]+in.count-1)%in.count];\r
117 \r
118         free (sorted);\r
119 \r
120         out.count = out_p - out.data;\r
121 \r
122         return out;\r
123 }\r
124 \r
125 //==========================================================================\r
126 \r
127 typedef struct hnode_s\r
128 {\r
129         int                     count;\r
130         qboolean        used;\r
131         int                     children[2];\r
132 } hnode_t;\r
133 \r
134 int                     numhnodes;\r
135 hnode_t         hnodes[512];\r
136 unsigned        charbits[256];\r
137 int                     charbitscount[256];\r
138 \r
139 int     SmallestNode (void)\r
140 {\r
141         int             i;\r
142         int             best, bestnode;\r
143 \r
144         best = 99999999;\r
145         bestnode = -1;\r
146         for (i=0 ; i<numhnodes ; i++)\r
147         {\r
148                 if (hnodes[i].used)\r
149                         continue;\r
150                 if (!hnodes[i].count)\r
151                         continue;\r
152                 if (hnodes[i].count < best)\r
153                 {\r
154                         best = hnodes[i].count;\r
155                         bestnode = i;\r
156                 }\r
157         }\r
158 \r
159         if (bestnode == -1)\r
160                 return -1;\r
161 \r
162         hnodes[bestnode].used = true;\r
163         return bestnode;\r
164 }\r
165 \r
166 void BuildChars (int nodenum, unsigned bits, int bitcount)\r
167 {\r
168         hnode_t *node;\r
169 \r
170         if (nodenum < 256)\r
171         {\r
172                 if (bitcount > 32)\r
173                         Error ("bitcount > 32");\r
174                 charbits[nodenum] = bits;\r
175                 charbitscount[nodenum] = bitcount;\r
176                 return;\r
177         }\r
178 \r
179         node = &hnodes[nodenum];\r
180         bits <<= 1;\r
181         BuildChars (node->children[0], bits, bitcount+1);\r
182         bits |= 1;\r
183         BuildChars (node->children[1], bits, bitcount+1);\r
184 }\r
185 \r
186 /*\r
187 ==================\r
188 Huffman\r
189 ==================\r
190 */\r
191 cblock_t Huffman (cblock_t in)\r
192 {\r
193         int                     i;\r
194         hnode_t         *node;\r
195         int                     outbits, c;\r
196         unsigned        bits;\r
197         byte            *out_p;\r
198         cblock_t        out;\r
199         int                     max, maxchar;\r
200 \r
201         // count\r
202         memset (hnodes, 0, sizeof(hnodes));\r
203         for (i=0 ; i<in.count ; i++)\r
204                 hnodes[in.data[i]].count++;\r
205 \r
206         // normalize counts\r
207         max = 0;\r
208         maxchar = 0;\r
209         for (i=0 ; i<256 ; i++)\r
210         {\r
211                 if (hnodes[i].count > max)\r
212                 {\r
213                         max = hnodes[i].count;\r
214                         maxchar = i;\r
215                 }\r
216         }\r
217         if (max == 0)\r
218                 Error ("Huffman: max == 0");\r
219 \r
220         for (i=0 ; i<256 ; i++)\r
221         {\r
222                 hnodes[i].count = (hnodes[i].count*255+max-1) / max;\r
223         }\r
224 \r
225         // build the nodes\r
226         numhnodes = 256;\r
227         while (numhnodes != 511)\r
228         {\r
229                 node = &hnodes[numhnodes];\r
230 \r
231                 // pick two lowest counts\r
232                 node->children[0] = SmallestNode ();\r
233                 if (node->children[0] == -1)\r
234                         break;  // no more\r
235 \r
236                 node->children[1] = SmallestNode ();\r
237                 if (node->children[1] == -1)\r
238                 {\r
239                         if (node->children[0] != numhnodes-1)\r
240                                 Error ("Bad smallestnode");\r
241                         break;\r
242                 }\r
243                 node->count = hnodes[node->children[0]].count + \r
244                         hnodes[node->children[1]].count;\r
245                 numhnodes++;\r
246         }\r
247 \r
248         BuildChars (numhnodes-1, 0, 0);\r
249 \r
250         out_p = out.data = malloc(in.count*2 + 1024);\r
251         memset (out_p, 0, in.count*2+1024);\r
252 \r
253         // write count\r
254         *out_p++ = in.count&255;\r
255         *out_p++ = (in.count>>8)&255;\r
256         *out_p++ = (in.count>>16)&255;\r
257         *out_p++ = (in.count>>24)&255;\r
258 \r
259         // save out the 256 normalized counts so the tree can be recreated\r
260         for (i=0 ; i<256 ; i++)\r
261                 *out_p++ = hnodes[i].count;\r
262 \r
263         // write bits\r
264         outbits = 0;\r
265         for (i=0 ; i<in.count ; i++)\r
266         {\r
267                 c = charbitscount[in.data[i]];\r
268                 bits = charbits[in.data[i]];\r
269                 while (c)\r
270                 {\r
271                         c--;\r
272                         if (bits & (1<<c))\r
273                                 out_p[outbits>>3] |= 1<<(outbits&7);\r
274                         outbits++;\r
275                 }\r
276         }\r
277 \r
278         out_p += (outbits+7)>>3;\r
279 \r
280         out.count = out_p - out.data;\r
281 \r
282         return out;\r
283 }\r
284 \r
285 //==========================================================================\r
286 \r
287 /*\r
288 ==================\r
289 RLE\r
290 ==================\r
291 */\r
292 #define RLE_CODE        0xe8\r
293 #define RLE_TRIPPLE     0xe9\r
294 \r
295 int     rle_counts[256];\r
296 int     rle_bytes[256];\r
297 \r
298 cblock_t RLE (cblock_t in)\r
299 {\r
300         int             i;\r
301         byte    *out_p;\r
302         int             val;\r
303         int             repeat;\r
304         cblock_t        out;\r
305 \r
306         out_p = out.data = malloc (in.count*2);\r
307 \r
308         // write count\r
309         *out_p++ = in.count&255;\r
310         *out_p++ = (in.count>>8)&255;\r
311         *out_p++ = (in.count>>16)&255;\r
312         *out_p++ = (in.count>>24)&255;\r
313 \r
314         for (i=0 ; i<in.count ; )\r
315         {\r
316                 val = in.data[i];\r
317                 rle_bytes[val]++;\r
318                 repeat = 1;\r
319                 i++;\r
320                 while (i<in.count && repeat < 255 && in.data[i] == val)\r
321                 {\r
322                         repeat++;\r
323                         i++;\r
324                 }\r
325 if (repeat < 256)\r
326 rle_counts[repeat]++;\r
327                 if (repeat > 3 || val == RLE_CODE)\r
328                 {\r
329                         *out_p++ = RLE_CODE;\r
330                         *out_p++ = val;\r
331                         *out_p++ = repeat;\r
332                 }\r
333                 else\r
334                 {\r
335                         while (repeat--)\r
336                                 *out_p++ = val;\r
337                 }\r
338         }\r
339 \r
340         out.count = out_p - out.data;\r
341         return out;\r
342 }\r
343 \r
344 //==========================================================================\r
345 \r
346 unsigned        lzss_head[256];\r
347 unsigned        lzss_next[0x20000];\r
348 \r
349 /*\r
350 ==================\r
351 LZSS\r
352 ==================\r
353 */\r
354 #define BACK_WINDOW             0x10000\r
355 #define BACK_BITS               16\r
356 #define FRONT_WINDOW    16\r
357 #define FRONT_BITS              4\r
358 cblock_t LZSS (cblock_t in)\r
359 {\r
360         int             i;\r
361         byte    *out_p;\r
362         cblock_t        out;\r
363         int             val;\r
364         int             j, start, max;\r
365         int             bestlength, beststart;\r
366         int             outbits;\r
367 \r
368 if (in.count >= sizeof(lzss_next)/4)\r
369 Error ("LZSS: too big");\r
370 \r
371         memset (lzss_head, -1, sizeof(lzss_head));\r
372 \r
373         out_p = out.data = malloc (in.count*2);\r
374         memset (out.data, 0, in.count*2);\r
375 \r
376         // write count\r
377         *out_p++ = in.count&255;\r
378         *out_p++ = (in.count>>8)&255;\r
379         *out_p++ = (in.count>>16)&255;\r
380         *out_p++ = (in.count>>24)&255;\r
381 \r
382         outbits = 0;\r
383         for (i=0 ; i<in.count ; )\r
384         {\r
385                 val = in.data[i];\r
386 #if 1\r
387 // chained search\r
388                 bestlength = 0;\r
389                 beststart = 0;\r
390 \r
391                 max = FRONT_WINDOW;\r
392                 if (i + max > in.count)\r
393                         max = in.count - i;\r
394 \r
395                 start = lzss_head[val];\r
396                 while (start != -1 && start >= i-BACK_WINDOW)\r
397                 {                       \r
398                         // count match length\r
399                         for (j=0 ; j<max ; j++)\r
400                                 if (in.data[start+j] != in.data[i+j])\r
401                                         break;\r
402                         if (j > bestlength)\r
403                         {\r
404                                 bestlength = j;\r
405                                 beststart = start;\r
406                         }\r
407                         start = lzss_next[start];\r
408                 }\r
409 \r
410 #else\r
411 // slow simple search\r
412                 // search for a match\r
413                 max = FRONT_WINDOW;\r
414                 if (i + max > in.count)\r
415                         max = in.count - i;\r
416 \r
417                 start = i - BACK_WINDOW;\r
418                 if (start < 0)\r
419                         start = 0;\r
420                 bestlength = 0;\r
421                 beststart = 0;\r
422                 for ( ; start < i ; start++)\r
423                 {\r
424                         if (in.data[start] != val)\r
425                                 continue;\r
426                         // count match length\r
427                         for (j=0 ; j<max ; j++)\r
428                                 if (in.data[start+j] != in.data[i+j])\r
429                                         break;\r
430                         if (j > bestlength)\r
431                         {\r
432                                 bestlength = j;\r
433                                 beststart = start;\r
434                         }\r
435                 }\r
436 #endif\r
437                 beststart = BACK_WINDOW - (i-beststart);\r
438 \r
439                 if (bestlength < 3)\r
440                 {       // output a single char\r
441                         bestlength = 1;\r
442 \r
443                         out_p[outbits>>3] |= 1<<(outbits&7);    // set bit to mark char\r
444                         outbits++;\r
445                         for (j=0 ; j<8 ; j++, outbits++)\r
446                                 if (val & (1<<j) )\r
447                                         out_p[outbits>>3] |= 1<<(outbits&7);\r
448                 }\r
449                 else\r
450                 {       // output a phrase\r
451                         outbits++;      // leave a 0 bit to mark phrase\r
452                         for (j=0 ; j<BACK_BITS ; j++, outbits++)\r
453                                 if (beststart & (1<<j) )\r
454                                         out_p[outbits>>3] |= 1<<(outbits&7);\r
455                         for (j=0 ; j<FRONT_BITS ; j++, outbits++)\r
456                                 if (bestlength & (1<<j) )\r
457                                         out_p[outbits>>3] |= 1<<(outbits&7);\r
458                 }\r
459 \r
460                 while (bestlength--)\r
461                 {\r
462                         val = in.data[i];\r
463                         lzss_next[i] = lzss_head[val];\r
464                         lzss_head[val] = i;\r
465                         i++;\r
466                 }\r
467         }\r
468 \r
469         out_p += (outbits+7)>>3;\r
470         out.count = out_p - out.data;\r
471         return out;\r
472 }\r
473 \r
474 //==========================================================================\r
475 \r
476 #define MIN_REPT        15\r
477 #define MAX_REPT        0\r
478 #define HUF_TOKENS      (256+MAX_REPT)\r
479 \r
480 unsigned        charbits1[256][HUF_TOKENS];\r
481 int                     charbitscount1[256][HUF_TOKENS];\r
482 \r
483 hnode_t         hnodes1[256][HUF_TOKENS*2];\r
484 int                     numhnodes1[256];\r
485 \r
486 int                     order0counts[256];\r
487 \r
488 /*\r
489 ==================\r
490 SmallestNode1\r
491 ==================\r
492 */\r
493 int     SmallestNode1 (hnode_t *hnodes, int numhnodes)\r
494 {\r
495         int             i;\r
496         int             best, bestnode;\r
497 \r
498         best = 99999999;\r
499         bestnode = -1;\r
500         for (i=0 ; i<numhnodes ; i++)\r
501         {\r
502                 if (hnodes[i].used)\r
503                         continue;\r
504                 if (!hnodes[i].count)\r
505                         continue;\r
506                 if (hnodes[i].count < best)\r
507                 {\r
508                         best = hnodes[i].count;\r
509                         bestnode = i;\r
510                 }\r
511         }\r
512 \r
513         if (bestnode == -1)\r
514                 return -1;\r
515 \r
516         hnodes[bestnode].used = true;\r
517         return bestnode;\r
518 }\r
519 \r
520 \r
521 /*\r
522 ==================\r
523 BuildChars1\r
524 ==================\r
525 */\r
526 void BuildChars1 (int prev, int nodenum, unsigned bits, int bitcount)\r
527 {\r
528         hnode_t *node;\r
529 \r
530         if (nodenum < HUF_TOKENS)\r
531         {\r
532                 if (bitcount > 32)\r
533                         Error ("bitcount > 32");\r
534                 charbits1[prev][nodenum] = bits;\r
535                 charbitscount1[prev][nodenum] = bitcount;\r
536                 return;\r
537         }\r
538 \r
539         node = &hnodes1[prev][nodenum];\r
540         bits <<= 1;\r
541         BuildChars1 (prev, node->children[0], bits, bitcount+1);\r
542         bits |= 1;\r
543         BuildChars1 (prev, node->children[1], bits, bitcount+1);\r
544 }\r
545 \r
546 \r
547 /*\r
548 ==================\r
549 BuildTree1\r
550 ==================\r
551 */\r
552 void BuildTree1 (int prev)\r
553 {\r
554         hnode_t         *node, *nodebase;\r
555         int                     numhnodes;\r
556 \r
557         // build the nodes\r
558         numhnodes = HUF_TOKENS;\r
559         nodebase = hnodes1[prev];\r
560         while (1)\r
561         {\r
562                 node = &nodebase[numhnodes];\r
563 \r
564                 // pick two lowest counts\r
565                 node->children[0] = SmallestNode1 (nodebase, numhnodes);\r
566                 if (node->children[0] == -1)\r
567                         break;  // no more\r
568 \r
569                 node->children[1] = SmallestNode1 (nodebase, numhnodes);\r
570                 if (node->children[1] == -1)\r
571                         break;\r
572 \r
573                 node->count = nodebase[node->children[0]].count + \r
574                         nodebase[node->children[1]].count;\r
575                 numhnodes++;\r
576         }\r
577         numhnodes1[prev] = numhnodes-1;\r
578         BuildChars1 (prev, numhnodes-1, 0, 0);\r
579 }\r
580 \r
581 \r
582 /*\r
583 ==================\r
584 Huffman1_Count\r
585 ==================\r
586 */\r
587 void Huffman1_Count (cblock_t in)\r
588 {\r
589         int             i;\r
590         int             prev;\r
591         int             v;\r
592         int             rept;\r
593 \r
594         prev = 0;\r
595         for (i=0 ; i<in.count ; i++)\r
596         {\r
597                 v = in.data[i];\r
598                 order0counts[v]++;\r
599                 hnodes1[prev][v].count++;\r
600                 prev = v;\r
601 #if 1\r
602                 for (rept=1 ; i+rept < in.count && rept < MAX_REPT ; rept++)\r
603                         if (in.data[i+rept] != v)\r
604                                 break;\r
605                 if (rept > MIN_REPT)\r
606                 {\r
607                         hnodes1[prev][255+rept].count++;\r
608                         i += rept-1;\r
609                 }\r
610 #endif\r
611         }\r
612 }\r
613 \r
614 \r
615 /*\r
616 ==================\r
617 Huffman1_Build\r
618 ==================\r
619 */\r
620 byte    scaled[256][HUF_TOKENS];\r
621 void Huffman1_Build (FILE *f)\r
622 {\r
623         int             i, j, v;\r
624         int             max;\r
625         int             total;\r
626 \r
627         for (i=0 ; i<256 ; i++)\r
628         {\r
629                 // normalize and save the counts\r
630                 max = 0;\r
631                 for (j=0 ; j<HUF_TOKENS ; j++)\r
632                 {\r
633                         if (hnodes1[i][j].count > max)\r
634                                 max = hnodes1[i][j].count;\r
635                 }\r
636                 if (max == 0)\r
637                         max = 1;\r
638                 total = 0;\r
639                 for (j=0 ; j<HUF_TOKENS ; j++)\r
640                 {       // easy to overflow 32 bits here!\r
641                         v = (hnodes1[i][j].count*(double)255+max-1)/max;\r
642                         if (v > 255)\r
643                                 Error ("v > 255");\r
644                         scaled[i][j] = hnodes1[i][j].count = v;\r
645                         if (v)\r
646                                 total++;\r
647                 }\r
648                 if (total == 1)\r
649                 {       // must have two tokens\r
650                         if (!scaled[i][0])\r
651                                 scaled[i][0] = hnodes1[i][0].count = 1;\r
652                         else\r
653                                 scaled[i][1] = hnodes1[i][1].count = 1;\r
654                 }\r
655 \r
656                 BuildTree1 (i);\r
657         }\r
658 \r
659 #if 0\r
660         // count up the total bits\r
661         total = 0;\r
662         for (i=0 ; i<256 ; i++)\r
663                 for (j=0 ; j<256 ; j++)\r
664                         total += charbitscount1[i][j] * hnodes1[i][j].count;\r
665 \r
666         total = (total+7)/8;\r
667         printf ("%i bytes huffman1 compressed\n", total);\r
668 #endif\r
669 \r
670         fwrite (scaled, 1, sizeof(scaled), f);\r
671 }\r
672 \r
673 /*\r
674 ==================\r
675 Huffman1\r
676 \r
677 Order 1 compression with pre-built table\r
678 ==================\r
679 */\r
680 cblock_t Huffman1 (cblock_t in)\r
681 {\r
682         int                     i;\r
683         int                     outbits, c;\r
684         unsigned        bits;\r
685         byte            *out_p;\r
686         cblock_t        out;\r
687         int                     prev;\r
688         int                     v;\r
689         int                     rept;\r
690 \r
691         out_p = out.data = malloc(in.count*2 + 1024);\r
692         memset (out_p, 0, in.count*2+1024);\r
693 \r
694         // write count\r
695         *out_p++ = in.count&255;\r
696         *out_p++ = (in.count>>8)&255;\r
697         *out_p++ = (in.count>>16)&255;\r
698         *out_p++ = (in.count>>24)&255;\r
699 \r
700         // write bits\r
701         outbits = 0;\r
702         prev = 0;\r
703         for (i=0 ; i<in.count ; i++)\r
704         {\r
705                 v = in.data[i];\r
706 \r
707                 c = charbitscount1[prev][v];\r
708                 bits = charbits1[prev][v];\r
709                 if (!c)\r
710                         Error ("!bits");\r
711                 while (c)\r
712                 {\r
713                         c--;\r
714                         if (bits & (1<<c))\r
715                                 out_p[outbits>>3] |= 1<<(outbits&7);\r
716                         outbits++;\r
717                 }\r
718 \r
719                 prev = v;\r
720 #if 1\r
721                 // check for repeat encodes\r
722                 for (rept=1 ; i+rept < in.count && rept < MAX_REPT ; rept++)\r
723                         if (in.data[i+rept] != v)\r
724                                 break;\r
725                 if (rept > MIN_REPT)\r
726                 {\r
727                         c = charbitscount1[prev][255+rept];\r
728                         bits = charbits1[prev][255+rept];\r
729                         if (!c)\r
730                                 Error ("!bits");\r
731                         while (c)\r
732                         {\r
733                                 c--;\r
734                                 if (bits & (1<<c))\r
735                                         out_p[outbits>>3] |= 1<<(outbits&7);\r
736                                 outbits++;\r
737                         }\r
738                         i += rept-1;\r
739                 }\r
740 #endif\r
741         }\r
742 \r
743         out_p += (outbits+7)>>3;\r
744 \r
745         out.count = out_p - out.data;\r
746 \r
747         return out;\r
748 }\r
749 \r
750 #endif\r