ViewVC Help
View File | Revision Log | Show Annotations | Root Listing
root/cvsroot/UserCode/MitCommon/OptIO/src/rzipunzip.c
Revision: 1.1
Committed: Tue Feb 24 11:56:44 2009 UTC (16 years, 2 months ago) by loizides
Content type: text/plain
Branch: MAIN
Log Message:
Preload lib for compression improvements.

File Contents

# User Rev Content
1 loizides 1.1 // $Id: Types.cc,v 1.1 2008/09/27 05:44:11 loizides Exp $
2    
3     #include "MitCommon/OptIO/src/rzipunzip.h"
4     #include "MitCommon/OptIO/src/zlib.h"
5     #include "MitCommon/OptIO/src/bzlib.h"
6     #include "MitCommon/OptIO/src/lzo/lzo1x.h"
7     #include "MitCommon/OptIO/src/rle.h"
8    
9     #include <stdio.h>
10     #include <stdlib.h>
11     #include <string.h>
12    
13     #ifndef HDRSIZE
14     #define HDRSIZE 9
15     #else
16     #error "HDRSIZE already defined."
17     #endif
18    
19     #ifndef MYSIZE
20     #define MYSIZE 5*1024*1024
21     #else
22     #error "MYSIZE already defined."
23     #endif
24    
25     typedef unsigned char uch; /* code assumes unsigned bytes; these type- */
26    
27     extern int R__ZipMode;
28     char mymem[MYSIZE];
29     char *myptr = mymem;
30    
31     // the following can be changed using the OptInt interface
32     double lzipfrac = 1;
33     double gzipfrac = 1;
34     double bzipfrac = 1;
35     int myverbose = 0;
36     int mystaticm = 0;
37    
38     //--------------------------------------------------------------------------------------------------
39     void *mymalloc(size_t size)
40     {
41     if(!mystaticm || size>MYSIZE)
42     return malloc(size);
43    
44     if(myptr-mymem>MYSIZE-size) {
45     myptr=mymem;
46     }
47    
48     void *v=(void*)myptr;
49     myptr+=size;
50     return v;
51     }
52    
53     //--------------------------------------------------------------------------------------------------
54     void myfree(void *ptr)
55     {
56     if (mystaticm && (char*)ptr>=mymem && (char*)ptr<mymem+MYSIZE)
57     return;
58     free(ptr);
59     }
60    
61     //--------------------------------------------------------------------------------------------------
62     void R__zip(int cxlevel, int *srcsize, char *src, int *tgtsize, char *tgt, int *irep)
63     /* int cxlevel; compression level */
64     /* int *srcsize, *tgtsize, *irep; source and target sizes, replay */
65     /* char *tgt, *src; source and target buffers */
66     {
67     int err = 0;
68     int method = 0;
69     unsigned int in_size = 0;
70     unsigned int out_size = 0;
71    
72     if (*tgtsize <= HDRSIZE) {
73     R__error("target buffer too small");
74     return;
75     }
76     if (*srcsize > 0xffffff) {
77     R__error("source buffer too big");
78     return;
79     }
80    
81     *irep = 0;
82     in_size = (unsigned)(*srcsize); /* decompressed size */
83     char *tgtptr = tgt+HDRSIZE; /* compress data */
84    
85     if (R__ZipMode == 5) { /*determine best of all methods*/
86     int cont = 1;
87     int msize = -1;
88     int zmode = -1;
89     int tgtlen = 2*in_size+64*1024;
90     char *mptr1 = mymalloc(tgtlen);
91     char *mptr2 = mymalloc(tgtlen);
92     char *mptr = 0;
93     char *mdum = 0;
94    
95     if (cont) {
96     zmode = 4;
97     msize = RLE_Compress(src, mptr1, in_size);
98     mptr = mptr1;
99     mdum = mptr2;
100     }
101    
102     if (cont && lzipfrac>0) {
103     err = lzo_init();
104     if ( err == LZO_E_OK) {
105     lzo_uint outlen = tgtlen;
106     char *lwmem = mymalloc(LZO1X_999_MEM_COMPRESS);
107     err = lzo1x_999_compress(src,in_size,mdum,&outlen,lwmem);
108     if (err == LZO_E_OK) {
109     if (outlen<lzipfrac*msize) {
110     msize = outlen;
111     zmode = 3;
112     char *tmp = mptr;
113     mptr = mdum;
114     mdum = tmp;
115     }
116     }
117     myfree(lwmem);
118     }
119     }
120    
121     if (cont && gzipfrac>0) {
122     z_stream stream;
123     stream.next_in = (Bytef*)src;
124     stream.avail_in = (uInt)(in_size);
125     stream.next_out = (Bytef*)(mdum);
126     stream.avail_out = (uInt)(*tgtsize);
127     stream.zalloc = (alloc_func)0;
128     stream.zfree = (free_func)0;
129     stream.opaque = (voidpf)0;
130    
131     err = deflateInit(&stream, cxlevel);
132     if (err == Z_OK) {
133     err = deflate(&stream, Z_FINISH);
134     if (err == Z_STREAM_END) {
135     err = deflateEnd(&stream);
136     int outlen = stream.total_out;
137     if (outlen<gzipfrac*msize) {
138     msize = outlen;
139     zmode = 1;
140     char *tmp = mptr;
141     mptr = mdum;
142     mdum = tmp;
143     }
144     }
145     }
146     }
147    
148     if (cont && bzipfrac>0) {
149     bz_stream stream;
150     stream.next_in = src;
151     stream.avail_in = (uInt)(in_size);
152     stream.next_out = mdum;
153     stream.avail_out = (uInt)(*tgtsize);
154     stream.bzalloc = 0;
155     stream.bzfree = 0;
156     stream.opaque = 0;
157    
158     err = BZ2_bzCompressInit(&stream, 9, 0, 1);
159     if (err == BZ_OK) {
160     err = BZ2_bzCompress(&stream, BZ_FINISH);
161     if (err == BZ_STREAM_END) {
162     BZ2_bzCompressEnd(&stream);
163     }
164     int outlen = stream.total_out_lo32;
165     if (outlen<bzipfrac*msize) {
166     msize = outlen;
167     zmode = 2;
168     char *tmp = mptr;
169     mptr = mdum;
170     mdum = tmp;
171     }
172     }
173     }
174    
175     if (msize>=in_size) {
176     out_size = in_size;
177     memcpy(tgtptr,src,out_size);
178     tgt[0] = 'X'; /* signature xx */
179     tgt[1] = 'X';
180     method = 0;
181     } else {
182     switch (zmode) {
183     case 1:
184     tgt[0] = 'Z'; /* signature zlib */
185     tgt[1] = 'L';
186     method = Z_DEFLATED; //==8
187     break;
188     case 2:
189     tgt[0] = 'B'; /* signature bzlib */
190     tgt[1] = 'Z';
191     method = 2;
192     break;
193     case 3:
194     tgt[0] = 'L'; /* signature lzolib */
195     tgt[1] = 'O';
196     method = 19;
197     break;
198     case 4:
199     tgt[0] = 'R'; /* signature rlelib */
200     tgt[1] = 'E';
201     method = 4;
202     break;
203     }
204     out_size = msize;
205     memcpy(tgtptr,mptr,out_size);
206     }
207     myfree(mptr1);
208     myfree(mptr2);
209     } else if (R__ZipMode == 4) { /*rle*/
210     int tgtlen = 2*in_size+1;
211     char *lmem1 = mymalloc(tgtlen);
212    
213     out_size = RLE_Compress(src, lmem1, in_size);
214     if (out_size>=in_size) {
215     out_size = in_size;
216     memcpy(tgtptr,src,out_size);
217     tgt[0] = 'X'; /* signature xx */
218     tgt[1] = 'X';
219     method = 0;
220     } else {
221     memcpy(tgtptr,lmem1,out_size);
222     tgt[0] = 'R'; /* signature rlelib */
223     tgt[1] = 'E';
224     method = 4;
225     }
226     myfree(lmem1);
227     } else if (R__ZipMode == 3) { /*lzo*/
228     err = lzo_init();
229     if ( err != LZO_E_OK) {
230     printf("error %d - lzo_init()\n", err);
231     return;
232     }
233    
234     lzo_uint tgtlen = 2*in_size+64*1024;
235     char *lmem1 = mymalloc(tgtlen);
236     char *lmem2 = 0;
237     if (cxlevel<=1) {
238     lmem2 = mymalloc(LZO1X_1_11_MEM_COMPRESS);
239     err = lzo1x_1_11_compress(src,in_size,lmem1,&tgtlen,lmem2);
240     method = 11;
241     } else if (cxlevel==2) {
242     lmem2 = mymalloc(LZO1X_1_12_MEM_COMPRESS);
243     err = lzo1x_1_12_compress(src,in_size,lmem1,&tgtlen,lmem2);
244     method = 12;
245     } else if (cxlevel<=5) {
246     lmem2 = mymalloc(LZO1X_1_15_MEM_COMPRESS);
247     err = lzo1x_1_15_compress(src,in_size,lmem1,&tgtlen,lmem2);
248     method = 15;
249     } else {
250     lmem2 = mymalloc(LZO1X_999_MEM_COMPRESS);
251     err = lzo1x_999_compress(src,in_size,lmem1,&tgtlen,lmem2);
252     method = 19;
253     }
254     if (err != LZO_E_OK) {
255     myfree(lmem1);
256     myfree(lmem2);
257     return;
258     }
259    
260     if (tgtlen>=lzipfrac*in_size) {
261     out_size = in_size;
262     memcpy(tgtptr,src,out_size);
263     tgt[0] = 'X'; /* signature xx */
264     tgt[1] = 'X';
265     method = 0;
266     } else {
267     out_size = tgtlen; /* compressed size */
268     memcpy(tgtptr,lmem1,out_size);
269     tgt[0] = 'L'; /* signature lzolib */
270     tgt[1] = 'O';
271     }
272     myfree(lmem1);
273     myfree(lmem2);
274     } else if (R__ZipMode == 2) { /*bzip*/
275     bz_stream stream;
276     stream.next_in = src;
277     stream.avail_in = (uInt)(in_size);
278     stream.next_out = tgtptr;
279     stream.avail_out = (uInt)(*tgtsize);
280     stream.bzalloc = 0;
281     stream.bzfree = 0;
282     stream.opaque = 0;
283    
284     err = BZ2_bzCompressInit(&stream, 9, 0, 1+(9-cxlevel)*25);
285     if (err != BZ_OK) {
286     printf("error %d in BZ2_bzCompressInit (bzlib)\n",err);
287     return;
288     }
289     err = BZ2_bzCompress(&stream, BZ_FINISH);
290     if (err != BZ_STREAM_END) {
291     BZ2_bzCompressEnd(&stream);
292     //printf("error %d in BZ2_bzCompress (bzlib) is not = %d\n",err,BZ_STREAM_END);
293     return;
294     }
295     err = BZ2_bzCompressEnd(&stream);
296    
297     out_size = stream.total_out_lo32; /* compressed size */
298     if (out_size>=bzipfrac*in_size) {
299     out_size = in_size;
300     memcpy(tgtptr,src,out_size);
301     tgt[0] = 'X'; /* signature xx */
302     tgt[1] = 'X';
303     method = 0;
304     } else {
305     tgt[0] = 'B'; /* signature bzlib */
306     tgt[1] = 'Z';
307     method = 2;
308     }
309     } else if (R__ZipMode == 1) { /*zip*/
310     z_stream stream;
311     stream.next_in = (Bytef*)src;
312     stream.avail_in = (uInt)(in_size);
313     stream.next_out = (Bytef*)(tgtptr);
314     stream.avail_out = (uInt)(*tgtsize);
315     stream.zalloc = (alloc_func)0;
316     stream.zfree = (free_func)0;
317     stream.opaque = (voidpf)0;
318    
319     err = deflateInit(&stream, cxlevel);
320     if (err != Z_OK) {
321     printf("error %d in deflateInit (zlib)\n",err);
322     return;
323     }
324     err = deflate(&stream, Z_FINISH);
325     if (err != Z_STREAM_END) {
326     deflateEnd(&stream);
327     //printf("error %d in deflate (zlib) is not = %d\n",err,Z_STREAM_END);
328     return;
329     }
330     err = deflateEnd(&stream);
331    
332     out_size = stream.total_out; /* compressed size */
333     if (out_size>=gzipfrac*in_size) {
334     out_size = in_size;
335     memcpy(tgtptr,src,out_size);
336     tgt[0] = 'X'; /* signature xx */
337     tgt[1] = 'X';
338     method = 0;
339     } else {
340     tgt[0] = 'Z'; /* signature zlib */
341     tgt[1] = 'L';
342     method = Z_DEFLATED;
343     }
344     } else if (R__ZipMode == 0) {
345     printf("error: Old zip method not supported in this patch.");
346     return;
347     }
348    
349     // fill rest of header
350     tgt[2] = (char)method;
351     tgt[3] = (char)(out_size & 0xff);
352     tgt[4] = (char)((out_size >> 8) & 0xff);
353     tgt[5] = (char)((out_size >> 16) & 0xff);
354     tgt[6] = (char)(in_size & 0xff);
355     tgt[7] = (char)((in_size >> 8) & 0xff);
356     tgt[8] = (char)((in_size >> 16) & 0xff);
357     *irep = out_size + HDRSIZE;
358    
359     if (myverbose==1||myverbose>9) {
360     printf("R__zip:: zm=%d m=%d cl=%d: %c%c compressed %lu bytes into %lu bytes -> %.3f%%\n",
361     R__ZipMode, method, cxlevel, tgt[0], tgt[1],
362     (unsigned long)in_size, (unsigned long)out_size, (double)out_size/in_size*100.);
363     }
364     }
365    
366     //--------------------------------------------------------------------------------------------------
367     void R__unzip(int *srcsize, uch *src, int *tgtsize, uch *tgt, int *irep)
368     /* Input: scrsize - size of input buffer */
369     /* src - input buffer */
370     /* tgtsize - size of target buffer */
371     /* Output: tgt - target buffer (decompressed) */
372     /* irep - size of decompressed data */
373     /* 0 - if error */
374     {
375     unsigned int osize = 0;
376     unsigned int ibufcnt = 0, obufcnt = 0;
377     uch *ibufptr = 0, *obufptr = 0;
378     *irep = 0L;
379    
380     /* C H E C K H E A D E R */
381     if (*srcsize < HDRSIZE) {
382     fprintf(stderr,"R__unzip: too small source\n");
383     return;
384     }
385    
386     char method = src[2];
387     if (method!=0 && method!=2 && method!=4 && method!=Z_DEFLATED && method!=11
388     && method!=12 && method!=15 && method!=19) {
389     fprintf(stderr,"R__unzip: error in header -> unknown method %d\n", method);
390     return;
391     }
392    
393     if ((method==0 && (src[0] != 'X' || src[1] != 'X')) ||
394     (method==2 && (src[0] != 'B' || src[1] != 'Z')) ||
395     (method==4 && (src[0] != 'R' || src[1] != 'E')) ||
396     (method==Z_DEFLATED && ((src[0] != 'C' || src[1] != 'S') &&
397     (src[0] != 'Z' || src[1] != 'L'))) ||
398     ((method==11 || method==12 || method==15 || method==19) &&
399     (src[0] != 'L' || src[1] != 'O'))) {
400     fprintf(stderr,"R__unzip: error in header -> m=%d with %c%c\n",
401     method, src[0], src[1]);
402     return;
403     }
404    
405     ibufptr = src + HDRSIZE;
406     ibufcnt = (unsigned int)src[3] | ((unsigned int)src[4] << 8) | ((unsigned int)src[5] << 16);
407     osize = (unsigned int)src[6] | ((unsigned int)src[7] << 8) | ((unsigned int)src[8] << 16);
408     obufptr = tgt;
409     obufcnt = *tgtsize;
410    
411     if (obufcnt < osize) {
412     fprintf(stderr,"R__unzip: too small target\n");
413     return;
414     }
415    
416     if (ibufcnt + HDRSIZE != *srcsize) {
417     fprintf(stderr,"R__unzip: discrepancy in source length\n");
418     return;
419     }
420    
421     if (myverbose==2 || myverbose>9) {
422     printf("R__unzip:: zm=%d m=%d: %c%c uncompressed %lu bytes from %lu bytes (%.3f%%)\n",
423     R__ZipMode, method, src[0], src[1], osize, ibufcnt, (double)ibufcnt/osize*100.);
424     }
425    
426     if (method==0) { // apparently this is not reached since underlying ROOT code catches this
427     if (ibufcnt!=osize) {
428     fprintf(stderr,"R__unzip: error in header -> input should be output %d!=%d\n",
429     ibufcnt, osize);
430     return;
431     }
432     memcpy(obufptr,ibufptr,osize);
433     *irep = obufcnt;
434     return;
435     }
436    
437     /* D E C O M P R E S S D A T A */
438     if (src[0] == 'R' && src[1] == 'E') { /* rle format */
439     RLE_Uncompress(ibufptr, obufptr, ibufcnt);
440     *irep = obufcnt;
441     return;
442     } else if (src[0] == 'L' && src[1] == 'O') { /* lolib format */
443    
444     int err = lzo_init();
445     if ( err != LZO_E_OK) {
446     fprintf(stderr,"R__unzip: error %d in lzo_init (lzolib)\n",err);
447     return;
448     }
449     lzo_uint new_len = obufcnt;
450     err = lzo1x_decompress(ibufptr,ibufcnt,obufptr,&new_len, NULL);
451     if ((err != LZO_E_OK) || (new_len!=osize)) {
452     fprintf(stderr,"R__unzip: error %d (%d,%d) in lzo1x_decompress (lzolib)\n",
453     err, new_len, osize);
454     return;
455     }
456     *irep = obufcnt;
457     return;
458     } else if (src[0] == 'B' && src[1] == 'Z') { /* bzlib format */
459     bz_stream stream; /* decompression stream */
460     stream.next_in = ibufptr;
461     stream.avail_in = ibufcnt;
462     stream.next_out = obufptr;
463     stream.avail_out = obufcnt;
464     stream.bzalloc = 0;//my_balloc;
465     stream.bzfree = 0;//my_bfree;
466     stream.opaque = 0;
467    
468     int err = BZ2_bzDecompressInit(&stream,0,0);
469     if (err != BZ_OK) {
470     fprintf(stderr,"R__unzip: error %d in BZ2_bzDecompressInit (bzlib)\n",err);
471     return;
472     }
473     err = BZ2_bzDecompress(&stream);
474     if (err != BZ_STREAM_END) {
475     fprintf(stderr,"R__unzip: error %d inBZ2_bzDecompress (bzlib)\n",err);
476     BZ2_bzDecompressEnd(&stream);
477     return;
478     }
479     BZ2_bzDecompressEnd(&stream);
480     *irep = stream.total_out_lo32;
481     return;
482     } else if (src[0] == 'Z' && src[1] == 'L') { /* zlib format */
483     z_stream stream; /* decompression stream */
484     stream.next_in = (Bytef*)(&src[HDRSIZE]);
485     stream.avail_in = (uInt)(*srcsize);
486     stream.next_out = (Bytef*)tgt;
487     stream.avail_out = (uInt)(*tgtsize);
488     stream.zalloc = 0; // my_zalloc;
489     stream.zfree = 0; //my_zfree;
490     stream.opaque = (voidpf)0;
491    
492     int err = inflateInit(&stream);
493     if (err != Z_OK) {
494     fprintf(stderr,"R__unzip: error %d in inflateInit (zlib)\n",err);
495     return;
496     }
497     err = inflate(&stream, Z_FINISH);
498     if (err != Z_STREAM_END) {
499     inflateEnd(&stream);
500     fprintf(stderr,"R__unzip: error %d in inflate (zlib)\n",err);
501     return;
502     }
503     inflateEnd(&stream);
504     *irep = stream.total_out;
505     return;
506     } else if (src[0] == 'C' && src[1] == 'S') { /* old zlib format */
507     if (R__Inflate(&ibufptr, &ibufcnt, &obufptr, &obufcnt)) {
508     fprintf(stderr,"R__unzip: error during decompression\n");
509     return;
510     }
511    
512     /* if (obufptr - tgt != osize) {
513     There are some rare cases when a few more bytes are required */
514     if (obufptr - tgt > *tgtsize) {
515     fprintf(stderr,"R__unzip: discrepancy (%ld) with initial size: %ld, tgtsize=%d\n",
516     (long)(obufptr - tgt),osize,*tgtsize);
517     *irep = obufptr - tgt;
518     return;
519     }
520     *irep = osize;
521     } else {
522     fprintf(stderr,"R__unzip: Format not supported -> m=%d with %d%d", method, src[0], src[1]);
523     return;
524     }
525     }