File: H5Zlzo.c

package info (click to toggle)
pytables 2.3.1-3
  • links: PTS, VCS
  • area: main
  • in suites: wheezy
  • size: 14,624 kB
  • sloc: python: 55,641; xml: 15,737; ansic: 5,805; sh: 460; makefile: 234
file content (317 lines) | stat: -rw-r--r-- 9,540 bytes parent folder | download | duplicates (2)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
#include <string.h>
#include <stdlib.h>
#include <hdf5.h>

#include "H5Zlzo.h"
#include "tables.h"

#ifdef HAVE_LZO_LIB
#   include "lzo1x.h"
#endif
#ifdef HAVE_LZO2_LIB
#   include "lzo/lzo1x.h"
#   define HAVE_LZO_LIB  /* The API for LZO and LZO2 is mostly identical */
#endif

/* #undef DEBUG */

/* Activate the checksum. It is safer and takes only a 1% more of
   space and a 2% more of CPU (but sometimes is faster than without
   checksum, which is almost negligible.  F. Alted 2003/07/22

   Added code for pytables 0.5 backward compatibility.
   F. Alted 2003/07/28

   Added code for saving the uncompressed length buffer as well.
   F. Alted 2003/07/29

*/

/* From pytables 0.8 on I decided to let the user select the
   fletcher32 checksum provided in HDF5 1.6 or higher. So, even though
   the CHECKSUM support here seems pretty stable it will be disabled.
   F. Alted 2004/01/02 */
#undef CHECKSUM

size_t lzo_deflate (unsigned flags, size_t cd_nelmts,
		    const unsigned cd_values[], size_t nbytes,
		    size_t *buf_size, void **buf);


int register_lzo(char **version, char **date) {

#ifdef HAVE_LZO_LIB

/* The conditional below is somewhat messy, but it is necessary because
  the THG team has decided to fix an API inconsistency in the definition
  of the H5Z_class_t structure in version 1.8.3 */
#if (H5_VERS_MAJOR == 1 && H5_VERS_MINOR < 7) || \
    (H5_USE_16_API && (H5_VERS_MAJOR > 1 || \
      (H5_VERS_MAJOR == 1 && (H5_VERS_MINOR > 8 || \
        (H5_VERS_MINOR == 8 && H5_VERS_RELEASE >= 3)))))
   /* 1.6.x */
  H5Z_class_t filter_class = {
    (H5Z_filter_t)(FILTER_LZO),    /* filter_id */
    "lzo",                         /* comment */
    NULL,                          /* can_apply_func */
    NULL,                          /* set_local_func */
    (H5Z_func_t)(lzo_deflate)      /* filter_func */
  };
#else
   /* 1.8.x where x < 3 */
  H5Z_class_t filter_class = {
    H5Z_CLASS_T_VERS,             /* H5Z_class_t version */
    (H5Z_filter_t)(FILTER_LZO),   /* filter_id */
    1, 1,                         /* Encoding and decoding enabled */
    "lzo",	 		  /* comment */
    NULL,                         /* can_apply_func */
    NULL,                         /* set_local_func */
    (H5Z_func_t)(lzo_deflate)     /* filter_func */
  };
#endif

  /* Init the LZO library */
  if (lzo_init()!=LZO_E_OK) {
    fprintf(stderr, "Problems initializing LZO library\n");
    *version = NULL;
    *date = NULL;
    return 0; /* lib is not available */
  }

  /* Register the lzo compressor */
  H5Zregister(&filter_class);

  *version = strdup(LZO_VERSION_STRING);
  *date = strdup(LZO_VERSION_DATE);
  return 1; /* lib is available */

#else
  *version = NULL;
  *date = NULL;
  return 0; /* lib is not available */
#endif /* HAVE_LZO_LIB */

}


size_t lzo_deflate (unsigned flags, size_t cd_nelmts,
		    const unsigned cd_values[], size_t nbytes,
		    size_t *buf_size, void **buf)
{
  size_t ret_value = 0;
#ifdef HAVE_LZO_LIB
  void *outbuf = NULL, *wrkmem = NULL;
  int status;
  size_t  nalloc = *buf_size;
  lzo_uint out_len = (lzo_uint) nalloc;
  /* max_len_buffer will keep the likely output buffer size
     after processing the first chunk */
  static unsigned int max_len_buffer = 0;
  int complevel = 1;
  int object_version = 10;    	/* Default version 1.0 */
  int object_type = Table;      /* Default object type */
#ifdef CHECKSUM
  lzo_uint32 checksum;
#endif

  /* Check arguments */
  /* For Table versions < 20, there were no parameters */
  if (cd_nelmts==1 ) {
    complevel = cd_values[0];	/* This do nothing right now */
  }
  else if (cd_nelmts==2 ) {
    complevel = cd_values[0];	/* This do nothing right now */
    object_version = cd_values[1]; /* The table VERSION attribute */
  }
  else if (cd_nelmts==3 ) {
    complevel = cd_values[0];	/* This do nothing right now */
    object_version = cd_values[1]; /* The table VERSION attribute */
    object_type = cd_values[2]; /* A tag for identifying the object 
				   (see tables.h) */
  }

#ifdef DEBUG
  printf("Object type: %d. ", object_type);
  printf("object_version:%d\n", object_version);
#endif

  if (flags & H5Z_FLAG_REVERSE) {
    /* Input */

/*     printf("Decompressing chunk with LZO\n"); */
#ifdef CHECKSUM
    if ((object_type == Table && object_version >= 20) ||
	object_type != Table) {
      nbytes -= 4; 		/* Point to uncompressed buffer length */
      memcpy(&nalloc, ((unsigned char *)(*buf)+nbytes), 4);
      out_len = nalloc;
      nbytes -= 4; 		/* Point to the checksum */
#ifdef DEBUG
      printf("Compressed bytes: %d. Uncompressed bytes: %d\n", nbytes, nalloc);
#endif
    }
#endif

    /* Only allocate the bytes for the outbuf */
    if (max_len_buffer == 0) {
      if (NULL==(outbuf = (void *)malloc(nalloc)))
	fprintf(stderr, "Memory allocation failed for lzo uncompression.\n");
    }
    else {
      if (NULL==(outbuf = (void *)malloc(max_len_buffer)))
	fprintf(stderr, "Memory allocation failed for lzo uncompression.\n");
      out_len = max_len_buffer;
      nalloc =  max_len_buffer;
    }

    while(1) {

#ifdef DEBUG
      printf("nbytes -->%d\n", nbytes);
      printf("nalloc -->%d\n", nalloc);
      printf("max_len_buffer -->%d\n", max_len_buffer);
#endif /* DEBUG */

      /* The assembler version is a 10% slower than the C version with 
	 gcc 3.2.2 and gcc 3.3.3 */
/*       status = lzo1x_decompress_asm_safe(*buf, (lzo_uint)nbytes, outbuf, */
/* 				     &out_len, NULL); */
      /* The safe and unsafe versions have the same speed more or less */
      status = lzo1x_decompress_safe(*buf, (lzo_uint)nbytes, outbuf,
				       &out_len, NULL);

      if (status == LZO_E_OK) {
#ifdef DEBUG
	printf("decompressed %lu bytes back into %lu bytes\n",
	       (long) nbytes, (long) out_len);
#endif
	max_len_buffer = out_len;
	break; /* done */
      }
      else if (status == LZO_E_OUTPUT_OVERRUN) {
	nalloc *= 2;
	out_len = (lzo_uint) nalloc;
	if (NULL==(outbuf = realloc(outbuf, nalloc))) {
	  fprintf(stderr, "Memory allocation failed for lzo uncompression\n");
	}
      }
      else {
	/* this should NEVER happen */
	fprintf(stderr, "internal error - decompression failed: %d\n", status);
	ret_value = 0; /* fail */
	goto done;
      }
    }

#ifdef CHECKSUM
    if ((object_type == Table && object_version >= 20) ||
	object_type != Table) {
#ifdef DEBUG
      printf("Checksum uncompressing...");
#endif
      /* Compute the checksum */
      checksum=lzo_adler32(lzo_adler32(0,NULL,0), outbuf, out_len);
  
      /* Compare */
      if (memcmp(&checksum, (unsigned char*)(*buf)+nbytes, 4)) {
	ret_value = 0; /*fail*/
	fprintf(stderr,"Checksum failed!.\n");
	goto done;
      }
    }
#endif /* CHECKSUM */

    free(*buf);
    *buf = outbuf;
    outbuf = NULL;
    *buf_size = nalloc;
    ret_value = out_len;

  } else {
    /*
     * Output; compress but fail if the result would be larger than the
     * input.  The library doesn't provide in-place compression, so we
     * must allocate a separate buffer for the result.
     */
    lzo_byte *z_src = (lzo_byte*)(*buf);
    lzo_byte *z_dst;         /*destination buffer            */
    lzo_uint z_src_nbytes = (lzo_uint)(nbytes);
    /* The next was the original computation for worst-case expansion */
    /* I don't know why the difference with LZO1*. Perhaps some wrong docs in
       LZO package? */
/*     lzo_uint z_dst_nbytes = (lzo_uint)(nbytes + (nbytes / 64) + 16 + 3); */
    /* The next is for LZO1* algorithms */
/*     lzo_uint z_dst_nbytes = (lzo_uint)(nbytes + (nbytes / 16) + 64 + 3); */
    /* The next is for LZO2* algorithms. This will be the default */
    lzo_uint z_dst_nbytes = (lzo_uint)(nbytes + (nbytes / 8) + 128 + 3);

#ifdef CHECKSUM
    if ((object_type == Table && object_version >= 20) ||
	object_type != Table) {
      z_dst_nbytes += 4+4;      /* Checksum + buffer size */
    }
#endif

    if (NULL==(z_dst=outbuf=(void *)malloc(z_dst_nbytes))) {
      fprintf(stderr, "Unable to allocate lzo destination buffer.\n");
      ret_value = 0; /* fail */
      goto done;
    }

    /* Compress this buffer */
    wrkmem = malloc(LZO1X_1_MEM_COMPRESS);
    if (wrkmem == NULL) {
      fprintf(stderr, "Memory allocation failed for lzo compression\n");
      ret_value = 0;
      goto done;
    }

    status = lzo1x_1_compress (z_src, z_src_nbytes, z_dst, &z_dst_nbytes,
			       wrkmem);

    free(wrkmem);
    wrkmem = NULL;

#ifdef CHECKSUM
    if ((object_type == Table && object_version >= 20) ||
	object_type != Table) {
#ifdef DEBUG
      printf("Checksum compressing ...");
      printf("src_nbytes: %d, dst_nbytes: %d\n", z_src_nbytes, z_dst_nbytes);
#endif
      /* Append checksum of *uncompressed* data at the end */
      checksum = lzo_adler32(lzo_adler32(0,NULL,0), *buf, nbytes);
      memcpy((unsigned char*)(z_dst)+z_dst_nbytes, &checksum, 4);
      memcpy((unsigned char*)(z_dst)+z_dst_nbytes+4, &nbytes, 4);
      z_dst_nbytes += (lzo_uint)4+4;
      nbytes += 4+4;
    }
#endif

    if (z_dst_nbytes >= nbytes) {
#ifdef DEBUG
      printf("The compressed buffer takes more space than uncompressed!.\n");
#endif
      ret_value = 0; /* fail */
      goto done;
    } else if (LZO_E_OK != status) {
      fprintf(stderr,"lzo library error in compression\n");
      ret_value = 0; /* fail */
      goto done;
    } else {
      free(*buf);
      *buf = outbuf;
      outbuf = NULL;
      *buf_size = z_dst_nbytes;
      ret_value = z_dst_nbytes;
    }
  }

done:
  if(outbuf)
    free(outbuf);

#endif  /* HAVE_LZO_LIB */

  return ret_value;
}