source: thirdparty/blosc/blosc.h @ 981e22c

Revision 981e22c, 16.4 KB checked in by Hal Finkel <hfinkel@…>, 8 years ago (diff)

Upgrade to latest blosc library

blosc git: e394f327ccc78319d90a06af0b88bce07034b8dd

  • Property mode set to 100644
RevLine 
[00587dc]1/*********************************************************************
[981e22c]2  Blosc - Blocked Shuffling and Compression Library
[00587dc]3
[981e22c]4  Author: Francesc Alted <[email protected]>
[00587dc]5
6  See LICENSES/BLOSC.txt for details about copyright and rights to use.
7**********************************************************************/
[981e22c]8#ifndef BLOSC_H
9#define BLOSC_H
[00587dc]10
11#include <limits.h>
[981e22c]12#include <stdlib.h>
13#include "blosc-export.h"
[00587dc]14
[981e22c]15#ifdef __cplusplus
16extern "C" {
17#endif
[00587dc]18
19/* Version numbers */
20#define BLOSC_VERSION_MAJOR    1    /* for major interface/format changes  */
[981e22c]21#define BLOSC_VERSION_MINOR    10   /* for minor interface/format changes  */
22#define BLOSC_VERSION_RELEASE  1    /* for tweaks, bug-fixes, or development */
[00587dc]23
[981e22c]24#define BLOSC_VERSION_STRING   "1.10.1.dev"  /* string version.  Sync with above! */
[00587dc]25#define BLOSC_VERSION_REVISION "$Rev$"   /* revision version */
[981e22c]26#define BLOSC_VERSION_DATE     "$Date:: 2016-07-20 #$"    /* date version */
[00587dc]27
[981e22c]28#define BLOSCLZ_VERSION_STRING "1.0.5"   /* the internal compressor version */
[00587dc]29
[981e22c]30/* The *_FORMAT symbols should be just 1-byte long */
31#define BLOSC_VERSION_FORMAT    2   /* Blosc format version, starting at 1 */
[00587dc]32
33/* Minimum header length */
34#define BLOSC_MIN_HEADER_LENGTH 16
35
36/* The maximum overhead during compression in bytes.  This equals to
37   BLOSC_MIN_HEADER_LENGTH now, but can be higher in future
38   implementations */
39#define BLOSC_MAX_OVERHEAD BLOSC_MIN_HEADER_LENGTH
40
[981e22c]41/* Maximum source buffer size to be compressed */
[00587dc]42#define BLOSC_MAX_BUFFERSIZE (INT_MAX - BLOSC_MAX_OVERHEAD)
43
[981e22c]44/* Maximum typesize before considering source buffer as a stream of bytes */
[00587dc]45#define BLOSC_MAX_TYPESIZE 255         /* Cannot be larger than 255 */
46
47/* The maximum number of threads (for some static arrays) */
48#define BLOSC_MAX_THREADS 256
49
[981e22c]50/* Codes for shuffling (see blosc_compress) */
51#define BLOSC_NOSHUFFLE   0  /* no shuffle */
52#define BLOSC_SHUFFLE     1  /* byte-wise shuffle */
53#define BLOSC_BITSHUFFLE  2  /* bit-wise shuffle */
[00587dc]54
[981e22c]55/* Codes for internal flags (see blosc_cbuffer_metainfo) */
56#define BLOSC_DOSHUFFLE    0x1  /* byte-wise shuffle */
57#define BLOSC_MEMCPYED     0x2  /* plain copy */
58#define BLOSC_DOBITSHUFFLE 0x4  /* bit-wise shuffle */
59
60/* Codes for the different compressors shipped with Blosc */
61#define BLOSC_BLOSCLZ   0
62#define BLOSC_LZ4       1
63#define BLOSC_LZ4HC     2
64#define BLOSC_SNAPPY    3
65#define BLOSC_ZLIB      4
66#define BLOSC_ZSTD      5
67
68/* Names for the different compressors shipped with Blosc */
69#define BLOSC_BLOSCLZ_COMPNAME   "blosclz"
70#define BLOSC_LZ4_COMPNAME       "lz4"
71#define BLOSC_LZ4HC_COMPNAME     "lz4hc"
72#define BLOSC_SNAPPY_COMPNAME    "snappy"
73#define BLOSC_ZLIB_COMPNAME      "zlib"
74#define BLOSC_ZSTD_COMPNAME      "zstd"
75
76/* Codes for compression libraries shipped with Blosc (code must be < 8) */
77#define BLOSC_BLOSCLZ_LIB   0
78#define BLOSC_LZ4_LIB       1
79#define BLOSC_SNAPPY_LIB    2
80#define BLOSC_ZLIB_LIB      3
81#define BLOSC_ZSTD_LIB      4
82
83/* Names for the different compression libraries shipped with Blosc */
84#define BLOSC_BLOSCLZ_LIBNAME   "BloscLZ"
85#define BLOSC_LZ4_LIBNAME       "LZ4"
86#define BLOSC_SNAPPY_LIBNAME    "Snappy"
87#define BLOSC_ZLIB_LIBNAME      "Zlib"
88#define BLOSC_ZSTD_LIBNAME      "Zstd"
89
90/* The codes for compressor formats shipped with Blosc */
91#define BLOSC_BLOSCLZ_FORMAT  BLOSC_BLOSCLZ_LIB
92#define BLOSC_LZ4_FORMAT      BLOSC_LZ4_LIB
93#define BLOSC_LZ4HC_FORMAT    BLOSC_LZ4_LIB /* LZ4HC and LZ4 share the same format */
94#define BLOSC_SNAPPY_FORMAT   BLOSC_SNAPPY_LIB
95#define BLOSC_ZLIB_FORMAT     BLOSC_ZLIB_LIB
96#define BLOSC_ZSTD_FORMAT     BLOSC_ZSTD_LIB
97
98
99/* The version formats for compressors shipped with Blosc */
100/* All versions here starts at 1 */
101#define BLOSC_BLOSCLZ_VERSION_FORMAT  1
102#define BLOSC_LZ4_VERSION_FORMAT      1
103#define BLOSC_LZ4HC_VERSION_FORMAT    1  /* LZ4HC and LZ4 share the same format */
104#define BLOSC_SNAPPY_VERSION_FORMAT   1
105#define BLOSC_ZLIB_VERSION_FORMAT     1
106#define BLOSC_ZSTD_VERSION_FORMAT     1
[00587dc]107
108
109/**
[981e22c]110  Initialize the Blosc library environment.
[00587dc]111
[981e22c]112  You must call this previous to any other Blosc call, unless you want
113  Blosc to be used simultaneously in a multi-threaded environment, in
114  which case you should *exclusively* use the
115  blosc_compress_ctx()/blosc_decompress_ctx() pair (see below).
116  */
117BLOSC_EXPORT void blosc_init(void);
[00587dc]118
119
120/**
[981e22c]121  Destroy the Blosc library environment.
[00587dc]122
[981e22c]123  You must call this after to you are done with all the Blosc calls,
124  unless you have not used blosc_init() before (see blosc_init()
125  above).
126  */
127BLOSC_EXPORT void blosc_destroy(void);
[00587dc]128
129
130/**
131  Compress a block of data in the `src` buffer and returns the size of
[981e22c]132  the compressed block.  The size of `src` buffer is specified by
[00587dc]133  `nbytes`.  There is not a minimum for `src` buffer size (`nbytes`).
134
135  `clevel` is the desired compression level and must be a number
136  between 0 (no compression) and 9 (maximum compression).
137
138  `doshuffle` specifies whether the shuffle compression preconditioner
[981e22c]139  should be applied or not.  BLOSC_NOSHUFFLE means not applying it,
140  BLOSC_SHUFFLE means applying it at a byte level and BLOSC_BITSHUFFLE
141  at a bit level (slower but may achieve better entropy alignment).
[00587dc]142
143  `typesize` is the number of bytes for the atomic type in binary
144  `src` buffer.  This is mainly useful for the shuffle preconditioner.
[981e22c]145  For implementation reasons, only a 1 < typesize < 256 will allow the
146  shuffle filter to work.  When typesize is not in this range, shuffle
147  will be silently disabled.
[00587dc]148
149  The `dest` buffer must have at least the size of `destsize`.  Blosc
150  guarantees that if you set `destsize` to, at least,
151  (`nbytes`+BLOSC_MAX_OVERHEAD), the compression will always succeed.
152  The `src` buffer and the `dest` buffer can not overlap.
153
[981e22c]154  Compression is memory safe and guaranteed not to write the `dest`
155  buffer more than what is specified in `destsize`.
156
[00587dc]157  If `src` buffer cannot be compressed into `destsize`, the return
158  value is zero and you should discard the contents of the `dest`
159  buffer.
160
161  A negative return value means that an internal error happened.  This
162  should never happen.  If you see this, please report it back
163  together with the buffer data causing this and compression settings.
164
[981e22c]165  Environment variables
166  ---------------------
167
168  blosc_compress() honors different environment variables to control
169  internal parameters without the need of doing that programatically.
170  Here are the ones supported:
171
172  BLOSC_CLEVEL=(INTEGER): This will overwrite the `clevel` parameter
173  before the compression process starts.
174
175  BLOSC_SHUFFLE=[NOSHUFFLE | SHUFFLE | BITSHUFFLE]: This will
176  overwrite the `doshuffle` parameter before the compression process
177  starts.
178
179  BLOSC_TYPESIZE=(INTEGER): This will overwrite the `typesize`
180  parameter before the compression process starts.
181
182  BLOSC_COMPRESSOR=[BLOSCLZ | LZ4 | LZ4HC | SNAPPY | ZLIB]: This will
183  call blosc_set_compressor(BLOSC_COMPRESSOR) before the compression
184  process starts.
185
186  BLOSC_NTHREADS=(INTEGER): This will call
187  blosc_set_nthreads(BLOSC_NTHREADS) before the compression process
188  starts.
189
190  BLOSC_BLOCKSIZE=(INTEGER): This will call
191  blosc_set_blocksize(BLOSC_BLOCKSIZE) before the compression process
192  starts.  *NOTE:* The blocksize is a critical parameter with
193  important restrictions in the allowed values, so use this with care.
[00587dc]194
[981e22c]195  BLOSC_NOLOCK=(ANY VALUE): This will call blosc_compress_ctx() under
196  the hood, with the `compressor`, `blocksize` and
197  `numinternalthreads` parameters set to the same as the last calls to
198  blosc_set_compressor(), blosc_set_blocksize() and
199  blosc_set_nthreads().  BLOSC_CLEVEL, BLOSC_SHUFFLE, BLOSC_TYPESIZE
200  environment vars will also be honored.
201  */
202BLOSC_EXPORT int blosc_compress(int clevel, int doshuffle, size_t typesize,
203                                size_t nbytes, const void *src, void *dest,
204                                size_t destsize);
[00587dc]205
206
207/**
[981e22c]208  Context interface to blosc compression. This does not require a call
209  to blosc_init() and can be called from multithreaded applications
210  without the global lock being used, so allowing Blosc be executed
211  simultaneously in those scenarios.
212
213  It uses the same parameters than the blosc_compress() function plus:
214
215  `compressor`: the string representing the type of compressor to use.
216
217  `blocksize`: the requested size of the compressed blocks.  If 0, an
218   automatic blocksize will be used.
219
220  `numinternalthreads`: the number of threads to use internally.
221
222  A negative return value means that an internal error happened.  This
223  should never happen.  If you see this, please report it back
224  together with the buffer data causing this and compression settings.
225*/
226BLOSC_EXPORT int blosc_compress_ctx(int clevel, int doshuffle, size_t typesize,
227                                    size_t nbytes, const void* src, void* dest,
228                                    size_t destsize, const char* compressor,
229                                    size_t blocksize, int numinternalthreads);
230
231/**
[00587dc]232  Decompress a block of compressed data in `src`, put the result in
[981e22c]233  `dest` and returns the size of the decompressed block.
[00587dc]234
235  The `src` buffer and the `dest` buffer can not overlap.
236
237  Decompression is memory safe and guaranteed not to write the `dest`
[981e22c]238  buffer more than what is specified in `destsize`.
239
240  If an error occurs, e.g. the compressed data is corrupted or the
241  output buffer is not large enough, then 0 (zero) or a negative value
242  will be returned instead.
243
244  Environment variables
245  ---------------------
246
247  blosc_decompress() honors different environment variables to control
248  internal parameters without the need of doing that programatically.
249  Here are the ones supported:
250
251  BLOSC_NTHREADS=(INTEGER): This will call
252  blosc_set_nthreads(BLOSC_NTHREADS) before the proper decompression
253  process starts.
254
255  BLOSC_NOLOCK=(ANY VALUE): This will call blosc_decompress_ctx()
256  under the hood, with the `numinternalthreads` parameter set to the
257  same value as the last call to blosc_set_nthreads().
[00587dc]258*/
[981e22c]259BLOSC_EXPORT int blosc_decompress(const void *src, void *dest, size_t destsize);
260
261
262/**
263  Context interface to blosc decompression. This does not require a
264  call to blosc_init() and can be called from multithreaded
265  applications without the global lock being used, so allowing Blosc
266  be executed simultaneously in those scenarios.
267
268  It uses the same parameters than the blosc_decompress() function plus:
269
270  `numinternalthreads`: number of threads to use internally.
[00587dc]271
[981e22c]272  Decompression is memory safe and guaranteed not to write the `dest`
273  buffer more than what is specified in `destsize`.
[00587dc]274
[981e22c]275  If an error occurs, e.g. the compressed data is corrupted or the
276  output buffer is not large enough, then 0 (zero) or a negative value
277  will be returned instead.
278*/
279BLOSC_EXPORT int blosc_decompress_ctx(const void *src, void *dest,
280                                      size_t destsize, int numinternalthreads);
[00587dc]281
282/**
283  Get `nitems` (of typesize size) in `src` buffer starting in `start`.
284  The items are returned in `dest` buffer, which has to have enough
[981e22c]285  space for storing all items.
[00587dc]286
[981e22c]287  Returns the number of bytes copied to `dest` or a negative value if
288  some error happens.
289  */
290BLOSC_EXPORT int blosc_getitem(const void *src, int start, int nitems, void *dest);
291
292
293/**
294  Returns the current number of threads that are used for
295  compression/decompression.
296  */
297BLOSC_EXPORT int blosc_get_nthreads(void);
[00587dc]298
299
300/**
301  Initialize a pool of threads for compression/decompression.  If
302  `nthreads` is 1, then the serial version is chosen and a possible
[981e22c]303  previous existing pool is ended.  If this is not called, `nthreads`
304  is set to 1 internally.
305
306  Returns the previous number of threads.
307  */
308BLOSC_EXPORT int blosc_set_nthreads(int nthreads);
309
310
311/**
312  Returns the current compressor that is used for compression.
313  */
314BLOSC_EXPORT char* blosc_get_compressor(void);
315
316
317/**
318  Select the compressor to be used.  The supported ones are "blosclz",
319  "lz4", "lz4hc", "snappy", "zlib" and "ztsd".  If this function is not
320  called, then "blosclz" will be used.
321
322  In case the compressor is not recognized, or there is not support
323  for it in this build, it returns a -1.  Else it returns the code for
324  the compressor (>=0).
325  */
326BLOSC_EXPORT int blosc_set_compressor(const char* compname);
327
328
329/**
330  Get the `compname` associated with the `compcode`.
331
332  If the compressor code is not recognized, or there is not support
333  for it in this build, -1 is returned.  Else, the compressor code is
334  returned.
335 */
336BLOSC_EXPORT int blosc_compcode_to_compname(int compcode, char **compname);
337
338
339/**
340  Return the compressor code associated with the compressor name.
341
342  If the compressor name is not recognized, or there is not support
343  for it in this build, -1 is returned instead.
344 */
345BLOSC_EXPORT int blosc_compname_to_compcode(const char *compname);
346
347
348/**
349  Get a list of compressors supported in the current build.  The
350  returned value is a string with a concatenation of "blosclz", "lz4",
351  "lz4hc", "snappy", "zlib" or "zstd "separated by commas, depending
352  on which ones are present in the build.
[00587dc]353
[981e22c]354  This function does not leak, so you should not free() the returned
355  list.
[00587dc]356
[981e22c]357  This function should always succeed.
358  */
359BLOSC_EXPORT char* blosc_list_compressors(void);
[00587dc]360
361/**
[981e22c]362  Return the version of blosc in string format.
363
364  Useful for dynamic libraries.
[00587dc]365*/
[981e22c]366BLOSC_EXPORT char* blosc_get_version_string(void);
367
368
369/**
370  Get info from compression libraries included in the current build.
371  In `compname` you pass the compressor name that you want info from.
372  In `complib` and `version` you get the compression library name and
373  version (if available) as output.
[00587dc]374
[981e22c]375  In `complib` and `version` you get a pointer to the compressor
376  library name and the version in string format respectively.  After
377  using the name and version, you should free() them so as to avoid
378  leaks.
379
380  If the compressor is supported, it returns the code for the library
381  (>=0).  If it is not supported, this function returns -1.
382  */
383BLOSC_EXPORT int blosc_get_complib_info(char *compname, char **complib, char **version);
384
385
386/**
387  Free possible memory temporaries and thread resources.  Use this
388  when you are not going to use Blosc for a long while.  In case of
389  problems releasing the resources, it returns a negative number, else
390  it returns 0.
391  */
392BLOSC_EXPORT int blosc_free_resources(void);
[00587dc]393
394
395/**
396  Return information about a compressed buffer, namely the number of
397  uncompressed bytes (`nbytes`) and compressed (`cbytes`).  It also
398  returns the `blocksize` (which is used internally for doing the
399  compression by blocks).
400
401  You only need to pass the first BLOSC_MIN_HEADER_LENGTH bytes of a
402  compressed buffer for this call to work.
403
404  This function should always succeed.
[981e22c]405  */
406BLOSC_EXPORT void blosc_cbuffer_sizes(const void *cbuffer, size_t *nbytes,
407                                      size_t *cbytes, size_t *blocksize);
[00587dc]408
409
410/**
411  Return information about a compressed buffer, namely the type size
412  (`typesize`), as well as some internal `flags`.
413
414  The `flags` is a set of bits, where the currently used ones are:
415    * bit 0: whether the shuffle filter has been applied or not
416    * bit 1: whether the internal buffer is a pure memcpy or not
417
[981e22c]418  You can use the `BLOSC_DOSHUFFLE`, `BLOSC_DOBITSHUFFLE` and
419  `BLOSC_MEMCPYED` symbols for extracting the interesting bits
420  (e.g. ``flags & BLOSC_DOSHUFFLE`` says whether the buffer is
421  byte-shuffled or not).
[00587dc]422
423  This function should always succeed.
[981e22c]424  */
425BLOSC_EXPORT void blosc_cbuffer_metainfo(const void *cbuffer, size_t *typesize,
426                                         int *flags);
[00587dc]427
428
429/**
430  Return information about a compressed buffer, namely the internal
431  Blosc format version (`version`) and the format for the internal
[981e22c]432  Lempel-Ziv compressor used (`versionlz`).
433
434  This function should always succeed.
435  */
436BLOSC_EXPORT void blosc_cbuffer_versions(const void *cbuffer, int *version,
437                                             int *versionlz);
438
[00587dc]439
[981e22c]440/**
441  Return the compressor library/format used in a compressed buffer.
442
443  This function should always succeed.
444  */
445BLOSC_EXPORT char *blosc_cbuffer_complib(const void *cbuffer);
[00587dc]446
447
448
449/*********************************************************************
450
451  Low-level functions follows.  Use them only if you are an expert!
452
453*********************************************************************/
454
[981e22c]455/* Get the internal blocksize to be used during compression.  0 means
456   that an automatic blocksize is computed internally. */
457BLOSC_EXPORT int blosc_get_blocksize(void);
[00587dc]458
459/**
460  Force the use of a specific blocksize.  If 0, an automatic
461  blocksize will be used (the default).
462
[981e22c]463  The blocksize is a critical parameter with important restrictions in
464  the allowed values, so use this with care.
465  */
466BLOSC_EXPORT void blosc_set_blocksize(size_t blocksize);
467
468#ifdef __cplusplus
469}
470#endif
[00587dc]471
472
473#endif
Note: See TracBrowser for help on using the repository browser.