source: thirdparty/blosc/internal-complibs/zstd-0.7.4/compress/zbuff_compress.c @ 8ebc79b

Revision 8ebc79b, 13.3 KB checked in by Hal Finkel <hfinkel@…>, 8 years ago (diff)

Add the other internal compression libraries from blocs

  • Property mode set to 100644
RevLine 
[8ebc79b]1/*
2    Buffered version of Zstd compression library
3    Copyright (C) 2015-2016, Yann Collet.
4
5    BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
6
7    Redistribution and use in source and binary forms, with or without
8    modification, are permitted provided that the following conditions are
9    met:
10    * Redistributions of source code must retain the above copyright
11    notice, this list of conditions and the following disclaimer.
12    * Redistributions in binary form must reproduce the above
13    copyright notice, this list of conditions and the following disclaimer
14    in the documentation and/or other materials provided with the
15    distribution.
16    THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
17    "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
18    LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
19    A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
20    OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
21    SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
22    LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
23    DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
24    THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
25    (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
26    OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
27
28    You can contact the author at :
29    - zstd homepage : http://www.zstd.net/
30*/
31
32
33/* *************************************
34*  Dependencies
35***************************************/
36#include <stdlib.h>
37#include "error_private.h"
38#include "zstd_internal.h"  /* MIN, ZSTD_BLOCKHEADERSIZE, defaultCustomMem */
39#define ZBUFF_STATIC_LINKING_ONLY
40#include "zbuff.h"
41
42
43/* *************************************
44*  Constants
45***************************************/
46static size_t const ZBUFF_endFrameSize = ZSTD_BLOCKHEADERSIZE;
47
48
49/*_**************************************************
50*  Streaming compression
51*
52*  A ZBUFF_CCtx object is required to track streaming operation.
53*  Use ZBUFF_createCCtx() and ZBUFF_freeCCtx() to create/release resources.
54*  Use ZBUFF_compressInit() to start a new compression operation.
55*  ZBUFF_CCtx objects can be reused multiple times.
56*
57*  Use ZBUFF_compressContinue() repetitively to consume your input.
58*  *srcSizePtr and *dstCapacityPtr can be any size.
59*  The function will report how many bytes were read or written by modifying *srcSizePtr and *dstCapacityPtr.
60*  Note that it may not consume the entire input, in which case it's up to the caller to call again the function with remaining input.
61*  The content of dst will be overwritten (up to *dstCapacityPtr) at each function call, so save its content if it matters or change dst .
62*  @return : a hint to preferred nb of bytes to use as input for next function call (it's only a hint, to improve latency)
63*            or an error code, which can be tested using ZBUFF_isError().
64*
65*  ZBUFF_compressFlush() can be used to instruct ZBUFF to compress and output whatever remains within its buffer.
66*  Note that it will not output more than *dstCapacityPtr.
67*  Therefore, some content might still be left into its internal buffer if dst buffer is too small.
68*  @return : nb of bytes still present into internal buffer (0 if it's empty)
69*            or an error code, which can be tested using ZBUFF_isError().
70*
71*  ZBUFF_compressEnd() instructs to finish a frame.
72*  It will perform a flush and write frame epilogue.
73*  Similar to ZBUFF_compressFlush(), it may not be able to output the entire internal buffer content if *dstCapacityPtr is too small.
74*  @return : nb of bytes still present into internal buffer (0 if it's empty)
75*            or an error code, which can be tested using ZBUFF_isError().
76*
77*  Hint : recommended buffer sizes (not compulsory)
78*  input : ZSTD_BLOCKSIZE_MAX (128 KB), internal unit size, it improves latency to use this value.
79*  output : ZSTD_compressBound(ZSTD_BLOCKSIZE_MAX) + ZSTD_blockHeaderSize + ZBUFF_endFrameSize : ensures it's always possible to write/flush/end a full block at best speed.
80* **************************************************/
81
82typedef enum { ZBUFFcs_init, ZBUFFcs_load, ZBUFFcs_flush, ZBUFFcs_final } ZBUFF_cStage;
83
84/* *** Resources *** */
85struct ZBUFF_CCtx_s {
86    ZSTD_CCtx* zc;
87    char*  inBuff;
88    size_t inBuffSize;
89    size_t inToCompress;
90    size_t inBuffPos;
91    size_t inBuffTarget;
92    size_t blockSize;
93    char*  outBuff;
94    size_t outBuffSize;
95    size_t outBuffContentSize;
96    size_t outBuffFlushedSize;
97    ZBUFF_cStage stage;
98    ZSTD_customMem customMem;
99};   /* typedef'd tp ZBUFF_CCtx within "zstd_buffered.h" */
100
101ZBUFF_CCtx* ZBUFF_createCCtx(void)
102{
103    return ZBUFF_createCCtx_advanced(defaultCustomMem);
104}
105
106ZBUFF_CCtx* ZBUFF_createCCtx_advanced(ZSTD_customMem customMem)
107{
108    ZBUFF_CCtx* zbc;
109
110    if (!customMem.customAlloc && !customMem.customFree)
111        customMem = defaultCustomMem;
112
113    if (!customMem.customAlloc || !customMem.customFree)
114        return NULL;
115
116    zbc = (ZBUFF_CCtx*)customMem.customAlloc(customMem.opaque, sizeof(ZBUFF_CCtx));
117    if (zbc==NULL) return NULL;
118    memset(zbc, 0, sizeof(ZBUFF_CCtx));
119    memcpy(&zbc->customMem, &customMem, sizeof(ZSTD_customMem));
120    zbc->zc = ZSTD_createCCtx_advanced(customMem);
121    if (zbc->zc == NULL) { ZBUFF_freeCCtx(zbc); return NULL; }
122    return zbc;
123}
124
125size_t ZBUFF_freeCCtx(ZBUFF_CCtx* zbc)
126{
127    if (zbc==NULL) return 0;   /* support free on NULL */
128    ZSTD_freeCCtx(zbc->zc);
129    if (zbc->inBuff) zbc->customMem.customFree(zbc->customMem.opaque, zbc->inBuff);
130    if (zbc->outBuff) zbc->customMem.customFree(zbc->customMem.opaque, zbc->outBuff);
131    zbc->customMem.customFree(zbc->customMem.opaque, zbc);
132    return 0;
133}
134
135
136/* *** Initialization *** */
137
138size_t ZBUFF_compressInit_advanced(ZBUFF_CCtx* zbc,
139                                   const void* dict, size_t dictSize,
140                                   ZSTD_parameters params, unsigned long long pledgedSrcSize)
141{
142    /* allocate buffers */
143    {   size_t const neededInBuffSize = (size_t)1 << params.cParams.windowLog;
144        if (zbc->inBuffSize < neededInBuffSize) {
145            zbc->inBuffSize = neededInBuffSize;
146            zbc->customMem.customFree(zbc->customMem.opaque, zbc->inBuff);   /* should not be necessary */
147            zbc->inBuff = (char*)zbc->customMem.customAlloc(zbc->customMem.opaque, neededInBuffSize);
148            if (zbc->inBuff == NULL) return ERROR(memory_allocation);
149        }
150        zbc->blockSize = MIN(ZSTD_BLOCKSIZE_MAX, neededInBuffSize);
151    }
152    if (zbc->outBuffSize < ZSTD_compressBound(zbc->blockSize)+1) {
153        zbc->outBuffSize = ZSTD_compressBound(zbc->blockSize)+1;
154        zbc->customMem.customFree(zbc->customMem.opaque, zbc->outBuff);   /* should not be necessary */
155        zbc->outBuff = (char*)zbc->customMem.customAlloc(zbc->customMem.opaque, zbc->outBuffSize);
156        if (zbc->outBuff == NULL) return ERROR(memory_allocation);
157    }
158
159    { size_t const errorCode = ZSTD_compressBegin_advanced(zbc->zc, dict, dictSize, params, pledgedSrcSize);
160      if (ZSTD_isError(errorCode)) return errorCode; }
161
162    zbc->inToCompress = 0;
163    zbc->inBuffPos = 0;
164    zbc->inBuffTarget = zbc->blockSize;
165    zbc->outBuffContentSize = zbc->outBuffFlushedSize = 0;
166    zbc->stage = ZBUFFcs_load;
167    return 0;   /* ready to go */
168}
169
170
171size_t ZBUFF_compressInitDictionary(ZBUFF_CCtx* zbc, const void* dict, size_t dictSize, int compressionLevel)
172{
173    ZSTD_parameters const params = ZSTD_getParams(compressionLevel, 0, dictSize);
174    return ZBUFF_compressInit_advanced(zbc, dict, dictSize, params, 0);
175}
176
177size_t ZBUFF_compressInit(ZBUFF_CCtx* zbc, int compressionLevel)
178{
179    return ZBUFF_compressInitDictionary(zbc, NULL, 0, compressionLevel);
180}
181
182
183/* internal util function */
184MEM_STATIC size_t ZBUFF_limitCopy(void* dst, size_t dstCapacity, const void* src, size_t srcSize)
185{
186    size_t const length = MIN(dstCapacity, srcSize);
187    memcpy(dst, src, length);
188    return length;
189}
190
191
192/* *** Compression *** */
193
194static size_t ZBUFF_compressContinue_generic(ZBUFF_CCtx* zbc,
195                              void* dst, size_t* dstCapacityPtr,
196                        const void* src, size_t* srcSizePtr,
197                              int flush)
198{
199    U32 notDone = 1;
200    const char* const istart = (const char*)src;
201    const char* const iend = istart + *srcSizePtr;
202    const char* ip = istart;
203    char* const ostart = (char*)dst;
204    char* const oend = ostart + *dstCapacityPtr;
205    char* op = ostart;
206
207    while (notDone) {
208        switch(zbc->stage)
209        {
210        case ZBUFFcs_init: return ERROR(init_missing);   /* call ZBUFF_compressInit() first ! */
211
212        case ZBUFFcs_load:
213            /* complete inBuffer */
214            {   size_t const toLoad = zbc->inBuffTarget - zbc->inBuffPos;
215                size_t const loaded = ZBUFF_limitCopy(zbc->inBuff + zbc->inBuffPos, toLoad, ip, iend-ip);
216                zbc->inBuffPos += loaded;
217                ip += loaded;
218                if ( (zbc->inBuffPos==zbc->inToCompress) || (!flush && (toLoad != loaded)) ) {
219                    notDone = 0; break;  /* not enough input to get a full block : stop there, wait for more */
220            }   }
221            /* compress current block (note : this stage cannot be stopped in the middle) */
222            {   void* cDst;
223                size_t cSize;
224                size_t const iSize = zbc->inBuffPos - zbc->inToCompress;
225                size_t oSize = oend-op;
226                if (oSize >= ZSTD_compressBound(iSize))
227                    cDst = op;   /* compress directly into output buffer (avoid flush stage) */
228                else
229                    cDst = zbc->outBuff, oSize = zbc->outBuffSize;
230                cSize = ZSTD_compressContinue(zbc->zc, cDst, oSize, zbc->inBuff + zbc->inToCompress, iSize);
231                if (ZSTD_isError(cSize)) return cSize;
232                /* prepare next block */
233                zbc->inBuffTarget = zbc->inBuffPos + zbc->blockSize;
234                if (zbc->inBuffTarget > zbc->inBuffSize)
235                    zbc->inBuffPos = 0, zbc->inBuffTarget = zbc->blockSize;   /* note : inBuffSize >= blockSize */
236                zbc->inToCompress = zbc->inBuffPos;
237                if (cDst == op) { op += cSize; break; }   /* no need to flush */
238                zbc->outBuffContentSize = cSize;
239                zbc->outBuffFlushedSize = 0;
240                zbc->stage = ZBUFFcs_flush;   /* continue to flush stage */
241            }
242
243        case ZBUFFcs_flush:
244            {   size_t const toFlush = zbc->outBuffContentSize - zbc->outBuffFlushedSize;
245                size_t const flushed = ZBUFF_limitCopy(op, oend-op, zbc->outBuff + zbc->outBuffFlushedSize, toFlush);
246                op += flushed;
247                zbc->outBuffFlushedSize += flushed;
248                if (toFlush!=flushed) { notDone = 0; break; } /* dst too small to store flushed data : stop there */
249                zbc->outBuffContentSize = zbc->outBuffFlushedSize = 0;
250                zbc->stage = ZBUFFcs_load;
251                break;
252            }
253
254        case ZBUFFcs_final:
255            notDone = 0;   /* do nothing */
256            break;
257
258        default:
259            return ERROR(GENERIC);   /* impossible */
260        }
261    }
262
263    *srcSizePtr = ip - istart;
264    *dstCapacityPtr = op - ostart;
265    {   size_t hintInSize = zbc->inBuffTarget - zbc->inBuffPos;
266        if (hintInSize==0) hintInSize = zbc->blockSize;
267        return hintInSize;
268    }
269}
270
271size_t ZBUFF_compressContinue(ZBUFF_CCtx* zbc,
272                              void* dst, size_t* dstCapacityPtr,
273                        const void* src, size_t* srcSizePtr)
274{
275    return ZBUFF_compressContinue_generic(zbc, dst, dstCapacityPtr, src, srcSizePtr, 0);
276}
277
278
279
280/* *** Finalize *** */
281
282size_t ZBUFF_compressFlush(ZBUFF_CCtx* zbc, void* dst, size_t* dstCapacityPtr)
283{
284    size_t srcSize = 0;
285    ZBUFF_compressContinue_generic(zbc, dst, dstCapacityPtr, &srcSize, &srcSize, 1);  /* use a valid src address instead of NULL */
286    return zbc->outBuffContentSize - zbc->outBuffFlushedSize;
287}
288
289
290size_t ZBUFF_compressEnd(ZBUFF_CCtx* zbc, void* dst, size_t* dstCapacityPtr)
291{
292    BYTE* const ostart = (BYTE*)dst;
293    BYTE* const oend = ostart + *dstCapacityPtr;
294    BYTE* op = ostart;
295
296    if (zbc->stage != ZBUFFcs_final) {
297        /* flush whatever remains */
298        size_t outSize = *dstCapacityPtr;
299        size_t const remainingToFlush = ZBUFF_compressFlush(zbc, dst, &outSize);
300        op += outSize;
301        if (remainingToFlush) {
302            *dstCapacityPtr = op-ostart;
303            return remainingToFlush + ZBUFF_endFrameSize;
304        }
305        /* create epilogue */
306        zbc->stage = ZBUFFcs_final;
307        zbc->outBuffContentSize = ZSTD_compressEnd(zbc->zc, zbc->outBuff, zbc->outBuffSize); /* epilogue into outBuff */
308    }
309
310    /* flush epilogue */
311    {   size_t const toFlush = zbc->outBuffContentSize - zbc->outBuffFlushedSize;
312        size_t const flushed = ZBUFF_limitCopy(op, oend-op, zbc->outBuff + zbc->outBuffFlushedSize, toFlush);
313        op += flushed;
314        zbc->outBuffFlushedSize += flushed;
315        *dstCapacityPtr = op-ostart;
316        if (toFlush==flushed) zbc->stage = ZBUFFcs_init;  /* end reached */
317        return toFlush - flushed;
318    }
319}
320
321
322
323/* *************************************
324*  Tool functions
325***************************************/
326size_t ZBUFF_recommendedCInSize(void)  { return ZSTD_BLOCKSIZE_MAX; }
327size_t ZBUFF_recommendedCOutSize(void) { return ZSTD_compressBound(ZSTD_BLOCKSIZE_MAX) + ZSTD_blockHeaderSize + ZBUFF_endFrameSize; }
Note: See TracBrowser for help on using the repository browser.