source: thirdparty/SZ/sz/include/sz.h @ 9ee2ce3

Revision 9ee2ce3, 13.0 KB checked in by Hal Finkel <hfinkel@…>, 6 years ago (diff)

importing new SZ files
  • Property mode set to 100644
Line 
1/**
2 *  @file sz.h
3 *  @author Sheng Di
4 *  @date April, 2015
5 *  @brief Header file for the whole detector.
6 *  (C) 2015 by Mathematics and Computer Science (MCS), Argonne National Laboratory.
7 *      See COPYRIGHT in top-level directory.
8 */
9
10#ifndef _SZ_H
11#define _SZ_H
12
13#include <stdio.h>
14#include <stdint.h>
15#include <sys/time.h>      /* For gettimeofday(), in microseconds */
16#include <time.h>          /* For time(), in seconds */
17#include "iniparser.h"
18#include "CompressElement.h"
19#include "DynamicByteArray.h"
20#include "DynamicIntArray.h"
21#include "VarSet.h"
22#include "Huffman.h"
23#include "TightDataPointStorageD.h"
24#include "TightDataPointStorageF.h"
25#include "TightDataPointStorageI.h"
26#include "conf.h"
27#include "dataCompression.h"
28#include "ByteToolkit.h"
29#include "TypeManager.h"
30#include "sz_int8.h"
31#include "sz_int16.h"
32#include "sz_int32.h"
33#include "sz_int64.h"
34#include "sz_uint8.h"
35#include "sz_uint16.h"
36#include "sz_uint32.h"
37#include "sz_uint64.h"
38#include "sz_float.h"
39#include "sz_double.h"
40#include "szd_int8.h"
41#include "szd_int16.h"
42#include "szd_int32.h"
43#include "szd_int64.h"
44#include "szd_uint8.h"
45#include "szd_uint16.h"
46#include "szd_uint32.h"
47#include "szd_uint64.h"
48#include "szd_float.h"
49#include "szd_double.h"
50#include "sz_float_pwr.h"
51#include "sz_double_pwr.h"
52#include "callZlib.h"
53#include "rw.h"
54#include "pastri.h"
55#include "sz_float_ts.h"
56#include "szd_float_ts.h"
57#include "utility.h"
58
59#ifdef _WIN32
60#define PATH_SEPARATOR ';'
61#else
62#define PATH_SEPARATOR ':'
63#endif
64
65#ifdef __cplusplus
66extern "C" {
67#endif
68
69//typedef char int8_t;
70//typedef unsigned char uint8_t;
71//typedef short int16_t;
72//typedef unsigned short uint16_t;
73//typedef int int32_t;
74//typedef unsigned int uint32_t;
75//typedef long int64_t;
76//typedef unsigned long uint64_t;
77
78#define SZ_VERNUM 0x0200
79#define SZ_VER_MAJOR 2
80#define SZ_VER_MINOR 0
81#define SZ_VER_BUILD 2
82#define SZ_VER_REVISION 0
83
84#define PASTRI 103
85#define HZ 102
86#define SZ 101
87
88//prediction mode of temporal dimension based compression
89#define SZ_PREVIOUS_VALUE_ESTIMATE 0
90
91#define MIN_NUM_OF_ELEMENTS 20 //if the # elements <= 20, skip the compression
92
93#define ABS 0
94#define REL 1
95#define ABS_AND_REL 2
96#define ABS_OR_REL 3
97#define PSNR 4
98
99#define PW_REL 10
100#define ABS_AND_PW_REL 11
101#define ABS_OR_PW_REL 12
102#define REL_AND_PW_REL 13
103#define REL_OR_PW_REL 14
104
105#define SZ_FLOAT 0
106#define SZ_DOUBLE 1
107#define SZ_UINT8 2
108#define SZ_INT8 3
109#define SZ_UINT16 4
110#define SZ_INT16 5
111#define SZ_UINT32 6
112#define SZ_INT32 7
113#define SZ_UINT64 8
114#define SZ_INT64 9
115
116#define LITTLE_ENDIAN_DATA 0 //refers to the endian type of the data read from the disk
117#define BIG_ENDIAN_DATA 1 //big_endian (ppc, max, etc.) ; little_endian (x86, x64, etc.)
118
119#define LITTLE_ENDIAN_SYSTEM 0 //refers to the endian type of the system
120#define BIG_ENDIAN_SYSTEM 1
121
122#define DynArrayInitLen 1024
123
124#define MIN_ZLIB_DEC_ALLOMEM_BYTES 1000000
125
126//#define maxRangeRadius 32768
127//#define maxRangeRadius 1048576//131072
128
129#define SZ_BEST_SPEED 0
130#define SZ_BEST_COMPRESSION 1
131#define SZ_DEFAULT_COMPRESSION 2
132#define SZ_TEMPORAL_COMPRESSION 3
133
134#define SZ_NO_REGRESSION 0
135#define SZ_WITH_LINEAR_REGRESSION 1
136
137#define SZ_PWR_MIN_TYPE 0
138#define SZ_PWR_AVG_TYPE 1
139#define SZ_PWR_MAX_TYPE 2
140
141//SUCCESS returning status
142#define SZ_SCES 0  //successful
143#define SZ_NSCS -1 //Not successful
144#define SZ_FERR -2 //Failed to open input file
145#define SZ_TERR -3 //wrong data type (should be only float or double)
146#define SZ_DERR -4 //dimension error
147#define SZ_MERR -5 //sz_mode error
148#define SZ_BERR -6 //bound-mode error (should be only ABS, REL, ABS_AND_REL, ABS_OR_REL, or PW_REL)
149
150#define SZ_MAINTAIN_VAR_DATA 0
151#define SZ_DESTROY_WHOLE_VARSET 1
152
153#define GROUP_COUNT 16 //2^{16}=65536
154       
155#define MetaDataByteLength 20   
156       
157#define numOfBufferedSteps 1 //the number of time steps in the buffer   
158
159
160#define GZIP_COMPRESSOR 0 //i.e., ZLIB_COMPRSSOR
161#define ZSTD_COMPRESSOR 1
162       
163//Note: the following setting should be consistent with stateNum in Huffman.h
164//#define intvCapacity 65536
165//#define intvRadius 32768
166//#define intvCapacity 131072
167//#define intvRadius 65536
168
169#define SZ_COMPUTE_1D_NUMBER_OF_BLOCKS( COUNT, NUM_BLOCKS, BLOCK_SIZE ) \
170    if (COUNT <= BLOCK_SIZE){                  \
171        NUM_BLOCKS = 1;             \
172    }                                   \
173    else{                               \
174        NUM_BLOCKS = COUNT / BLOCK_SIZE;       \
175    }                                   \
176
177#define SZ_COMPUTE_2D_NUMBER_OF_BLOCKS( COUNT, NUM_BLOCKS, BLOCK_SIZE ) \
178    if (COUNT <= BLOCK_SIZE){                   \
179        NUM_BLOCKS = 1;             \
180    }                                   \
181    else{                               \
182        NUM_BLOCKS = COUNT / BLOCK_SIZE;        \
183    }                                   \
184
185#define SZ_COMPUTE_3D_NUMBER_OF_BLOCKS( COUNT, NUM_BLOCKS, BLOCK_SIZE ) \
186    if (COUNT <= BLOCK_SIZE){                   \
187        NUM_BLOCKS = 1;             \
188    }                                   \
189    else{                               \
190        NUM_BLOCKS = COUNT / BLOCK_SIZE;        \
191    }                                   \
192
193#define SZ_COMPUTE_BLOCKCOUNT( COUNT, NUM_BLOCKS, SPLIT_INDEX,       \
194                                       EARLY_BLOCK_COUNT, LATE_BLOCK_COUNT ) \
195    EARLY_BLOCK_COUNT = LATE_BLOCK_COUNT = COUNT / NUM_BLOCKS;               \
196    SPLIT_INDEX = COUNT % NUM_BLOCKS;                                        \
197    if (0 != SPLIT_INDEX) {                                                  \
198        EARLY_BLOCK_COUNT = EARLY_BLOCK_COUNT + 1;                           \
199    }                                                                        \
200
201//typedef unsigned long unsigned long;
202//typedef unsigned int uint;
203
204typedef union lint16
205{
206        unsigned short usvalue;
207        short svalue;
208        unsigned char byte[2];
209} lint16;
210
211typedef union lint32
212{
213        int ivalue;
214        unsigned int uivalue;
215        unsigned char byte[4];
216} lint32;
217
218typedef union lint64
219{
220        long lvalue;
221        unsigned long ulvalue;
222        unsigned char byte[8];
223} lint64;
224
225typedef union ldouble
226{
227    double value;
228    unsigned long lvalue;
229    unsigned char byte[8];
230} ldouble;
231
232typedef union lfloat
233{
234    float value;
235    unsigned int ivalue;
236    unsigned char byte[4];
237} lfloat;
238
239/* array meta data and compression parameters for SZ_Init_Params() */
240typedef struct sz_params
241{
242        int dataType;
243        unsigned int max_quant_intervals; //max number of quantization intervals for quantization
244        unsigned int quantization_intervals; 
245        unsigned int maxRangeRadius;
246        int sol_ID;// it's always SZ, unless the setting is PASTRI compression mode (./configure --enable-pastri)
247        int losslessCompressor;
248        int sampleDistance; //2 bytes
249        float predThreshold;  // 2 bytes
250        int szMode; //* 0 (best speed) or 1 (better compression with Gzip) or 3 temporal-dimension based compression
251        int gzipMode; //* four options: Z_NO_COMPRESSION, or Z_BEST_SPEED, Z_BEST_COMPRESSION, Z_DEFAULT_COMPRESSION
252        int  errorBoundMode; //4bits (0.5byte), //ABS, REL, ABS_AND_REL, or ABS_OR_REL, PSNR, or PW_REL, PSNR
253        double absErrBound; //absolute error bound
254        double relBoundRatio; //value range based relative error bound ratio
255        double psnr; //PSNR
256        double pw_relBoundRatio; //point-wise relative error bound
257        int segment_size; //only used for 2D/3D data compression with pw_relBoundRatio
258        int pwr_type; //only used for 2D/3D data compression with pw_relBoundRatio
259       
260        int snapshotCmprStep; //perform single-snapshot-based compression if time_step == snapshotCmprStep
261        int predictionMode;
262} sz_params;
263
264typedef struct sz_metadata
265{
266        int versionNumber[3]; //only used for checking the version by calling SZ_GetMetaData()
267        int isConstant; //only used for checking if the data are constant values by calling SZ_GetMetaData()
268        int isLossless; //only used for checking if the data compression was lossless, used only by calling SZ_GetMetaData()
269        int sizeType; //only used for checking whether the size type is "int" or "long" in the compression, used only by calling SZ_GetMetaData()
270        size_t dataSeriesLength; //# number of data points in the dataset
271        int defactoNBBins; //real number of quantization bins
272        struct sz_params* conf_params; //configuration parameters
273} sz_metadata;
274
275typedef struct sz_exedata
276{
277        char optQuantMode;      //opt Quantization (0: fixed ; 1: optimized)   
278        int intvCapacity; // the number of intervals for the linear-scaling quantization
279        int intvRadius;  // the number of intervals for the radius of the quantization range (intvRadius=intvCapacity/2)
280        int SZ_SIZE_TYPE; //the length (# bytes) of the size_t in the system at runtime //4 or 8: sizeof(size_t)
281} sz_exedata;
282
283/*We use a linked list to maintain time-step meta info for time-step based compression*/
284typedef struct sz_tsc_metainfo
285{
286        int totalNumOfSteps;
287        int currentStep;
288        char metadata_filename[256];
289        FILE *metadata_file;
290        unsigned char* bit_array; //sihuan added
291        size_t intersect_size; //sihuan added
292        int64_t* hist_index; //sihuan added: prestep index
293
294} sz_tsc_metadata;
295
296extern int versionNumber[4];
297
298//-------------------key global variables--------------
299extern int dataEndianType; //*endian type of the data read from disk
300extern int sysEndianType; //*sysEndianType is actually set automatically.
301
302extern sz_params *confparams_cpr;
303extern sz_params *confparams_dec;
304extern sz_exedata *exe_params;
305extern int sz_with_regression;
306
307//------------------------------------------------
308extern SZ_VarSet* sz_varset;
309extern sz_multisteps *multisteps; //compression based on multiple time steps (time-dimension based compression)
310extern sz_tsc_metadata *sz_tsc;
311
312//for pastri
313#ifdef PASTRI
314extern pastri_params pastri_par; 
315#endif
316
317//sz.h
318HuffmanTree* SZ_Reset();
319
320int SZ_Init(const char *configFilePath);
321
322int SZ_Init_Params(sz_params *params);
323
324size_t computeDataLength(size_t r5, size_t r4, size_t r3, size_t r2, size_t r1);
325
326int computeDimension(size_t r5, size_t r4, size_t r3, size_t r2, size_t r1);
327
328int SZ_compress_args_float_subblock(unsigned char* compressedBytes, float *oriData,
329size_t r5, size_t r4, size_t r3, size_t r2, size_t r1,
330size_t s5, size_t s4, size_t s3, size_t s2, size_t s1,
331size_t e5, size_t e4, size_t e3, size_t e2, size_t e1,
332size_t *outSize, int errBoundMode, double absErr_Bound, double relBoundRatio);
333
334int SZ_compress_args_double_subblock(unsigned char* compressedBytes, double *oriData,
335size_t r5, size_t r4, size_t r3, size_t r2, size_t r1,
336size_t s5, size_t s4, size_t s3, size_t s2, size_t s1,
337size_t e5, size_t e4, size_t e3, size_t e2, size_t e1,
338size_t *outSize, int errBoundMode, double absErr_Bound, double relBoundRatio);
339
340unsigned char *SZ_compress(int dataType, void *data, size_t *outSize, size_t r5, size_t r4, size_t r3, size_t r2, size_t r1);
341
342unsigned char* SZ_compress_args(int dataType, void *data, size_t *outSize, int errBoundMode, double absErrBound, 
343double relBoundRatio, double pwrBoundRatio, size_t r5, size_t r4, size_t r3, size_t r2, size_t r1);
344
345int SZ_compress_args2(int dataType, void *data, unsigned char* compressed_bytes, size_t *outSize, 
346int errBoundMode, double absErrBound, double relBoundRatio, double pwrBoundRatio, 
347size_t r5, size_t r4, size_t r3, size_t r2, size_t r1);
348
349int SZ_compress_args3(int dataType, void *data, unsigned char* compressed_bytes, size_t *outSize, int errBoundMode, double absErrBound, double relBoundRatio, 
350size_t r5, size_t r4, size_t r3, size_t r2, size_t r1,
351size_t s5, size_t s4, size_t s3, size_t s2, size_t s1,
352size_t e5, size_t e4, size_t e3, size_t e2, size_t e1);
353
354unsigned char *SZ_compress_rev_args(int dataType, void *data, void *reservedValue, size_t *outSize, int errBoundMode, double absErrBound, double relBoundRatio, 
355size_t r5, size_t r4, size_t r3, size_t r2, size_t r1);
356
357int SZ_compress_rev_args2(int dataType, void *data, void *reservedValue, unsigned char* compressed_bytes, size_t *outSize, int errBoundMode, double absErrBound, double relBoundRatio, 
358size_t r5, size_t r4, size_t r3, size_t r2, size_t r1);
359unsigned char *SZ_compress_rev(int dataType, void *data, void *reservedValue, size_t *outSize, size_t r5, size_t r4, size_t r3, size_t r2, size_t r1);
360
361void SZ_Create_ParamsExe(sz_params** conf_params, sz_exedata** exe_params);
362
363void *SZ_decompress(int dataType, unsigned char *bytes, size_t byteLength, size_t r5, size_t r4, size_t r3, size_t r2, size_t r1);
364size_t SZ_decompress_args(int dataType, unsigned char *bytes, size_t byteLength, void* decompressed_array, size_t r5, size_t r4, size_t r3, size_t r2, size_t r1);
365
366sz_metadata* SZ_getMetadata(unsigned char* bytes);
367void SZ_printMetadata(sz_metadata* metadata);
368
369
370void filloutDimArray(size_t* dim, size_t r5, size_t r4, size_t r3, size_t r2, size_t r1);
371
372size_t compute_total_batch_size();
373
374void SZ_registerVar(char* varName, int dataType, void* data, 
375                        int errBoundMode, double absErrBound, double relBoundRatio, double pwRelBoundRatio, 
376                        size_t r5, size_t r4, size_t r3, size_t r2, size_t r1);
377int SZ_deregisterVar(char* varName);
378int SZ_deregisterAllVars();
379
380int SZ_compress_ts(unsigned char** newByteData, size_t *outSize);
381void SZ_decompress_ts(unsigned char *bytes, size_t byteLength);
382
383void SZ_Finalize();
384
385void convertSZParamsToBytes(sz_params* params, unsigned char* result);
386sz_params* convertBytesToSZParams(unsigned char* bytes);
387
388#ifdef __cplusplus
389}
390#endif
391
392#endif /* ----- #ifndef _SZ_H  ----- */
Note: See TracBrowser for help on using the repository browser.