source: thirdparty/SZ/sz/include/sz.h @ 2c47b73

Revision 2c47b73, 12.7 KB checked in by Hal Finkel <hfinkel@…>, 6 years ago (diff)

more work on adding SZ (latest version)
  • Property mode set to 100644
Line 
1/**
2 *  @file sz.h
3 *  @author Sheng Di
4 *  @date April, 2015
5 *  @brief Header file for the whole detector.
6 *  (C) 2015 by Mathematics and Computer Science (MCS), Argonne National Laboratory.
7 *      See COPYRIGHT in top-level directory.
8 */
9
10#ifndef _SZ_H
11#define _SZ_H
12
13#include <stdio.h>
14#include <stdint.h>
15#include <sys/time.h>      /* For gettimeofday(), in microseconds */
16#include <time.h>          /* For time(), in seconds */
17#include "iniparser.h"
18#include "CompressElement.h"
19#include "DynamicByteArray.h"
20#include "DynamicIntArray.h"
21#include "VarSet.h"
22#include "Huffman.h"
23#include "TightDataPointStorageD.h"
24#include "TightDataPointStorageF.h"
25#include "TightDataPointStorageI.h"
26#include "conf.h"
27#include "dataCompression.h"
28#include "ByteToolkit.h"
29#include "TypeManager.h"
30#include "sz_int8.h"
31#include "sz_int16.h"
32#include "sz_int32.h"
33#include "sz_int64.h"
34#include "sz_uint8.h"
35#include "sz_uint16.h"
36#include "sz_uint32.h"
37#include "sz_uint64.h"
38#include "sz_float.h"
39#include "sz_double.h"
40#include "szd_int8.h"
41#include "szd_int16.h"
42#include "szd_int32.h"
43#include "szd_int64.h"
44#include "szd_uint8.h"
45#include "szd_uint16.h"
46#include "szd_uint32.h"
47#include "szd_uint64.h"
48#include "szd_float.h"
49#include "szd_double.h"
50#include "sz_float_pwr.h"
51#include "sz_double_pwr.h"
52#include "callZlib.h"
53#include "rw.h"
54#include "pastri.h"
55#include "sz_float_ts.h"
56#include "szd_float_ts.h"
57
58#ifdef _WIN32
59#define PATH_SEPARATOR ';'
60#else
61#define PATH_SEPARATOR ':'
62#endif
63
64#ifdef __cplusplus
65extern "C" {
66#endif
67
68//typedef char int8_t;
69//typedef unsigned char uint8_t;
70//typedef short int16_t;
71//typedef unsigned short uint16_t;
72//typedef int int32_t;
73//typedef unsigned int uint32_t;
74//typedef long int64_t;
75//typedef unsigned long uint64_t;
76
77#define SZ_VERNUM 0x0140
78#define SZ_VER_MAJOR 1
79#define SZ_VER_MINOR 4
80#define SZ_VER_BUILD 13
81#define SZ_VER_REVISION 5
82
83#define PASTRI 103
84#define HZ 102
85#define SZ 101
86
87//prediction mode of temporal dimension based compression
88#define SZ_PREVIOUS_VALUE_ESTIMATE 0
89
90#define MIN_NUM_OF_ELEMENTS 20 //if the # elements <= 20, skip the compression
91
92#define ABS 0
93#define REL 1
94#define ABS_AND_REL 2
95#define ABS_OR_REL 3
96#define PSNR 4
97
98#define PW_REL 10
99#define ABS_AND_PW_REL 11
100#define ABS_OR_PW_REL 12
101#define REL_AND_PW_REL 13
102#define REL_OR_PW_REL 14
103
104#define SZ_FLOAT 0
105#define SZ_DOUBLE 1
106#define SZ_UINT8 2
107#define SZ_INT8 3
108#define SZ_UINT16 4
109#define SZ_INT16 5
110#define SZ_UINT32 6
111#define SZ_INT32 7
112#define SZ_UINT64 8
113#define SZ_INT64 9
114
115#define LITTLE_ENDIAN_DATA 0 //refers to the endian type of the data read from the disk
116#define BIG_ENDIAN_DATA 1 //big_endian (ppc, max, etc.) ; little_endian (x86, x64, etc.)
117
118#define LITTLE_ENDIAN_SYSTEM 0 //refers to the endian type of the system
119#define BIG_ENDIAN_SYSTEM 1
120
121#define DynArrayInitLen 1024
122
123#define MIN_ZLIB_DEC_ALLOMEM_BYTES 1000000
124
125//#define maxRangeRadius 32768
126//#define maxRangeRadius 1048576//131072
127
128#define SZ_BEST_SPEED 0
129#define SZ_BEST_COMPRESSION 1
130#define SZ_DEFAULT_COMPRESSION 2
131#define SZ_TEMPORAL_COMPRESSION 3
132
133#define SZ_PWR_MIN_TYPE 0
134#define SZ_PWR_AVG_TYPE 1
135#define SZ_PWR_MAX_TYPE 2
136
137//SUCCESS returning status
138#define SZ_SCES 0  //successful
139#define SZ_NSCS -1 //Not successful
140#define SZ_FERR -2 //Failed to open input file
141#define SZ_TERR -3 //wrong data type (should be only float or double)
142#define SZ_DERR -4 //dimension error
143#define SZ_MERR -5 //sz_mode error
144#define SZ_BERR -6 //bound-mode error (should be only ABS, REL, ABS_AND_REL, ABS_OR_REL, or PW_REL)
145
146#define SZ_MAINTAIN_VAR_DATA 0
147#define SZ_DESTROY_WHOLE_VARSET 1
148
149#define GROUP_COUNT 16 //2^{16}=65536
150       
151#define MetaDataByteLength 20   
152       
153#define numOfBufferedSteps 1 //the number of time steps in the buffer   
154       
155//Note: the following setting should be consistent with stateNum in Huffman.h
156//#define intvCapacity 65536
157//#define intvRadius 32768
158//#define intvCapacity 131072
159//#define intvRadius 65536
160
161#define SZ_COMPUTE_1D_NUMBER_OF_BLOCKS( COUNT, NUM_BLOCKS, BLOCK_SIZE ) \
162    if (COUNT <= BLOCK_SIZE){                  \
163        NUM_BLOCKS = 1;             \
164    }                                   \
165    else{                               \
166        NUM_BLOCKS = COUNT / BLOCK_SIZE;       \
167    }                                   \
168
169#define SZ_COMPUTE_2D_NUMBER_OF_BLOCKS( COUNT, NUM_BLOCKS, BLOCK_SIZE ) \
170    if (COUNT <= BLOCK_SIZE){                   \
171        NUM_BLOCKS = 1;             \
172    }                                   \
173    else{                               \
174        NUM_BLOCKS = COUNT / BLOCK_SIZE;        \
175    }                                   \
176
177#define SZ_COMPUTE_3D_NUMBER_OF_BLOCKS( COUNT, NUM_BLOCKS, BLOCK_SIZE ) \
178    if (COUNT <= BLOCK_SIZE){                   \
179        NUM_BLOCKS = 1;             \
180    }                                   \
181    else{                               \
182        NUM_BLOCKS = COUNT / BLOCK_SIZE;        \
183    }                                   \
184
185#define SZ_COMPUTE_BLOCKCOUNT( COUNT, NUM_BLOCKS, SPLIT_INDEX,       \
186                                       EARLY_BLOCK_COUNT, LATE_BLOCK_COUNT ) \
187    EARLY_BLOCK_COUNT = LATE_BLOCK_COUNT = COUNT / NUM_BLOCKS;               \
188    SPLIT_INDEX = COUNT % NUM_BLOCKS;                                        \
189    if (0 != SPLIT_INDEX) {                                                  \
190        EARLY_BLOCK_COUNT = EARLY_BLOCK_COUNT + 1;                           \
191    }                                                                        \
192
193//typedef unsigned long unsigned long;
194//typedef unsigned int uint;
195
196typedef union lint16
197{
198        unsigned short usvalue;
199        short svalue;
200        unsigned char byte[2];
201} lint16;
202
203typedef union lint32
204{
205        int ivalue;
206        unsigned int uivalue;
207        unsigned char byte[4];
208} lint32;
209
210typedef union lint64
211{
212        long lvalue;
213        unsigned long ulvalue;
214        unsigned char byte[8];
215} lint64;
216
217typedef union ldouble
218{
219    double value;
220    unsigned long lvalue;
221    unsigned char byte[8];
222} ldouble;
223
224typedef union lfloat
225{
226    float value;
227    unsigned int ivalue;
228    unsigned char byte[4];
229} lfloat;
230
231/* array meta data and compression parameters for SZ_Init_Params() */
232typedef struct sz_params
233{
234        int dataType;
235        unsigned int max_quant_intervals; //max number of quantization intervals for quantization
236        unsigned int quantization_intervals; 
237        unsigned int maxRangeRadius;
238        int sol_ID;// it's always SZ, unless the setting is PASTRI compression mode (./configure --enable-pastri)
239        int sampleDistance; //2 bytes
240        float predThreshold;  // 2 bytes
241        int szMode; //* 0 (best speed) or 1 (better compression with Gzip) or 3 temporal-dimension based compression
242        int gzipMode; //* four options: Z_NO_COMPRESSION, or Z_BEST_SPEED, Z_BEST_COMPRESSION, Z_DEFAULT_COMPRESSION
243        int  errorBoundMode; //4bits (0.5byte), //ABS, REL, ABS_AND_REL, or ABS_OR_REL, PSNR, or PW_REL, PSNR
244        double absErrBound; //absolute error bound
245        double relBoundRatio; //value range based relative error bound ratio
246        double psnr; //PSNR
247        double pw_relBoundRatio; //point-wise relative error bound
248        int segment_size; //only used for 2D/3D data compression with pw_relBoundRatio
249        int pwr_type; //only used for 2D/3D data compression with pw_relBoundRatio
250       
251        int snapshotCmprStep; //perform single-snapshot-based compression if time_step == snapshotCmprStep
252        int predictionMode;
253} sz_params;
254
255typedef struct sz_metadata
256{
257        int versionNumber[3]; //only used for checking the version by calling SZ_GetMetaData()
258        int isConstant; //only used for checking if the data are constant values by calling SZ_GetMetaData()
259        int isLossless; //only used for checking if the data compression was lossless, used only by calling SZ_GetMetaData()
260        int sizeType; //only used for checking whether the size type is "int" or "long" in the compression, used only by calling SZ_GetMetaData()
261        size_t dataSeriesLength; //# number of data points in the dataset
262        int defactoNBBins; //real number of quantization bins
263        struct sz_params* conf_params; //configuration parameters
264} sz_metadata;
265
266typedef struct sz_exedata
267{
268        char optQuantMode;      //opt Quantization (0: fixed ; 1: optimized)   
269        int intvCapacity; // the number of intervals for the linear-scaling quantization
270        int intvRadius;  // the number of intervals for the radius of the quantization range (intvRadius=intvCapacity/2)
271        int SZ_SIZE_TYPE; //the length (# bytes) of the size_t in the system at runtime //4 or 8: sizeof(size_t)
272} sz_exedata;
273
274/*We use a linked list to maintain time-step meta info for time-step based compression*/
275typedef struct sz_tsc_metainfo
276{
277        int totalNumOfSteps;
278        int currentStep;
279        char metadata_filename[256];
280        FILE *metadata_file;
281} sz_tsc_metadata;
282
283extern int versionNumber[4];
284
285//-------------------key global variables--------------
286extern int dataEndianType; //*endian type of the data read from disk
287extern int sysEndianType; //*sysEndianType is actually set automatically.
288
289extern sz_params *confparams_cpr;
290extern sz_params *confparams_dec;
291extern sz_exedata *exe_params;
292//------------------------------------------------
293extern SZ_VarSet* sz_varset;
294extern sz_multisteps *multisteps; //compression based on multiple time steps (time-dimension based compression)
295extern sz_tsc_metadata *sz_tsc;
296
297//for pastri
298#ifdef PASTRI
299extern pastri_params pastri_par; 
300#endif
301
302//sz.h
303HuffmanTree* SZ_Reset();
304
305int SZ_Init(const char *configFilePath);
306
307int SZ_Init_Params(sz_params *params);
308
309size_t computeDataLength(size_t r5, size_t r4, size_t r3, size_t r2, size_t r1);
310
311int computeDimension(size_t r5, size_t r4, size_t r3, size_t r2, size_t r1);
312
313int SZ_compress_args_float_subblock(unsigned char* compressedBytes, float *oriData,
314size_t r5, size_t r4, size_t r3, size_t r2, size_t r1,
315size_t s5, size_t s4, size_t s3, size_t s2, size_t s1,
316size_t e5, size_t e4, size_t e3, size_t e2, size_t e1,
317size_t *outSize, int errBoundMode, double absErr_Bound, double relBoundRatio);
318
319int SZ_compress_args_double_subblock(unsigned char* compressedBytes, double *oriData,
320size_t r5, size_t r4, size_t r3, size_t r2, size_t r1,
321size_t s5, size_t s4, size_t s3, size_t s2, size_t s1,
322size_t e5, size_t e4, size_t e3, size_t e2, size_t e1,
323size_t *outSize, int errBoundMode, double absErr_Bound, double relBoundRatio);
324
325unsigned char *SZ_compress(int dataType, void *data, size_t *outSize, size_t r5, size_t r4, size_t r3, size_t r2, size_t r1);
326
327unsigned char* SZ_compress_args(int dataType, void *data, size_t *outSize, int errBoundMode, double absErrBound, 
328double relBoundRatio, double pwrBoundRatio, size_t r5, size_t r4, size_t r3, size_t r2, size_t r1);
329
330int SZ_compress_args2(int dataType, void *data, unsigned char* compressed_bytes, size_t *outSize, 
331int errBoundMode, double absErrBound, double relBoundRatio, double pwrBoundRatio, 
332size_t r5, size_t r4, size_t r3, size_t r2, size_t r1);
333
334int SZ_compress_args3(int dataType, void *data, unsigned char* compressed_bytes, size_t *outSize, int errBoundMode, double absErrBound, double relBoundRatio, 
335size_t r5, size_t r4, size_t r3, size_t r2, size_t r1,
336size_t s5, size_t s4, size_t s3, size_t s2, size_t s1,
337size_t e5, size_t e4, size_t e3, size_t e2, size_t e1);
338
339unsigned char *SZ_compress_rev_args(int dataType, void *data, void *reservedValue, size_t *outSize, int errBoundMode, double absErrBound, double relBoundRatio, 
340size_t r5, size_t r4, size_t r3, size_t r2, size_t r1);
341
342int SZ_compress_rev_args2(int dataType, void *data, void *reservedValue, unsigned char* compressed_bytes, size_t *outSize, int errBoundMode, double absErrBound, double relBoundRatio, 
343size_t r5, size_t r4, size_t r3, size_t r2, size_t r1);
344unsigned char *SZ_compress_rev(int dataType, void *data, void *reservedValue, size_t *outSize, size_t r5, size_t r4, size_t r3, size_t r2, size_t r1);
345
346void SZ_Create_ParamsExe(sz_params** conf_params, sz_exedata** exe_params);
347
348void *SZ_decompress(int dataType, unsigned char *bytes, size_t byteLength, size_t r5, size_t r4, size_t r3, size_t r2, size_t r1);
349size_t SZ_decompress_args(int dataType, unsigned char *bytes, size_t byteLength, void* decompressed_array, size_t r5, size_t r4, size_t r3, size_t r2, size_t r1);
350
351sz_metadata* SZ_getMetadata(unsigned char* bytes);
352void SZ_printMetadata(sz_metadata* metadata);
353
354
355void filloutDimArray(size_t* dim, size_t r5, size_t r4, size_t r3, size_t r2, size_t r1);
356
357size_t compute_total_batch_size();
358
359int isZlibFormat(unsigned char magic1, unsigned char magic2);
360
361void SZ_registerVar(char* varName, int dataType, void* data, 
362                        int errBoundMode, double absErrBound, double relBoundRatio, double pwRelBoundRatio, 
363                        size_t r5, size_t r4, size_t r3, size_t r2, size_t r1);
364int SZ_deregisterVar(char* varName);
365int SZ_deregisterAllVars();
366
367int SZ_compress_ts(unsigned char** newByteData, size_t *outSize);
368void SZ_decompress_ts(unsigned char *bytes, size_t byteLength);
369
370void SZ_Finalize();
371
372void convertSZParamsToBytes(sz_params* params, unsigned char* result);
373sz_params* convertBytesToSZParams(unsigned char* bytes);
374
375#ifdef __cplusplus
376}
377#endif
378
379#endif /* ----- #ifndef _SZ_H  ----- */
Note: See TracBrowser for help on using the repository browser.