Changeset eeacdad

Timestamp:

09/28/18 12:44:11 (6 years ago)

Author:

Hal Finkel <hfinkel@…>

Branches:

master, pympi

Children:

b060d3c

Parents:

2c47b73

git-author:

Hal Finkel <hfinkel@…> (09/28/18 12:44:11)

git-committer:

Hal Finkel <hfinkel@…> (09/28/18 12:44:11)

Message:

initial SZ integration - it compiles

Files:

• GenericIO.cxx (modified) (7 diffs)
• GenericIO.h (modified) (1 diff)

GenericIO.cxx

@@ -368 +368 @@
 static bool sz_initialized = false;
 
+static int GetSZDT(GenericIO::Variable &Var) {
+  if (Var.hasElementType<float>())
+    return SZ_FLOAT;
+  else if (Var.hasElementType<double>())
+    return SZ_DOUBLE;
+  else if (Var.hasElementType<uint8_t>())
+    return SZ_UINT8;
+  else if (Var.hasElementType<int8_t>())
+    return SZ_INT8;
+  else if (Var.hasElementType<uint16_t>())
+    return SZ_UINT16;
+  else if (Var.hasElementType<int16_t>())
+    return SZ_INT16;
+  else if (Var.hasElementType<uint32_t>())
+    return SZ_UINT32;
+  else if (Var.hasElementType<int32_t>())
+    return SZ_INT32;
+  else if (Var.hasElementType<uint64_t>())
+    return SZ_UINT64;
+  else if (Var.hasElementType<int64_t>())
+    return SZ_INT64;
+  else
+    return -1;
+}
+
 #ifndef GENERICIO_NO_MPI
 void GenericIO::write() {
@@ -501 +526 @@
       memset(&LocalBlockHeaders[i], 0, sizeof(BlockHeader<IsBigEndian>));
       if (ShouldCompress) {
+        void *OrigData = Vars[i].Data;
+        bool FreeOrigData = false;
+        size_t OrigUnitSize = Vars[i].Size;
+        size_t OrigDataSize = NElems*Vars[i].Size;
+
+        int FilterIdx = 0;
+        if (Vars[i].LCI.Mode != LossyCompressionInfo::LCModeNone) {
+          int SZDT = GetSZDT(Vars[i]);
+          if (SZDT == -1)
+            goto nosz;
+
+          int EBM;
+          switch (Vars[i].LCI.Mode) {
+          case LossyCompressionInfo::LCModeAbs:
+            EBM = ABS;
+            break;
+          case LossyCompressionInfo::LCModeRel:
+            EBM = REL;
+            break;
+          case LossyCompressionInfo::LCModeAbsAndRel:
+            EBM = ABS_AND_REL;
+            break;
+          case LossyCompressionInfo::LCModeAbsOrRel:
+            EBM = ABS_OR_REL;
+            break;
+          case LossyCompressionInfo::LCModePSNR:
+            EBM = PSNR;
+            break;
+          }
+
+          size_t LOutSize;
+          unsigned char *LCompressedData = SZ_compress_args(SZDT, Vars[i].Data, &LOutSize, EBM,
+                                                            Vars[i].LCI.AbsErrThreshold, Vars[i].LCI.RelErrThreshold,
+                                                            Vars[i].LCI.PSNRThreshold, 0, 0, 0, 0, NElems);
+          if (!LCompressedData)
+            goto nosz;
+          if (LOutSize >= NElems*Vars[i].Size) {
+            free(LCompressedData);
+            goto nosz;
+          }
+
+          OrigData = LCompressedData;
+          FreeOrigData = true;
+          OrigUnitSize = 1;
+          OrigDataSize = LOutSize;
+
+          strncpy(LocalBlockHeaders[i].Filters[FilterIdx++], LossyCompressName, FilterNameSize);
+        }
+nosz:
+
         LocalCData[i].resize(sizeof(CompressHeader<IsBigEndian>));
 
         CompressHeader<IsBigEndian> *CH = (CompressHeader<IsBigEndian>*) &LocalCData[i][0];
-        CH->OrigCRC = crc64_omp(Vars[i].Data, Vars[i].Size*NElems);
+        CH->OrigCRC = crc64_omp(OrigData, OrigDataSize);
 
 #ifdef _OPENMP
@@ -526 +601 @@
 #endif
 
-        LocalCData[i].resize(LocalCData[i].size() + NElems*Vars[i].Size);
-        if (blosc_compress(9, 1, Vars[i].Size, NElems*Vars[i].Size, Vars[i].Data,
+        LocalCData[i].resize(LocalCData[i].size() + OrigDataSize);
+        if (blosc_compress(9, 1, OrigUnitSize, OrigDataSize, OrigData,
                            &LocalCData[i][0] + sizeof(CompressHeader<IsBigEndian>),
-                           NElems*Vars[i].Size) <= 0)
+                           OrigDataSize) <= 0) {
+          if (FreeOrigData)
+            free(OrigData);
+
           goto nocomp;
-
-        strncpy(LocalBlockHeaders[i].Filters[0], CompressName, FilterNameSize);
+        }
+
+        if (FreeOrigData)
+          free(OrigData);
+
+        strncpy(LocalBlockHeaders[i].Filters[FilterIdx++], CompressName, FilterNameSize);
         size_t CNBytes, CCBytes, CBlockSize;
         blosc_cbuffer_sizes(&LocalCData[i][0] + sizeof(CompressHeader<IsBigEndian>),
@@ -1435 +1517 @@
 
       vector<unsigned char> LData;
+      bool HasSZ = false;
       void *Data = VarData;
       bool HasExtraSpace = Vars[i].HasExtraSpace;
@@ -1445 +1528 @@
         Offset = BH->Start;
 
-        if (strncmp(BH->Filters[0], CompressName, FilterNameSize) == 0) {
+        int FilterIdx = 0;
+
+        if (strncmp(BH->Filters[FilterIdx], LossyCompressName, FilterNameSize) == 0) {
+          ++FilterIdx;
+          HasSZ = true;
+        }
+
+        if (strncmp(BH->Filters[FilterIdx], CompressName, FilterNameSize) == 0) {
           LData.resize(ReadSize);
           Data = &LData[0];
           HasExtraSpace = true;
-        } else if (BH->Filters[0][0] != '\0') {
+        } else if (BH->Filters[FilterIdx][0] != '\0') {
           stringstream ss;
           ss << "Unknown filter \"" << BH->Filters[0] << "\" on variable " << Vars[i].Name;
@@ -1574 +1664 @@
        }
 
+       if (!sz_initialized) {
+         SZ_Init(NULL);
+         sz_initialized = true;
+       }
+
 #ifdef _OPENMP
        blosc_set_nthreads(omp_get_max_threads());
@@ -1579 +1674 @@
 #endif
 
+        void *OrigData = VarData;
+        size_t OrigDataSize = Vars[i].Size*RH->NElems;
+
+        if (HasSZ) {
+          size_t CNBytes, CCBytes, CBlockSize;
+          blosc_cbuffer_sizes(&LData[0] + sizeof(CompressHeader<IsBigEndian>),
+                              &CNBytes, &CCBytes, &CBlockSize);
+
+          OrigData = malloc(CNBytes);
+          OrigDataSize = CNBytes;
+        }
+
         blosc_decompress(&LData[0] + sizeof(CompressHeader<IsBigEndian>),
-                         VarData, Vars[i].Size*RH->NElems);
-
-        if (CH->OrigCRC != crc64_omp(VarData, Vars[i].Size*RH->NElems)) {
+                         OrigData, OrigDataSize);
+
+        if (CH->OrigCRC != crc64_omp(OrigData, OrigDataSize)) {
           ++NErrs[2];
           break;
         }
+
+        if (HasSZ) {
+          int SZDT = GetSZDT(Vars[i]);
+          size_t LDSz = SZ_decompress_args(SZDT, (unsigned char *)OrigData, OrigDataSize,
+                                           VarData, 0, 0, 0, 0, RH->NElems);
+          free(OrigData);
+
+          if (LDSz != Vars[i].Size*RH->NElems)
+            throw runtime_error("Variable " + Vars[i].Name +
+                                ": SZ decompression yielded the wrong amount of data");
+        }
       }
 
       // Byte swap the data if necessary.
-      if (IsBigEndian != isBigEndian())
+      if (IsBigEndian != isBigEndian() && !HasSZ)
         for (size_t j = 0;
              j < RH->NElems*(Vars[i].Size/Vars[i].ElementSize); ++j) {

GenericIO.h

@@ -288 +288 @@
         ElementSize(VI.ElementSize), LCI(LCI) {}
 
+    template <typename ET>
+    bool hasElementType() {
+      if (ElementSize != sizeof(ET))
+        return false;
+      if (IsFloat != !std::numeric_limits<ET>::is_integer)
+        return false;
+      if (IsSigned != std::numeric_limits<ET>::is_signed)
+        return false;
+
+      return true;
+    }
+
     std::string Name;
     std::size_t Size;