Merge branch 'master' into newTestForGradient

Author: krzysg

src/io/APRWriter.hpp

@@ -15,6 +15,7 @@
 
 struct AprType {hid_t hdf5type; const char * const typeName;};
 namespace AprTypes  {
+
     const AprType TotalNumberOfParticlesType = {H5T_NATIVE_UINT64, "total_number_particles"};
     const AprType TotalNumberOfGapsType = {H5T_NATIVE_UINT64, "total_number_gaps"};
     const AprType TotalNumberOfNonEmptyRowsType = {H5T_NATIVE_UINT64, "total_number_non_empty_rows"};
@@ -42,7 +43,6 @@ namespace AprTypes  {
     const AprType NumberOfLevelXType = {H5T_NATIVE_INT, "x_num_"};
     const AprType NumberOfLevelYType = {H5T_NATIVE_INT, "y_num_"};
     const AprType NumberOfLevelZType = {H5T_NATIVE_INT, "z_num_"};
-    const AprType ParticleIntensitiesDataType = {H5T_NATIVE_INT64, "data_type"};
     const AprType MapGlobalIndexType = {H5T_NATIVE_INT16, "map_global_index"};
     const AprType MapYendType = {H5T_NATIVE_INT16, "map_y_end"};
     const AprType MapYbeginType = {H5T_NATIVE_INT16, "map_y_begin"};
@@ -128,11 +128,9 @@ class APRWriter {
         }
 
         // ------------- read data ------------------------------
-        hid_t dataType;
-        readAttr(AprTypes::ParticleIntensitiesDataType, f.groupId, &dataType);
         apr.particles_intensities.data.resize(apr.apr_access.total_number_particles);
         if (apr.particles_intensities.data.size() > 0) {
-            readData({dataType, AprTypes::ParticleIntensitiesType}, f.objectId, apr.particles_intensities.data.data());
+            readData(AprTypes::ParticleIntensitiesType, f.objectId, apr.particles_intensities.data.data());
         }
         apr.apr_access.y_num[apr.apr_access.level_max] = apr.apr_access.org_dims[0];
         apr.apr_access.x_num[apr.apr_access.level_max] = apr.apr_access.org_dims[1];
@@ -233,7 +231,6 @@ class APRWriter {
             apr_compressor.compress(apr,apr.particles_intensities);
         }
         hid_t type = Hdf5Type<ImageType>::type();
-        writeAttr(AprTypes::ParticleIntensitiesDataType, f.groupId, &type);
         writeData({type, AprTypes::ParticleIntensitiesType}, f.objectId, apr.particles_intensities.data, blosc_comp_type, blosc_comp_level, blosc_shuffle);
         write_timer.stop_timer();
 
@@ -341,8 +338,6 @@ class APRWriter {
         if (!f.isOpened()) return 0;
 
         // ------------- write metadata -------------------------
-        hid_t type = Hdf5Type<S>::type();
-        writeAttr(AprTypes::ParticleIntensitiesDataType, f.groupId, &type);
         uint64_t total_number_parts = parts_extra.data.size();
         writeAttr(AprTypes::TotalNumberOfParticlesType, f.groupId, &total_number_parts);
         writeString(AprTypes::GitType, f.groupId, ConfigAPR::APR_GIT_HASH);
@@ -351,6 +346,7 @@ class APRWriter {
         unsigned int blosc_comp_type = BLOSC_ZSTD;
         unsigned int blosc_comp_level = 3;
         unsigned int blosc_shuffle = 2;
+        hid_t type = Hdf5Type<S>::type();
         writeData({type, AprTypes::ExtraParticleDataType}, f.objectId, parts_extra.data, blosc_comp_type, blosc_comp_level, blosc_shuffle);
 
         // ------------- output the file size -------------------
@@ -369,12 +365,10 @@ class APRWriter {
         // ------------- read metadata --------------------------
         uint64_t numberOfParticles;
         readAttr(AprTypes::TotalNumberOfParticlesType, f.groupId, &numberOfParticles);
-        hid_t data_type;
-        readAttr(AprTypes::ParticleIntensitiesDataType, f.groupId, &data_type);
 
         // ------------- read data -----------------------------
         extra_parts.data.resize(numberOfParticles);
-        readData({data_type, AprTypes::ExtraParticleDataType}, f.objectId, extra_parts.data.data());
+        readData(AprTypes::ExtraParticleDataType, f.objectId, extra_parts.data.data());
     }
 
 private:
@@ -455,6 +449,10 @@ class APRWriter {
         hdf5_load_data_blosc(aObjectId, aType.hdf5type, aDest, aType.typeName);
     }
 
+    void readData(const char * const aAprTypeName, hid_t aObjectId, void *aDest) {
+        hdf5_load_data_blosc(aObjectId, aDest, aAprTypeName);
+    }
+
     template<typename T>
     void writeData(const AprType &aType, hid_t aObjectId, T aContainer, unsigned int blosc_comp_type, unsigned int blosc_comp_level,unsigned int blosc_shuffle) {
         hsize_t dims[] = {aContainer.size()};
@@ -478,6 +476,7 @@ class APRWriter {
     template<typename T> struct Hdf5Type {static hid_t type() {return  T::CANNOT_DETECT_TYPE_AND_WILL_NOT_COMPILE;}};
 };
 
+
 template<> struct APRWriter::Hdf5Type<int8_t> {static hid_t type() {return H5T_NATIVE_INT8;}};
 template<> struct APRWriter::Hdf5Type<uint8_t> {static hid_t type() {return H5T_NATIVE_UINT8;}};
 template<> struct APRWriter::Hdf5Type<int16_t> {static hid_t type() {return H5T_NATIVE_INT16;}};
@@ -489,4 +488,5 @@ template<> struct APRWriter::Hdf5Type<uint64_t> {static hid_t type() {return H5T
 template<> struct APRWriter::Hdf5Type<float> {static hid_t type() {return H5T_NATIVE_FLOAT;}};
 template<> struct APRWriter::Hdf5Type<double> {static hid_t type() {return H5T_NATIVE_DOUBLE;}};
 
+
 #endif //APRWRITER_HPP

src/io/hdf5functions_blosc.cpp

@@ -21,9 +21,20 @@ void register_blosc(){
 /**
  * reads data from hdf5
  */
-void hdf5_load_data_blosc(hid_t obj_id, hid_t data_type, void* buff, const char* data_name) {
+void hdf5_load_data_blosc(hid_t obj_id, hid_t dataType, void* buff, const char* data_name) {
     hid_t data_id =  H5Dopen2(obj_id, data_name ,H5P_DEFAULT);
-    H5Dread(data_id, data_type, H5S_ALL, H5S_ALL, H5P_DEFAULT, buff);
+    H5Dread(data_id, dataType, H5S_ALL, H5S_ALL, H5P_DEFAULT, buff);
+    H5Dclose(data_id);
+}
+
+/**
+ * reads data from hdf5 (data type auto-detection)
+ */
+void hdf5_load_data_blosc(hid_t obj_id, void* buff, const char* data_name) {
+    hid_t data_id =  H5Dopen2(obj_id, data_name ,H5P_DEFAULT);
+    hid_t dataType = H5Dget_type(data_id);
+    H5Dread(data_id, dataType, H5S_ALL, H5S_ALL, H5P_DEFAULT, buff);
+    H5Tclose(dataType);
     H5Dclose(data_id);
 }
 

src/io/hdf5functions_blosc.h

@@ -24,9 +24,9 @@ extern "C" {
 
 void register_blosc();
 hid_t hdf5_create_file_blosc(std::string file_name);
-void hdf5_load_data_blosc(hid_t obj_id,hid_t data_type,void* buff, const char* data_name);
+void hdf5_load_data_blosc(hid_t obj_id, void* buff, const char* data_name);
+void hdf5_load_data_blosc(hid_t obj_id, hid_t dataType, void* buff, const char* data_name);
 void hdf5_write_attribute_blosc(hid_t obj_id,hid_t type_id,const char* attr_name,hsize_t rank,hsize_t* dims, const void * const data );
-void hdf5_write_string_blosc(hid_t obj_id,const char* attr_name, const std::string &output_str);
 void hdf5_write_data_blosc(hid_t obj_id,hid_t type_id,const char* ds_name,hsize_t rank,hsize_t* dims, void* data ,unsigned int comp_type,unsigned int comp_level,unsigned int shuffle);
 
 void write_main_paraview_xdmf_xml(std::string save_loc,std::string file_name,uint64_t num_parts);