From 3b718eeb59ee2bdd252c03e1a58338c432dafc9d Mon Sep 17 00:00:00 2001 From: aothms Date: Fri, 5 Mar 2021 19:22:36 +0000 Subject: [PATCH 001/112] msvc 2019 and win build script fixes --- CMakeLists.txt | 32 +++++++++++++++++++++----------- json_logger.cpp | 1 + 2 files changed, 22 insertions(+), 11 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 951e04e..a96f267 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -2,17 +2,20 @@ cmake_minimum_required (VERSION 2.8.11) project (voxel) OPTION(IFC_SUPPORT "Build Volization Toolkit with IFC support (requires IfcOpenShell/master)." ON) -OPTION(USE_STATIC_MSVC_RUNTIME "Link to the static runtime on MSVC." ON) +OPTION(USE_STATIC_MSVC_RUNTIME "Link to the static runtime on MSVC." OFF) OPTION(ENABLE_TESTS "Enable tests." ON) include(CTest) set(CMAKE_BUILD_TYPE "RelWithDebInfo") +set(Boost_NO_BOOST_CMAKE On) if(CMAKE_SIZEOF_VOID_P EQUAL 8) set(WIN_ARCH x64) + set(BITS 64) elseif(CMAKE_SIZEOF_VOID_P EQUAL 4) set(WIN_ARCH Win32) + set(BITS 32) endif() set(OCC_LIBRARY_NAMES TKernel TKMath TKBRep TKGeomBase TKGeomAlgo TKG3d TKG2d TKShHealing TKTopAlgo TKMesh TKOffset @@ -57,20 +60,25 @@ if(UNIX) add_definitions(-Wno-deprecated-declarations) else() if(IFC_SUPPORT) + # 1900 = VS 14.0 (v140 toolset) + # 1910-1919 = VS 15.0 (v141 toolset) + # 1920-1929 = VS 16.0 (v142 toolset) set(MSVC_YEAR 2017) if ("${MSVC_VERSION}" STREQUAL "1900") set(MSVC_YEAR 2015) + elseif(MSVC_VERSION GREATER 1919) + set(MSVC_YEAR 2019) endif() file(TO_CMAKE_PATH "${IFCOPENSHELL_ROOT}" IFCOPENSHELL_ROOT) - if (EXISTS ${IFCOPENSHELL_ROOT}/deps-vs${MSVC_YEAR}-x86-installed/oce/${WIN_ARCH}/lib) - set(OCC_LIBRARY_DIR ${IFCOPENSHELL_ROOT}/deps-vs${MSVC_YEAR}-x86-installed/oce/${WIN_ARCH}/lib) + if (EXISTS ${IFCOPENSHELL_ROOT}/_deps-vs${MSVC_YEAR}-${WIN_ARCH}-installed/oce/${WIN_ARCH}/lib) + set(OCC_LIBRARY_DIR ${IFCOPENSHELL_ROOT}/_deps-vs${MSVC_YEAR}-${WIN_ARCH}-installed/oce/${WIN_ARCH}/lib) else() - set(OCC_LIBRARY_DIR ${IFCOPENSHELL_ROOT}/deps-vs${MSVC_YEAR}-x86-installed/opencascade-7.3.0p3/win32/lib) + set(OCC_LIBRARY_DIR ${IFCOPENSHELL_ROOT}/_deps-vs${MSVC_YEAR}-${WIN_ARCH}-installed/opencascade-7.3.0p3/win${BITS}/lib) endif() - set(OCC_INCLUDE_DIR ${IFCOPENSHELL_ROOT}/deps-vs${MSVC_YEAR}-x86-installed/oce/include/oce - ${IFCOPENSHELL_ROOT}/deps-vs${MSVC_YEAR}-x86-installed/opencascade-7.3.0p3/inc) + set(OCC_INCLUDE_DIR ${IFCOPENSHELL_ROOT}/_deps-vs${MSVC_YEAR}-${WIN_ARCH}-installed/oce/include/oce + ${IFCOPENSHELL_ROOT}/_deps-vs${MSVC_YEAR}-${WIN_ARCH}-installed/opencascade-7.3.0p3/inc) else() file(TO_CMAKE_PATH "${OCC_LIBRARY_DIR}" OCC_LIBRARY_DIR) endif() @@ -140,18 +148,18 @@ if(UNIX) string(REGEX REPLACE "([^;]+)" "${ICU_LIBRARY_DIR}/lib\\1.a" ICU_LIBRARIES "${ICU_LIBRARY_NAMES}") endif() else() - set(BOOST_ROOT ${IFCOPENSHELL_ROOT}/deps/boost_1_67_0) - set(BOOST_LIBRARYDIR ${IFCOPENSHELL_ROOT}/deps/boost_1_67_0/stage/vs${MSVC_YEAR}-${WIN_ARCH}/lib) + set(BOOST_ROOT ${IFCOPENSHELL_ROOT}/_deps/boost_1_74_0) + set(BOOST_LIBRARYDIR ${IFCOPENSHELL_ROOT}/_deps/boost_1_74_0/stage/vs${MSVC_YEAR}-${WIN_ARCH}/lib) set(IFC_INCLUDE_DIR ${IFCOPENSHELL_ROOT}/src) - set(IFC_LIBRARY_DIR ${IFCOPENSHELL_ROOT}/build-vs${MSVC_YEAR}-x86/${CMAKE_BUILD_TYPE}) + set(IFC_LIBRARY_DIR ${IFCOPENSHELL_ROOT}/_build-vs${MSVC_YEAR}-${WIN_ARCH}/${CMAKE_BUILD_TYPE}) string(REGEX REPLACE "([^;]+)" "${IFC_LIBRARY_DIR}/\\1.lib" IFC_LIBRARIES "${IFC_LIBRARY_NAMES}") if (${ifcopenshell_major_version} STREQUAL "0.5") set(ICU_LIBRARY_NAMES icuuc icudt) - set(ICU_LIBRARY_DIR ${IFCOPENSHELL_ROOT}/deps-vs${MSVC_YEAR}-x86-installed/icu/lib) - set(ICU_INCLUDE_DIR ${IFCOPENSHELL_ROOT}/deps-vs${MSVC_YEAR}-x86-installed/icu/include) + set(ICU_LIBRARY_DIR ${IFCOPENSHELL_ROOT}/_deps-vs${MSVC_YEAR}-${WIN_ARCH}-installed/icu/lib) + set(ICU_INCLUDE_DIR ${IFCOPENSHELL_ROOT}/_deps-vs${MSVC_YEAR}-${WIN_ARCH}-installed/icu/include) string(REGEX REPLACE "([^;]+)" "${ICU_LIBRARY_DIR}/\\1.lib" ICU_LIBRARIES "${ICU_LIBRARY_NAMES}") endif() endif() @@ -166,6 +174,7 @@ include_directories(${IFC_INCLUDE_DIR} ${OCC_INCLUDE_DIR} ${Boost_INCLUDE_DIRS} link_directories(${Boost_LIBRARY_DIRS}) include_directories(${INCLUDE_DIRECTORIES} ${OCC_INCLUDE_DIR} ${Boost_INCLUDE_DIRS} ${IFC_INCLUDE_DIR}) +if (NOT MSVC) include(CheckCXXCompilerFlag) CHECK_CXX_COMPILER_FLAG("-std=c++14" COMPILER_SUPPORTS_CXX11) if(COMPILER_SUPPORTS_CXX11) @@ -173,6 +182,7 @@ if(COMPILER_SUPPORTS_CXX11) else() message(FATAL_ERROR "requires a compiler with C++11 support") endif() +endif() if (ENABLE_TESTS) diff --git a/json_logger.cpp b/json_logger.cpp index d811189..7573738 100644 --- a/json_logger.cpp +++ b/json_logger.cpp @@ -9,6 +9,7 @@ #include #include #include +#include namespace { struct ptree_writer { From 60f309d342865128e10f2a9e734ea3d0377b1437 Mon Sep 17 00:00:00 2001 From: johltn Date: Thu, 18 Mar 2021 15:10:47 +0100 Subject: [PATCH 002/112] Create test for HDF libs --- .gitignore | 2 ++ CMakeLists.txt | 19 ++++++++++++++++--- tests/test_hdf.cpp | 39 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 57 insertions(+), 3 deletions(-) create mode 100644 tests/test_hdf.cpp diff --git a/.gitignore b/.gitignore index de7f0f3..78c0dd4 100644 --- a/.gitignore +++ b/.gitignore @@ -31,3 +31,5 @@ *.out *.app build/ + +test diff --git a/CMakeLists.txt b/CMakeLists.txt index a96f267..2fea951 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -111,6 +111,19 @@ endif() set(BOOST_COMPONENTS regex program_options iostreams system) + +SET(HDF5_INCLUDE_DIR ${IFCOPENSHELL_ROOT}/_deps-vs2017-x64-installed/HDF5-1.8.22-win64/include) +SET(HDF5_LIBRARY_DIR ${IFCOPENSHELL_ROOT}/_deps-vs2017-x64-installed/HDF5-1.8.22-win64/lib) + + +SET(HDF5_LIBRARIES + "${HDF5_LIBRARY_DIR}/libhdf5_cpp.lib" + "${HDF5_LIBRARY_DIR}/libhdf5.lib" + "${HDF5_LIBRARY_DIR}/libzlib.lib" + "${HDF5_LIBRARY_DIR}/libsz.lib" + "${HDF5_LIBRARY_DIR}/libaec.lib") + + if (IFC_SUPPORT) # Find IfcOpenShell version, v0.6.0 does not have the IfcSchema namespace anymore file(READ "${IFCOPENSHELL_ROOT}/src/ifcparse/IfcParse.h" header) @@ -170,10 +183,10 @@ find_package(Boost REQUIRED COMPONENTS ${BOOST_COMPONENTS}) message(STATUS "Boost include files found in ${Boost_INCLUDE_DIRS}") message(STATUS "Boost libraries found in ${Boost_LIBRARY_DIRS}") -include_directories(${IFC_INCLUDE_DIR} ${OCC_INCLUDE_DIR} ${Boost_INCLUDE_DIRS} ${CMAKE_CURRENT_SOURCE_DIR}/3rdparty/eigen) +include_directories(${IFC_INCLUDE_DIR} ${OCC_INCLUDE_DIR} ${Boost_INCLUDE_DIRS} ${HDF5_INCLUDE_DIR} ${CMAKE_CURRENT_SOURCE_DIR}/3rdparty/eigen) link_directories(${Boost_LIBRARY_DIRS}) -include_directories(${INCLUDE_DIRECTORIES} ${OCC_INCLUDE_DIR} ${Boost_INCLUDE_DIRS} ${IFC_INCLUDE_DIR}) +include_directories(${INCLUDE_DIRECTORIES} ${OCC_INCLUDE_DIR} ${Boost_INCLUDE_DIRS} ${HDF5_INCLUDE_DIR} ${IFC_INCLUDE_DIR}) if (NOT MSVC) include(CheckCXXCompilerFlag) CHECK_CXX_COMPILER_FLAG("-std=c++14" COMPILER_SUPPORTS_CXX11) @@ -221,7 +234,7 @@ ENDIF() set(LIBVOXELIZER_HEADER_FILES progress_writer.h dim3.h sweep.h shift.h collapse.h edge_detect.h fill_gaps.h offset.h polyfill.h resample.h storage.h writer.h factory.h processor.h volume.h voxelizer.h traversal.h util.h json_logger.h) set(LIBVOXELIZER_SOURCE_FILES tribox3.cpp polyfill.cpp progress_writer.cpp storage.cpp factory.cpp json_logger.cpp) add_library(libvoxel STATIC ${LIBVOXELIZER_HEADER_FILES} ${LIBVOXELIZER_SOURCE_FILES}) -target_link_libraries(libvoxel ${IFC_LIBRARIES} ${ICU_LIBRARIES} ${Boost_LIBRARIES} ${BCRYPT_LIBRARIES} ${OCC_LIBRARIES} ${LIB_RT} ${dl} ${CMAKE_THREAD_LIBS_INIT} ${WS2_LIBRARIES}) +target_link_libraries(libvoxel ${IFC_LIBRARIES} ${ICU_LIBRARIES} ${Boost_LIBRARIES} ${BCRYPT_LIBRARIES} ${OCC_LIBRARIES} ${LIB_RT} ${dl} ${CMAKE_THREAD_LIBS_INIT} ${WS2_LIBRARIES} ${HDF5_LIBRARIES}) set(LIBVOXEC_HEADER_FILES voxelfile.h voxec.h) set(LIBVOXEC_SOURCE_FILES voxec.cpp) diff --git a/tests/test_hdf.cpp b/tests/test_hdf.cpp new file mode 100644 index 0000000..ffe96b9 --- /dev/null +++ b/tests/test_hdf.cpp @@ -0,0 +1,39 @@ +#include "../voxelizer.h" +#include "../writer.h" +#include "H5Cpp.h" +#include + +#include +#include + +TEST(HdfFileName, HDF) { + + const H5std_string FILE_NAME("SDS.h5"); + const H5std_string DATASET_NAME("IntArray"); + const int NX = 5; + const int NY = 6; + const int RANK = 2; + + int i, j; + int data[NX][NY]; + for (j = 0; j < NX; j++) + { + for (i = 0; i < NY; i++) + data[j][i] = i + j; + } + + H5::H5File file(FILE_NAME, H5F_ACC_TRUNC); + hsize_t dimsf[2]; + dimsf[0] = NX; + dimsf[1] = NY; + H5::DataSpace dataspace(RANK, dimsf); + H5::IntType datatype(H5::PredType::NATIVE_INT); + datatype.setOrder(H5T_ORDER_LE); + + H5::DataSet dataset = file.createDataSet(DATASET_NAME, datatype, dataspace); + dataset.write(data, H5::PredType::NATIVE_INT); + + std::string file_name = file.getFileName(); + EXPECT_TRUE(file_name == FILE_NAME); + +} From daf138f5064e5faacd5cfe05806b242301dbc89a Mon Sep 17 00:00:00 2001 From: johltn Date: Mon, 12 Apr 2021 08:31:20 +0200 Subject: [PATCH 003/112] Abstract writer class with Hdf writer subclass --- tests/test_hdf.cpp | 23 ++++++++++++++++++--- writer.h | 51 +++++++++++++++++++++++++++++++++++++++------- 2 files changed, 64 insertions(+), 10 deletions(-) diff --git a/tests/test_hdf.cpp b/tests/test_hdf.cpp index ffe96b9..54b9927 100644 --- a/tests/test_hdf.cpp +++ b/tests/test_hdf.cpp @@ -8,14 +8,31 @@ TEST(HdfFileName, HDF) { + auto storage = new chunked_voxel_storage(0., 0., 0., 0.1, 200, 150, 10, 32); + + + BRepBuilderAPI_MakePolygon mp(gp_Pnt(1, 1, 0), gp_Pnt(16, 1, 0), gp_Pnt(16, 9.8, 0), gp_Pnt(1, 9.8, 0), true); + BRepBuilderAPI_MakeFace mf(mp.Wire()); + TopoDS_Face face = mf.Face(); + + auto vox = voxelizer(face, storage); + vox.Convert(); + + /*HDF WRITER*/ + hdf_writer writer; + writer.SetVoxels(storage); + writer.Write("test_hdf_writer.vox"); + /*HDF WRITER*/ + + const H5std_string FILE_NAME("SDS.h5"); const H5std_string DATASET_NAME("IntArray"); - const int NX = 5; + const int NX = 5; const int NY = 6; const int RANK = 2; int i, j; - int data[NX][NY]; + int data[NX][NY]; for (j = 0; j < NX; j++) { for (i = 0; i < NY; i++) @@ -23,7 +40,7 @@ TEST(HdfFileName, HDF) { } H5::H5File file(FILE_NAME, H5F_ACC_TRUNC); - hsize_t dimsf[2]; + hsize_t dimsf[2]; dimsf[0] = NX; dimsf[1] = NY; H5::DataSpace dataspace(RANK, dimsf); diff --git a/writer.h b/writer.h index 2ea3842..a082fb7 100644 --- a/writer.h +++ b/writer.h @@ -2,13 +2,27 @@ #define WRITER_H #include "storage.h" +#include "H5Cpp.h" #include #include -class voxel_writer { -private: + +class abstract_writer { + +public: abstract_voxel_storage* voxels_; + void SetVoxels(abstract_voxel_storage* voxels) { + voxels_ = voxels; + } + + virtual void Write(const std::string& fnc) = 0; + + +}; + +class voxel_writer :public abstract_writer { +private: std::ofstream& assert_good_(const std::string& fn, std::ofstream& fs) { if (!fs.good()) { @@ -16,12 +30,7 @@ class voxel_writer { } return fs; } - public: - void SetVoxels(abstract_voxel_storage* voxels) { - voxels_ = voxels; - } - void Write(const std::string& fnc) { { std::string fn = fnc + std::string(".index"); @@ -46,4 +55,32 @@ class voxel_writer { } }; + + +class hdf_writer :public abstract_writer { + +public: + void Write(const std::string& fnc) { + + chunked_voxel_storage* storage = (chunked_voxel_storage*)voxels_; + + for (int i = 0; i < storage->num_chunks().get(1); i++) { + + auto c = storage->get_chunk(make_vec(i, 1, 0)); + + if (c && c->is_explicit()) { + continuous_voxel_storage* convox = (continuous_voxel_storage*)voxels_; + bit_t::storage_type* data = convox->data(); + + + } + } + } + + +}; + + + + #endif From f559f0c6b5493c0ec58cf159a2ba195637934ee5 Mon Sep 17 00:00:00 2001 From: johltn Date: Mon, 12 Apr 2021 19:34:24 +0200 Subject: [PATCH 004/112] Start iteration through continuous chunk --- writer.h | 27 ++++++++++++++++++++------- 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/writer.h b/writer.h index a082fb7..622e758 100644 --- a/writer.h +++ b/writer.h @@ -64,16 +64,29 @@ class hdf_writer :public abstract_writer { chunked_voxel_storage* storage = (chunked_voxel_storage*)voxels_; - for (int i = 0; i < storage->num_chunks().get(1); i++) { - - auto c = storage->get_chunk(make_vec(i, 1, 0)); + for (int i = 0; i < storage->num_chunks().get(0); i++) { + auto c = storage->get_chunk(make_vec(i, 1, 0)); if (c && c->is_explicit()) { - continuous_voxel_storage* convox = (continuous_voxel_storage*)voxels_; - bit_t::storage_type* data = convox->data(); - - + continuous_voxel_storage* convox = (continuous_voxel_storage*)c; + auto d = convox->data(); + //std::cout << c->value_bits(); + for (int j = 0; j < convox->size(); j++) { + auto vd = convox->data()[j]; + std::bitset<8> b(convox->data()[j]); + std::cout << (int)convox->data()[j]<is_constant() ){ + std::cout << "Constant handling to implement."< Date: Mon, 12 Apr 2021 22:20:49 +0200 Subject: [PATCH 005/112] Read bits stream and start with HDF5 --- tests/test_hdf.cpp | 16 ++++++++------- writer.h | 49 +++++++++++++++++++++++++++++++++++++--------- 2 files changed, 49 insertions(+), 16 deletions(-) diff --git a/tests/test_hdf.cpp b/tests/test_hdf.cpp index 54b9927..bed2e5c 100644 --- a/tests/test_hdf.cpp +++ b/tests/test_hdf.cpp @@ -5,18 +5,20 @@ #include #include +#include TEST(HdfFileName, HDF) { - auto storage = new chunked_voxel_storage(0., 0., 0., 0.1, 200, 150, 10, 32); + auto storage = new chunked_voxel_storage(0., 0., 0., 0.1, 1000, 1000, 100, 32); + for (int i = 0; i < 10; ++i) { + for (int j = 0; j < 10; ++j) { + BRepPrimAPI_MakeBox mb(gp_Pnt(10 * i, 10 * j, 0.), 8., 8., 8.); - BRepBuilderAPI_MakePolygon mp(gp_Pnt(1, 1, 0), gp_Pnt(16, 1, 0), gp_Pnt(16, 9.8, 0), gp_Pnt(1, 9.8, 0), true); - BRepBuilderAPI_MakeFace mf(mp.Wire()); - TopoDS_Face face = mf.Face(); - - auto vox = voxelizer(face, storage); - vox.Convert(); + auto vox = voxelizer(mb.Solid(), storage); + vox.Convert(); + } + } /*HDF WRITER*/ hdf_writer writer; diff --git a/writer.h b/writer.h index 622e758..59c48e4 100644 --- a/writer.h +++ b/writer.h @@ -4,6 +4,7 @@ #include "storage.h" #include "H5Cpp.h" +#include #include #include @@ -61,33 +62,63 @@ class hdf_writer :public abstract_writer { public: void Write(const std::string& fnc) { + const int RANK = 2; + const H5std_string FILE_NAME(fnc); + const H5std_string DATASET_NAME("Continuous"); + + H5::H5File file(FILE_NAME, H5F_ACC_TRUNC); + + hsize_t dims[2] = { 3, 3 }; // dataset dimensions at creation + hsize_t maxdims[2] = { H5S_UNLIMITED, H5S_UNLIMITED }; + H5::DataSpace mspace1(RANK, dims, maxdims); + + H5::DSetCreatPropList cparms; + hsize_t chunk_dims[2] = { 2, 5 }; + cparms.setChunk(RANK, chunk_dims); + + H5::DataSet dataset = file.createDataSet(DATASET_NAME, H5::PredType::NATIVE_INT, mspace1, cparms); chunked_voxel_storage* storage = (chunked_voxel_storage*)voxels_; for (int i = 0; i < storage->num_chunks().get(0); i++) { - auto c = storage->get_chunk(make_vec(i, 1, 0)); + auto c = storage->get_chunk(make_vec(i, 0, 0)); if (c && c->is_explicit()) { continuous_voxel_storage* convox = (continuous_voxel_storage*)c; auto d = convox->data(); - //std::cout << c->value_bits(); + for (int j = 0; j < convox->size(); j++) { auto vd = convox->data()[j]; - std::bitset<8> b(convox->data()[j]); - std::cout << (int)convox->data()[j]< b(convox->data()[j]); + + /*std::cout << b<data()[j] << std::endl; + for (int k = 0; k < 8; k++) { + int bitmask = 1 << k; + int masked = vd & bitmask; + int voxbit = masked >> vd; + std::cout << voxbit; + } + } - if (c && c->is_constant() ){ - std::cout << "Constant handling to implement."<is_constant()) { + std::cout << "Constant handling to implement." << std::endl; + } else { - std::cout << "Plane handling to implement."< Date: Mon, 26 Apr 2021 09:20:55 +0200 Subject: [PATCH 006/112] Store bit values into HDF file --- tests/test_hdf.cpp | 67 +++++++++---------------- writer.h | 120 ++++++++++++++++++++++++++++++++++----------- 2 files changed, 114 insertions(+), 73 deletions(-) diff --git a/tests/test_hdf.cpp b/tests/test_hdf.cpp index bed2e5c..3109e76 100644 --- a/tests/test_hdf.cpp +++ b/tests/test_hdf.cpp @@ -6,53 +6,30 @@ #include #include #include +#include +#include +#include +#include + + TEST(HdfFileName, HDF) { - auto storage = new chunked_voxel_storage(0., 0., 0., 0.1, 1000, 1000, 100, 32); - - for (int i = 0; i < 10; ++i) { - for (int j = 0; j < 10; ++j) { - BRepPrimAPI_MakeBox mb(gp_Pnt(10 * i, 10 * j, 0.), 8., 8., 8.); - - auto vox = voxelizer(mb.Solid(), storage); - vox.Convert(); - } - } - - /*HDF WRITER*/ - hdf_writer writer; - writer.SetVoxels(storage); - writer.Write("test_hdf_writer.vox"); - /*HDF WRITER*/ - - - const H5std_string FILE_NAME("SDS.h5"); - const H5std_string DATASET_NAME("IntArray"); - const int NX = 5; - const int NY = 6; - const int RANK = 2; - - int i, j; - int data[NX][NY]; - for (j = 0; j < NX; j++) - { - for (i = 0; i < NY; i++) - data[j][i] = i + j; - } - - H5::H5File file(FILE_NAME, H5F_ACC_TRUNC); - hsize_t dimsf[2]; - dimsf[0] = NX; - dimsf[1] = NY; - H5::DataSpace dataspace(RANK, dimsf); - H5::IntType datatype(H5::PredType::NATIVE_INT); - datatype.setOrder(H5T_ORDER_LE); - - H5::DataSet dataset = file.createDataSet(DATASET_NAME, datatype, dataspace); - dataset.write(data, H5::PredType::NATIVE_INT); - - std::string file_name = file.getFileName(); - EXPECT_TRUE(file_name == FILE_NAME); + auto storage2 = new chunked_voxel_storage(0., 0., 0., 1, 32, 32, 32, 32); + + BRepPrimAPI_MakeBox mb(gp_Pnt(1, 1, 1), gp_Pnt(10, 10, 10)); + auto x = mb.Solid(); + BRepMesh_IncrementalMesh(x, 0.001); + auto vox = voxelizer(x, storage2); + vox.Convert(); + + hdf_writer writer; + writer.SetVoxels(storage2); + writer.Write("voxels.h5"); + + std::ofstream fs("voxobj.obj"); + obj_export_helper oeh{ fs }; + storage2->obj_export(oeh, false, false); + } diff --git a/writer.h b/writer.h index 59c48e4..51ee20c 100644 --- a/writer.h +++ b/writer.h @@ -31,6 +31,7 @@ class voxel_writer :public abstract_writer { } return fs; } + public: void Write(const std::string& fnc) { { @@ -62,64 +63,127 @@ class hdf_writer :public abstract_writer { public: void Write(const std::string& fnc) { - const int RANK = 2; - const H5std_string FILE_NAME(fnc); - const H5std_string DATASET_NAME("Continuous"); - H5::H5File file(FILE_NAME, H5F_ACC_TRUNC); + chunked_voxel_storage* storage = (chunked_voxel_storage*)voxels_; + + int continuous_count = 0; + int planar_count = 0; + int constant_count = 0; + + for (int i = 0; i < storage->num_chunks().get(0); i++) { + auto chunk = storage->get_chunk(make_vec(i, 0, 0)); + + if (chunk && chunk->is_explicit()) { + continuous_count++; + + } + else { + if (chunk && chunk->is_constant()) { + constant_count++; + } + + else { + planar_count++; + } + } + } + //std::cout << continuous_count << std::endl; + //std::cout << planar_count << std::endl; + //std::cout << constant_count << std::endl; + + int chunk_stream = storage->chunk_size() * storage->voxel_size(); + + const H5std_string FILE_NAME(fnc); + const H5std_string DATASET_NAME("continuous_chunks"); + const int NX = 10; + const int NY = 5; + const int RANK = 2; hsize_t dims[2] = { 3, 3 }; // dataset dimensions at creation hsize_t maxdims[2] = { H5S_UNLIMITED, H5S_UNLIMITED }; H5::DataSpace mspace1(RANK, dims, maxdims); + + H5::H5File file(FILE_NAME, H5F_ACC_TRUNC); H5::DSetCreatPropList cparms; hsize_t chunk_dims[2] = { 2, 5 }; cparms.setChunk(RANK, chunk_dims); - + + int fill_val = 0; + cparms.setFillValue(H5::PredType::NATIVE_INT, &fill_val); + H5::DataSet dataset = file.createDataSet(DATASET_NAME, H5::PredType::NATIVE_INT, mspace1, cparms); - - chunked_voxel_storage* storage = (chunked_voxel_storage*)voxels_; + + hsize_t size[2]; + size[0] = continuous_count; + size[1] = 32 * 32 * 32; + dataset.extend(size); + + H5::DataSpace fspace1 = dataset.getSpace(); + hsize_t offset[2]; + offset[0] = 1; + offset[1] = 0; + hsize_t dims1[2] = { 3, 3 }; /* data1 dimensions */ + fspace1.selectHyperslab(H5S_SELECT_SET, dims1, offset); + + H5::DataSpace fspace2 = dataset.getSpace(); + offset[0] = 1; + offset[1] = 0; + hsize_t dims2[2] = { 1, 32*32*32 }; + + fspace2.selectHyperslab(H5S_SELECT_SET, dims2, offset); + + H5::DataSpace mspace2(RANK, dims2); + + int cont_count = 0; for (int i = 0; i < storage->num_chunks().get(0); i++) { + + std::vector bits_container; auto c = storage->get_chunk(make_vec(i, 0, 0)); if (c && c->is_explicit()) { + + offset[0] = cont_count; + cont_count++; + offset[1] = 0; + fspace2.selectHyperslab(H5S_SELECT_SET, dims2, offset); + continuous_voxel_storage* convox = (continuous_voxel_storage*)c; auto d = convox->data(); + //std::cout << convox->size(); for (int j = 0; j < convox->size(); j++) { auto vd = convox->data()[j]; + // consider using boost::dynamic_bitset std::bitset<8> b(convox->data()[j]); + + for (int k = 0; k < 8; k++) { + bits_container.push_back(b[k]); + + } + } - /*std::cout << b<is_constant()) { + std::cout << "Constant handling to implement." << std::endl; - std::cout << (int)convox->data()[j] << std::endl; - for (int k = 0; k < 8; k++) { - int bitmask = 1 << k; - int masked = vd & bitmask; - int voxbit = masked >> vd; - std::cout << voxbit; - } - - } + } + else { + std::cout << "Plane handling to implement." << std::endl; - if (c && c->is_constant()) { - std::cout << "Constant handling to implement." << std::endl; + } - } - else { - std::cout << "Plane handling to implement." << std::endl; - } } } - std:cout << std::endl; - } + + + + }; From dbe4e0023458ad5edc740861a93f726f7c042a22 Mon Sep 17 00:00:00 2001 From: johltn Date: Wed, 28 Apr 2021 16:29:53 +0200 Subject: [PATCH 007/112] Hyperslab writing on fixed dataset dimensions and add test case --- tests/test_hdf.cpp | 14 +++++++- writer.h | 82 +++++++++++++++++++--------------------------- 2 files changed, 47 insertions(+), 49 deletions(-) diff --git a/tests/test_hdf.cpp b/tests/test_hdf.cpp index 3109e76..38591c3 100644 --- a/tests/test_hdf.cpp +++ b/tests/test_hdf.cpp @@ -23,9 +23,21 @@ TEST(HdfFileName, HDF) { auto vox = voxelizer(x, storage2); vox.Convert(); + + auto storage = new chunked_voxel_storage(0., 0., 0., 0.1, 200, 150, 10, 32); + + { + BRepBuilderAPI_MakePolygon mp(gp_Pnt(1, 1, 0), gp_Pnt(16, 1, 0), gp_Pnt(16, 9.8, 0), gp_Pnt(1, 9.8, 0), true); + BRepBuilderAPI_MakeFace mf(mp.Wire()); + TopoDS_Face face = mf.Face(); + + auto vox = voxelizer(face, storage); + vox.Convert(); + } + hdf_writer writer; writer.SetVoxels(storage2); - writer.Write("voxels.h5"); + writer.Write("voxels2.h5"); std::ofstream fs("voxobj.obj"); obj_export_helper oeh{ fs }; diff --git a/writer.h b/writer.h index 51ee20c..2f3ef33 100644 --- a/writer.h +++ b/writer.h @@ -66,14 +66,21 @@ class hdf_writer :public abstract_writer { chunked_voxel_storage* storage = (chunked_voxel_storage*)voxels_; - int continuous_count = 0; - int planar_count = 0; - int constant_count = 0; + int continuous_count = 0; + int planar_count = 0; + int constant_count = 0; + + int col_n = 0; for (int i = 0; i < storage->num_chunks().get(0); i++) { auto chunk = storage->get_chunk(make_vec(i, 0, 0)); - + if (chunk && chunk->is_explicit()) { + if (col_n == 0) { + continuous_voxel_storage* casted = (continuous_voxel_storage*)chunk; + col_n = casted->size() * casted->value_bits() * 8; + } + continuous_count++; } @@ -88,52 +95,35 @@ class hdf_writer :public abstract_writer { } } - //std::cout << continuous_count << std::endl; - //std::cout << planar_count << std::endl; - //std::cout << constant_count << std::endl; - - int chunk_stream = storage->chunk_size() * storage->voxel_size(); + + int chunk_stream = storage->chunk_size() * storage->voxel_size(); + + std::vector bits_container; const H5std_string FILE_NAME(fnc); const H5std_string DATASET_NAME("continuous_chunks"); - const int NX = 10; - const int NY = 5; + const int NX = continuous_count; + const int NY = col_n; const int RANK = 2; - hsize_t dims[2] = { 3, 3 }; // dataset dimensions at creation - hsize_t maxdims[2] = { H5S_UNLIMITED, H5S_UNLIMITED }; - H5::DataSpace mspace1(RANK, dims, maxdims); - H5::H5File file(FILE_NAME, H5F_ACC_TRUNC); - H5::DSetCreatPropList cparms; - hsize_t chunk_dims[2] = { 2, 5 }; - cparms.setChunk(RANK, chunk_dims); - - int fill_val = 0; - cparms.setFillValue(H5::PredType::NATIVE_INT, &fill_val); - - H5::DataSet dataset = file.createDataSet(DATASET_NAME, H5::PredType::NATIVE_INT, mspace1, cparms); - - hsize_t size[2]; - size[0] = continuous_count; - size[1] = 32 * 32 * 32; - dataset.extend(size); - - H5::DataSpace fspace1 = dataset.getSpace(); - hsize_t offset[2]; - offset[0] = 1; - offset[1] = 0; - hsize_t dims1[2] = { 3, 3 }; /* data1 dimensions */ - fspace1.selectHyperslab(H5S_SELECT_SET, dims1, offset); + hsize_t dimsf[2]; + dimsf[0] = NX; + dimsf[1] = NY; + H5::DataSpace dataspace(RANK, dimsf); + H5::IntType datatype(H5::PredType::NATIVE_INT); + datatype.setOrder(H5T_ORDER_LE); + H5::DataSet dataset = file.createDataSet(DATASET_NAME, datatype, dataspace); - H5::DataSpace fspace2 = dataset.getSpace(); - offset[0] = 1; + int column_number = col_n; + hsize_t offset[2]; + offset[0] = 0; offset[1] = 0; - hsize_t dims2[2] = { 1, 32*32*32 }; - fspace2.selectHyperslab(H5S_SELECT_SET, dims2, offset); + hsize_t slab_dimsf[2] = { 1, col_n }; + dataspace.selectHyperslab(H5S_SELECT_SET, slab_dimsf, offset); + H5::DataSpace mspace2(RANK, slab_dimsf); - H5::DataSpace mspace2(RANK, dims2); int cont_count = 0; @@ -147,12 +137,11 @@ class hdf_writer :public abstract_writer { offset[0] = cont_count; cont_count++; offset[1] = 0; - fspace2.selectHyperslab(H5S_SELECT_SET, dims2, offset); + dataspace.selectHyperslab(H5S_SELECT_SET, slab_dimsf, offset); continuous_voxel_storage* convox = (continuous_voxel_storage*)c; auto d = convox->data(); - //std::cout << convox->size(); - + for (int j = 0; j < convox->size(); j++) { auto vd = convox->data()[j]; // consider using boost::dynamic_bitset @@ -164,8 +153,7 @@ class hdf_writer :public abstract_writer { } } - - dataset.write(bits_container.data(), H5::PredType::NATIVE_INT, mspace2, fspace2); + dataset.write(bits_container.data(), H5::PredType::NATIVE_INT, mspace2, dataspace); } if (c && c->is_constant()) { @@ -182,12 +170,10 @@ class hdf_writer :public abstract_writer { } - + }; -}; - From 60d8a6e65b6c2e0d42e93c898d1823f90aa6a900 Mon Sep 17 00:00:00 2001 From: johltn Date: Wed, 28 Apr 2021 21:27:07 +0200 Subject: [PATCH 008/112] Write multidimensional dataset for continuous chunks --- tests/test_hdf.cpp | 43 +++++++++++++++++++++++++++++++++++++------ writer.h | 45 ++++++++++++++++++++++++++++++--------------- 2 files changed, 67 insertions(+), 21 deletions(-) diff --git a/tests/test_hdf.cpp b/tests/test_hdf.cpp index 38591c3..cbc6aef 100644 --- a/tests/test_hdf.cpp +++ b/tests/test_hdf.cpp @@ -23,7 +23,6 @@ TEST(HdfFileName, HDF) { auto vox = voxelizer(x, storage2); vox.Convert(); - auto storage = new chunked_voxel_storage(0., 0., 0., 0.1, 200, 150, 10, 32); { @@ -36,12 +35,44 @@ TEST(HdfFileName, HDF) { } hdf_writer writer; - writer.SetVoxels(storage2); - writer.Write("voxels2.h5"); + writer.SetVoxels(storage); + writer.Write("voxels.h5"); + + //std::ofstream fs("voxobj.obj"); + //obj_export_helper oeh{ fs }; + //storage2->obj_export(oeh, false, false); + + // Write a 4D dataset + + const H5std_string FILE_NAME("multidim.h5"); + const H5std_string DATASET_NAME("continuous_chunks"); + const int NX = 32; // dataset dimensions + const int NY = 32; + const int NZ = 32; + const int NC = 3; + + const int RANK = 4; + H5::H5File file(FILE_NAME, H5F_ACC_TRUNC); - std::ofstream fs("voxobj.obj"); - obj_export_helper oeh{ fs }; - storage2->obj_export(oeh, false, false); + hsize_t dimsf[4]; // dataset dimensions + dimsf[0] = NC; + dimsf[1] = NX; + dimsf[2] = NY; + dimsf[3] = NZ; + + H5::DataSpace dataspace(RANK, dimsf); + + H5::IntType datatype(H5::PredType::NATIVE_INT); + datatype.setOrder(H5T_ORDER_LE); + + H5::DataSet dataset = file.createDataSet(DATASET_NAME, datatype, dataspace); + + std::vector data; + + for (int i = 0; i < NX*NY*NZ*NC; i++) { + data.push_back(0); + } + dataset.write(data.data(), H5::PredType::NATIVE_INT); } diff --git a/writer.h b/writer.h index 2f3ef33..12e9552 100644 --- a/writer.h +++ b/writer.h @@ -66,12 +66,19 @@ class hdf_writer :public abstract_writer { chunked_voxel_storage* storage = (chunked_voxel_storage*)voxels_; + std::cout<num_chunks().get(0)<num_chunks().get(1) << std::endl; + std::cout << storage->num_chunks().get(2) << std::endl; + + std::cout << storage->chunk_size() << std::endl; + int continuous_count = 0; int planar_count = 0; int constant_count = 0; int col_n = 0; + for (int i = 0; i < storage->num_chunks().get(0); i++) { auto chunk = storage->get_chunk(make_vec(i, 0, 0)); @@ -95,30 +102,38 @@ class hdf_writer :public abstract_writer { } } - int chunk_stream = storage->chunk_size() * storage->voxel_size(); std::vector bits_container; const H5std_string FILE_NAME(fnc); const H5std_string DATASET_NAME("continuous_chunks"); - const int NX = continuous_count; - const int NY = col_n; - const int RANK = 2; + + const int NC = continuous_count; + const int NX = storage->num_chunks().get(0); + const int NY = storage->num_chunks().get(1); + const int NZ = storage->num_chunks().get(2); + + const int RANK = 4; H5::H5File file(FILE_NAME, H5F_ACC_TRUNC); - hsize_t dimsf[2]; - dimsf[0] = NX; - dimsf[1] = NY; + hsize_t dimsf[4]; + dimsf[0] = NC; + dimsf[1] = NX; + dimsf[2] = NY; + dimsf[3] = NZ; + H5::DataSpace dataspace(RANK, dimsf); H5::IntType datatype(H5::PredType::NATIVE_INT); datatype.setOrder(H5T_ORDER_LE); H5::DataSet dataset = file.createDataSet(DATASET_NAME, datatype, dataspace); int column_number = col_n; - hsize_t offset[2]; + hsize_t offset[4]; offset[0] = 0; offset[1] = 0; + offset[2] = 0; + offset[3] = 0; hsize_t slab_dimsf[2] = { 1, col_n }; dataspace.selectHyperslab(H5S_SELECT_SET, slab_dimsf, offset); @@ -134,21 +149,21 @@ class hdf_writer :public abstract_writer { if (c && c->is_explicit()) { - offset[0] = cont_count; + /*offset[0] = cont_count; cont_count++; offset[1] = 0; - dataspace.selectHyperslab(H5S_SELECT_SET, slab_dimsf, offset); + dataspace.selectHyperslab(H5S_SELECT_SET, slab_dimsf, offset);*/ continuous_voxel_storage* convox = (continuous_voxel_storage*)c; auto d = convox->data(); - for (int j = 0; j < convox->size(); j++) { - auto vd = convox->data()[j]; + for (int v = 0; v < convox->size(); v++) { + auto vd = convox->data()[v]; // consider using boost::dynamic_bitset - std::bitset<8> b(convox->data()[j]); + std::bitset<8> b(convox->data()[v]); - for (int k = 0; k < 8; k++) { - bits_container.push_back(b[k]); + for (int l = 0; l< 8; l++) { + bits_container.push_back(b[l]); } } From a1ba1c1f8a27be6fbab9dc0129550409e2eddbaa Mon Sep 17 00:00:00 2001 From: johltn Date: Sat, 1 May 2021 11:37:22 +0200 Subject: [PATCH 009/112] Use extents to write multidimensional HDF --- writer.h | 119 +++++++++++++++++++++++++------------------------------ 1 file changed, 55 insertions(+), 64 deletions(-) diff --git a/writer.h b/writer.h index 12e9552..bd07fea 100644 --- a/writer.h +++ b/writer.h @@ -9,6 +9,27 @@ #include +#define BEGIN_LOOP(i0, i1, j0, j1, k0, k1) {\ + vec_n<3, typename std::remove_reference::type> ijk;\ + for (ijk.get(0) = i0; ijk.get(0) < i1; ++ijk.get(0)) {\ + for (ijk.get(1) = j0; ijk.get(1) < j1; ++ijk.get(1)) {\ + for (ijk.get(2) = k0; ijk.get(2) < k1; ++ijk.get(2)) { + +#define BEGIN_LOOP_I(i0, i1, j0, j1, k0, k1) {\ + vec_n<3, typename std::remove_reference::type> ijk;\ + for (ijk.get(0) = i0; ijk.get(0) <= i1; ++ijk.get(0)) {\ + for (ijk.get(1) = j0; ijk.get(1) <= j1; ++ijk.get(1)) {\ + for (ijk.get(2) = k0; ijk.get(2) <= k1; ++ijk.get(2)) { + +#define BEGIN_LOOP2(v0, v1) BEGIN_LOOP(v0.template get<0>(), v1.template get<0>(), v0.template get<1>(), v1.template get<1>(), v0.template get<2>(), v1.template get<2>()) + +#define BEGIN_LOOP_I2(v0, v1) BEGIN_LOOP_I(v0.template get<0>(), v1.template get<0>(), v0.template get<1>(), v1.template get<1>(), v0.template get<2>(), v1.template get<2>()) + +#define BEGIN_LOOP_ZERO_2(v1) BEGIN_LOOP2(make_vec((decltype(v1)::element_type)0, (decltype(v1)::element_type)0, (decltype(v1)::element_type)0), v1) + +#define END_LOOP }}}} + + class abstract_writer { public: @@ -58,7 +79,6 @@ class voxel_writer :public abstract_writer { }; - class hdf_writer :public abstract_writer { public: @@ -66,28 +86,21 @@ class hdf_writer :public abstract_writer { chunked_voxel_storage* storage = (chunked_voxel_storage*)voxels_; - std::cout<num_chunks().get(0)<num_chunks().get(1) << std::endl; - std::cout << storage->num_chunks().get(2) << std::endl; - - std::cout << storage->chunk_size() << std::endl; - int continuous_count = 0; int planar_count = 0; int constant_count = 0; int col_n = 0; - for (int i = 0; i < storage->num_chunks().get(0); i++) { auto chunk = storage->get_chunk(make_vec(i, 0, 0)); - + if (chunk && chunk->is_explicit()) { if (col_n == 0) { continuous_voxel_storage* casted = (continuous_voxel_storage*)chunk; col_n = casted->size() * casted->value_bits() * 8; } - + continuous_count++; } @@ -102,17 +115,14 @@ class hdf_writer :public abstract_writer { } } - int chunk_stream = storage->chunk_size() * storage->voxel_size(); - - std::vector bits_container; const H5std_string FILE_NAME(fnc); const H5std_string DATASET_NAME("continuous_chunks"); - + const int NC = continuous_count; - const int NX = storage->num_chunks().get(0); - const int NY = storage->num_chunks().get(1); - const int NZ = storage->num_chunks().get(2); + const int NX = storage->chunk_size(); + const int NY = storage->chunk_size(); + const int NZ = storage->chunk_size(); const int RANK = 4; H5::H5File file(FILE_NAME, H5F_ACC_TRUNC); @@ -128,65 +138,46 @@ class hdf_writer :public abstract_writer { datatype.setOrder(H5T_ORDER_LE); H5::DataSet dataset = file.createDataSet(DATASET_NAME, datatype, dataspace); - int column_number = col_n; - hsize_t offset[4]; - offset[0] = 0; - offset[1] = 0; - offset[2] = 0; - offset[3] = 0; - - hsize_t slab_dimsf[2] = { 1, col_n }; - dataspace.selectHyperslab(H5S_SELECT_SET, slab_dimsf, offset); - H5::DataSpace mspace2(RANK, slab_dimsf); - - int cont_count = 0; - for (int i = 0; i < storage->num_chunks().get(0); i++) { + std::vector bits_container; - std::vector bits_container; - auto c = storage->get_chunk(make_vec(i, 0, 0)); - - if (c && c->is_explicit()) { - - /*offset[0] = cont_count; - cont_count++; - offset[1] = 0; - dataspace.selectHyperslab(H5S_SELECT_SET, slab_dimsf, offset);*/ - - continuous_voxel_storage* convox = (continuous_voxel_storage*)c; - auto d = convox->data(); - - for (int v = 0; v < convox->size(); v++) { - auto vd = convox->data()[v]; - // consider using boost::dynamic_bitset - std::bitset<8> b(convox->data()[v]); - - for (int l = 0; l< 8; l++) { - bits_container.push_back(b[l]); - + for (int i = 0; i < storage->num_chunks().get(0); i++) { + for (int j = 0; j < storage->num_chunks().get(1); j++) { + for (int k = 0; k < storage->num_chunks().get(2); k++) { + auto c = storage->get_chunk(make_vec(i, j, k)); + if (c && c->is_explicit()) { + + continuous_voxel_storage* convox = (continuous_voxel_storage*)c; + size_t i0, j0, k0, i1, j1, k1; + i0 = 0; + j0 = 0; + k0 = 0; + convox->extents().tie(i1, j1, k1); + + BEGIN_LOOP_ZERO_2(make_vec(i1, j1, k1)) + bits_container.push_back(convox->Get(ijk)); + END_LOOP; } - } - dataset.write(bits_container.data(), H5::PredType::NATIVE_INT, mspace2, dataspace); - } - - if (c && c->is_constant()) { - std::cout << "Constant handling to implement." << std::endl; + if (c && c->is_constant()) { + std::cout << "Constant handling to implement." << std::endl; + } + else { + std::cout << "Plane handling to implement." << std::endl; + } + } } - else { - std::cout << "Plane handling to implement." << std::endl; + } - } + dataset.write(bits_container.data(), H5::PredType::NATIVE_INT); + } - } - } +}; - }; - From 4e8868a385af1aecb5cf8bdb41bfb8b35df4dd1f Mon Sep 17 00:00:00 2001 From: johltn Date: Tue, 4 May 2021 09:15:52 +0200 Subject: [PATCH 010/112] Handle constant null storage chunks --- writer.h | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/writer.h b/writer.h index bd07fea..2e078ce 100644 --- a/writer.h +++ b/writer.h @@ -160,12 +160,14 @@ class hdf_writer :public abstract_writer { END_LOOP; } - if (c && c->is_constant()) { + if (c && c->is_constant() || c==nullptr ){ std::cout << "Constant handling to implement." << std::endl; } else { std::cout << "Plane handling to implement." << std::endl; - + planar_voxel_storage* planvox = (planar_voxel_storage*)c; + continuous_voxel_storage* continuoused = planvox->make_explicit(); + } } } From d27df8846b487f809071ef06bfabf068bba2c5f1 Mon Sep 17 00:00:00 2001 From: johltn Date: Sat, 15 May 2021 21:51:09 +0200 Subject: [PATCH 011/112] Better control flow --- tests/test_hdf.cpp | 57 +++++++++++++++++------ writer.h | 111 ++++++++++++++++++++++++++++++++------------- 2 files changed, 123 insertions(+), 45 deletions(-) diff --git a/tests/test_hdf.cpp b/tests/test_hdf.cpp index cbc6aef..92f7e8a 100644 --- a/tests/test_hdf.cpp +++ b/tests/test_hdf.cpp @@ -1,24 +1,51 @@ #include "../voxelizer.h" #include "../writer.h" +#include "../processor.h" + #include "H5Cpp.h" #include #include #include #include +#include #include #include #include #include +#include + +#ifdef WITH_IFC +#include +#ifdef IFCOPENSHELL_05 +#include +using namespace Ifc2x3; +#else +#include +#endif +#endif + +#ifdef WIN32 +#define DIRSEP "\\" +#else +#define DIRSEP "/" +#endif TEST(HdfFileName, HDF) { - auto storage2 = new chunked_voxel_storage(0., 0., 0., 1, 32, 32, 32, 32); - BRepPrimAPI_MakeBox mb(gp_Pnt(1, 1, 1), gp_Pnt(10, 10, 10)); - auto x = mb.Solid(); + BRepPrimAPI_MakeBox mb(gp_Pnt(0, 0, 0), gp_Pnt(10, 10, 10)); + BRepPrimAPI_MakeSphere s(gp_Pnt(10, 10, 10), 7); + + auto x = s.Solid(); + + auto storage2 = new chunked_voxel_storage( 0, 0, 0, 1, 32, 32, 32, 32); + BRepTools breptools; + + breptools.Write(x, "sphere.brep"); + BRepMesh_IncrementalMesh(x, 0.001); auto vox = voxelizer(x, storage2); vox.Convert(); @@ -26,7 +53,7 @@ TEST(HdfFileName, HDF) { auto storage = new chunked_voxel_storage(0., 0., 0., 0.1, 200, 150, 10, 32); { - BRepBuilderAPI_MakePolygon mp(gp_Pnt(1, 1, 0), gp_Pnt(16, 1, 0), gp_Pnt(16, 9.8, 0), gp_Pnt(1, 9.8, 0), true); + BRepBuilderAPI_MakePolygon mp(gp_Pnt(1, 1, 2), gp_Pnt(16, 1, 2), gp_Pnt(16, 9.8, 2), gp_Pnt(1, 9.8, 2), true); BRepBuilderAPI_MakeFace mf(mp.Wire()); TopoDS_Face face = mf.Face(); @@ -34,9 +61,11 @@ TEST(HdfFileName, HDF) { vox.Convert(); } - hdf_writer writer; - writer.SetVoxels(storage); - writer.Write("voxels.h5"); + + + hdf_writer writer; + writer.SetVoxels(storage2); + writer.Write("multi_dim_vox.h5"); //std::ofstream fs("voxobj.obj"); //obj_export_helper oeh{ fs }; @@ -49,7 +78,7 @@ TEST(HdfFileName, HDF) { const int NX = 32; // dataset dimensions const int NY = 32; const int NZ = 32; - const int NC = 3; + const int NC = 3; const int RANK = 4; H5::H5File file(FILE_NAME, H5F_ACC_TRUNC); @@ -61,18 +90,18 @@ TEST(HdfFileName, HDF) { dimsf[3] = NZ; H5::DataSpace dataspace(RANK, dimsf); - + H5::IntType datatype(H5::PredType::NATIVE_INT); datatype.setOrder(H5T_ORDER_LE); H5::DataSet dataset = file.createDataSet(DATASET_NAME, datatype, dataspace); - std::vector data; + //std::vector data; - for (int i = 0; i < NX*NY*NZ*NC; i++) { - data.push_back(0); - } + //for (int i = 0; i < NX*NY*NZ*NC; i++) { + // data.push_back(0); + //} - dataset.write(data.data(), H5::PredType::NATIVE_INT); + //dataset.write(data.data(), H5::PredType::NATIVE_INT); } diff --git a/writer.h b/writer.h index 2e078ce..7e7e3bd 100644 --- a/writer.h +++ b/writer.h @@ -93,23 +93,28 @@ class hdf_writer :public abstract_writer { int col_n = 0; for (int i = 0; i < storage->num_chunks().get(0); i++) { - auto chunk = storage->get_chunk(make_vec(i, 0, 0)); - - if (chunk && chunk->is_explicit()) { - if (col_n == 0) { - continuous_voxel_storage* casted = (continuous_voxel_storage*)chunk; - col_n = casted->size() * casted->value_bits() * 8; - } - - continuous_count++; + auto c = storage->get_chunk(make_vec(i, 0, 0)); + if (c == nullptr) { + std::cout << "Null pointer" << std::endl; } + else { - if (chunk && chunk->is_constant()) { - constant_count++; + if (c->is_explicit() || c->is_constant()) { + if (c->is_constant()) { + std::cout << "Constant chunk" << std::endl; + constant_count++; + } + else { + std::cout << "Continuous chunk" << std::endl; + continuous_count++; + + } } else { + + std::cout << "Planar chunk" << std::endl; planar_count++; } } @@ -138,36 +143,80 @@ class hdf_writer :public abstract_writer { datatype.setOrder(H5T_ORDER_LE); H5::DataSet dataset = file.createDataSet(DATASET_NAME, datatype, dataspace); - int cont_count = 0; + + //const H5std_string DATASET_NAME2("planar_chunks"); + + //const int NP = planar_count; + //const int NXX = storage->chunk_size(); + //const int NYY = storage->chunk_size(); + + //const int RANK = 3; + //H5::H5File file(FILE_NAME, H5F_ACC_TRUNC); + + //hsize_t dimsf2[3]; + //dimsf2[0] = NP; + //dimsf2[1] = NXX; + //dimsf2[2] = NYY; + // + //H5::DataSpace dataspace2(RANK, dimsf2); + //H5::IntType datatype2(H5::PredType::NATIVE_INT); + //datatype.setOrder(H5T_ORDER_LE); + //H5::DataSet dataset2 = file.createDataSet(DATASET_NAME2, datatype2, dataspace2); + + + std::vector planar_container; std::vector bits_container; for (int i = 0; i < storage->num_chunks().get(0); i++) { for (int j = 0; j < storage->num_chunks().get(1); j++) { for (int k = 0; k < storage->num_chunks().get(2); k++) { auto c = storage->get_chunk(make_vec(i, j, k)); - if (c && c->is_explicit()) { - - continuous_voxel_storage* convox = (continuous_voxel_storage*)c; - size_t i0, j0, k0, i1, j1, k1; - i0 = 0; - j0 = 0; - k0 = 0; - convox->extents().tie(i1, j1, k1); - - BEGIN_LOOP_ZERO_2(make_vec(i1, j1, k1)) - bits_container.push_back(convox->Get(ijk)); - END_LOOP; + if (c == nullptr) { + std::cout << "Null pointer" << std::endl; } - if (c && c->is_constant() || c==nullptr ){ - std::cout << "Constant handling to implement." << std::endl; - } else { - std::cout << "Plane handling to implement." << std::endl; - planar_voxel_storage* planvox = (planar_voxel_storage*)c; - continuous_voxel_storage* continuoused = planvox->make_explicit(); - + if (c->is_explicit() || c->is_constant()) { + if (c->is_constant()) { + std::cout << "Constant chunk" << std::endl; + } + else { + std::cout << "Continuous chunk" << std::endl; + continuous_voxel_storage* convox = (continuous_voxel_storage*)c; + size_t i0, j0, k0, i1, j1, k1; + i0 = 0; + j0 = 0; + k0 = 0; + convox->extents().tie(i1, j1, k1); + + BEGIN_LOOP_ZERO_2(make_vec(i1, j1, k1)) + bits_container.push_back(convox->Get(ijk)); + END_LOOP; + } + } + + else { + std::cout << "Planar chunk" << std::endl; + planar_voxel_storage* planvox = (planar_voxel_storage*)c; + auto off = planvox->offsets(); + auto axis = planvox->axis(); + + bool make_explicit = 0; + if (make_explicit) { + continuous_voxel_storage* convox = planvox->make_explicit(); + size_t i0, j0, k0, i1, j1, k1; + i0 = 0; + j0 = 0; + k0 = 0; + convox->extents().tie(i1, j1, k1); + + BEGIN_LOOP_ZERO_2(make_vec(i1, j1, k1)) + bits_container.push_back(convox->Get(ijk)); + END_LOOP; + + } + } } } } From 9ff08b993a21266e668787caba1d63eb90fa0cfc Mon Sep 17 00:00:00 2001 From: johltn Date: Sun, 16 May 2021 13:20:41 +0200 Subject: [PATCH 012/112] Count all chunks and make planar explicit --- writer.h | 43 +++++++++++++++++++++++++------------------ 1 file changed, 25 insertions(+), 18 deletions(-) diff --git a/writer.h b/writer.h index 7e7e3bd..cf3005f 100644 --- a/writer.h +++ b/writer.h @@ -93,32 +93,39 @@ class hdf_writer :public abstract_writer { int col_n = 0; for (int i = 0; i < storage->num_chunks().get(0); i++) { - auto c = storage->get_chunk(make_vec(i, 0, 0)); - - if (c == nullptr) { - std::cout << "Null pointer" << std::endl; - } + for (int j = 0; j < storage->num_chunks().get(1); j++) { + for (int k = 0; k < storage->num_chunks().get(2); k++) { + auto c = storage->get_chunk(make_vec(i, j, k)); - else { - if (c->is_explicit() || c->is_constant()) { - if (c->is_constant()) { - std::cout << "Constant chunk" << std::endl; + if (c == nullptr) { + std::cout << "Null pointer" << std::endl; constant_count++; } + else { - std::cout << "Continuous chunk" << std::endl; - continuous_count++; - - } - } + if (c->is_explicit() || c->is_constant()) { + if (c->is_constant()) { + std::cout << "Count Constant chunk" << std::endl; + constant_count++; + } + else { + std::cout << "Count Continuous chunk" << std::endl; + continuous_count++; - else { + } + } - std::cout << "Planar chunk" << std::endl; - planar_count++; + else { + + std::cout << " Count Planar chunk" << std::endl; + planar_count++; + } + } } } } + + const H5std_string FILE_NAME(fnc); @@ -202,7 +209,7 @@ class hdf_writer :public abstract_writer { auto off = planvox->offsets(); auto axis = planvox->axis(); - bool make_explicit = 0; + bool make_explicit = 1; if (make_explicit) { continuous_voxel_storage* convox = planvox->make_explicit(); size_t i0, j0, k0, i1, j1, k1; From e83e84a99d896c80dce58706304a88a27b8b7873 Mon Sep 17 00:00:00 2001 From: johltn Date: Tue, 1 Jun 2021 08:09:39 +0200 Subject: [PATCH 013/112] Write bits incrementally --- writer.h | 76 +++++++++++++++++++++----------------------------------- 1 file changed, 29 insertions(+), 47 deletions(-) diff --git a/writer.h b/writer.h index cf3005f..ba793e4 100644 --- a/writer.h +++ b/writer.h @@ -90,8 +90,6 @@ class hdf_writer :public abstract_writer { int planar_count = 0; int constant_count = 0; - int col_n = 0; - for (int i = 0; i < storage->num_chunks().get(0); i++) { for (int j = 0; j < storage->num_chunks().get(1); j++) { for (int k = 0; k < storage->num_chunks().get(2); k++) { @@ -111,7 +109,6 @@ class hdf_writer :public abstract_writer { else { std::cout << "Count Continuous chunk" << std::endl; continuous_count++; - } } @@ -127,11 +124,13 @@ class hdf_writer :public abstract_writer { - const H5std_string FILE_NAME(fnc); - const H5std_string DATASET_NAME("continuous_chunks"); + const H5std_string CONTINUOUS_DATASET_NAME("continuous_chunks"); + const H5std_string PLANAR_DATASET_NAME("continuous_chunks"); const int NC = continuous_count; + const int NP = planar_count; + const int NX = storage->chunk_size(); const int NY = storage->chunk_size(); const int NZ = storage->chunk_size(); @@ -148,48 +147,42 @@ class hdf_writer :public abstract_writer { H5::DataSpace dataspace(RANK, dimsf); H5::IntType datatype(H5::PredType::NATIVE_INT); datatype.setOrder(H5T_ORDER_LE); - H5::DataSet dataset = file.createDataSet(DATASET_NAME, datatype, dataspace); + H5::DataSet dataset = file.createDataSet(CONTINUOUS_DATASET_NAME, datatype, dataspace); + //Hyperslab prep + hsize_t offset[4]; + offset[0] = -1; + offset[1] = 0; + offset[2] = 0; + offset[3] = 0; - //const H5std_string DATASET_NAME2("planar_chunks"); + hsize_t slab_dimsf[4] = { 1, 1, 1, 1 }; + H5::DataSpace mspace2(RANK, slab_dimsf); - //const int NP = planar_count; - //const int NXX = storage->chunk_size(); - //const int NYY = storage->chunk_size(); - - //const int RANK = 3; - //H5::H5File file(FILE_NAME, H5F_ACC_TRUNC); - - //hsize_t dimsf2[3]; - //dimsf2[0] = NP; - //dimsf2[1] = NXX; - //dimsf2[2] = NYY; - // - //H5::DataSpace dataspace2(RANK, dimsf2); - //H5::IntType datatype2(H5::PredType::NATIVE_INT); - //datatype.setOrder(H5T_ORDER_LE); - //H5::DataSet dataset2 = file.createDataSet(DATASET_NAME2, datatype2, dataspace2); + for (int x = 0; x < storage->num_chunks().get(0); x++) { + for (int y = 0; y < storage->num_chunks().get(1); y++) { + for (int z = 0; z < storage->num_chunks().get(2); z++) { + auto c = storage->get_chunk(make_vec(x, y, z)); - std::vector planar_container; - std::vector bits_container; - for (int i = 0; i < storage->num_chunks().get(0); i++) { - for (int j = 0; j < storage->num_chunks().get(1); j++) { - for (int k = 0; k < storage->num_chunks().get(2); k++) { - auto c = storage->get_chunk(make_vec(i, j, k)); if (c == nullptr) { std::cout << "Null pointer" << std::endl; + } else { if (c->is_explicit() || c->is_constant()) { if (c->is_constant()) { std::cout << "Constant chunk" << std::endl; + } else { std::cout << "Continuous chunk" << std::endl; + + offset[0]++; + continuous_voxel_storage* convox = (continuous_voxel_storage*)c; size_t i0, j0, k0, i1, j1, k1; i0 = 0; @@ -198,7 +191,13 @@ class hdf_writer :public abstract_writer { convox->extents().tie(i1, j1, k1); BEGIN_LOOP_ZERO_2(make_vec(i1, j1, k1)) - bits_container.push_back(convox->Get(ijk)); + offset[1] = ijk.get(0); + offset[2] = ijk.get(1); + offset[3] = ijk.get(2); + dataspace.selectHyperslab(H5S_SELECT_SET, slab_dimsf, offset); + + std::vector hslab = { convox->Get(ijk) }; + dataset.write(hslab.data(), H5::PredType::NATIVE_INT, mspace2, dataspace); END_LOOP; } } @@ -209,28 +208,11 @@ class hdf_writer :public abstract_writer { auto off = planvox->offsets(); auto axis = planvox->axis(); - bool make_explicit = 1; - if (make_explicit) { - continuous_voxel_storage* convox = planvox->make_explicit(); - size_t i0, j0, k0, i1, j1, k1; - i0 = 0; - j0 = 0; - k0 = 0; - convox->extents().tie(i1, j1, k1); - - BEGIN_LOOP_ZERO_2(make_vec(i1, j1, k1)) - bits_container.push_back(convox->Get(ijk)); - END_LOOP; - - } } } } } } - - dataset.write(bits_container.data(), H5::PredType::NATIVE_INT); - } From 35298b1c18035a9c100a8d0963546fe1da7ec404 Mon Sep 17 00:00:00 2001 From: johltn Date: Tue, 1 Jun 2021 20:47:34 +0200 Subject: [PATCH 014/112] Incrementally write dataset regions in Region dataset --- tests/test_hdf.cpp | 8 ++++++-- writer.h | 46 ++++++++++++++++++++++++++++++++++++++++------ 2 files changed, 46 insertions(+), 8 deletions(-) diff --git a/tests/test_hdf.cpp b/tests/test_hdf.cpp index 92f7e8a..7e51745 100644 --- a/tests/test_hdf.cpp +++ b/tests/test_hdf.cpp @@ -61,10 +61,14 @@ TEST(HdfFileName, HDF) { vox.Convert(); } + const double dim = 0.1; + auto storage3 = new chunked_voxel_storage(-dim, -dim, -dim, dim, 100, 100, 100, 16); + BRepPrimAPI_MakeBox make_box(8., 8., 8.); + auto vox3 = voxelizer(make_box.Solid(), storage3); + vox3.Convert(); - hdf_writer writer; - writer.SetVoxels(storage2); + writer.SetVoxels(storage3); writer.Write("multi_dim_vox.h5"); //std::ofstream fs("voxobj.obj"); diff --git a/writer.h b/writer.h index ba793e4..d0c1ba8 100644 --- a/writer.h +++ b/writer.h @@ -123,10 +123,9 @@ class hdf_writer :public abstract_writer { } - const H5std_string FILE_NAME(fnc); const H5std_string CONTINUOUS_DATASET_NAME("continuous_chunks"); - const H5std_string PLANAR_DATASET_NAME("continuous_chunks"); + const H5std_string PLANAR_DATASET_NAME("planar_chunks"); const int NC = continuous_count; const int NP = planar_count; @@ -149,7 +148,6 @@ class hdf_writer :public abstract_writer { datatype.setOrder(H5T_ORDER_LE); H5::DataSet dataset = file.createDataSet(CONTINUOUS_DATASET_NAME, datatype, dataspace); - //Hyperslab prep hsize_t offset[4]; offset[0] = -1; @@ -160,12 +158,39 @@ class hdf_writer :public abstract_writer { hsize_t slab_dimsf[4] = { 1, 1, 1, 1 }; H5::DataSpace mspace2(RANK, slab_dimsf); + //Regions dataset + const H5std_string REGIONS_DATASET_NAME("regions"); + const int REGION_RANK = 2; + hsize_t regions_dimsf[2]; + regions_dimsf[0] = NC; + regions_dimsf[1] = 1; + H5::DataSpace regions_dataspace(REGION_RANK, regions_dimsf); + H5::PredType regions_datatype(H5::PredType::STD_REF_DSETREG); + regions_datatype.setOrder(H5T_ORDER_LE); + + H5::DataSet regions_dataset = file.createDataSet(REGIONS_DATASET_NAME, regions_datatype, regions_dataspace); + // Regions hyperlab + hsize_t region_offset[2]; + region_offset[0] = -1; + region_offset[1] = 0; + hsize_t region_slab_dimsf[2] = { 1, 1 }; + H5::DataSpace mspace3(REGION_RANK, region_slab_dimsf); + + //Chunk hyperslab + hsize_t chunk_offset[4]; + const int CHUNK_RANK = 4; + chunk_offset[0] = -1; + chunk_offset[1] = 0; + chunk_offset[2] = 0; + chunk_offset[3] = 0; + hsize_t chunk_dimsf[4] = { 1, NX, NY, NZ }; + H5::DataSpace chunk_space(CHUNK_RANK, chunk_dimsf); for (int x = 0; x < storage->num_chunks().get(0); x++) { for (int y = 0; y < storage->num_chunks().get(1); y++) { for (int z = 0; z < storage->num_chunks().get(2); z++) { auto c = storage->get_chunk(make_vec(x, y, z)); - + if (c == nullptr) { std::cout << "Null pointer" << std::endl; @@ -199,6 +224,17 @@ class hdf_writer :public abstract_writer { std::vector hslab = { convox->Get(ijk) }; dataset.write(hslab.data(), H5::PredType::NATIVE_INT, mspace2, dataspace); END_LOOP; + + region_offset[0]++; + regions_dataspace.selectHyperslab(H5S_SELECT_SET, region_slab_dimsf, region_offset); + + chunk_offset[0]++; + + dataspace.selectHyperslab(H5S_SELECT_SET, chunk_dimsf, chunk_offset); + + hobj_ref_t inter[1]; + file.reference(&inter[0], "/continuous_chunks", dataspace, H5R_DATASET_REGION); + regions_dataset.write(inter, H5::PredType::STD_REF_DSETREG, mspace3, regions_dataspace); } } @@ -214,8 +250,6 @@ class hdf_writer :public abstract_writer { } } } - - }; From 85e98206cc010d73aafbb8d514c4e82550f77729 Mon Sep 17 00:00:00 2001 From: johltn Date: Wed, 2 Jun 2021 09:35:59 +0200 Subject: [PATCH 015/112] New test case --- tests/test_hdf_2.cpp | 102 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 102 insertions(+) create mode 100644 tests/test_hdf_2.cpp diff --git a/tests/test_hdf_2.cpp b/tests/test_hdf_2.cpp new file mode 100644 index 0000000..4583968 --- /dev/null +++ b/tests/test_hdf_2.cpp @@ -0,0 +1,102 @@ +#include "../voxelizer.h" +#include "../writer.h" +#include "../processor.h" + +#ifdef WITH_IFC +#include +#ifdef IFCOPENSHELL_05 +#include +using namespace Ifc2x3; +#else +#include +#endif +#endif + +#include +#include +#include + +#include + +#ifdef WIN32 +#define DIRSEP "\\" +#else +#define DIRSEP "/" +#endif + + +#if defined(WITH_IFC) && !defined(IFCOPENSHELL_05) +TEST(Voxelization, IfcWallIds) { + const std::string input_filename = ".." DIRSEP "tests" DIRSEP "fixtures" DIRSEP "duplex.ifc"; + + IfcParse::IfcFile ifc_file(input_filename); + + const double d = 0.5; + + ASSERT_TRUE(ifc_file.good()); + + IfcGeom::IteratorSettings settings_surface; + settings_surface.set(IfcGeom::IteratorSettings::DISABLE_TRIANGULATION, true); + settings_surface.set(IfcGeom::IteratorSettings::USE_WORLD_COORDS, true); + + IfcGeom::entity_filter ef; + ef.include = true; + ef.entity_names = { "IfcWall" }; + ef.traverse = false; + + auto filters = std::vector({ ef }); + + IfcGeom::Iterator it(settings_surface, &ifc_file, filters, 1); + + ASSERT_TRUE(it.initialize()); + + geometry_collection_t geoms; + + while (true) { + TopoDS_Compound C = it.get_native()->geometry().as_compound(); + BRepMesh_IncrementalMesh(C, 0.001); + geoms.push_back({ it.get_native()->id(), C }); + if (!it.next()) { + break; + } + } + + Bnd_Box global_bounds; + for (auto& P : geoms) { + BRepBndLib::Add(P.second, global_bounds); + } + + double x1, y1, z1, x2, y2, z2; + global_bounds.Get(x1, y1, z1, x2, y2, z2); + int nx = (int)ceil((x2 - x1) / d) + 10; + int ny = (int)ceil((y2 - y1) / d) + 10; + int nz = (int)ceil((z2 - z1) / d) + 10; + + x1 -= d * 5; + y1 -= d * 5; + z1 -= d * 5; + + chunked_voxel_storage* storage = new chunked_voxel_storage(x1, y1, z1, d, nx, ny, nz, 64); + + { + progress_writer progress_1("test_wall_ids"); + processor pr(storage, progress_1); + pr.use_scanline() = false; + pr.process(geoms.begin(), geoms.end(), VOLUME_PRODUCT_ID(), output(MERGED(), "test_wall_ids.vox")); + } + + // PRINT WORLD BOUNDS OF STORAGE...e b + + //std::ofstream fs("boundaries.obj"); + //obj_export_helper oeh{ fs }; + //storage->obj_export(oeh, false, true); + + + hdf_writer writer; + writer.SetVoxels(storage); + writer.Write("test_walls.h5"); + +} +#else +TEST(Voxelization, DISABLED_IfcSpaceIds) {} +#endif \ No newline at end of file From 5433a717b98472e468f41d56423c7bb1ea348f96 Mon Sep 17 00:00:00 2001 From: johltn Date: Wed, 2 Jun 2021 09:59:53 +0200 Subject: [PATCH 016/112] Install hdf5 --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 2974493..899d3c5 100644 --- a/.travis.yml +++ b/.travis.yml @@ -12,6 +12,7 @@ install: - | sudo apt-get install -y \ libocct-* \ + libhdf5-dev \ libboost-all-dev script: From e8537f59bf40a3f1f1d6345ada31d1685acd27bd Mon Sep 17 00:00:00 2001 From: johltn Date: Wed, 2 Jun 2021 10:17:04 +0200 Subject: [PATCH 017/112] Add pathes to include and lib dirs --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 899d3c5..9740af7 100644 --- a/.travis.yml +++ b/.travis.yml @@ -18,7 +18,7 @@ install: script: - mkdir build && cd build - | - cmake .. \ + cmake -DHDF5_INCLUDE_DIR=/usr/include/hdf5 -DHDF5_LIBRARY_DIR=/usr/lib/x86_64-linux-gnu/libhdf5_cpp.a;/usr/lib/x86_64-linux-gnu/libhdf5.a;/usr/lib/x86_64-linux-gnu/libzlib.a;/usr/lib/x86_64-linux-gnu/libsz.a;/usr/lib/x86_64-linux-gnu/libaec.a .. \ -DIFC_SUPPORT=Off \ -DOCC_INCLUDE_DIR=/usr/include/opencascade \ -DOCC_LIBRARY_DIR=/usr/lib/x86_64-linux-gnu \ From 290fc6793ddd3589e469b406d50c6155628d6ee0 Mon Sep 17 00:00:00 2001 From: johltn Date: Wed, 2 Jun 2021 10:33:33 +0200 Subject: [PATCH 018/112] Add quotes --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 9740af7..68b12fa 100644 --- a/.travis.yml +++ b/.travis.yml @@ -18,7 +18,7 @@ install: script: - mkdir build && cd build - | - cmake -DHDF5_INCLUDE_DIR=/usr/include/hdf5 -DHDF5_LIBRARY_DIR=/usr/lib/x86_64-linux-gnu/libhdf5_cpp.a;/usr/lib/x86_64-linux-gnu/libhdf5.a;/usr/lib/x86_64-linux-gnu/libzlib.a;/usr/lib/x86_64-linux-gnu/libsz.a;/usr/lib/x86_64-linux-gnu/libaec.a .. \ + cmake -DHDF5_INCLUDE_DIR=/usr/include/hdf5 -DHDF5_LIBRARY_DIR="/usr/lib/x86_64-linux-gnu/libhdf5_cpp.a;/usr/lib/x86_64-linux-gnu/libhdf5.a;/usr/lib/x86_64-linux-gnu/libzlib.a;/usr/lib/x86_64-linux-gnu/libsz.a;/usr/lib/x86_64-linux-gnu/libaec.a" .. \ -DIFC_SUPPORT=Off \ -DOCC_INCLUDE_DIR=/usr/include/opencascade \ -DOCC_LIBRARY_DIR=/usr/lib/x86_64-linux-gnu \ From 162fd65b4d66b776d7983574331483b7a6448c56 Mon Sep 17 00:00:00 2001 From: johltn Date: Wed, 2 Jun 2021 11:02:04 +0200 Subject: [PATCH 019/112] Add subdir for include path --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 68b12fa..1c71903 100644 --- a/.travis.yml +++ b/.travis.yml @@ -18,7 +18,7 @@ install: script: - mkdir build && cd build - | - cmake -DHDF5_INCLUDE_DIR=/usr/include/hdf5 -DHDF5_LIBRARY_DIR="/usr/lib/x86_64-linux-gnu/libhdf5_cpp.a;/usr/lib/x86_64-linux-gnu/libhdf5.a;/usr/lib/x86_64-linux-gnu/libzlib.a;/usr/lib/x86_64-linux-gnu/libsz.a;/usr/lib/x86_64-linux-gnu/libaec.a" .. \ + cmake -DHDF5_INCLUDE_DIR=/usr/include/hdf5/serial -DHDF5_LIBRARY_DIR="/usr/lib/x86_64-linux-gnu/libhdf5_cpp.a;/usr/lib/x86_64-linux-gnu/libhdf5.a;/usr/lib/x86_64-linux-gnu/libzlib.a;/usr/lib/x86_64-linux-gnu/libsz.a;/usr/lib/x86_64-linux-gnu/libaec.a" .. \ -DIFC_SUPPORT=Off \ -DOCC_INCLUDE_DIR=/usr/include/opencascade \ -DOCC_LIBRARY_DIR=/usr/lib/x86_64-linux-gnu \ From 8ea19515a15002635c4c5c6d00b6f2d9922edae3 Mon Sep 17 00:00:00 2001 From: johltn Date: Wed, 2 Jun 2021 14:17:37 +0200 Subject: [PATCH 020/112] Move HDF installation variables --- CMakeLists.txt | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 2fea951..12a0244 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -112,18 +112,6 @@ endif() set(BOOST_COMPONENTS regex program_options iostreams system) -SET(HDF5_INCLUDE_DIR ${IFCOPENSHELL_ROOT}/_deps-vs2017-x64-installed/HDF5-1.8.22-win64/include) -SET(HDF5_LIBRARY_DIR ${IFCOPENSHELL_ROOT}/_deps-vs2017-x64-installed/HDF5-1.8.22-win64/lib) - - -SET(HDF5_LIBRARIES - "${HDF5_LIBRARY_DIR}/libhdf5_cpp.lib" - "${HDF5_LIBRARY_DIR}/libhdf5.lib" - "${HDF5_LIBRARY_DIR}/libzlib.lib" - "${HDF5_LIBRARY_DIR}/libsz.lib" - "${HDF5_LIBRARY_DIR}/libaec.lib") - - if (IFC_SUPPORT) # Find IfcOpenShell version, v0.6.0 does not have the IfcSchema namespace anymore file(READ "${IFCOPENSHELL_ROOT}/src/ifcparse/IfcParse.h" header) @@ -161,6 +149,17 @@ if(UNIX) string(REGEX REPLACE "([^;]+)" "${ICU_LIBRARY_DIR}/lib\\1.a" ICU_LIBRARIES "${ICU_LIBRARY_NAMES}") endif() else() + + SET(HDF5_INCLUDE_DIR ${IFCOPENSHELL_ROOT}/_deps-vs2017-x64-installed/HDF5-1.8.22-win64/include) + SET(HDF5_LIBRARY_DIR ${IFCOPENSHELL_ROOT}/_deps-vs2017-x64-installed/HDF5-1.8.22-win64/lib) + + SET(HDF5_LIBRARIES + "${HDF5_LIBRARY_DIR}/libhdf5_cpp.lib" + "${HDF5_LIBRARY_DIR}/libhdf5.lib" + "${HDF5_LIBRARY_DIR}/libzlib.lib" + "${HDF5_LIBRARY_DIR}/libsz.lib" + "${HDF5_LIBRARY_DIR}/libaec.lib") + set(BOOST_ROOT ${IFCOPENSHELL_ROOT}/_deps/boost_1_74_0) set(BOOST_LIBRARYDIR ${IFCOPENSHELL_ROOT}/_deps/boost_1_74_0/stage/vs${MSVC_YEAR}-${WIN_ARCH}/lib) From e7a3c45803d507ab326071154791a60ef7f7533a Mon Sep 17 00:00:00 2001 From: johltn Date: Wed, 2 Jun 2021 15:04:49 +0200 Subject: [PATCH 021/112] Libraries variable --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 1c71903..b383928 100644 --- a/.travis.yml +++ b/.travis.yml @@ -18,7 +18,7 @@ install: script: - mkdir build && cd build - | - cmake -DHDF5_INCLUDE_DIR=/usr/include/hdf5/serial -DHDF5_LIBRARY_DIR="/usr/lib/x86_64-linux-gnu/libhdf5_cpp.a;/usr/lib/x86_64-linux-gnu/libhdf5.a;/usr/lib/x86_64-linux-gnu/libzlib.a;/usr/lib/x86_64-linux-gnu/libsz.a;/usr/lib/x86_64-linux-gnu/libaec.a" .. \ + cmake -DHDF5_INCLUDE_DIR=/usr/include/hdf5/serial -DHDF5_LIBRARIES="/usr/lib/x86_64-linux-gnu/libhdf5_cpp.a;/usr/lib/x86_64-linux-gnu/libhdf5.a;/usr/lib/x86_64-linux-gnu/libzlib.a;/usr/lib/x86_64-linux-gnu/libsz.a;/usr/lib/x86_64-linux-gnu/libaec.a" .. \ -DIFC_SUPPORT=Off \ -DOCC_INCLUDE_DIR=/usr/include/opencascade \ -DOCC_LIBRARY_DIR=/usr/lib/x86_64-linux-gnu \ From 5db0d4e2f901d83e74b555427dacf92ad0fae3b0 Mon Sep 17 00:00:00 2001 From: johltn Date: Wed, 2 Jun 2021 15:17:58 +0200 Subject: [PATCH 022/112] Update Travis file --- .travis.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index b383928..38dd2fd 100644 --- a/.travis.yml +++ b/.travis.yml @@ -18,7 +18,9 @@ install: script: - mkdir build && cd build - | - cmake -DHDF5_INCLUDE_DIR=/usr/include/hdf5/serial -DHDF5_LIBRARIES="/usr/lib/x86_64-linux-gnu/libhdf5_cpp.a;/usr/lib/x86_64-linux-gnu/libhdf5.a;/usr/lib/x86_64-linux-gnu/libzlib.a;/usr/lib/x86_64-linux-gnu/libsz.a;/usr/lib/x86_64-linux-gnu/libaec.a" .. \ + cmake .. \ + -DHDF5_INCLUDE_DIR=/usr/include/hdf5/serial + -DHDF5_LIBRARIES="/usr/lib/x86_64-linux-gnu/libhdf5_cpp.a;/usr/lib/x86_64-linux-gnu/libhdf5.a;/usr/lib/x86_64-linux-gnu/libzlib.a;/usr/lib/x86_64-linux-gnu/libsz.a;/usr/lib/x86_64-linux-gnu/libaec.a" -DIFC_SUPPORT=Off \ -DOCC_INCLUDE_DIR=/usr/include/opencascade \ -DOCC_LIBRARY_DIR=/usr/lib/x86_64-linux-gnu \ From 413febfe36f1e2e298c3aac7b01e4601ef1c3028 Mon Sep 17 00:00:00 2001 From: johltn Date: Wed, 2 Jun 2021 15:22:35 +0200 Subject: [PATCH 023/112] Update Travis file --- .travis.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 38dd2fd..b383928 100644 --- a/.travis.yml +++ b/.travis.yml @@ -18,9 +18,7 @@ install: script: - mkdir build && cd build - | - cmake .. \ - -DHDF5_INCLUDE_DIR=/usr/include/hdf5/serial - -DHDF5_LIBRARIES="/usr/lib/x86_64-linux-gnu/libhdf5_cpp.a;/usr/lib/x86_64-linux-gnu/libhdf5.a;/usr/lib/x86_64-linux-gnu/libzlib.a;/usr/lib/x86_64-linux-gnu/libsz.a;/usr/lib/x86_64-linux-gnu/libaec.a" + cmake -DHDF5_INCLUDE_DIR=/usr/include/hdf5/serial -DHDF5_LIBRARIES="/usr/lib/x86_64-linux-gnu/libhdf5_cpp.a;/usr/lib/x86_64-linux-gnu/libhdf5.a;/usr/lib/x86_64-linux-gnu/libzlib.a;/usr/lib/x86_64-linux-gnu/libsz.a;/usr/lib/x86_64-linux-gnu/libaec.a" .. \ -DIFC_SUPPORT=Off \ -DOCC_INCLUDE_DIR=/usr/include/opencascade \ -DOCC_LIBRARY_DIR=/usr/lib/x86_64-linux-gnu \ From 273444dfea2f4ba5316a8e884c75f3c9e90582e9 Mon Sep 17 00:00:00 2001 From: johltn Date: Wed, 2 Jun 2021 15:40:08 +0200 Subject: [PATCH 024/112] serial --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index b383928..b134f0b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -18,7 +18,7 @@ install: script: - mkdir build && cd build - | - cmake -DHDF5_INCLUDE_DIR=/usr/include/hdf5/serial -DHDF5_LIBRARIES="/usr/lib/x86_64-linux-gnu/libhdf5_cpp.a;/usr/lib/x86_64-linux-gnu/libhdf5.a;/usr/lib/x86_64-linux-gnu/libzlib.a;/usr/lib/x86_64-linux-gnu/libsz.a;/usr/lib/x86_64-linux-gnu/libaec.a" .. \ + cmake -DHDF5_INCLUDE_DIR=/usr/include/hdf5/serial -DHDF5_LIBRARIES="/usr/lib/x86_64-linux-gnu/hdf5/serial/libhdf5_cpp.a;/usr/lib/x86_64-linux-gnu/hdf5/serial/libhdf5.a;/usr/lib/x86_64-linux-gnu/hdf5/serial/libzlib.a;/usr/lib/x86_64-linux-gnu/hdf5/serial/libsz.a;/usr/lib/x86_64-linux-gnu/hdf5/serial/libaec.a" .. \ -DIFC_SUPPORT=Off \ -DOCC_INCLUDE_DIR=/usr/include/opencascade \ -DOCC_LIBRARY_DIR=/usr/lib/x86_64-linux-gnu \ From d4c996b2c5dc0239d3f4898f2e5cb39bff251d40 Mon Sep 17 00:00:00 2001 From: johltn Date: Wed, 2 Jun 2021 16:11:44 +0200 Subject: [PATCH 025/112] Replace lib name --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index b134f0b..8ba56df 100644 --- a/.travis.yml +++ b/.travis.yml @@ -18,7 +18,7 @@ install: script: - mkdir build && cd build - | - cmake -DHDF5_INCLUDE_DIR=/usr/include/hdf5/serial -DHDF5_LIBRARIES="/usr/lib/x86_64-linux-gnu/hdf5/serial/libhdf5_cpp.a;/usr/lib/x86_64-linux-gnu/hdf5/serial/libhdf5.a;/usr/lib/x86_64-linux-gnu/hdf5/serial/libzlib.a;/usr/lib/x86_64-linux-gnu/hdf5/serial/libsz.a;/usr/lib/x86_64-linux-gnu/hdf5/serial/libaec.a" .. \ + cmake -DHDF5_INCLUDE_DIR=/usr/include/hdf5/serial -DHDF5_LIBRARIES="/usr/lib/x86_64-linux-gnu/hdf5/serial/libhdf5_cpp.a;/usr/lib/x86_64-linux-gnu/hdf5/serial/libhdf5.a;/usr/lib/x86_64-linux-gnu/libz.a;/usr/lib/x86_64-linux-gnu/hdf5/serial/libsz.a;/usr/lib/x86_64-linux-gnu/hdf5/serial/libaec.a" .. \ -DIFC_SUPPORT=Off \ -DOCC_INCLUDE_DIR=/usr/include/opencascade \ -DOCC_LIBRARY_DIR=/usr/lib/x86_64-linux-gnu \ From 39889c019de5f02c70c77cc95bd4b57e05f038db Mon Sep 17 00:00:00 2001 From: johltn Date: Wed, 2 Jun 2021 16:25:28 +0200 Subject: [PATCH 026/112] serial --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 8ba56df..3f294f6 100644 --- a/.travis.yml +++ b/.travis.yml @@ -18,7 +18,7 @@ install: script: - mkdir build && cd build - | - cmake -DHDF5_INCLUDE_DIR=/usr/include/hdf5/serial -DHDF5_LIBRARIES="/usr/lib/x86_64-linux-gnu/hdf5/serial/libhdf5_cpp.a;/usr/lib/x86_64-linux-gnu/hdf5/serial/libhdf5.a;/usr/lib/x86_64-linux-gnu/libz.a;/usr/lib/x86_64-linux-gnu/hdf5/serial/libsz.a;/usr/lib/x86_64-linux-gnu/hdf5/serial/libaec.a" .. \ + cmake -DHDF5_INCLUDE_DIR=/usr/include/hdf5/serial -DHDF5_LIBRARIES="/usr/lib/x86_64-linux-gnu/hdf5/serial/libhdf5_cpp.a;/usr/lib/x86_64-linux-gnu/hdf5/serial/libhdf5.a;/usr/lib/x86_64-linux-gnu/libz.a;/usr/lib/x86_64-linux-gnu/libsz.a;/usr/lib/x86_64-linux-gnu/hdf5/serial/libaec.a" .. \ -DIFC_SUPPORT=Off \ -DOCC_INCLUDE_DIR=/usr/include/opencascade \ -DOCC_LIBRARY_DIR=/usr/lib/x86_64-linux-gnu \ From 55e002ab3c909d908a5018a3cd28d7f743952903 Mon Sep 17 00:00:00 2001 From: johltn Date: Wed, 2 Jun 2021 19:24:04 +0200 Subject: [PATCH 027/112] Update Travis file --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 3f294f6..4a50ad4 100644 --- a/.travis.yml +++ b/.travis.yml @@ -18,7 +18,7 @@ install: script: - mkdir build && cd build - | - cmake -DHDF5_INCLUDE_DIR=/usr/include/hdf5/serial -DHDF5_LIBRARIES="/usr/lib/x86_64-linux-gnu/hdf5/serial/libhdf5_cpp.a;/usr/lib/x86_64-linux-gnu/hdf5/serial/libhdf5.a;/usr/lib/x86_64-linux-gnu/libz.a;/usr/lib/x86_64-linux-gnu/libsz.a;/usr/lib/x86_64-linux-gnu/hdf5/serial/libaec.a" .. \ + cmake -DHDF5_INCLUDE_DIR=/usr/include/hdf5/serial -DHDF5_LIBRARIES="/usr/lib/x86_64-linux-gnu/hdf5/serial/libhdf5_cpp.a;/usr/lib/x86_64-linux-gnu/hdf5/serial/libhdf5.a;/usr/lib/x86_64-linux-gnu/libz.a;/usr/lib/x86_64-linux-gnu/libsz.a;/usr/lib/x86_64-linux-gnu/libaec.a" .. \ -DIFC_SUPPORT=Off \ -DOCC_INCLUDE_DIR=/usr/include/opencascade \ -DOCC_LIBRARY_DIR=/usr/lib/x86_64-linux-gnu \ From 99e967b26ad73299542e4f0295d754e5213688fb Mon Sep 17 00:00:00 2001 From: johltn Date: Thu, 3 Jun 2021 10:17:51 +0200 Subject: [PATCH 028/112] Add dl --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 4a50ad4..b52625d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -18,7 +18,7 @@ install: script: - mkdir build && cd build - | - cmake -DHDF5_INCLUDE_DIR=/usr/include/hdf5/serial -DHDF5_LIBRARIES="/usr/lib/x86_64-linux-gnu/hdf5/serial/libhdf5_cpp.a;/usr/lib/x86_64-linux-gnu/hdf5/serial/libhdf5.a;/usr/lib/x86_64-linux-gnu/libz.a;/usr/lib/x86_64-linux-gnu/libsz.a;/usr/lib/x86_64-linux-gnu/libaec.a" .. \ + cmake -DHDF5_INCLUDE_DIR=/usr/include/hdf5/serial -DHDF5_LIBRARIES="/usr/lib/x86_64-linux-gnu/hdf5/serial/libhdf5_cpp.a;/usr/lib/x86_64-linux-gnu/hdf5/serial/libhdf5.a;/usr/lib/x86_64-linux-gnu/libz.a;/usr/lib/x86_64-linux-gnu/libsz.a;/usr/lib/x86_64-linux-gnu/libaec.a;dl" .. \ -DIFC_SUPPORT=Off \ -DOCC_INCLUDE_DIR=/usr/include/opencascade \ -DOCC_LIBRARY_DIR=/usr/lib/x86_64-linux-gnu \ From e5602bb7f1d2ddabb32903b4fe923efeee51c3a1 Mon Sep 17 00:00:00 2001 From: johltn Date: Sun, 6 Jun 2021 13:12:53 +0200 Subject: [PATCH 029/112] Fix iterations over chunks --- writer.h | 159 +++++++++++++++++++++++++++---------------------------- 1 file changed, 77 insertions(+), 82 deletions(-) diff --git a/writer.h b/writer.h index d0c1ba8..699b8bd 100644 --- a/writer.h +++ b/writer.h @@ -90,38 +90,37 @@ class hdf_writer :public abstract_writer { int planar_count = 0; int constant_count = 0; - for (int i = 0; i < storage->num_chunks().get(0); i++) { - for (int j = 0; j < storage->num_chunks().get(1); j++) { - for (int k = 0; k < storage->num_chunks().get(2); k++) { - auto c = storage->get_chunk(make_vec(i, j, k)); + hsize_t nchunks_x = storage->num_chunks().get(0); + hsize_t nchunks_y = storage->num_chunks().get(1); + hsize_t nchunks_z = storage->num_chunks().get(2); - if (c == nullptr) { - std::cout << "Null pointer" << std::endl; + BEGIN_LOOP(size_t(0), nchunks_x, 0U, nchunks_y, 0U, nchunks_z) + auto c = storage->get_chunk(ijk); + + if (c == nullptr) { + std::cout << "Null pointer" << std::endl; + constant_count++; + } + + else { + if (c->is_explicit() || c->is_constant()) { + if (c->is_constant()) { + std::cout << "Count Constant chunk" << std::endl; constant_count++; } - else { - if (c->is_explicit() || c->is_constant()) { - if (c->is_constant()) { - std::cout << "Count Constant chunk" << std::endl; - constant_count++; - } - else { - std::cout << "Count Continuous chunk" << std::endl; - continuous_count++; - } - } - - else { - - std::cout << " Count Planar chunk" << std::endl; - planar_count++; - } + std::cout << "Count Continuous chunk" << std::endl; + continuous_count++; } } + + else { + + std::cout << " Count Planar chunk" << std::endl; + planar_count++; + } } - } - + END_LOOP; const H5std_string FILE_NAME(fnc); const H5std_string CONTINUOUS_DATASET_NAME("continuous_chunks"); @@ -129,7 +128,7 @@ class hdf_writer :public abstract_writer { const int NC = continuous_count; const int NP = planar_count; - + const int NX = storage->chunk_size(); const int NY = storage->chunk_size(); const int NZ = storage->chunk_size(); @@ -186,69 +185,65 @@ class hdf_writer :public abstract_writer { hsize_t chunk_dimsf[4] = { 1, NX, NY, NZ }; H5::DataSpace chunk_space(CHUNK_RANK, chunk_dimsf); - for (int x = 0; x < storage->num_chunks().get(0); x++) { - for (int y = 0; y < storage->num_chunks().get(1); y++) { - for (int z = 0; z < storage->num_chunks().get(2); z++) { - auto c = storage->get_chunk(make_vec(x, y, z)); - - if (c == nullptr) { - std::cout << "Null pointer" << std::endl; + BEGIN_LOOP(size_t(0), nchunks_x, 0U, nchunks_y, 0U, nchunks_z) + auto c = storage->get_chunk(ijk); - } + if (c == nullptr) { + std::cout << "Null pointer" << std::endl; + + } + + else { + if (c->is_explicit() || c->is_constant()) { + if (c->is_constant()) { + std::cout << "Constant chunk" << std::endl; + } else { - if (c->is_explicit() || c->is_constant()) { - if (c->is_constant()) { - std::cout << "Constant chunk" << std::endl; - - } - else { - std::cout << "Continuous chunk" << std::endl; - - offset[0]++; - - continuous_voxel_storage* convox = (continuous_voxel_storage*)c; - size_t i0, j0, k0, i1, j1, k1; - i0 = 0; - j0 = 0; - k0 = 0; - convox->extents().tie(i1, j1, k1); - - BEGIN_LOOP_ZERO_2(make_vec(i1, j1, k1)) - offset[1] = ijk.get(0); - offset[2] = ijk.get(1); - offset[3] = ijk.get(2); - dataspace.selectHyperslab(H5S_SELECT_SET, slab_dimsf, offset); - - std::vector hslab = { convox->Get(ijk) }; - dataset.write(hslab.data(), H5::PredType::NATIVE_INT, mspace2, dataspace); - END_LOOP; - - region_offset[0]++; - regions_dataspace.selectHyperslab(H5S_SELECT_SET, region_slab_dimsf, region_offset); - - chunk_offset[0]++; - - dataspace.selectHyperslab(H5S_SELECT_SET, chunk_dimsf, chunk_offset); - - hobj_ref_t inter[1]; - file.reference(&inter[0], "/continuous_chunks", dataspace, H5R_DATASET_REGION); - regions_dataset.write(inter, H5::PredType::STD_REF_DSETREG, mspace3, regions_dataspace); - } - } - - else { - std::cout << "Planar chunk" << std::endl; - planar_voxel_storage* planvox = (planar_voxel_storage*)c; - auto off = planvox->offsets(); - auto axis = planvox->axis(); - - } + std::cout << "Continuous chunk" << std::endl; + + offset[0]++; + + continuous_voxel_storage* convox = (continuous_voxel_storage*)c; + size_t i0, j0, k0, i1, j1, k1; + i0 = 0; + j0 = 0; + k0 = 0; + convox->extents().tie(i1, j1, k1); + + BEGIN_LOOP_ZERO_2(make_vec(i1, j1, k1)) + offset[1] = ijk.get(0); + offset[2] = ijk.get(1); + offset[3] = ijk.get(2); + dataspace.selectHyperslab(H5S_SELECT_SET, slab_dimsf, offset); + + std::vector hslab = { convox->Get(ijk) }; + dataset.write(hslab.data(), H5::PredType::NATIVE_INT, mspace2, dataspace); + END_LOOP; + + region_offset[0]++; + regions_dataspace.selectHyperslab(H5S_SELECT_SET, region_slab_dimsf, region_offset); + + chunk_offset[0]++; + + dataspace.selectHyperslab(H5S_SELECT_SET, chunk_dimsf, chunk_offset); + + hobj_ref_t inter[1]; + file.reference(&inter[0], "/continuous_chunks", dataspace, H5R_DATASET_REGION); + //regions_dataset.write(inter, H5::PredType::STD_REF_DSETREG, mspace3, regions_dataspace); } } + + else { + std::cout << "Planar chunk" << std::endl; + planar_voxel_storage* planvox = (planar_voxel_storage*)c; + auto off = planvox->offsets(); + auto axis = planvox->axis(); + + } } - } + END_LOOP } }; From 952c57e291134735b6bc1c1ef7ee466ddab38aeb Mon Sep 17 00:00:00 2001 From: johltn Date: Sun, 6 Jun 2021 13:17:57 +0200 Subject: [PATCH 030/112] Fix region slabs dimensions --- writer.h | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/writer.h b/writer.h index 699b8bd..ec6d5cd 100644 --- a/writer.h +++ b/writer.h @@ -159,20 +159,20 @@ class hdf_writer :public abstract_writer { //Regions dataset const H5std_string REGIONS_DATASET_NAME("regions"); - const int REGION_RANK = 2; - hsize_t regions_dimsf[2]; + const int REGION_RANK = 1; + hsize_t regions_dimsf[1]; regions_dimsf[0] = NC; - regions_dimsf[1] = 1; + //regions_dimsf[1] = 1; H5::DataSpace regions_dataspace(REGION_RANK, regions_dimsf); H5::PredType regions_datatype(H5::PredType::STD_REF_DSETREG); regions_datatype.setOrder(H5T_ORDER_LE); H5::DataSet regions_dataset = file.createDataSet(REGIONS_DATASET_NAME, regions_datatype, regions_dataspace); // Regions hyperlab - hsize_t region_offset[2]; + hsize_t region_offset[1]; region_offset[0] = -1; - region_offset[1] = 0; - hsize_t region_slab_dimsf[2] = { 1, 1 }; + //region_offset[1] = 0; + hsize_t region_slab_dimsf[1] = { 1}; H5::DataSpace mspace3(REGION_RANK, region_slab_dimsf); //Chunk hyperslab @@ -231,7 +231,7 @@ class hdf_writer :public abstract_writer { hobj_ref_t inter[1]; file.reference(&inter[0], "/continuous_chunks", dataspace, H5R_DATASET_REGION); - //regions_dataset.write(inter, H5::PredType::STD_REF_DSETREG, mspace3, regions_dataspace); + regions_dataset.write(inter, H5::PredType::STD_REF_DSETREG, mspace3, regions_dataspace); } } From c4e64e035e26cf640db7ed1d5438e2aa2d20f280 Mon Sep 17 00:00:00 2001 From: johltn Date: Mon, 7 Jun 2021 12:00:52 +0200 Subject: [PATCH 031/112] Tentative to incrementally write planar chunks to compound typed dataset --- writer.h | 189 ++++++++++++++++++++++++++++++++++++------------------- 1 file changed, 126 insertions(+), 63 deletions(-) diff --git a/writer.h b/writer.h index ec6d5cd..811c8fe 100644 --- a/writer.h +++ b/writer.h @@ -4,6 +4,7 @@ #include "storage.h" #include "H5Cpp.h" + #include #include #include @@ -84,6 +85,10 @@ class hdf_writer :public abstract_writer { public: void Write(const std::string& fnc) { + + + + chunked_voxel_storage* storage = (chunked_voxel_storage*)voxels_; int continuous_count = 0; @@ -97,34 +102,34 @@ class hdf_writer :public abstract_writer { BEGIN_LOOP(size_t(0), nchunks_x, 0U, nchunks_y, 0U, nchunks_z) auto c = storage->get_chunk(ijk); - if (c == nullptr) { - std::cout << "Null pointer" << std::endl; - constant_count++; - } + if (c == nullptr) { + std::cout << "Null pointer" << std::endl; + constant_count++; + } - else { - if (c->is_explicit() || c->is_constant()) { - if (c->is_constant()) { - std::cout << "Count Constant chunk" << std::endl; - constant_count++; - } - else { - std::cout << "Count Continuous chunk" << std::endl; - continuous_count++; - } + else { + if (c->is_explicit() || c->is_constant()) { + if (c->is_constant()) { + std::cout << "Count Constant chunk" << std::endl; + constant_count++; } - else { - - std::cout << " Count Planar chunk" << std::endl; - planar_count++; + std::cout << "Count Continuous chunk" << std::endl; + continuous_count++; } } + + else { + + std::cout << " Count Planar chunk" << std::endl; + planar_count++; + } + } END_LOOP; const H5std_string FILE_NAME(fnc); const H5std_string CONTINUOUS_DATASET_NAME("continuous_chunks"); - const H5std_string PLANAR_DATASET_NAME("planar_chunks"); + const int NC = continuous_count; const int NP = planar_count; @@ -147,7 +152,7 @@ class hdf_writer :public abstract_writer { datatype.setOrder(H5T_ORDER_LE); H5::DataSet dataset = file.createDataSet(CONTINUOUS_DATASET_NAME, datatype, dataspace); - //Hyperslab prep + // Hyperslab prep hsize_t offset[4]; offset[0] = -1; offset[1] = 0; @@ -157,11 +162,11 @@ class hdf_writer :public abstract_writer { hsize_t slab_dimsf[4] = { 1, 1, 1, 1 }; H5::DataSpace mspace2(RANK, slab_dimsf); - //Regions dataset + // Regions dataset const H5std_string REGIONS_DATASET_NAME("regions"); const int REGION_RANK = 1; hsize_t regions_dimsf[1]; - regions_dimsf[0] = NC; + regions_dimsf[0] = continuous_count + constant_count + planar_count; //regions_dimsf[1] = 1; H5::DataSpace regions_dataspace(REGION_RANK, regions_dimsf); H5::PredType regions_datatype(H5::PredType::STD_REF_DSETREG); @@ -172,10 +177,10 @@ class hdf_writer :public abstract_writer { hsize_t region_offset[1]; region_offset[0] = -1; //region_offset[1] = 0; - hsize_t region_slab_dimsf[1] = { 1}; + hsize_t region_slab_dimsf[1] = { 1 }; H5::DataSpace mspace3(REGION_RANK, region_slab_dimsf); - //Chunk hyperslab + // Chunk hyperslab hsize_t chunk_offset[4]; const int CHUNK_RANK = 4; chunk_offset[0] = -1; @@ -186,63 +191,121 @@ class hdf_writer :public abstract_writer { H5::DataSpace chunk_space(CHUNK_RANK, chunk_dimsf); + typedef struct s2_t { + double a; + hvl_t b; + } s2_t; + + const H5std_string AXIS("axis"); + const H5std_string OFFSETS("offsets"); + + H5::VarLenType varlen_type(&H5::PredType::NATIVE_INT); + H5::CompType mtype2(sizeof(s2_t)); + mtype2.insertMember(AXIS, HOFFSET(s2_t, a), H5::PredType::NATIVE_INT); + mtype2.insertMember(OFFSETS, HOFFSET(s2_t, b), varlen_type); + const H5std_string PLANAR_DATASET_NAME("planar_chunks"); + + + const int PLANAR_RANK = 2; + hsize_t planar_dimsf[1]; + planar_dimsf[0] = planar_count; + planar_dimsf[1] = 1; + + + H5::DataSpace planar_dataspace(PLANAR_RANK, planar_dimsf); + + H5::DataSet planar_dataset = file.createDataSet(PLANAR_DATASET_NAME, mtype2, planar_dataspace); + + + + hsize_t planar_offset[2]; + planar_offset[0] = 1; + planar_offset[1] = 0; + + hsize_t planar_dims[2] = { 1,1 }; + H5::DataSpace planar_space(PLANAR_RANK, planar_dims); + + planar_dataspace.selectHyperslab(H5S_SELECT_SET, planar_dims, planar_offset); + + + + std::vector input_vec; + + hvl_t varlen_offsets; + std::vector tt = { 1,2,4,4,4,4 }; + varlen_offsets.p = &tt; + + + // No bug but nothing written when inspecting output file + s2_t ss; + ss.a = 4; + ss.b = varlen_offsets; + input_vec.push_back(ss); + planar_dataset.write(input_vec.data(), mtype2, planar_space, planar_dataspace); + + + BEGIN_LOOP(size_t(0), nchunks_x, 0U, nchunks_y, 0U, nchunks_z) auto c = storage->get_chunk(ijk); - if (c == nullptr) { - std::cout << "Null pointer" << std::endl; + if (c == nullptr) { + std::cout << "Null pointer" << std::endl; - } + } - else { - if (c->is_explicit() || c->is_constant()) { - if (c->is_constant()) { - std::cout << "Constant chunk" << std::endl; + else { + if (c->is_explicit() || c->is_constant()) { + if (c->is_constant()) { + std::cout << "Constant chunk" << std::endl; - } - else { - std::cout << "Continuous chunk" << std::endl; + } + else { + std::cout << "Continuous chunk" << std::endl; - offset[0]++; + offset[0]++; - continuous_voxel_storage* convox = (continuous_voxel_storage*)c; - size_t i0, j0, k0, i1, j1, k1; - i0 = 0; - j0 = 0; - k0 = 0; - convox->extents().tie(i1, j1, k1); + continuous_voxel_storage* convox = (continuous_voxel_storage*)c; + size_t i0, j0, k0, i1, j1, k1; + i0 = 0; + j0 = 0; + k0 = 0; + convox->extents().tie(i1, j1, k1); - BEGIN_LOOP_ZERO_2(make_vec(i1, j1, k1)) - offset[1] = ijk.get(0); - offset[2] = ijk.get(1); - offset[3] = ijk.get(2); - dataspace.selectHyperslab(H5S_SELECT_SET, slab_dimsf, offset); + BEGIN_LOOP_ZERO_2(make_vec(i1, j1, k1)) + offset[1] = ijk.get(0); + offset[2] = ijk.get(1); + offset[3] = ijk.get(2); + dataspace.selectHyperslab(H5S_SELECT_SET, slab_dimsf, offset); - std::vector hslab = { convox->Get(ijk) }; - dataset.write(hslab.data(), H5::PredType::NATIVE_INT, mspace2, dataspace); - END_LOOP; + std::vector hslab = { convox->Get(ijk) }; + dataset.write(hslab.data(), H5::PredType::NATIVE_INT, mspace2, dataspace); + END_LOOP; - region_offset[0]++; - regions_dataspace.selectHyperslab(H5S_SELECT_SET, region_slab_dimsf, region_offset); + region_offset[0]++; + regions_dataspace.selectHyperslab(H5S_SELECT_SET, region_slab_dimsf, region_offset); - chunk_offset[0]++; + chunk_offset[0]++; - dataspace.selectHyperslab(H5S_SELECT_SET, chunk_dimsf, chunk_offset); + dataspace.selectHyperslab(H5S_SELECT_SET, chunk_dimsf, chunk_offset); - hobj_ref_t inter[1]; - file.reference(&inter[0], "/continuous_chunks", dataspace, H5R_DATASET_REGION); - regions_dataset.write(inter, H5::PredType::STD_REF_DSETREG, mspace3, regions_dataspace); - } + hobj_ref_t inter[1]; + file.reference(&inter[0], "/continuous_chunks", dataspace, H5R_DATASET_REGION); + regions_dataset.write(inter, H5::PredType::STD_REF_DSETREG, mspace3, regions_dataspace); } + } + + else { + std::cout << "Planar chunk" << std::endl; + planar_voxel_storage* planvox = (planar_voxel_storage*)c; + + auto off = planvox->offsets(); + auto axis = planvox->axis(); + + - else { - std::cout << "Planar chunk" << std::endl; - planar_voxel_storage* planvox = (planar_voxel_storage*)c; - auto off = planvox->offsets(); - auto axis = planvox->axis(); - } } + } END_LOOP } }; From 367f4bab199c578c0c82791a584a7d1726b184f3 Mon Sep 17 00:00:00 2001 From: johltn Date: Mon, 7 Jun 2021 13:16:48 +0200 Subject: [PATCH 032/112] Planar dataset writing ok for simple example --- writer.h | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/writer.h b/writer.h index 811c8fe..60df017 100644 --- a/writer.h +++ b/writer.h @@ -192,7 +192,7 @@ class hdf_writer :public abstract_writer { typedef struct s2_t { - double a; + int a; hvl_t b; } s2_t; @@ -206,10 +206,10 @@ class hdf_writer :public abstract_writer { const H5std_string PLANAR_DATASET_NAME("planar_chunks"); - const int PLANAR_RANK = 2; + const int PLANAR_RANK = 1; hsize_t planar_dimsf[1]; planar_dimsf[0] = planar_count; - planar_dimsf[1] = 1; + //planar_dimsf[1] = 1; H5::DataSpace planar_dataspace(PLANAR_RANK, planar_dimsf); @@ -218,11 +218,11 @@ class hdf_writer :public abstract_writer { - hsize_t planar_offset[2]; + hsize_t planar_offset[1]; planar_offset[0] = 1; - planar_offset[1] = 0; + - hsize_t planar_dims[2] = { 1,1 }; + hsize_t planar_dims[2] = { 1,1}; H5::DataSpace planar_space(PLANAR_RANK, planar_dims); planar_dataspace.selectHyperslab(H5S_SELECT_SET, planar_dims, planar_offset); @@ -233,7 +233,7 @@ class hdf_writer :public abstract_writer { hvl_t varlen_offsets; std::vector tt = { 1,2,4,4,4,4 }; - varlen_offsets.p = &tt; + varlen_offsets.p = tt.data(); // No bug but nothing written when inspecting output file @@ -241,7 +241,7 @@ class hdf_writer :public abstract_writer { ss.a = 4; ss.b = varlen_offsets; input_vec.push_back(ss); - planar_dataset.write(input_vec.data(), mtype2, planar_space, planar_dataspace); + planar_dataset.write(&ss, mtype2, planar_space, planar_dataspace); From e018ef7a79882fa23e7a79332014a018b1a47683 Mon Sep 17 00:00:00 2001 From: johltn Date: Mon, 7 Jun 2021 21:44:17 +0200 Subject: [PATCH 033/112] Start cleaning before cosntant chunks handling --- writer.h | 159 +++++++++++++++++++++++++++++-------------------------- 1 file changed, 83 insertions(+), 76 deletions(-) diff --git a/writer.h b/writer.h index 60df017..7b08e71 100644 --- a/writer.h +++ b/writer.h @@ -85,10 +85,6 @@ class hdf_writer :public abstract_writer { public: void Write(const std::string& fnc) { - - - - chunked_voxel_storage* storage = (chunked_voxel_storage*)voxels_; int continuous_count = 0; @@ -127,20 +123,20 @@ class hdf_writer :public abstract_writer { } END_LOOP; + const H5std_string FILE_NAME(fnc); - const H5std_string CONTINUOUS_DATASET_NAME("continuous_chunks"); + H5::H5File file(FILE_NAME, H5F_ACC_TRUNC); + // Continuous chunks dataset + const H5std_string CONTINUOUS_DATASET_NAME("continuous_chunks"); const int NC = continuous_count; - const int NP = planar_count; - const int NX = storage->chunk_size(); const int NY = storage->chunk_size(); const int NZ = storage->chunk_size(); const int RANK = 4; - H5::H5File file(FILE_NAME, H5F_ACC_TRUNC); - + hsize_t dimsf[4]; dimsf[0] = NC; dimsf[1] = NX; @@ -152,7 +148,7 @@ class hdf_writer :public abstract_writer { datatype.setOrder(H5T_ORDER_LE); H5::DataSet dataset = file.createDataSet(CONTINUOUS_DATASET_NAME, datatype, dataspace); - // Hyperslab prep + // Continuous dataset hyperslab preparation hsize_t offset[4]; offset[0] = -1; offset[1] = 0; @@ -162,38 +158,12 @@ class hdf_writer :public abstract_writer { hsize_t slab_dimsf[4] = { 1, 1, 1, 1 }; H5::DataSpace mspace2(RANK, slab_dimsf); - // Regions dataset - const H5std_string REGIONS_DATASET_NAME("regions"); - const int REGION_RANK = 1; - hsize_t regions_dimsf[1]; - regions_dimsf[0] = continuous_count + constant_count + planar_count; - //regions_dimsf[1] = 1; - H5::DataSpace regions_dataspace(REGION_RANK, regions_dimsf); - H5::PredType regions_datatype(H5::PredType::STD_REF_DSETREG); - regions_datatype.setOrder(H5T_ORDER_LE); - - H5::DataSet regions_dataset = file.createDataSet(REGIONS_DATASET_NAME, regions_datatype, regions_dataspace); - // Regions hyperlab - hsize_t region_offset[1]; - region_offset[0] = -1; - //region_offset[1] = 0; - hsize_t region_slab_dimsf[1] = { 1 }; - H5::DataSpace mspace3(REGION_RANK, region_slab_dimsf); - - // Chunk hyperslab - hsize_t chunk_offset[4]; - const int CHUNK_RANK = 4; - chunk_offset[0] = -1; - chunk_offset[1] = 0; - chunk_offset[2] = 0; - chunk_offset[3] = 0; - hsize_t chunk_dimsf[4] = { 1, NX, NY, NZ }; - H5::DataSpace chunk_space(CHUNK_RANK, chunk_dimsf); + //Planar chunks dataset typedef struct s2_t { - int a; - hvl_t b; + int axis; + hvl_t offsets; } s2_t; const H5std_string AXIS("axis"); @@ -201,64 +171,74 @@ class hdf_writer :public abstract_writer { H5::VarLenType varlen_type(&H5::PredType::NATIVE_INT); H5::CompType mtype2(sizeof(s2_t)); - mtype2.insertMember(AXIS, HOFFSET(s2_t, a), H5::PredType::NATIVE_INT); - mtype2.insertMember(OFFSETS, HOFFSET(s2_t, b), varlen_type); + mtype2.insertMember(AXIS, HOFFSET(s2_t, axis), H5::PredType::NATIVE_INT); + mtype2.insertMember(OFFSETS, HOFFSET(s2_t, offsets), varlen_type); const H5std_string PLANAR_DATASET_NAME("planar_chunks"); - const int PLANAR_RANK = 1; hsize_t planar_dimsf[1]; planar_dimsf[0] = planar_count; - //planar_dimsf[1] = 1; - - H5::DataSpace planar_dataspace(PLANAR_RANK, planar_dimsf); - H5::DataSet planar_dataset = file.createDataSet(PLANAR_DATASET_NAME, mtype2, planar_dataspace); - - - hsize_t planar_offset[1]; - planar_offset[0] = 1; - - - hsize_t planar_dims[2] = { 1,1}; + planar_offset[0] = -1; + hsize_t planar_dims[1] = { 1 }; H5::DataSpace planar_space(PLANAR_RANK, planar_dims); - planar_dataspace.selectHyperslab(H5S_SELECT_SET, planar_dims, planar_offset); + // Regions dataset + const H5std_string REGIONS_DATASET_NAME("regions"); - std::vector input_vec; + hsize_t regions_dimsf[1]; + regions_dimsf[0] = continuous_count + constant_count + planar_count; - hvl_t varlen_offsets; - std::vector tt = { 1,2,4,4,4,4 }; - varlen_offsets.p = tt.data(); + const int REGION_RANK = 1; + H5::DataSpace regions_dataspace(REGION_RANK, regions_dimsf); + H5::PredType regions_datatype(H5::PredType::STD_REF_DSETREG); + regions_datatype.setOrder(H5T_ORDER_LE); + H5::DataSet regions_dataset = file.createDataSet(REGIONS_DATASET_NAME, regions_datatype, regions_dataspace); + + // Regions dataset hyperslab preparation + hsize_t region_offset[1]; + region_offset[0] = -1; + hsize_t region_slab_dimsf[1] = { 1 }; + H5::DataSpace mspace3(REGION_RANK, region_slab_dimsf); - // No bug but nothing written when inspecting output file - s2_t ss; - ss.a = 4; - ss.b = varlen_offsets; - input_vec.push_back(ss); - planar_dataset.write(&ss, mtype2, planar_space, planar_dataspace); + // Continuous chunks region hyperslab preparation + hsize_t chunk_offset[4]; + const int CHUNK_RANK = 4; + chunk_offset[0] = -1; + chunk_offset[1] = 0; + chunk_offset[2] = 0; + chunk_offset[3] = 0; + hsize_t chunk_dimsf[4] = { 1, NX, NY, NZ }; + H5::DataSpace chunk_space(CHUNK_RANK, chunk_dimsf); + // Planar chunks region hyperslab preparation + hsize_t planar_chunk_offset[1]; + const int PLANAR_CHUNK_RANK = 1; + planar_chunk_offset[0] = 1; + hsize_t planar_chunk_dimsf[1] = { 1 }; + H5::DataSpace planar_chunk_space(PLANAR_CHUNK_RANK, planar_chunk_dimsf); BEGIN_LOOP(size_t(0), nchunks_x, 0U, nchunks_y, 0U, nchunks_z) auto c = storage->get_chunk(ijk); - if (c == nullptr) { - std::cout << "Null pointer" << std::endl; + if (c == nullptr) { + std::cout << "Null pointer" << std::endl; - } + } - else { - if (c->is_explicit() || c->is_constant()) { - if (c->is_constant()) { + else { + if (c->is_explicit() || c->is_constant()) { + if (c->is_constant()) { std::cout << "Constant chunk" << std::endl; } + else { std::cout << "Continuous chunk" << std::endl; @@ -294,18 +274,45 @@ class hdf_writer :public abstract_writer { } } - else { - std::cout << "Planar chunk" << std::endl; - planar_voxel_storage* planvox = (planar_voxel_storage*)c; + else { + std::cout << "Planar chunk" << std::endl; + planar_voxel_storage* planvox = (planar_voxel_storage*)c; + + auto axis = planvox->axis(); + auto off = planvox->offsets(); + + std::vector offsets(off.begin(), off.end()); - auto off = planvox->offsets(); - auto axis = planvox->axis(); + planar_offset[0]++; + + planar_dataspace.selectHyperslab(H5S_SELECT_SET, planar_dims, planar_offset); + std::vector input_vec; + hvl_t varlen_offsets; + + varlen_offsets.len = offsets.size(); + varlen_offsets.p = offsets.data(); + s2_t compound_data; + compound_data.axis = axis; + compound_data.offsets = varlen_offsets; + input_vec.push_back(compound_data); + planar_dataset.write(&compound_data, mtype2, planar_space, planar_dataspace); + region_offset[0]++; + regions_dataspace.selectHyperslab(H5S_SELECT_SET, region_slab_dimsf, region_offset); + + planar_chunk_offset[0]++; + + dataspace.selectHyperslab(H5S_SELECT_SET, planar_chunk_dimsf, planar_chunk_offset); + + hobj_ref_t inter[1]; + file.reference(&inter[0], "/planar_chunks", planar_dataspace, H5R_DATASET_REGION); + regions_dataset.write(inter, H5::PredType::STD_REF_DSETREG, mspace3, regions_dataspace); + + } } - } END_LOOP } }; From 7a41f0f069874f1beb061decdf85c11ec62eb4a1 Mon Sep 17 00:00:00 2001 From: johltn Date: Mon, 7 Jun 2021 22:12:06 +0200 Subject: [PATCH 034/112] Writing preparation to constant chunks dataset --- writer.h | 96 +++++++++++++++++++++++++++++++++++--------------------- 1 file changed, 61 insertions(+), 35 deletions(-) diff --git a/writer.h b/writer.h index 7b08e71..275de10 100644 --- a/writer.h +++ b/writer.h @@ -136,7 +136,7 @@ class hdf_writer :public abstract_writer { const int NZ = storage->chunk_size(); const int RANK = 4; - + hsize_t dimsf[4]; dimsf[0] = NC; dimsf[1] = NX; @@ -187,6 +187,21 @@ class hdf_writer :public abstract_writer { + //Constant chunks dataset + const H5std_string CONSTANT_DATASET_NAME("constant_chunks"); + + const int CONSTANT_RANK = 1; + hsize_t constant_dimsf[1]; + constant_dimsf[0] = constant_count; + H5::DataSpace constant_dataspace(CONSTANT_RANK, constant_dimsf); + H5::DataSet constant_dataset = file.createDataSet(CONSTANT_DATASET_NAME, H5::PredType::NATIVE_INT, constant_dataspace); + hsize_t constant_offset[1]; + constant_offset[0] = -1; + hsize_t constant_dims[1] = { 1 }; + H5::DataSpace constant_space(CONSTANT_RANK, constant_dims); + + + // Regions dataset const H5std_string REGIONS_DATASET_NAME("regions"); @@ -223,20 +238,31 @@ class hdf_writer :public abstract_writer { hsize_t planar_chunk_dimsf[1] = { 1 }; H5::DataSpace planar_chunk_space(PLANAR_CHUNK_RANK, planar_chunk_dimsf); + // Constant chunks region hyperslab preparation + hsize_t constant_chunk_offset[1]; + const int CONSTANT_CHUNK_RANK = 1; + constant_chunk_offset[0] = 1; + hsize_t constant_chunk_dimsf[1] = { 1 }; + H5::DataSpace constant_chunk_space(CONSTANT_CHUNK_RANK, constant_chunk_dimsf); + BEGIN_LOOP(size_t(0), nchunks_x, 0U, nchunks_y, 0U, nchunks_z) auto c = storage->get_chunk(ijk); - if (c == nullptr) { - std::cout << "Null pointer" << std::endl; - - } + if (c == nullptr) { + std::cout << "Null pointer" << std::endl; + } - else { - if (c->is_explicit() || c->is_constant()) { - if (c->is_constant()) { + else { + if (c->is_explicit() || c->is_constant()) { + if (c->is_constant()) { std::cout << "Constant chunk" << std::endl; + constant_offset[0]++; + constant_dataspace.selectHyperslab(H5S_SELECT_SET, constant_dims, constant_offset); + int inter[1] = { 1 }; + dataset.write(inter, H5::PredType::NATIVE_INT, constant_space, constant_dataspace); + } else { @@ -274,45 +300,45 @@ class hdf_writer :public abstract_writer { } } - else { - std::cout << "Planar chunk" << std::endl; - planar_voxel_storage* planvox = (planar_voxel_storage*)c; + else { + std::cout << "Planar chunk" << std::endl; + planar_voxel_storage* planvox = (planar_voxel_storage*)c; - auto axis = planvox->axis(); - auto off = planvox->offsets(); + auto axis = planvox->axis(); + auto off = planvox->offsets(); - std::vector offsets(off.begin(), off.end()); + std::vector offsets(off.begin(), off.end()); - planar_offset[0]++; - - planar_dataspace.selectHyperslab(H5S_SELECT_SET, planar_dims, planar_offset); - std::vector input_vec; - hvl_t varlen_offsets; + planar_offset[0]++; - - varlen_offsets.len = offsets.size(); + planar_dataspace.selectHyperslab(H5S_SELECT_SET, planar_dims, planar_offset); + std::vector input_vec; + hvl_t varlen_offsets; - varlen_offsets.p = offsets.data(); - s2_t compound_data; - compound_data.axis = axis; - compound_data.offsets = varlen_offsets; - input_vec.push_back(compound_data); - planar_dataset.write(&compound_data, mtype2, planar_space, planar_dataspace); + varlen_offsets.len = offsets.size(); - region_offset[0]++; - regions_dataspace.selectHyperslab(H5S_SELECT_SET, region_slab_dimsf, region_offset); + varlen_offsets.p = offsets.data(); + s2_t compound_data; + compound_data.axis = axis; + compound_data.offsets = varlen_offsets; + input_vec.push_back(compound_data); + planar_dataset.write(&compound_data, mtype2, planar_space, planar_dataspace); - planar_chunk_offset[0]++; - dataspace.selectHyperslab(H5S_SELECT_SET, planar_chunk_dimsf, planar_chunk_offset); + region_offset[0]++; + regions_dataspace.selectHyperslab(H5S_SELECT_SET, region_slab_dimsf, region_offset); - hobj_ref_t inter[1]; - file.reference(&inter[0], "/planar_chunks", planar_dataspace, H5R_DATASET_REGION); - regions_dataset.write(inter, H5::PredType::STD_REF_DSETREG, mspace3, regions_dataspace); + planar_chunk_offset[0]++; + + dataspace.selectHyperslab(H5S_SELECT_SET, planar_chunk_dimsf, planar_chunk_offset); + + hobj_ref_t inter[1]; + file.reference(&inter[0], "/planar_chunks", planar_dataspace, H5R_DATASET_REGION); + regions_dataset.write(inter, H5::PredType::STD_REF_DSETREG, mspace3, regions_dataspace); - } } + } END_LOOP } }; From 95d7c0dc59e71e70332925f3fadb45d0bfeb97ba Mon Sep 17 00:00:00 2001 From: johltn Date: Mon, 7 Jun 2021 22:19:41 +0200 Subject: [PATCH 035/112] Write constant chunks + region references --- writer.h | 37 ++++++++++++++++++++++++++++++++++--- 1 file changed, 34 insertions(+), 3 deletions(-) diff --git a/writer.h b/writer.h index 275de10..efb2761 100644 --- a/writer.h +++ b/writer.h @@ -251,6 +251,23 @@ class hdf_writer :public abstract_writer { if (c == nullptr) { std::cout << "Null pointer" << std::endl; + + constant_offset[0]++; + constant_dataspace.selectHyperslab(H5S_SELECT_SET, constant_dims, constant_offset); + int constant_value[1] = { 0 }; + dataset.write(constant_value, H5::PredType::NATIVE_INT, constant_space, constant_dataspace); + + + region_offset[0]++; + regions_dataspace.selectHyperslab(H5S_SELECT_SET, region_slab_dimsf, region_offset); + + constant_chunk_offset[0]++; + + //dataspace.selectHyperslab(H5S_SELECT_SET, planar_chunk_dimsf, planar_chunk_offset); + + hobj_ref_t inter[1]; + file.reference(&inter[0], "/constant_chunks", constant_dataspace, H5R_DATASET_REGION); + regions_dataset.write(inter, H5::PredType::STD_REF_DSETREG, mspace3, regions_dataspace); } else { @@ -260,8 +277,22 @@ class hdf_writer :public abstract_writer { constant_offset[0]++; constant_dataspace.selectHyperslab(H5S_SELECT_SET, constant_dims, constant_offset); - int inter[1] = { 1 }; - dataset.write(inter, H5::PredType::NATIVE_INT, constant_space, constant_dataspace); + int constant_value[1] = { 1 }; + dataset.write(constant_value , H5::PredType::NATIVE_INT, constant_space, constant_dataspace); + + + region_offset[0]++; + regions_dataspace.selectHyperslab(H5S_SELECT_SET, region_slab_dimsf, region_offset); + + constant_chunk_offset[0]++; + + //dataspace.selectHyperslab(H5S_SELECT_SET, planar_chunk_dimsf, planar_chunk_offset); + + hobj_ref_t inter[1]; + file.reference(&inter[0], "/constant_chunks", constant_dataspace, H5R_DATASET_REGION); + regions_dataset.write(inter, H5::PredType::STD_REF_DSETREG, mspace3, regions_dataspace); + + } @@ -331,7 +362,7 @@ class hdf_writer :public abstract_writer { planar_chunk_offset[0]++; - dataspace.selectHyperslab(H5S_SELECT_SET, planar_chunk_dimsf, planar_chunk_offset); + //dataspace.selectHyperslab(H5S_SELECT_SET, planar_chunk_dimsf, planar_chunk_offset); hobj_ref_t inter[1]; file.reference(&inter[0], "/planar_chunks", planar_dataspace, H5R_DATASET_REGION); From b17dd731c26dca125a27c21cd98163a6d8e17b6f Mon Sep 17 00:00:00 2001 From: johltn Date: Mon, 7 Jun 2021 22:29:41 +0200 Subject: [PATCH 036/112] Change dimensions types to comply with clang compilator --- writer.h | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/writer.h b/writer.h index efb2761..71ad373 100644 --- a/writer.h +++ b/writer.h @@ -130,10 +130,10 @@ class hdf_writer :public abstract_writer { // Continuous chunks dataset const H5std_string CONTINUOUS_DATASET_NAME("continuous_chunks"); - const int NC = continuous_count; - const int NX = storage->chunk_size(); - const int NY = storage->chunk_size(); - const int NZ = storage->chunk_size(); + const hsize_t NC = continuous_count; + const hsize_t NX = storage->chunk_size(); + const hsize_t NY = storage->chunk_size(); + const hsize_t NZ = storage->chunk_size(); const int RANK = 4; From 237ddb7d115e74e4b217c9417d149894a58fc939 Mon Sep 17 00:00:00 2001 From: johltn Date: Tue, 8 Jun 2021 10:14:57 +0200 Subject: [PATCH 037/112] Modify control flow --- writer.h | 146 ++++++++++++++++++++++++++++++------------------------- 1 file changed, 79 insertions(+), 67 deletions(-) diff --git a/writer.h b/writer.h index 71ad373..aa95a68 100644 --- a/writer.h +++ b/writer.h @@ -99,28 +99,35 @@ class hdf_writer :public abstract_writer { auto c = storage->get_chunk(ijk); if (c == nullptr) { - std::cout << "Null pointer" << std::endl; + std::cout << "Null pointer count" << std::endl; constant_count++; + } else { - if (c->is_explicit() || c->is_constant()) { + + if (c->is_explicit()) { + std::cout << "Continuous chunk count" << std::endl; + continuous_count++; + } + + else { if (c->is_constant()) { - std::cout << "Count Constant chunk" << std::endl; + std::cout << "Constant chunk count" << std::endl; constant_count++; } + + else { - std::cout << "Count Continuous chunk" << std::endl; - continuous_count++; + std::cout << "Planar chunk count" << std::endl; + planar_count++; + } } - else { - std::cout << " Count Planar chunk" << std::endl; - planar_count++; - } } + END_LOOP; @@ -271,14 +278,50 @@ class hdf_writer :public abstract_writer { } else { - if (c->is_explicit() || c->is_constant()) { + + if (c->is_explicit()) { + std::cout << "Continuous chunk" << std::endl; + + offset[0]++; + + continuous_voxel_storage* convox = (continuous_voxel_storage*)c; + size_t i0, j0, k0, i1, j1, k1; + i0 = 0; + j0 = 0; + k0 = 0; + convox->extents().tie(i1, j1, k1); + + BEGIN_LOOP_ZERO_2(make_vec(i1, j1, k1)) + offset[1] = ijk.get(0); + offset[2] = ijk.get(1); + offset[3] = ijk.get(2); + dataspace.selectHyperslab(H5S_SELECT_SET, slab_dimsf, offset); + + std::vector hslab = { convox->Get(ijk) }; + dataset.write(hslab.data(), H5::PredType::NATIVE_INT, mspace2, dataspace); + END_LOOP; + + region_offset[0]++; + regions_dataspace.selectHyperslab(H5S_SELECT_SET, region_slab_dimsf, region_offset); + + chunk_offset[0]++; + + dataspace.selectHyperslab(H5S_SELECT_SET, chunk_dimsf, chunk_offset); + + hobj_ref_t inter[1]; + file.reference(&inter[0], "/continuous_chunks", dataspace, H5R_DATASET_REGION); + regions_dataset.write(inter, H5::PredType::STD_REF_DSETREG, mspace3, regions_dataspace); + + } + + else { if (c->is_constant()) { std::cout << "Constant chunk" << std::endl; constant_offset[0]++; constant_dataspace.selectHyperslab(H5S_SELECT_SET, constant_dims, constant_offset); int constant_value[1] = { 1 }; - dataset.write(constant_value , H5::PredType::NATIVE_INT, constant_space, constant_dataspace); + dataset.write(constant_value, H5::PredType::NATIVE_INT, constant_space, constant_dataspace); region_offset[0]++; @@ -296,79 +339,48 @@ class hdf_writer :public abstract_writer { } + else { - std::cout << "Continuous chunk" << std::endl; + std::cout << "Planar chunk" << std::endl; + planar_voxel_storage* planvox = (planar_voxel_storage*)c; + + auto axis = planvox->axis(); + auto off = planvox->offsets(); + + std::vector offsets(off.begin(), off.end()); - offset[0]++; + planar_offset[0]++; - continuous_voxel_storage* convox = (continuous_voxel_storage*)c; - size_t i0, j0, k0, i1, j1, k1; - i0 = 0; - j0 = 0; - k0 = 0; - convox->extents().tie(i1, j1, k1); + planar_dataspace.selectHyperslab(H5S_SELECT_SET, planar_dims, planar_offset); + std::vector input_vec; + hvl_t varlen_offsets; - BEGIN_LOOP_ZERO_2(make_vec(i1, j1, k1)) - offset[1] = ijk.get(0); - offset[2] = ijk.get(1); - offset[3] = ijk.get(2); - dataspace.selectHyperslab(H5S_SELECT_SET, slab_dimsf, offset); - std::vector hslab = { convox->Get(ijk) }; - dataset.write(hslab.data(), H5::PredType::NATIVE_INT, mspace2, dataspace); - END_LOOP; + varlen_offsets.len = offsets.size(); + + varlen_offsets.p = offsets.data(); + s2_t compound_data; + compound_data.axis = axis; + compound_data.offsets = varlen_offsets; + input_vec.push_back(compound_data); + planar_dataset.write(&compound_data, mtype2, planar_space, planar_dataspace); + region_offset[0]++; regions_dataspace.selectHyperslab(H5S_SELECT_SET, region_slab_dimsf, region_offset); - chunk_offset[0]++; + planar_chunk_offset[0]++; - dataspace.selectHyperslab(H5S_SELECT_SET, chunk_dimsf, chunk_offset); + //dataspace.selectHyperslab(H5S_SELECT_SET, planar_chunk_dimsf, planar_chunk_offset); hobj_ref_t inter[1]; - file.reference(&inter[0], "/continuous_chunks", dataspace, H5R_DATASET_REGION); + file.reference(&inter[0], "/planar_chunks", planar_dataspace, H5R_DATASET_REGION); regions_dataset.write(inter, H5::PredType::STD_REF_DSETREG, mspace3, regions_dataspace); + } } - else { - std::cout << "Planar chunk" << std::endl; - planar_voxel_storage* planvox = (planar_voxel_storage*)c; - - auto axis = planvox->axis(); - auto off = planvox->offsets(); - - std::vector offsets(off.begin(), off.end()); - - planar_offset[0]++; - - planar_dataspace.selectHyperslab(H5S_SELECT_SET, planar_dims, planar_offset); - std::vector input_vec; - hvl_t varlen_offsets; - - varlen_offsets.len = offsets.size(); - - varlen_offsets.p = offsets.data(); - s2_t compound_data; - compound_data.axis = axis; - compound_data.offsets = varlen_offsets; - input_vec.push_back(compound_data); - planar_dataset.write(&compound_data, mtype2, planar_space, planar_dataspace); - - - region_offset[0]++; - regions_dataspace.selectHyperslab(H5S_SELECT_SET, region_slab_dimsf, region_offset); - - planar_chunk_offset[0]++; - - //dataspace.selectHyperslab(H5S_SELECT_SET, planar_chunk_dimsf, planar_chunk_offset); - - hobj_ref_t inter[1]; - file.reference(&inter[0], "/planar_chunks", planar_dataspace, H5R_DATASET_REGION); - regions_dataset.write(inter, H5::PredType::STD_REF_DSETREG, mspace3, regions_dataspace); - - } } END_LOOP } From 1a6c32642bb117eb17e18f1bc0e96e36be2f6bfb Mon Sep 17 00:00:00 2001 From: johltn Date: Thu, 17 Jun 2021 13:50:14 +0200 Subject: [PATCH 038/112] Start handling voxels stored as integers --- writer.h | 37 ++++++++++++++++++++++++++----------- 1 file changed, 26 insertions(+), 11 deletions(-) diff --git a/writer.h b/writer.h index aa95a68..c70f0a0 100644 --- a/writer.h +++ b/writer.h @@ -85,8 +85,8 @@ class hdf_writer :public abstract_writer { public: void Write(const std::string& fnc) { - chunked_voxel_storage* storage = (chunked_voxel_storage*)voxels_; - + abstract_chunked_voxel_storage* storage = (abstract_chunked_voxel_storage*)voxels_; + int continuous_count = 0; int planar_count = 0; int constant_count = 0; @@ -255,6 +255,8 @@ class hdf_writer :public abstract_writer { BEGIN_LOOP(size_t(0), nchunks_x, 0U, nchunks_y, 0U, nchunks_z) auto c = storage->get_chunk(ijk); + + if (c == nullptr) { std::cout << "Null pointer" << std::endl; @@ -283,22 +285,35 @@ class hdf_writer :public abstract_writer { std::cout << "Continuous chunk" << std::endl; offset[0]++; - - continuous_voxel_storage* convox = (continuous_voxel_storage*)c; + size_t i0, j0, k0, i1, j1, k1; i0 = 0; j0 = 0; k0 = 0; - convox->extents().tie(i1, j1, k1); + c->extents().tie(i1, j1, k1); BEGIN_LOOP_ZERO_2(make_vec(i1, j1, k1)) offset[1] = ijk.get(0); - offset[2] = ijk.get(1); - offset[3] = ijk.get(2); - dataspace.selectHyperslab(H5S_SELECT_SET, slab_dimsf, offset); - - std::vector hslab = { convox->Get(ijk) }; - dataset.write(hslab.data(), H5::PredType::NATIVE_INT, mspace2, dataspace); + offset[2] = ijk.get(1); + offset[3] = ijk.get(2); + dataspace.selectHyperslab(H5S_SELECT_SET, slab_dimsf, offset); + + if (c->value_bits() == 32) { + + uint32_t v; + std::vector hslab; + c->Get(ijk, &v); + hslab.push_back(v); + dataset.write(hslab.data(), H5::PredType::NATIVE_INT, mspace2, dataspace); + + } + + else { + std::vector hslab = { c->Get(ijk) }; + dataset.write(hslab.data(), H5::PredType::NATIVE_INT, mspace2, dataspace); + } + + END_LOOP; region_offset[0]++; From 7bb7411a45576a504e52abca89b0ca60b0411cbf Mon Sep 17 00:00:00 2001 From: johltn Date: Mon, 21 Jun 2021 09:37:10 +0200 Subject: [PATCH 039/112] Store operations as groups containing chunk datasets --- writer.h | 52 ++++++++++++++++++++++------------------------------ 1 file changed, 22 insertions(+), 30 deletions(-) diff --git a/writer.h b/writer.h index c70f0a0..af67b39 100644 --- a/writer.h +++ b/writer.h @@ -39,7 +39,7 @@ class abstract_writer { voxels_ = voxels; } - virtual void Write(const std::string& fnc) = 0; + virtual void Write(const std::string& fnc, std::string statement="default") = 0; }; @@ -55,7 +55,7 @@ class voxel_writer :public abstract_writer { } public: - void Write(const std::string& fnc) { + void Write(const std::string& fnc, std::string statement = "default") { { std::string fn = fnc + std::string(".index"); std::ofstream fs(fn.c_str()); @@ -83,10 +83,10 @@ class voxel_writer :public abstract_writer { class hdf_writer :public abstract_writer { public: - void Write(const std::string& fnc) { + void Write(const std::string& fnc, std::string statement="default") { abstract_chunked_voxel_storage* storage = (abstract_chunked_voxel_storage*)voxels_; - + int continuous_count = 0; int planar_count = 0; int constant_count = 0; @@ -99,7 +99,6 @@ class hdf_writer :public abstract_writer { auto c = storage->get_chunk(ijk); if (c == nullptr) { - std::cout << "Null pointer count" << std::endl; constant_count++; } @@ -107,19 +106,16 @@ class hdf_writer :public abstract_writer { else { if (c->is_explicit()) { - std::cout << "Continuous chunk count" << std::endl; continuous_count++; } else { if (c->is_constant()) { - std::cout << "Constant chunk count" << std::endl; constant_count++; } else { - std::cout << "Planar chunk count" << std::endl; planar_count++; } @@ -131,9 +127,14 @@ class hdf_writer :public abstract_writer { END_LOOP; + + //TODO: Handle existing file (with group) case const H5std_string FILE_NAME(fnc); - H5::H5File file(FILE_NAME, H5F_ACC_TRUNC); + H5::H5File file(FILE_NAME, H5F_ACC_RDWR); + + H5::Group operation_group = H5::Group(file.createGroup(statement)); + // Continuous chunks dataset const H5std_string CONTINUOUS_DATASET_NAME("continuous_chunks"); @@ -153,7 +154,7 @@ class hdf_writer :public abstract_writer { H5::DataSpace dataspace(RANK, dimsf); H5::IntType datatype(H5::PredType::NATIVE_INT); datatype.setOrder(H5T_ORDER_LE); - H5::DataSet dataset = file.createDataSet(CONTINUOUS_DATASET_NAME, datatype, dataspace); + H5::DataSet dataset = operation_group.createDataSet(CONTINUOUS_DATASET_NAME, datatype, dataspace); // Continuous dataset hyperslab preparation hsize_t offset[4]; @@ -186,7 +187,7 @@ class hdf_writer :public abstract_writer { hsize_t planar_dimsf[1]; planar_dimsf[0] = planar_count; H5::DataSpace planar_dataspace(PLANAR_RANK, planar_dimsf); - H5::DataSet planar_dataset = file.createDataSet(PLANAR_DATASET_NAME, mtype2, planar_dataspace); + H5::DataSet planar_dataset = operation_group.createDataSet(PLANAR_DATASET_NAME, mtype2, planar_dataspace); hsize_t planar_offset[1]; planar_offset[0] = -1; hsize_t planar_dims[1] = { 1 }; @@ -201,7 +202,7 @@ class hdf_writer :public abstract_writer { hsize_t constant_dimsf[1]; constant_dimsf[0] = constant_count; H5::DataSpace constant_dataspace(CONSTANT_RANK, constant_dimsf); - H5::DataSet constant_dataset = file.createDataSet(CONSTANT_DATASET_NAME, H5::PredType::NATIVE_INT, constant_dataspace); + H5::DataSet constant_dataset = operation_group.createDataSet(CONSTANT_DATASET_NAME, H5::PredType::NATIVE_INT, constant_dataspace); hsize_t constant_offset[1]; constant_offset[0] = -1; hsize_t constant_dims[1] = { 1 }; @@ -220,7 +221,7 @@ class hdf_writer :public abstract_writer { H5::DataSpace regions_dataspace(REGION_RANK, regions_dimsf); H5::PredType regions_datatype(H5::PredType::STD_REF_DSETREG); regions_datatype.setOrder(H5T_ORDER_LE); - H5::DataSet regions_dataset = file.createDataSet(REGIONS_DATASET_NAME, regions_datatype, regions_dataspace); + H5::DataSet regions_dataset = operation_group.createDataSet(REGIONS_DATASET_NAME, regions_datatype, regions_dataspace); // Regions dataset hyperslab preparation hsize_t region_offset[1]; @@ -259,8 +260,7 @@ class hdf_writer :public abstract_writer { if (c == nullptr) { - std::cout << "Null pointer" << std::endl; - + constant_offset[0]++; constant_dataspace.selectHyperslab(H5S_SELECT_SET, constant_dims, constant_offset); int constant_value[1] = { 0 }; @@ -272,18 +272,15 @@ class hdf_writer :public abstract_writer { constant_chunk_offset[0]++; - //dataspace.selectHyperslab(H5S_SELECT_SET, planar_chunk_dimsf, planar_chunk_offset); - hobj_ref_t inter[1]; - file.reference(&inter[0], "/constant_chunks", constant_dataspace, H5R_DATASET_REGION); + file.reference(&inter[0], "/" + statement + "/constant_chunks", constant_dataspace, H5R_DATASET_REGION); regions_dataset.write(inter, H5::PredType::STD_REF_DSETREG, mspace3, regions_dataspace); } else { if (c->is_explicit()) { - std::cout << "Continuous chunk" << std::endl; - + offset[0]++; size_t i0, j0, k0, i1, j1, k1; @@ -324,15 +321,14 @@ class hdf_writer :public abstract_writer { dataspace.selectHyperslab(H5S_SELECT_SET, chunk_dimsf, chunk_offset); hobj_ref_t inter[1]; - file.reference(&inter[0], "/continuous_chunks", dataspace, H5R_DATASET_REGION); + file.reference(&inter[0], "/" + statement + "/continuous_chunks", dataspace, H5R_DATASET_REGION); regions_dataset.write(inter, H5::PredType::STD_REF_DSETREG, mspace3, regions_dataspace); } else { if (c->is_constant()) { - std::cout << "Constant chunk" << std::endl; - + constant_offset[0]++; constant_dataspace.selectHyperslab(H5S_SELECT_SET, constant_dims, constant_offset); int constant_value[1] = { 1 }; @@ -344,19 +340,16 @@ class hdf_writer :public abstract_writer { constant_chunk_offset[0]++; - //dataspace.selectHyperslab(H5S_SELECT_SET, planar_chunk_dimsf, planar_chunk_offset); hobj_ref_t inter[1]; - file.reference(&inter[0], "/constant_chunks", constant_dataspace, H5R_DATASET_REGION); + file.reference(&inter[0], "/" + statement + "/constant_chunks", constant_dataspace, H5R_DATASET_REGION); regions_dataset.write(inter, H5::PredType::STD_REF_DSETREG, mspace3, regions_dataspace); - - } else { - std::cout << "Planar chunk" << std::endl; + planar_voxel_storage* planvox = (planar_voxel_storage*)c; auto axis = planvox->axis(); @@ -386,10 +379,9 @@ class hdf_writer :public abstract_writer { planar_chunk_offset[0]++; - //dataspace.selectHyperslab(H5S_SELECT_SET, planar_chunk_dimsf, planar_chunk_offset); hobj_ref_t inter[1]; - file.reference(&inter[0], "/planar_chunks", planar_dataspace, H5R_DATASET_REGION); + file.reference(&inter[0], "/" + statement + "/planar_chunks", planar_dataspace, H5R_DATASET_REGION); regions_dataset.write(inter, H5::PredType::STD_REF_DSETREG, mspace3, regions_dataspace); } From 47ed3ad7b5660788fed0b55a575cfb7f50f97389 Mon Sep 17 00:00:00 2001 From: Thomas Krijnen Date: Sun, 14 Mar 2021 11:07:02 +0100 Subject: [PATCH 040/112] option to specify padding --- voxec.cpp | 8 ++++++++ voxec.h | 17 +++++++++-------- voxec_main.cpp | 5 +++++ 3 files changed, 22 insertions(+), 8 deletions(-) diff --git a/voxec.cpp b/voxec.cpp index da733ad..35ba9ff 100644 --- a/voxec.cpp +++ b/voxec.cpp @@ -247,3 +247,11 @@ scope_map run(const std::vector& statements, double size, size_t return context; } + +size_t padding_ = 32; +size_t get_padding() { + return padding_; +} +void set_padding(size_t v) { + padding_ = v; +} \ No newline at end of file diff --git a/voxec.h b/voxec.h index 11ecf54..b70385c 100644 --- a/voxec.h +++ b/voxec.h @@ -56,6 +56,9 @@ struct filtered_files_t {}; #define DIRSEP "/" #endif +size_t get_padding(); +void set_padding(size_t); + typedef boost::variant symbol_value; class voxel_operation; @@ -435,19 +438,17 @@ namespace { BRepBndLib::Add(p.second, global_bounds); } - const size_t PADDING = 32U; - global_bounds.Get(x1, y1, z1, x2, y2, z2); nx = (int)ceil((x2 - x1) / vsize); ny = (int)ceil((y2 - y1) / vsize); nz = (int)ceil((z2 - z1) / vsize); - x1 -= vsize * PADDING; - y1 -= vsize * PADDING; - z1 -= vsize * PADDING; - nx += PADDING * 2; - ny += PADDING * 2; - nz += PADDING * 2; + x1 -= vsize * get_padding(); + y1 -= vsize * get_padding(); + z1 -= vsize * get_padding(); + nx += get_padding() * 2; + ny += get_padding() * 2; + nz += get_padding() * 2; std::unique_ptr method; if (use_volume) { diff --git a/voxec_main.cpp b/voxec_main.cpp index 5821e71..c63071e 100644 --- a/voxec_main.cpp +++ b/voxec_main.cpp @@ -18,6 +18,7 @@ int main(int argc, char** argv) { ("log-file", po::value(), "filename to log json message") ("threads,t", po::value(), "number of parallel processing threads") ("size,d", po::value(), "voxel size in meters") + ("padding,p", po::value(), "padding around geometry bounding box (default=32)") ("chunk,c", po::value(), "chunk size in number of voxels") ("mmap,m", "use memory-mapped files instead of pure RAM") ("mesh", "emit obj mesh for the last instruction") @@ -53,6 +54,10 @@ int main(int argc, char** argv) { d = vmap["size"].as(); } + if (vmap.count("padding")) { + set_padding(vmap["padding"].as()); + } + boost::optional threads, chunk; if (vmap.count("threads")) { From a24873d40fa8936c1022f6a145498195353c8947 Mon Sep 17 00:00:00 2001 From: Thomas Krijnen Date: Sun, 14 Mar 2021 12:14:26 +0100 Subject: [PATCH 041/112] Delete appveyor.yml --- appveyor.yml | 20 -------------------- 1 file changed, 20 deletions(-) delete mode 100644 appveyor.yml diff --git a/appveyor.yml b/appveyor.yml deleted file mode 100644 index 0094646..0000000 --- a/appveyor.yml +++ /dev/null @@ -1,20 +0,0 @@ -version: 1.0.{build}-{branch} -image: Visual Studio 2015 -clone_folder: C:\projects\voxelization_toolkit_build - -build_script: - - mkdir occt - - cd occt - - curl -fsSL -o opencascade.tar.bz2 https://anaconda.org/conda-forge/occt/7.3.0/download/win-64/occt-7.3.0-hd7db75f_1002.tar.bz2 - - 7z x opencascade.tar.bz2 - - 7z x opencascade.tar - - cd .. - - - mkdir build - - cd build - - cmake .. -G "Visual Studio 14 2015 Win64" -DIFC_SUPPORT=Off -DBOOST_ROOT=C:\Libraries\boost_1_67_0 -DOCC_INCLUDE_DIR=C:\projects\voxelization_toolkit_build\occt\Library\include\opencascade\ -DOCC_LIBRARY_DIR=C:\projects\voxelization_toolkit_build\occt\Library\lib - - cmake --build . - -test_script: - - for %%i in (C:\projects\voxelization_toolkit_build\build\Debug\test_*.exe) do ("%%i") - From c7143b95a28fba1a62cca980a87de552b0f4765d Mon Sep 17 00:00:00 2001 From: johltn Date: Thu, 18 Mar 2021 15:10:47 +0100 Subject: [PATCH 042/112] Create test for HDF libs --- .gitignore | 2 ++ CMakeLists.txt | 19 ++++++++++++++++--- tests/test_hdf.cpp | 39 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 57 insertions(+), 3 deletions(-) create mode 100644 tests/test_hdf.cpp diff --git a/.gitignore b/.gitignore index de7f0f3..78c0dd4 100644 --- a/.gitignore +++ b/.gitignore @@ -31,3 +31,5 @@ *.out *.app build/ + +test diff --git a/CMakeLists.txt b/CMakeLists.txt index a96f267..2fea951 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -111,6 +111,19 @@ endif() set(BOOST_COMPONENTS regex program_options iostreams system) + +SET(HDF5_INCLUDE_DIR ${IFCOPENSHELL_ROOT}/_deps-vs2017-x64-installed/HDF5-1.8.22-win64/include) +SET(HDF5_LIBRARY_DIR ${IFCOPENSHELL_ROOT}/_deps-vs2017-x64-installed/HDF5-1.8.22-win64/lib) + + +SET(HDF5_LIBRARIES + "${HDF5_LIBRARY_DIR}/libhdf5_cpp.lib" + "${HDF5_LIBRARY_DIR}/libhdf5.lib" + "${HDF5_LIBRARY_DIR}/libzlib.lib" + "${HDF5_LIBRARY_DIR}/libsz.lib" + "${HDF5_LIBRARY_DIR}/libaec.lib") + + if (IFC_SUPPORT) # Find IfcOpenShell version, v0.6.0 does not have the IfcSchema namespace anymore file(READ "${IFCOPENSHELL_ROOT}/src/ifcparse/IfcParse.h" header) @@ -170,10 +183,10 @@ find_package(Boost REQUIRED COMPONENTS ${BOOST_COMPONENTS}) message(STATUS "Boost include files found in ${Boost_INCLUDE_DIRS}") message(STATUS "Boost libraries found in ${Boost_LIBRARY_DIRS}") -include_directories(${IFC_INCLUDE_DIR} ${OCC_INCLUDE_DIR} ${Boost_INCLUDE_DIRS} ${CMAKE_CURRENT_SOURCE_DIR}/3rdparty/eigen) +include_directories(${IFC_INCLUDE_DIR} ${OCC_INCLUDE_DIR} ${Boost_INCLUDE_DIRS} ${HDF5_INCLUDE_DIR} ${CMAKE_CURRENT_SOURCE_DIR}/3rdparty/eigen) link_directories(${Boost_LIBRARY_DIRS}) -include_directories(${INCLUDE_DIRECTORIES} ${OCC_INCLUDE_DIR} ${Boost_INCLUDE_DIRS} ${IFC_INCLUDE_DIR}) +include_directories(${INCLUDE_DIRECTORIES} ${OCC_INCLUDE_DIR} ${Boost_INCLUDE_DIRS} ${HDF5_INCLUDE_DIR} ${IFC_INCLUDE_DIR}) if (NOT MSVC) include(CheckCXXCompilerFlag) CHECK_CXX_COMPILER_FLAG("-std=c++14" COMPILER_SUPPORTS_CXX11) @@ -221,7 +234,7 @@ ENDIF() set(LIBVOXELIZER_HEADER_FILES progress_writer.h dim3.h sweep.h shift.h collapse.h edge_detect.h fill_gaps.h offset.h polyfill.h resample.h storage.h writer.h factory.h processor.h volume.h voxelizer.h traversal.h util.h json_logger.h) set(LIBVOXELIZER_SOURCE_FILES tribox3.cpp polyfill.cpp progress_writer.cpp storage.cpp factory.cpp json_logger.cpp) add_library(libvoxel STATIC ${LIBVOXELIZER_HEADER_FILES} ${LIBVOXELIZER_SOURCE_FILES}) -target_link_libraries(libvoxel ${IFC_LIBRARIES} ${ICU_LIBRARIES} ${Boost_LIBRARIES} ${BCRYPT_LIBRARIES} ${OCC_LIBRARIES} ${LIB_RT} ${dl} ${CMAKE_THREAD_LIBS_INIT} ${WS2_LIBRARIES}) +target_link_libraries(libvoxel ${IFC_LIBRARIES} ${ICU_LIBRARIES} ${Boost_LIBRARIES} ${BCRYPT_LIBRARIES} ${OCC_LIBRARIES} ${LIB_RT} ${dl} ${CMAKE_THREAD_LIBS_INIT} ${WS2_LIBRARIES} ${HDF5_LIBRARIES}) set(LIBVOXEC_HEADER_FILES voxelfile.h voxec.h) set(LIBVOXEC_SOURCE_FILES voxec.cpp) diff --git a/tests/test_hdf.cpp b/tests/test_hdf.cpp new file mode 100644 index 0000000..ffe96b9 --- /dev/null +++ b/tests/test_hdf.cpp @@ -0,0 +1,39 @@ +#include "../voxelizer.h" +#include "../writer.h" +#include "H5Cpp.h" +#include + +#include +#include + +TEST(HdfFileName, HDF) { + + const H5std_string FILE_NAME("SDS.h5"); + const H5std_string DATASET_NAME("IntArray"); + const int NX = 5; + const int NY = 6; + const int RANK = 2; + + int i, j; + int data[NX][NY]; + for (j = 0; j < NX; j++) + { + for (i = 0; i < NY; i++) + data[j][i] = i + j; + } + + H5::H5File file(FILE_NAME, H5F_ACC_TRUNC); + hsize_t dimsf[2]; + dimsf[0] = NX; + dimsf[1] = NY; + H5::DataSpace dataspace(RANK, dimsf); + H5::IntType datatype(H5::PredType::NATIVE_INT); + datatype.setOrder(H5T_ORDER_LE); + + H5::DataSet dataset = file.createDataSet(DATASET_NAME, datatype, dataspace); + dataset.write(data, H5::PredType::NATIVE_INT); + + std::string file_name = file.getFileName(); + EXPECT_TRUE(file_name == FILE_NAME); + +} From 475fa6ce5fdd61a6e73e8f3c0eecb3135a676ad1 Mon Sep 17 00:00:00 2001 From: Thomas Krijnen Date: Sun, 21 Mar 2021 19:58:23 +0100 Subject: [PATCH 043/112] boolean operation fixes --- storage.h | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/storage.h b/storage.h index 9f81f51..4197e08 100644 --- a/storage.h +++ b/storage.h @@ -1116,7 +1116,10 @@ class abstract_chunked_voxel_storage : public regular_voxel_storage { continue; } } else if (mode == OP_SUBTRACTION) { - if (Sa == CK_EMPTY || Sb == CK_FULL) { + if (Sa == CK_EMPTY) { + continue; + } else if (Sb == CK_FULL) { + set_chunk(n, ijk, nullptr); continue; } else if (Sb == CK_EMPTY) { if (!inplace) { @@ -1126,6 +1129,7 @@ class abstract_chunked_voxel_storage : public regular_voxel_storage { } } else if (mode == OP_INTERSECTION) { if (Sa == CK_EMPTY || Sb == CK_EMPTY) { + // @todo this does not appear correct. When B is empty A needs to be *cleared* when not empty and in-place. continue; } else if (Sb == CK_FULL) { if (!inplace) { From d447d40bd8d5e2258040f68efd21d67ddc8530ad Mon Sep 17 00:00:00 2001 From: Thomas Krijnen Date: Sun, 21 Mar 2021 19:59:42 +0100 Subject: [PATCH 044/112] decribe_components() zero() plane() --- json_logger.cpp | 11 +++ json_logger.h | 1 + voxec.cpp | 3 + voxec.h | 175 +++++++++++++++++++++++++++++++++++++++++------- 4 files changed, 167 insertions(+), 23 deletions(-) diff --git a/json_logger.cpp b/json_logger.cpp index 7573738..b968eab 100644 --- a/json_logger.cpp +++ b/json_logger.cpp @@ -42,6 +42,17 @@ namespace { } } +std::string json_logger::to_json_string(const meta_data& m) { + boost::property_tree::ptree pt; + for (auto& p : m) { + ptree_writer vis{ pt, p.first }; + boost::apply_visitor(vis, p.second); + } + std::ostringstream oss; + boost::property_tree::write_json(oss, pt, false); + return oss.str(); +} + void json_logger::message(severity s, const std::string & message, const std::map& meta) { static const std::array severity_strings = { "", "debug", "notice", "warning", "error", "fatal" }; diff --git a/json_logger.h b/json_logger.h index 83b8263..4e975b8 100644 --- a/json_logger.h +++ b/json_logger.h @@ -14,6 +14,7 @@ class json_logger { typedef enum { FMT_TEXT, FMT_JSON } format; typedef boost::variant meta_value; typedef std::map meta_data; + static std::string to_json_string(const meta_data&); private: static std::vector> log_output; static severity severity_; diff --git a/voxec.cpp b/voxec.cpp index 35ba9ff..c5eff7e 100644 --- a/voxec.cpp +++ b/voxec.cpp @@ -36,6 +36,7 @@ voxel_operation_map::map_t& voxel_operation_map::map() { m.insert(std::make_pair(std::string("collapse"), &instantiate)); m.insert(std::make_pair(std::string("collapse_count"), &instantiate)); m.insert(std::make_pair(std::string("print_components"), &instantiate)); + m.insert(std::make_pair(std::string("describe_components"), &instantiate)); m.insert(std::make_pair(std::string("keep_components"), &instantiate)); m.insert(std::make_pair(std::string("dump_surfaces"), &instantiate)); m.insert(std::make_pair(std::string("json_stats"), &instantiate)); @@ -47,6 +48,8 @@ voxel_operation_map::map_t& voxel_operation_map::map() { m.insert(std::make_pair(std::string("filter_attributes"), &instantiate)); m.insert(std::make_pair(std::string("filter_properties"), &instantiate)); #endif + m.insert(std::make_pair(std::string("zeros"), &instantiate)); + m.insert(std::make_pair(std::string("plane"), &instantiate)); m.insert(std::make_pair(std::string("mesh"), &instantiate)); initialized = true; } diff --git a/voxec.h b/voxec.h index b70385c..6029f4d 100644 --- a/voxec.h +++ b/voxec.h @@ -39,6 +39,11 @@ struct filtered_files_t {}; #include #include #include +#include +#include +#include +#include +#include #include #include @@ -155,6 +160,29 @@ class voxel_operation { virtual ~voxel_operation() {} }; +namespace { + json_logger::meta_data dump_info(abstract_voxel_storage* voxels) { + if (dynamic_cast(voxels)) { + auto left = dynamic_cast(voxels)->grid_offset(); + auto nc = dynamic_cast(voxels)->num_chunks(); + auto right = (left + nc.as()) - (decltype(left)::element_type)1; + auto sz = dynamic_cast(voxels)->voxel_size(); + auto szl = (long)dynamic_cast(voxels)->chunk_size(); + auto left_world = ((voxels->bounds()[0].as() + left * szl).as() * sz); + auto right_world = ((voxels->bounds()[1].as() + left * szl).as() * sz); + + return { + {"count", (long)voxels->count()}, + {"grid", left.format() + " - " + right.format()}, + {"bounds", voxels->bounds()[0].format() + " - " + voxels->bounds()[1].format()}, + {"world", left_world.format() + " - " + right_world.format()}, + {"bits", (long)voxels->value_bits()} + }; + } + return {}; + } +} + #ifdef WITH_IFC class op_parse_ifc_file : public voxel_operation { public: @@ -537,6 +565,32 @@ class op_print_components : public voxel_operation { } }; +class op_describe_components : public voxel_operation { +public: + const std::vector& arg_names() const { + static std::vector nm_ = { { true, "output_path", "string" }, { true, "input", "voxels" } }; + return nm_; + } + symbol_value invoke(const scope_map& scope) const { + const std::string output_path = scope.get_value("output_path"); + std::ofstream ofs(output_path.c_str()); + ofs << "["; + bool first = true; + abstract_voxel_storage* voxels = scope.get_value("input"); + connected_components((regular_voxel_storage*)voxels, [&ofs, &first](regular_voxel_storage* c) { + if (!first) { + ofs << ","; + } + auto info = dump_info(c); + ofs << json_logger::to_json_string(info); + first = false; + }); + ofs << "]"; + symbol_value v; + return v; + } +}; + class op_keep_components : public voxel_operation { public: const std::vector& arg_names() const { @@ -559,27 +613,6 @@ class op_keep_components : public voxel_operation { }; namespace { - json_logger::meta_data dump_info(abstract_voxel_storage* voxels) { - if (dynamic_cast(voxels)) { - auto left = dynamic_cast(voxels)->grid_offset(); - auto nc = dynamic_cast(voxels)->num_chunks(); - auto right = (left + nc.as()) - (decltype(left)::element_type)1; - auto sz = dynamic_cast(voxels)->voxel_size(); - auto szl = (long)dynamic_cast(voxels)->chunk_size(); - auto left_world = ((voxels->bounds()[0].as() + left * szl).as() * sz); - auto right_world = ((voxels->bounds()[1].as() + left * szl).as() * sz); - - return { - {"count", (long)voxels->count()}, - {"grid", left.format() + " - " + right.format()}, - {"bounds", voxels->bounds()[0].format() + " - " + voxels->bounds()[1].format()}, - {"world", left_world.format() + " - " + right_world.format()}, - {"bits", (long)voxels->value_bits()} - }; - } - return {}; - } - template void revoxelize_and_check_overlap(abstract_voxel_storage* voxels, const geometry_collection_t& surfaces, Fn fn, Fn2 fn2) { BRep_Builder B; @@ -899,6 +932,101 @@ class op_offset : public voxel_operation { } }; +class op_zeros : public voxel_operation { +public: + const std::vector& arg_names() const { + static std::vector nm_ = { { true, "input", "voxels" } }; + return nm_; + } + symbol_value invoke(const scope_map& scope) const { + abstract_voxel_storage* voxels = scope.get_value("input"); + // @todo also implement empty_copy_as() here. + return voxels->empty_copy(); + } +}; + +class op_plane : public voxel_operation { +public: + const std::vector& arg_names() const { + static std::vector nm_ = { { true, "input", "voxels" }, {true, "a", "real"}, {true, "b", "real"}, {true, "c", "real"}, {true, "d", "real"} }; + return nm_; + } + symbol_value invoke(const scope_map& scope) const { + abstract_voxel_storage* v = scope.get_value("input"); + + auto voxels = dynamic_cast(v); + if (voxels == nullptr) { + throw std::runtime_error("expected chunked storage"); + } + + auto result = voxels->empty_copy(); + + if (voxels->count() == 0) { + return result; + } + + gp_Pln pln( + scope.get_value("a"), + scope.get_value("b"), + scope.get_value("c"), + scope.get_value("d") + ); + + auto left = voxels->grid_offset(); + auto sz = dynamic_cast(voxels)->voxel_size(); + auto szl = (long)dynamic_cast(voxels)->chunk_size(); + auto left_world = ((voxels->bounds()[0].as() + left * szl).as() * sz); + auto right_world = ((voxels->bounds()[1].as() + left * szl).as() * sz); + + BRepPrimAPI_MakeBox mb(gp_Pnt( + left_world.get<0>(), + left_world.get<1>(), + left_world.get<2>() + ), gp_Pnt( + right_world.get<0>(), + right_world.get<1>(), + right_world.get<2>() + )); + + auto box = mb.Solid(); + static double inf = std::numeric_limits::infinity(); + std::array, 2> uv_min_max = {{ {{+inf, +inf}}, {{-inf,-inf}} }}; + + TopExp_Explorer exp(box, TopAbs_VERTEX); + for (; exp.More(); exp.Next()) { + auto p = BRep_Tool::Pnt(TopoDS::Vertex(exp.Current())); + auto p2d = ProjLib::Project(pln, p); + for (int i = 0; i < 2; ++i) { + auto v = p2d.ChangeCoord().ChangeCoord(i + 1); + auto& min_v = uv_min_max[0][i]; + auto& max_v = uv_min_max[1][i]; + if (v < min_v) min_v = v; + if (v > max_v) max_v = v; + } + } + + auto face = BRepBuilderAPI_MakeFace(pln, + uv_min_max[0][0], uv_min_max[1][0], + uv_min_max[0][1], uv_min_max[1][1] + ); + + auto face_inside = BRepAlgoAPI_Common(face, box).Shape(); + TopoDS_Compound C; + if (face_inside.ShapeType() == TopAbs_COMPOUND) { + C = TopoDS::Compound(face_inside); + } else { + BRep_Builder B; + B.MakeCompound(C); + B.Add(C, face_inside); + } + + BRepMesh_IncrementalMesh(C, 0.001); + + geometry_collection_t* single = new geometry_collection_t{ { 0, C} }; + return single; + } +}; + class op_offset_xy : public voxel_operation { public: const std::vector& arg_names() const { @@ -1315,7 +1443,7 @@ class op_create_prop_filter : public voxel_operation { class op_mesh : public voxel_operation { public: const std::vector& arg_names() const { - static std::vector nm_ = { { true, "input", "voxels" }, { true, "filename", "string"}, {false, "use_value", "integer"} }; + static std::vector nm_ = { { true, "input", "voxels" }, { true, "filename", "string"}, {false, "use_value", "integer"}, {false, "with_components", "integer"} }; return nm_; } symbol_value invoke(const scope_map& scope) const { @@ -1324,7 +1452,8 @@ class op_mesh : public voxel_operation { auto filename = scope.get_value("filename"); std::ofstream ofs(filename.c_str()); if (voxels->value_bits() == 1) { - ((regular_voxel_storage*)voxels)->obj_export(ofs); + auto with_components = scope.get_value_or("with_components", -1); + ((regular_voxel_storage*)voxels)->obj_export(ofs, with_components != 0); } else { ((regular_voxel_storage*)voxels)->obj_export(ofs, use_value != 1, use_value == 1); } From 86bfbf4cae3a56721751b3d3d0235662b7b20097 Mon Sep 17 00:00:00 2001 From: Thomas Krijnen Date: Sun, 21 Mar 2021 20:15:06 +0100 Subject: [PATCH 045/112] Add missing include --- voxec.h | 1 + 1 file changed, 1 insertion(+) diff --git a/voxec.h b/voxec.h index 6029f4d..52a0411 100644 --- a/voxec.h +++ b/voxec.h @@ -44,6 +44,7 @@ struct filtered_files_t {}; #include #include #include +#include #include #include From 7e54760e39606608238738d469451bd07562a1f1 Mon Sep 17 00:00:00 2001 From: Thomas Krijnen Date: Thu, 25 Mar 2021 21:27:06 +0100 Subject: [PATCH 046/112] intersection fixes --- storage.h | 35 +++++++++++++++++++++++++++-------- 1 file changed, 27 insertions(+), 8 deletions(-) diff --git a/storage.h b/storage.h index 4197e08..beb6c5b 100644 --- a/storage.h +++ b/storage.h @@ -541,9 +541,20 @@ class continuous_voxel_storage : public regular_voxel_storage { } void boolean_intersection_inplace(const abstract_voxel_storage* other_) { - const continuous_voxel_storage* other = (const continuous_voxel_storage*) other_; - for (size_t i = 0; i < size(); ++i) { - data_[i] &= other->data_[i]; + if (this->value_bits() == other_->value_bits()) { + const continuous_voxel_storage* other = (const continuous_voxel_storage*) other_; + for (size_t i = 0; i < size(); ++i) { + data_[i] &= other->data_[i]; + } + } else if (this->value_bits() == 32 && other_->value_bits() == 1) { + uint32_t zero = 0; + BEGIN_LOOP(size_t(0), dimx_, 0U, dimy_, 0U, dimz_) + if (!other_->Get(ijk)) { + Set(ijk, &zero); + } + END_LOOP; + } else { + throw std::runtime_error("Not implemented"); } calculate_count_(); calculate_bounds_(); @@ -1111,7 +1122,8 @@ class abstract_chunked_voxel_storage : public regular_voxel_storage { set_chunk(n, ijk, a->copy(n->next_slot())); } continue; - } else if (Sa == CK_EMPTY) { + } else if (Sa == CK_EMPTY && n->value_bits() == b->value_bits()) { + // When copying from b we need to check whether datatypes match set_chunk(n, ijk, b->copy(n->next_slot())); continue; } @@ -1128,15 +1140,17 @@ class abstract_chunked_voxel_storage : public regular_voxel_storage { continue; } } else if (mode == OP_INTERSECTION) { - if (Sa == CK_EMPTY || Sb == CK_EMPTY) { - // @todo this does not appear correct. When B is empty A needs to be *cleared* when not empty and in-place. + if (Sa == CK_EMPTY) { + continue; + } else if (Sb == CK_EMPTY) { + set_chunk(n, ijk, nullptr); continue; } else if (Sb == CK_FULL) { if (!inplace) { set_chunk(n, ijk, a->copy(n->next_slot())); } continue; - } else if (Sa == CK_FULL) { + } else if (Sa == CK_FULL && n->value_bits() == b->value_bits()) { set_chunk(n, ijk, b->copy(n->next_slot())); continue; } @@ -1148,7 +1162,12 @@ class abstract_chunked_voxel_storage : public regular_voxel_storage { release(get_chunk(this, ijk)); } - abstract_voxel_storage* a_explicit = a->is_explicit() ? (inplace ? a : a->copy(n->next_slot())) : a->make_explicit(n->next_slot()); + abstract_voxel_storage* a_explicit; + if (a == nullptr) { + a_explicit = a = n->get_or_create_chunk(ijk.as()); + } else { + a_explicit = a->is_explicit() ? (inplace ? a : a->copy(n->next_slot())) : a->make_explicit(n->next_slot()); + } // b is temporary, so not assigned next_slot abstract_voxel_storage* b_temp = b->is_explicit() ? nullptr : b->make_explicit(); abstract_voxel_storage* b_forced_explicit = b_temp ? b_temp : b; From 5794a4d950234ede36adbdc1cad22e9f75d600ba Mon Sep 17 00:00:00 2001 From: Thomas Krijnen Date: Thu, 25 Mar 2021 21:27:29 +0100 Subject: [PATCH 047/112] describe_group_by() --- voxec.cpp | 1 + voxec.h | 61 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 62 insertions(+) diff --git a/voxec.cpp b/voxec.cpp index c5eff7e..f76c4fc 100644 --- a/voxec.cpp +++ b/voxec.cpp @@ -37,6 +37,7 @@ voxel_operation_map::map_t& voxel_operation_map::map() { m.insert(std::make_pair(std::string("collapse_count"), &instantiate)); m.insert(std::make_pair(std::string("print_components"), &instantiate)); m.insert(std::make_pair(std::string("describe_components"), &instantiate)); + m.insert(std::make_pair(std::string("describe_group_by"), &instantiate)); m.insert(std::make_pair(std::string("keep_components"), &instantiate)); m.insert(std::make_pair(std::string("dump_surfaces"), &instantiate)); m.insert(std::make_pair(std::string("json_stats"), &instantiate)); diff --git a/voxec.h b/voxec.h index 52a0411..d58553d 100644 --- a/voxec.h +++ b/voxec.h @@ -592,6 +592,67 @@ class op_describe_components : public voxel_operation { } }; +class op_describe_group_by : public voxel_operation { +public: + const std::vector& arg_names() const { + static std::vector nm_ = { { true, "output_path", "string" }, { true, "input", "voxels" }, { true, "groups", "voxels" } }; + return nm_; + } + symbol_value invoke(const scope_map& scope) const { + const std::string output_path = scope.get_value("output_path"); + std::ofstream ofs(output_path.c_str()); + ofs << "["; + bool first = true; + abstract_voxel_storage* voxels = scope.get_value("input"); + + auto groups = (regular_voxel_storage*) scope.get_value("groups"); + + if (groups->value_bits() != 32) { + throw std::runtime_error("Expected a uint stored dataset for groups"); + } + + uint32_t v; + std::set vs; + for (auto& ijk : *groups) { + groups->Get(ijk, &v); + vs.insert(v); + } + + static bit_t desc_bits; + + for (auto& id : vs) { + if (!first) { + ofs << ","; + } + + // A {0,1} dataset of `groups`==`id` + auto where_id = groups->empty_copy_as(&desc_bits); + for (auto& ijk : *groups) { + groups->Get(ijk, &v); + if (v == id) { + where_id->Set(ijk); + } + } + + auto c = voxels->boolean_intersection(where_id); + + auto info = dump_info(c); + info["id"] = static_cast(id); + ofs << json_logger::to_json_string(info); + + delete c; + delete where_id; + + first = false; + } + + ofs << "]"; + + symbol_value v_null; + return v_null; + } +}; + class op_keep_components : public voxel_operation { public: const std::vector& arg_names() const { From b8eeb3309e663bcdca8d0d560f104154f2902974 Mon Sep 17 00:00:00 2001 From: Thomas Krijnen Date: Fri, 26 Mar 2021 14:06:35 +0100 Subject: [PATCH 048/112] Use {0,1} voxelset for traversal on PRODUCT_ID method --- processor.h | 31 +++++++++---------------------- 1 file changed, 9 insertions(+), 22 deletions(-) diff --git a/processor.h b/processor.h index edf5bd6..39ff1c6 100644 --- a/processor.h +++ b/processor.h @@ -155,6 +155,7 @@ class abstract_processor { class processor : public abstract_processor { private: + bit_t use_bits; factory factory_; abstract_voxel_storage* voxels_; abstract_voxel_storage* voxels_temp_; @@ -175,7 +176,7 @@ class processor : public abstract_processor { processor(abstract_voxel_storage* storage, const std::function& progress) : voxels_(storage) - , voxels_temp_(storage->empty_copy()) + , voxels_temp_(storage->empty_copy_as(&use_bits)) , progress_(progress) , use_copy_(true) , use_scanline_(true) {} @@ -205,33 +206,19 @@ class processor : public abstract_processor { to_write->boolean_union_inplace(filled); delete filled; - output.intermediate_result(it->first, voxels_, voxels_temp_); - - int n = voxels_->value_bits(); if (volume.value == PRODUCT_ID) { + int n = voxels_->value_bits(); // @todo this still needs to be generalized if (n != 32) { throw std::runtime_error("Unable to assign product ids to this voxel value type"); } - if (auto cvs32 = dynamic_cast*>(voxels_)) { - size_t nchunksx, nchunksy, nchunksz; - cvs32->num_chunks().tie(nchunksx, nchunksy, nchunksz); - BEGIN_LOOP(size_t(0), nchunksx, 0U, nchunksy, 0U, nchunksz) - auto c = cvs32->get_chunk(ijk); - if (c != nullptr) { - if (!c->is_explicit()) { - throw std::runtime_error("Not implemented"); - } - auto c32 = (continuous_voxel_storage*) c; - auto d = c32->data(); - for (size_t i = 0; i < c32->size(); ++i) { - if (d[i] == 1) { - d[i] = it->first; - } - } - } - END_LOOP; + auto input = (regular_voxel_storage*)to_write; + uint32_t v = it->first; + for (auto& ijk : *input) { + voxels_->Set(ijk, &v); } + } else { + output.intermediate_result(it->first, voxels_, voxels_temp_); } if (use_copy_) { From 562fa5a298cd4357730f17c73e23ba36bd3a961d Mon Sep 17 00:00:00 2001 From: Thomas Krijnen Date: Fri, 26 Mar 2021 14:06:57 +0100 Subject: [PATCH 049/112] Fix hardcoded chunk size --- voxec.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/voxec.h b/voxec.h index d58553d..13449c1 100644 --- a/voxec.h +++ b/voxec.h @@ -487,7 +487,7 @@ namespace { } if (std::is_same::value) { - chunked_voxel_storage* storage = new chunked_voxel_storage(x1, y1, z1, vsize, nx, ny, nz, 64); + chunked_voxel_storage* storage = new chunked_voxel_storage(x1, y1, z1, vsize, nx, ny, nz, chunksize); progress_writer progress("voxelize"); processor pr(storage, progress); pr.use_scanline() = false; From 98e17129e2b72ff9f8282cfb5b0ade6778be1acd Mon Sep 17 00:00:00 2001 From: Thomas Krijnen Date: Fri, 26 Mar 2021 14:07:16 +0100 Subject: [PATCH 050/112] Add chunk size to log output --- voxec.h | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/voxec.h b/voxec.h index 13449c1..c7b6c34 100644 --- a/voxec.h +++ b/voxec.h @@ -164,6 +164,7 @@ class voxel_operation { namespace { json_logger::meta_data dump_info(abstract_voxel_storage* voxels) { if (dynamic_cast(voxels)) { + auto csize = dynamic_cast(voxels)->chunk_size(); auto left = dynamic_cast(voxels)->grid_offset(); auto nc = dynamic_cast(voxels)->num_chunks(); auto right = (left + nc.as()) - (decltype(left)::element_type)1; @@ -177,7 +178,8 @@ namespace { {"grid", left.format() + " - " + right.format()}, {"bounds", voxels->bounds()[0].format() + " - " + voxels->bounds()[1].format()}, {"world", left_world.format() + " - " + right_world.format()}, - {"bits", (long)voxels->value_bits()} + {"bits", (long)voxels->value_bits()}, + {"chunk_size", (long) csize} }; } return {}; From 9ad66d4eabf6ff109bdfae6b2186cf946358df2e Mon Sep 17 00:00:00 2001 From: Thomas Krijnen Date: Sat, 10 Apr 2021 13:22:31 +0200 Subject: [PATCH 051/112] shift() on uint storage --- shift.h | 30 +++++++++++++++++++++++------- 1 file changed, 23 insertions(+), 7 deletions(-) diff --git a/shift.h b/shift.h index cecb80b..c08221b 100644 --- a/shift.h +++ b/shift.h @@ -14,14 +14,30 @@ class shift { auto dxyz = make_vec(dx, dy, dz); - BEGIN_LOOP_I2(bounds[0], bounds[1]) - if (storage->Get(ijk)) { - auto ijk2 = ijk.as() + dxyz; - if ((ijk2 >= zero).all() && (ijk2 < extents).all()) { - shifted->Set(ijk2.as()); + bool use_value = storage->value_bits() == 32; + + if (use_value) { + uint32_t v; + BEGIN_LOOP_I2(bounds[0], bounds[1]) + storage->Get(ijk, &v); + if (v) { + auto ijk2 = ijk.as() + dxyz; + if ((ijk2 >= zero).all() && (ijk2 < extents).all()) { + shifted->Set(ijk2.as(), &v); + } + } + END_LOOP; + + } else { + BEGIN_LOOP_I2(bounds[0], bounds[1]) + if (storage->Get(ijk)) { + auto ijk2 = ijk.as() + dxyz; + if ((ijk2 >= zero).all() && (ijk2 < extents).all()) { + shifted->Set(ijk2.as()); + } } - } - END_LOOP; + END_LOOP; + } return shifted; } From c0b773d3f4db2b581df52dcf19fe40d5f99e289a Mon Sep 17 00:00:00 2001 From: Thomas Krijnen Date: Sat, 10 Apr 2021 13:23:45 +0200 Subject: [PATCH 052/112] sweep until --- collapse.h | 4 ++++ shift.h | 2 ++ sweep.h | 20 ++++++++++++++++++-- voxec.h | 8 +++++++- 4 files changed, 31 insertions(+), 3 deletions(-) diff --git a/collapse.h b/collapse.h index e249516..bd31772 100644 --- a/collapse.h +++ b/collapse.h @@ -5,6 +5,8 @@ class collapse { public: + abstract_voxel_storage* until = nullptr; + regular_voxel_storage* operator()(abstract_voxel_storage* storage, int dx, int dy, int dz) { int d[3] = { dx, dy, dz }; int nonzero = 0; @@ -53,6 +55,8 @@ class collapse { // @todo merge with above class collapse_count { public: + abstract_voxel_storage* until = nullptr; + regular_voxel_storage* operator()(abstract_voxel_storage* storage, int dx, int dy, int dz) { int d[3] = { dx, dy, dz }; int nonzero = 0; diff --git a/shift.h b/shift.h index c08221b..6338dfa 100644 --- a/shift.h +++ b/shift.h @@ -5,6 +5,8 @@ class shift { public: + abstract_voxel_storage* until = nullptr; + regular_voxel_storage* operator()(abstract_voxel_storage* storage, int dx, int dy, int dz) { regular_voxel_storage* shifted = (regular_voxel_storage*)storage->empty_copy(); auto bounds = storage->bounds(); diff --git a/sweep.h b/sweep.h index a636e01..d716ac9 100644 --- a/sweep.h +++ b/sweep.h @@ -5,6 +5,8 @@ class sweep { public: + abstract_voxel_storage* until = nullptr; + regular_voxel_storage* operator()(abstract_voxel_storage* storage, int dx, int dy, int dz) { int d[3] = { dx, dy, dz }; int nonzero = 0; @@ -37,15 +39,29 @@ class sweep { storage->Get(ijk, &v); } auto ijk2 = ijk.as(); - for (int i = 1; i < std::abs(d[D]) * v; ++i) { + int i = 1; + while (until || (i < std::abs(d[D]) * v)) { + if (pos) { ijk2.get(D)++; - } else { + } + else { ijk2.get(D)--; } + + if (until) { + if (until->Get(ijk2.as())) { + break; + } + } + if ((ijk2 >= zero).all() && (ijk2 < extents).all()) { swepts->Set(ijk2.as()); + } else { + break; } + + ++i; } } END_LOOP; diff --git a/voxec.h b/voxec.h index c7b6c34..b2f7e13 100644 --- a/voxec.h +++ b/voxec.h @@ -1276,7 +1276,7 @@ template class op_geom : public voxel_operation { public: const std::vector& arg_names() const { - static std::vector nm_ = { { true, "input", "voxels" }, { true, "dx", "integer|real" },{ true, "dy", "integer|real" },{ true, "dz", "integer|real" } }; + static std::vector nm_ = { { true, "input", "voxels" }, { true, "dx", "integer|real" },{ true, "dy", "integer|real" },{ true, "dz", "integer|real" }, { false, "until", "voxels" } }; return nm_; } symbol_value invoke(const scope_map& scope) const { @@ -1286,7 +1286,13 @@ class op_geom : public voxel_operation { int dy = scope.get_length("dy"); int dz = scope.get_length("dz"); + abstract_voxel_storage* until = nullptr; + try { + until = scope.get_value("until"); + } catch (scope_map::not_in_scope&) { } + T s; + s.until = until; return s(voxels, dx, dy, dz); } }; From 9d08e2c7afd03b3274701ecfc42bae802b8d7194 Mon Sep 17 00:00:00 2001 From: Thomas Krijnen Date: Sat, 10 Apr 2021 13:24:06 +0200 Subject: [PATCH 053/112] constant_like(type=) --- voxec.h | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/voxec.h b/voxec.h index b2f7e13..bc41ad1 100644 --- a/voxec.h +++ b/voxec.h @@ -1391,13 +1391,26 @@ class op_traverse : public voxel_operation { class op_constant_like : public voxel_operation { public: const std::vector& arg_names() const { - static std::vector nm_ = { { true, "input", "voxels" }, { true, "value", "integer" } }; + static std::vector nm_ = { { true, "input", "voxels" }, { true, "value", "integer" }, { false, "type", "string"} }; return nm_; } symbol_value invoke(const scope_map& scope) const { regular_voxel_storage* voxels = (regular_voxel_storage*)scope.get_value("input"); int value = scope.get_value("value"); - abstract_chunked_voxel_storage* output = (abstract_chunked_voxel_storage*)voxels->empty_copy(); + + abstract_chunked_voxel_storage* output; + + if (scope.has("type")) { + if (scope.get_value("type") == "uint") { + voxel_uint32_t fmt; + output = (abstract_chunked_voxel_storage*)voxels->empty_copy_as(&fmt); + } else { + throw std::runtime_error("not implemented"); + } + } else { + output = (abstract_chunked_voxel_storage*)voxels->empty_copy(); + } + if (value == 1) { auto i = make_vec(0U, 0U, 0U); auto j = output->num_chunks(); From 758b9e5eb7a241aa419ef0d7b286ff9e5c388eb6 Mon Sep 17 00:00:00 2001 From: Thomas Krijnen Date: Sat, 10 Apr 2021 13:24:29 +0200 Subject: [PATCH 054/112] mesh(groups=) --- voxec.h | 94 +++++++++++++++++++++++++++++++++++++++------------------ 1 file changed, 65 insertions(+), 29 deletions(-) diff --git a/voxec.h b/voxec.h index bc41ad1..9de10db 100644 --- a/voxec.h +++ b/voxec.h @@ -594,24 +594,9 @@ class op_describe_components : public voxel_operation { } }; -class op_describe_group_by : public voxel_operation { -public: - const std::vector& arg_names() const { - static std::vector nm_ = { { true, "output_path", "string" }, { true, "input", "voxels" }, { true, "groups", "voxels" } }; - return nm_; - } - symbol_value invoke(const scope_map& scope) const { - const std::string output_path = scope.get_value("output_path"); - std::ofstream ofs(output_path.c_str()); - ofs << "["; - bool first = true; - abstract_voxel_storage* voxels = scope.get_value("input"); - - auto groups = (regular_voxel_storage*) scope.get_value("groups"); - - if (groups->value_bits() != 32) { - throw std::runtime_error("Expected a uint stored dataset for groups"); - } +namespace { + template + void group_by(regular_voxel_storage* groups, abstract_voxel_storage* voxels, Fn fn) { uint32_t v; std::set vs; @@ -622,11 +607,12 @@ class op_describe_group_by : public voxel_operation { static bit_t desc_bits; - for (auto& id : vs) { - if (!first) { - ofs << ","; - } + std::cout << "voxels " << voxels->count() << std::endl; + + std::ofstream ofs0("debug-0.obj"); + ((regular_voxel_storage*)voxels)->obj_export(ofs0, false, false); + for (auto& id : vs) { // A {0,1} dataset of `groups`==`id` auto where_id = groups->empty_copy_as(&desc_bits); for (auto& ijk : *groups) { @@ -638,15 +624,52 @@ class op_describe_group_by : public voxel_operation { auto c = voxels->boolean_intersection(where_id); + std::cout << id << " " << where_id->count() << " " << c->count() << std::endl; + + std::ofstream ofs1("debug-1-" + boost::lexical_cast(id) + ".obj"); + std::ofstream ofs2("debug-2-" + boost::lexical_cast(id) + ".obj"); + + ((regular_voxel_storage*)where_id)->obj_export(ofs1, false, false); + ((regular_voxel_storage*)c)->obj_export(ofs2, false, false); + + delete where_id; + + fn(id, c); + } + } +} + +class op_describe_group_by : public voxel_operation { +public: + const std::vector& arg_names() const { + static std::vector nm_ = { { true, "output_path", "string" }, { true, "input", "voxels" }, { true, "groups", "voxels" } }; + return nm_; + } + symbol_value invoke(const scope_map& scope) const { + const std::string output_path = scope.get_value("output_path"); + std::ofstream ofs(output_path.c_str()); + ofs << "["; + bool first = true; + abstract_voxel_storage* voxels = scope.get_value("input"); + + auto groups = (regular_voxel_storage*) scope.get_value("groups"); + + if (groups->value_bits() != 32) { + throw std::runtime_error("Expected a uint stored dataset for groups"); + } + + group_by(groups, voxels, [&ofs, &first](uint32_t id, abstract_voxel_storage* c) { + if (!first) { + ofs << ","; + } auto info = dump_info(c); info["id"] = static_cast(id); ofs << json_logger::to_json_string(info); delete c; - delete where_id; first = false; - } + }); ofs << "]"; @@ -1526,19 +1549,32 @@ class op_create_prop_filter : public voxel_operation { class op_mesh : public voxel_operation { public: const std::vector& arg_names() const { - static std::vector nm_ = { { true, "input", "voxels" }, { true, "filename", "string"}, {false, "use_value", "integer"}, {false, "with_components", "integer"} }; + static std::vector nm_ = { { true, "input", "voxels" }, { true, "filename", "string"}, {false, "use_value", "integer"}, {false, "with_components", "integer"}, { false, "groups", "voxels" } }; return nm_; } symbol_value invoke(const scope_map& scope) const { auto voxels = scope.get_value("input"); auto use_value = scope.get_value_or("use_value", -1); auto filename = scope.get_value("filename"); + + auto groups = (regular_voxel_storage*) scope.get_value_or("groups", nullptr); + std::ofstream ofs(filename.c_str()); - if (voxels->value_bits() == 1) { - auto with_components = scope.get_value_or("with_components", -1); - ((regular_voxel_storage*)voxels)->obj_export(ofs, with_components != 0); + + if (groups) { + obj_export_helper helper(ofs); + group_by(groups, voxels, [&helper, &ofs](uint32_t id, abstract_voxel_storage* c) { + ofs << "g id-" << id << "\n"; + ((regular_voxel_storage*)c)->obj_export(helper, false, false); + }); } else { - ((regular_voxel_storage*)voxels)->obj_export(ofs, use_value != 1, use_value == 1); + + if (voxels->value_bits() == 1) { + auto with_components = scope.get_value_or("with_components", -1); + ((regular_voxel_storage*)voxels)->obj_export(ofs, with_components != 0); + } else { + ((regular_voxel_storage*)voxels)->obj_export(ofs, use_value != 1, use_value == 1); + } } symbol_value v; return v; From 20fdc74dd1815e0757783382777d0597d41aca1a Mon Sep 17 00:00:00 2001 From: Thomas Krijnen Date: Sat, 10 Apr 2021 15:23:50 +0200 Subject: [PATCH 055/112] Remove debugging --- voxec.h | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/voxec.h b/voxec.h index 9de10db..4c75034 100644 --- a/voxec.h +++ b/voxec.h @@ -607,11 +607,6 @@ namespace { static bit_t desc_bits; - std::cout << "voxels " << voxels->count() << std::endl; - - std::ofstream ofs0("debug-0.obj"); - ((regular_voxel_storage*)voxels)->obj_export(ofs0, false, false); - for (auto& id : vs) { // A {0,1} dataset of `groups`==`id` auto where_id = groups->empty_copy_as(&desc_bits); @@ -624,14 +619,6 @@ namespace { auto c = voxels->boolean_intersection(where_id); - std::cout << id << " " << where_id->count() << " " << c->count() << std::endl; - - std::ofstream ofs1("debug-1-" + boost::lexical_cast(id) + ".obj"); - std::ofstream ofs2("debug-2-" + boost::lexical_cast(id) + ".obj"); - - ((regular_voxel_storage*)where_id)->obj_export(ofs1, false, false); - ((regular_voxel_storage*)c)->obj_export(ofs2, false, false); - delete where_id; fn(id, c); From 7d0b1cf846a9ca6081896854656a8c0faa9d25ad Mon Sep 17 00:00:00 2001 From: Thomas Krijnen Date: Sun, 11 Apr 2021 13:45:40 +0200 Subject: [PATCH 056/112] don't traverse chunks on max_depth --- traversal.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/traversal.h b/traversal.h index 94bea10..16644fb 100644 --- a/traversal.h +++ b/traversal.h @@ -341,7 +341,7 @@ class visitor { } auto c = pos.second / chunk_size_; - if (DofT::use_chunks && is_chunked_ && chunks_.get(c.template get<0>(), c.template get<1>(), c.template get<2>()) != CHUNK_MIXED) { + if (!max_depth && DofT::use_chunks && is_chunked_ && chunks_.get(c.template get<0>(), c.template get<1>(), c.template get<2>()) != CHUNK_MIXED) { auto lower = c * chunk_size_; auto upper = (c + 1U) * chunk_size_; From 8269dd4c09d80a692ad465d36f8ae89832b7f66a Mon Sep 17 00:00:00 2001 From: Thomas Krijnen Date: Sun, 11 Apr 2021 13:46:20 +0200 Subject: [PATCH 057/112] uint storage option for traversal --- traversal.h | 3 +++ voxec.h | 46 +++++++++++++++++++++++++++++++++------------- 2 files changed, 36 insertions(+), 13 deletions(-) diff --git a/traversal.h b/traversal.h index 16644fb..a841c99 100644 --- a/traversal.h +++ b/traversal.h @@ -350,6 +350,7 @@ class visitor { chunk_neighbours_queue_add_(pos.first, pos.second, c); set_visited_chunk_(c); } else if (post_condition_(pos.second)) { + depth = pos.first; fn(tagged_index{ tagged_index::VOXEL, pos.second }); neighbours_queue_add_(pos); @@ -358,6 +359,8 @@ class visitor { } public: + double depth = std::numeric_limits::quiet_NaN(); + // @todo max_depth doesn't work correctly with implicit voxel storage boost::optional max_depth; typename queue_type::type queue; diff --git a/voxec.h b/voxec.h index 4c75034..0d44cc0 100644 --- a/voxec.h +++ b/voxec.h @@ -1343,23 +1343,13 @@ class op_less : public op_compare> {}; class op_traverse : public voxel_operation { public: const std::vector& arg_names() const { - static std::vector nm_ = { { true, "input", "voxels" }, { true, "seed", "voxels" }, { false, "depth", "integer|real" }, { false, "connectedness", "integer" } }; + static std::vector nm_ = { { true, "input", "voxels" }, { true, "seed", "voxels" }, { false, "depth", "integer|real" }, { false, "connectedness", "integer" }, {false, "type", "string"} }; return nm_; } symbol_value invoke(const scope_map& scope) const { regular_voxel_storage* voxels = (regular_voxel_storage*) scope.get_value("input"); regular_voxel_storage* seed = (regular_voxel_storage*) scope.get_value("seed"); - regular_voxel_storage* output = (regular_voxel_storage*)voxels->empty_copy(); - - auto callback = [this, output](const tagged_index& pos) { - if (pos.which == tagged_index::VOXEL) { - output->Set(pos.pos); - } else { - ((abstract_chunked_voxel_storage*)output)->create_constant(pos.pos, 1U); - } - }; - boost::optional max_depth; try { max_depth = scope.get_length("depth"); @@ -1367,8 +1357,7 @@ class op_traverse : public voxel_operation { // traversal with unconstrained depth } - // Type erasure to get a runtime argument into a template arg. - std::function run_visitor; + bool use_value = scope.has("type") && scope.get_value("type") == "uint"; int C = 6; try { @@ -1380,12 +1369,40 @@ class op_traverse : public voxel_operation { if (C != 6 && C != 26) { throw std::runtime_error("Connectedness should be 6 or 26"); } + + if (use_value && C != 26) { + throw std::runtime_error("Connectedness should be 26 when using value"); + } + + regular_voxel_storage* output; + if (use_value) { + voxel_uint32_t fmt; + output = (regular_voxel_storage*)voxels->empty_copy_as(&fmt); + } else { + output = (regular_voxel_storage*)voxels->empty_copy(); + } visitor<6> v6; visitor<26> v26; v6.max_depth = max_depth; v26.max_depth = max_depth; + uint32_t val; + + auto callback = [this, output, &v26, &val, &use_value](const tagged_index& pos) { + if (use_value) { + val = v26.depth * 10. + 1; + output->Set(pos.pos, &val); + } else if (pos.which == tagged_index::VOXEL) { + output->Set(pos.pos); + } else { + ((abstract_chunked_voxel_storage*)output)->create_constant(pos.pos, 1U); + } + }; + + // Type erasure to get a runtime argument into a template arg. + std::function run_visitor; + if (C == 6) { run_visitor = std::ref(v6); } else { @@ -1414,6 +1431,9 @@ class op_constant_like : public voxel_operation { if (scope.get_value("type") == "uint") { voxel_uint32_t fmt; output = (abstract_chunked_voxel_storage*)voxels->empty_copy_as(&fmt); + } else if (scope.get_value("type") == "bit") { + bit_t fmt; + output = (abstract_chunked_voxel_storage*)voxels->empty_copy_as(&fmt); } else { throw std::runtime_error("not implemented"); } From c6fc6fa7ed0a066b0b25bb230324f5b672199a56 Mon Sep 17 00:00:00 2001 From: Thomas Krijnen Date: Sun, 11 Apr 2021 13:46:43 +0200 Subject: [PATCH 058/112] export_csv() --- voxec.cpp | 1 + voxec.h | 46 ++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 47 insertions(+) diff --git a/voxec.cpp b/voxec.cpp index f76c4fc..4ac1c66 100644 --- a/voxec.cpp +++ b/voxec.cpp @@ -52,6 +52,7 @@ voxel_operation_map::map_t& voxel_operation_map::map() { m.insert(std::make_pair(std::string("zeros"), &instantiate)); m.insert(std::make_pair(std::string("plane"), &instantiate)); m.insert(std::make_pair(std::string("mesh"), &instantiate)); + m.insert(std::make_pair(std::string("export_csv"), &instantiate)); initialized = true; } return m; diff --git a/voxec.h b/voxec.h index 0d44cc0..df79cbe 100644 --- a/voxec.h +++ b/voxec.h @@ -1588,6 +1588,52 @@ class op_mesh : public voxel_operation { } }; +class op_export_csv : public voxel_operation { +public: + const std::vector& arg_names() const { + static std::vector nm_ = { { true, "input", "voxels" }, { true, "filename", "string"} }; + return nm_; + } + symbol_value invoke(const scope_map& scope) const { + auto voxels = (regular_voxel_storage*) scope.get_value("input"); + auto filename = scope.get_value("filename"); + std::ofstream ofs(filename.c_str()); + + bool use_value = voxels->value_bits() == 32; + + auto sz = dynamic_cast(voxels)->voxel_size(); + auto szl = (long)dynamic_cast(voxels)->chunk_size(); + auto left = dynamic_cast(voxels)->grid_offset(); + + uint32_t v; + + for (auto ijk : *voxels) { + if (use_value) { + voxels->Get(ijk, &v); + if (v == 0) { + // @todo why is this needed? + continue; + } + } + + auto xyz = (ijk.as() + left * szl).as() * sz; + + ofs << xyz.get<0>() << "," + << xyz.get<1>() << "," + << xyz.get<2>(); + + if (use_value) { + ofs << "," << v; + } + + ofs << "\n"; + } + + symbol_value v_null; + return v_null; + } +}; + template T* instantiate() { return new T(); From acba092f2bb8d10271e2bfe3a0e2e5e8342a3bb2 Mon Sep 17 00:00:00 2001 From: johltn Date: Mon, 12 Apr 2021 08:31:20 +0200 Subject: [PATCH 059/112] Abstract writer class with Hdf writer subclass --- tests/test_hdf.cpp | 23 ++++++++++++++++++--- writer.h | 51 +++++++++++++++++++++++++++++++++++++++------- 2 files changed, 64 insertions(+), 10 deletions(-) diff --git a/tests/test_hdf.cpp b/tests/test_hdf.cpp index ffe96b9..54b9927 100644 --- a/tests/test_hdf.cpp +++ b/tests/test_hdf.cpp @@ -8,14 +8,31 @@ TEST(HdfFileName, HDF) { + auto storage = new chunked_voxel_storage(0., 0., 0., 0.1, 200, 150, 10, 32); + + + BRepBuilderAPI_MakePolygon mp(gp_Pnt(1, 1, 0), gp_Pnt(16, 1, 0), gp_Pnt(16, 9.8, 0), gp_Pnt(1, 9.8, 0), true); + BRepBuilderAPI_MakeFace mf(mp.Wire()); + TopoDS_Face face = mf.Face(); + + auto vox = voxelizer(face, storage); + vox.Convert(); + + /*HDF WRITER*/ + hdf_writer writer; + writer.SetVoxels(storage); + writer.Write("test_hdf_writer.vox"); + /*HDF WRITER*/ + + const H5std_string FILE_NAME("SDS.h5"); const H5std_string DATASET_NAME("IntArray"); - const int NX = 5; + const int NX = 5; const int NY = 6; const int RANK = 2; int i, j; - int data[NX][NY]; + int data[NX][NY]; for (j = 0; j < NX; j++) { for (i = 0; i < NY; i++) @@ -23,7 +40,7 @@ TEST(HdfFileName, HDF) { } H5::H5File file(FILE_NAME, H5F_ACC_TRUNC); - hsize_t dimsf[2]; + hsize_t dimsf[2]; dimsf[0] = NX; dimsf[1] = NY; H5::DataSpace dataspace(RANK, dimsf); diff --git a/writer.h b/writer.h index 2ea3842..a082fb7 100644 --- a/writer.h +++ b/writer.h @@ -2,13 +2,27 @@ #define WRITER_H #include "storage.h" +#include "H5Cpp.h" #include #include -class voxel_writer { -private: + +class abstract_writer { + +public: abstract_voxel_storage* voxels_; + void SetVoxels(abstract_voxel_storage* voxels) { + voxels_ = voxels; + } + + virtual void Write(const std::string& fnc) = 0; + + +}; + +class voxel_writer :public abstract_writer { +private: std::ofstream& assert_good_(const std::string& fn, std::ofstream& fs) { if (!fs.good()) { @@ -16,12 +30,7 @@ class voxel_writer { } return fs; } - public: - void SetVoxels(abstract_voxel_storage* voxels) { - voxels_ = voxels; - } - void Write(const std::string& fnc) { { std::string fn = fnc + std::string(".index"); @@ -46,4 +55,32 @@ class voxel_writer { } }; + + +class hdf_writer :public abstract_writer { + +public: + void Write(const std::string& fnc) { + + chunked_voxel_storage* storage = (chunked_voxel_storage*)voxels_; + + for (int i = 0; i < storage->num_chunks().get(1); i++) { + + auto c = storage->get_chunk(make_vec(i, 1, 0)); + + if (c && c->is_explicit()) { + continuous_voxel_storage* convox = (continuous_voxel_storage*)voxels_; + bit_t::storage_type* data = convox->data(); + + + } + } + } + + +}; + + + + #endif From abe3c74afe210d9b6742c286f81ff3bbefff9572 Mon Sep 17 00:00:00 2001 From: johltn Date: Mon, 12 Apr 2021 19:34:24 +0200 Subject: [PATCH 060/112] Start iteration through continuous chunk --- writer.h | 27 ++++++++++++++++++++------- 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/writer.h b/writer.h index a082fb7..622e758 100644 --- a/writer.h +++ b/writer.h @@ -64,16 +64,29 @@ class hdf_writer :public abstract_writer { chunked_voxel_storage* storage = (chunked_voxel_storage*)voxels_; - for (int i = 0; i < storage->num_chunks().get(1); i++) { - - auto c = storage->get_chunk(make_vec(i, 1, 0)); + for (int i = 0; i < storage->num_chunks().get(0); i++) { + auto c = storage->get_chunk(make_vec(i, 1, 0)); if (c && c->is_explicit()) { - continuous_voxel_storage* convox = (continuous_voxel_storage*)voxels_; - bit_t::storage_type* data = convox->data(); - - + continuous_voxel_storage* convox = (continuous_voxel_storage*)c; + auto d = convox->data(); + //std::cout << c->value_bits(); + for (int j = 0; j < convox->size(); j++) { + auto vd = convox->data()[j]; + std::bitset<8> b(convox->data()[j]); + std::cout << (int)convox->data()[j]<is_constant() ){ + std::cout << "Constant handling to implement."< Date: Mon, 12 Apr 2021 22:20:49 +0200 Subject: [PATCH 061/112] Read bits stream and start with HDF5 --- tests/test_hdf.cpp | 16 ++++++++------- writer.h | 49 +++++++++++++++++++++++++++++++++++++--------- 2 files changed, 49 insertions(+), 16 deletions(-) diff --git a/tests/test_hdf.cpp b/tests/test_hdf.cpp index 54b9927..bed2e5c 100644 --- a/tests/test_hdf.cpp +++ b/tests/test_hdf.cpp @@ -5,18 +5,20 @@ #include #include +#include TEST(HdfFileName, HDF) { - auto storage = new chunked_voxel_storage(0., 0., 0., 0.1, 200, 150, 10, 32); + auto storage = new chunked_voxel_storage(0., 0., 0., 0.1, 1000, 1000, 100, 32); + for (int i = 0; i < 10; ++i) { + for (int j = 0; j < 10; ++j) { + BRepPrimAPI_MakeBox mb(gp_Pnt(10 * i, 10 * j, 0.), 8., 8., 8.); - BRepBuilderAPI_MakePolygon mp(gp_Pnt(1, 1, 0), gp_Pnt(16, 1, 0), gp_Pnt(16, 9.8, 0), gp_Pnt(1, 9.8, 0), true); - BRepBuilderAPI_MakeFace mf(mp.Wire()); - TopoDS_Face face = mf.Face(); - - auto vox = voxelizer(face, storage); - vox.Convert(); + auto vox = voxelizer(mb.Solid(), storage); + vox.Convert(); + } + } /*HDF WRITER*/ hdf_writer writer; diff --git a/writer.h b/writer.h index 622e758..59c48e4 100644 --- a/writer.h +++ b/writer.h @@ -4,6 +4,7 @@ #include "storage.h" #include "H5Cpp.h" +#include #include #include @@ -61,33 +62,63 @@ class hdf_writer :public abstract_writer { public: void Write(const std::string& fnc) { + const int RANK = 2; + const H5std_string FILE_NAME(fnc); + const H5std_string DATASET_NAME("Continuous"); + + H5::H5File file(FILE_NAME, H5F_ACC_TRUNC); + + hsize_t dims[2] = { 3, 3 }; // dataset dimensions at creation + hsize_t maxdims[2] = { H5S_UNLIMITED, H5S_UNLIMITED }; + H5::DataSpace mspace1(RANK, dims, maxdims); + + H5::DSetCreatPropList cparms; + hsize_t chunk_dims[2] = { 2, 5 }; + cparms.setChunk(RANK, chunk_dims); + + H5::DataSet dataset = file.createDataSet(DATASET_NAME, H5::PredType::NATIVE_INT, mspace1, cparms); chunked_voxel_storage* storage = (chunked_voxel_storage*)voxels_; for (int i = 0; i < storage->num_chunks().get(0); i++) { - auto c = storage->get_chunk(make_vec(i, 1, 0)); + auto c = storage->get_chunk(make_vec(i, 0, 0)); if (c && c->is_explicit()) { continuous_voxel_storage* convox = (continuous_voxel_storage*)c; auto d = convox->data(); - //std::cout << c->value_bits(); + for (int j = 0; j < convox->size(); j++) { auto vd = convox->data()[j]; - std::bitset<8> b(convox->data()[j]); - std::cout << (int)convox->data()[j]< b(convox->data()[j]); + + /*std::cout << b<data()[j] << std::endl; + for (int k = 0; k < 8; k++) { + int bitmask = 1 << k; + int masked = vd & bitmask; + int voxbit = masked >> vd; + std::cout << voxbit; + } + } - if (c && c->is_constant() ){ - std::cout << "Constant handling to implement."<is_constant()) { + std::cout << "Constant handling to implement." << std::endl; + } else { - std::cout << "Plane handling to implement."< Date: Sun, 25 Apr 2021 11:39:27 +0200 Subject: [PATCH 062/112] initialize num_threads --- processor.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/processor.h b/processor.h index 39ff1c6..08e44ef 100644 --- a/processor.h +++ b/processor.h @@ -267,8 +267,9 @@ class threaded_processor : public abstract_processor { threaded_processor(double x1, double y1, double z1, double d, int nx, int ny, int nz, size_t chunk_size, size_t num_threads, progress_writer& p) : x1_(x1), y1_(y1), z1_(z1), d_(d), nx_(nx), ny_(ny), nz_(nz), chunk_size_(chunk_size), num_threads_(num_threads > 0 ? num_threads : std::thread::hardware_concurrency()), p_(p), result_(nullptr), voxels_(nullptr) {} - threaded_processor(abstract_voxel_storage* storage, progress_writer& progress) + threaded_processor(abstract_voxel_storage* storage, size_t num_threads, progress_writer& progress) : voxels_(storage) + , num_threads_(num_threads) , p_(progress) , result_(nullptr) {} From d96acbfec1e2bb988105239a5fd9c1c853379a49 Mon Sep 17 00:00:00 2001 From: Thomas Krijnen Date: Sun, 25 Apr 2021 11:39:53 +0200 Subject: [PATCH 063/112] don't assign to first when skipped over interval --- processor.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/processor.h b/processor.h index 08e44ef..d3d68ad 100644 --- a/processor.h +++ b/processor.h @@ -290,9 +290,10 @@ class threaded_processor : public abstract_processor { auto progress = p_.thread(num_threads_); size_t x0 = 0; bool first = true; - for (size_t i = 0; i < num_threads_; ++i, first = false) { + for (size_t i = 0; i < num_threads_; ++i) { size_t x1 = (size_t)floor((i + 1) * d); if (x1 == x0) continue; + first = false; if (i == num_threads_ - 1) { x1 = N; } From e5863c75a5954b4e26422ce17b9b19cc7e0ad4fa Mon Sep 17 00:00:00 2001 From: Thomas Krijnen Date: Sun, 25 Apr 2021 11:40:26 +0200 Subject: [PATCH 064/112] voxel storage copy_as() method --- storage.h | 70 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 70 insertions(+) diff --git a/storage.h b/storage.h index beb6c5b..68e1365 100644 --- a/storage.h +++ b/storage.h @@ -140,6 +140,7 @@ class abstract_voxel_storage { virtual abstract_voxel_storage* empty_copy() const = 0; virtual abstract_voxel_storage* empty_copy_as(voxel_desc_t* fmt) const = 0; virtual abstract_voxel_storage* copy(void* location = nullptr) const = 0; + virtual abstract_voxel_storage* copy_as(voxel_desc_t* fmt, void* location = nullptr) const = 0; virtual size_t ray_intersect_n(const vec_n<3, size_t>& pos, const vec_n<3, size_t>& dir) { // @todo implement more efficiently for subtypes @@ -393,6 +394,20 @@ class continuous_voxel_storage : public regular_voxel_storage { return c; } + abstract_voxel_storage* copy_as(voxel_desc_t* fmt, void* location = nullptr) const { + if (fmt->get_size_in_bits() == 1) { + continuous_voxel_storage* c = new continuous_voxel_storage(ox_, oy_, oz_, d_, dimx_, dimy_, dimz_, location); + BEGIN_LOOP(size_t(0), dimx_, 0U, dimy_, 0U, dimz_) + if (Get(ijk)) { + c->Set(ijk); + } + END_LOOP; + return c; + } else { + throw std::runtime_error("Not implemented"); + } + } + abstract_voxel_storage* empty_copy() const { // @todo is this safe? return nullptr; @@ -628,6 +643,19 @@ class planar_voxel_storage : public regular_voxel_storage { return c; } + abstract_voxel_storage* copy_as(voxel_desc_t* fmt, void* location = nullptr) const { + if (fmt->get_size_in_bits() == 1) { + planar_voxel_storage* c = new planar_voxel_storage(ox_, oy_, oz_, d_, dimx_, dimy_, dimz_, axis_, *offsets_.begin()); + for (auto it = offsets_.begin(); it != offsets_.end(); ++it) { + c->add(*it); + } + return c; + } + else { + throw std::runtime_error("Not implemented"); + } + } + abstract_voxel_storage* empty_copy() const { // @todo is this safe? return nullptr; @@ -788,6 +816,14 @@ class constant_voxel_storage : public regular_voxel_storage { return new constant_voxel_storage(ox_, oy_, oz_, d_, dimx_, dimy_, dimz_, value_); } + abstract_voxel_storage* copy_as(voxel_desc_t* fmt, void* location = nullptr) const { + if (fmt->get_size_in_bits() == 1) { + return new constant_voxel_storage(ox_, oy_, oz_, d_, dimx_, dimy_, dimz_, value_ > 1 ? 1 : value_); + } else { + throw std::runtime_error("Not implemented"); + } + } + abstract_voxel_storage* empty_copy() const { // @todo is this safe? return nullptr; @@ -1602,6 +1638,19 @@ class memory_mapped_chunked_voxel_storage : public abstract_chunked_voxel_storag return c; } + abstract_voxel_storage* copy_as(voxel_desc_t* fmt, void* location = nullptr) const { + auto nc = num_chunks(); + memory_mapped_chunked_voxel_storage* c = new memory_mapped_chunked_voxel_storage(grid_offset_, d_, chunk_size_, nc, factory::mmap_filename()); + + BEGIN_LOOP(size_t(0), nchunksx_, 0U, nchunksy_, 0U, nchunksz_) + if (get_chunk(ijk) != nullptr) { + c->set_chunk(ijk, get_chunk(ijk)->copy(c->next_slot())); + } + END_LOOP; + + return c; + } + abstract_voxel_storage* get_or_create_chunk(const vec_n<3, size_t>& ijk) { const size_t& i = ijk.get<0>(); const size_t& j = ijk.get<1>(); @@ -1706,6 +1755,26 @@ class chunked_voxel_storage : public abstract_chunked_voxel_storage { return c; } + abstract_voxel_storage* copy_as(voxel_desc_t* fmt, void* location = nullptr) const { + if (fmt->get_size_in_bits() == 1) { + + auto nc = num_chunks(); + chunked_voxel_storage* c = new chunked_voxel_storage(grid_offset_, d_, chunk_size_, nc); + + BEGIN_LOOP(size_t(0), nchunksx_, 0U, nchunksy_, 0U, nchunksz_) + if (get_chunk(ijk) != nullptr) { + c->set_chunk(ijk, get_chunk(ijk)->copy_as(fmt)); + } + END_LOOP; + + return c; + + } + else { + throw std::runtime_error("Not implemented"); + } + } + abstract_voxel_storage* get_or_create_chunk(const vec_n<3, size_t>& ijk) { validate_chunk_index(ijk); const size_t& i = ijk.get<0>(); @@ -2003,6 +2072,7 @@ class voxel_region : public regular_voxel_storage { virtual abstract_voxel_storage* copy(void* location = nullptr) const { return base_->copy(); } virtual abstract_voxel_storage* empty_copy_as(voxel_desc_t* fmt) const { throw std::runtime_error("Not implemented"); } + virtual abstract_voxel_storage* copy_as(voxel_desc_t* fmt, void* location = nullptr) const { throw std::runtime_error("Not implemented"); } virtual long long unsigned int count() const { unsigned long long n = 0; From 7fc4a332c43cb6c26c6772dd9c522559cff7724a Mon Sep 17 00:00:00 2001 From: Thomas Krijnen Date: Sun, 25 Apr 2021 11:41:16 +0200 Subject: [PATCH 065/112] multi-threaded geometry creation --- voxec.h | 31 +++++++++++++++++++++++++------ 1 file changed, 25 insertions(+), 6 deletions(-) diff --git a/voxec.h b/voxec.h index df79cbe..be22963 100644 --- a/voxec.h +++ b/voxec.h @@ -268,6 +268,14 @@ class op_create_geometry : public voxel_operation { IfcGeom::entity_filter ef; + boost::optional threads; + if (scope.has("THREADS")) { + int t = scope.get_value("THREADS"); + if (t > 1) { + threads = (size_t)t; + } + } + #ifdef IFCOPENSHELL_05 auto ifc_roof = IfcSchema::Type::IfcRoof; auto ifc_slab = IfcSchema::Type::IfcSlab; @@ -360,12 +368,23 @@ class op_create_geometry : public voxel_operation { for (auto ifc_file : ifc_files.files) { - IfcGeom::Iterator iterator(settings_surface, ifc_file, filters_surface); + std::unique_ptr> iterator; + +#ifdef IFCOPENSHELL_05 + iterator.reset(IfcGeom::Iterator(settings_surface, ifc_file, filters_surface)); +#else + if (threads) { + iterator.reset(new IfcGeom::Iterator(settings_surface, ifc_file, filters_surface, *threads)); + } else { + iterator.reset(new IfcGeom::Iterator(settings_surface, ifc_file, filters_surface)); + } +#endif + // For debugging geometry creation from IfcOpenShell // Logger::SetOutput(&std::cout, &std::cout); - if (!iterator.initialize()) { + if (!iterator->initialize()) { continue; } @@ -374,7 +393,7 @@ class op_create_geometry : public voxel_operation { int old_progress = -1; for (;;) { - elem_t* elem = (elem_t*)iterator.get(); + elem_t* elem = (elem_t*)iterator->get(); bool process = true; if (boost::to_lower_copy(elem->name()).find("nulpunt") != std::string::npos) { @@ -425,14 +444,14 @@ class op_create_geometry : public voxel_operation { geometries->push_back(std::make_pair(elem->id(), compound)); } - if (old_progress != iterator.progress()) { - old_progress = iterator.progress(); + if (old_progress != iterator->progress()) { + old_progress = iterator->progress(); if (application_progress_callback) { (*application_progress_callback)(old_progress / 100.f); } } - if (!iterator.next()) { + if (!iterator->next()) { break; } } From 8507748bc412ed38a01a7a3029f87a6bb9e3491b Mon Sep 17 00:00:00 2001 From: Thomas Krijnen Date: Sun, 25 Apr 2021 11:41:57 +0200 Subject: [PATCH 066/112] multi threaded VOLUME_PRODUCT_ID --- voxec.h | 26 ++++++++++++++++++++------ 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/voxec.h b/voxec.h index be22963..ea987d4 100644 --- a/voxec.h +++ b/voxec.h @@ -508,13 +508,27 @@ namespace { } if (std::is_same::value) { - chunked_voxel_storage* storage = new chunked_voxel_storage(x1, y1, z1, vsize, nx, ny, nz, chunksize); - progress_writer progress("voxelize"); - processor pr(storage, progress); - pr.use_scanline() = false; + // @todo, uint32 defaults to VOLUME_PRODUCT_ID, make this explicit - pr.process(surfaces->begin(), surfaces->end(), VOLUME_PRODUCT_ID(), output(MERGED())); - return storage; + // @why + // pr.use_scanline() = false; + + chunked_voxel_storage* storage = new chunked_voxel_storage(x1, y1, z1, vsize, nx, ny, nz, chunksize); + + if (threads) { + progress_writer progress("voxelize", silent); + threaded_processor p(storage, *threads, progress); + p.process(surfaces->begin(), surfaces->end(), VOLUME_PRODUCT_ID(), output(MERGED())); + // @todo what actually happens to storage? + delete storage; + return p.voxels(); + } else { + progress_writer progress("voxelize"); + processor pr(storage, progress); + pr.process(surfaces->begin(), surfaces->end(), VOLUME_PRODUCT_ID(), output(MERGED())); + return storage; + } + } else { if (threads) { progress_writer progress("voxelize", silent); From 9e5e7d36cc8893eddbe29af7665e42734b6074ef Mon Sep 17 00:00:00 2001 From: Thomas Krijnen Date: Sun, 25 Apr 2021 11:42:29 +0200 Subject: [PATCH 067/112] multi-threaded group_by() --- voxec.h | 57 ++++++++++++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 48 insertions(+), 9 deletions(-) diff --git a/voxec.h b/voxec.h index ea987d4..9ef9a8c 100644 --- a/voxec.h +++ b/voxec.h @@ -627,9 +627,24 @@ class op_describe_components : public voxel_operation { } }; +#include +#include +#include +#include + namespace { + template + class mt_list : public std::list { + std::mutex m; + public: + void push_back(const T& t) { + std::lock_guard lock{ m }; + std::list::push_back(t); + } + }; + template - void group_by(regular_voxel_storage* groups, abstract_voxel_storage* voxels, Fn fn) { + void group_by(regular_voxel_storage* groups, abstract_voxel_storage* voxels, Fn fn, int threads=1) { uint32_t v; std::set vs; @@ -638,23 +653,47 @@ namespace { vs.insert(v); } - static bit_t desc_bits; + bit_t desc_bits; - for (auto& id : vs) { + mt_list> results; + boost::asio::thread_pool pool(threads); // 4 threads + + auto process = [&fn, &groups, &desc_bits, &voxels, &results, &threads](uint32_t id) { + uint32_t vv; // A {0,1} dataset of `groups`==`id` auto where_id = groups->empty_copy_as(&desc_bits); for (auto& ijk : *groups) { - groups->Get(ijk, &v); - if (v == id) { + groups->Get(ijk, &vv); + if (vv == id) { where_id->Set(ijk); } } auto c = voxels->boolean_intersection(where_id); - delete where_id; + delete where_id; - fn(id, c); + if (threads <= 1) { + fn(id, c); + } else { + results.push_back({ id, c }); + } + }; + + for (auto& id : vs) { + if (threads > 1) { + boost::asio::post(pool, std::bind(process, id)); + } else { + process(id); + } + } + + if (threads > 1) { + + pool.join(); + for (auto& r : results) { + fn(r.first, r.second); + } } } } @@ -689,7 +728,7 @@ class op_describe_group_by : public voxel_operation { delete c; first = false; - }); + }, scope.get_value_or("THREADS", 1)); ofs << "]"; @@ -1606,7 +1645,7 @@ class op_mesh : public voxel_operation { group_by(groups, voxels, [&helper, &ofs](uint32_t id, abstract_voxel_storage* c) { ofs << "g id-" << id << "\n"; ((regular_voxel_storage*)c)->obj_export(helper, false, false); - }); + }, scope.get_value_or("THREADS", 1)); } else { if (voxels->value_bits() == 1) { From cbfa656bd1de8ac3c2748de7371ffbe7f18b86c2 Mon Sep 17 00:00:00 2001 From: Thomas Krijnen Date: Sun, 25 Apr 2021 11:42:48 +0200 Subject: [PATCH 068/112] respect with_components in mesh() --- voxec.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/voxec.h b/voxec.h index 9ef9a8c..0c0ca93 100644 --- a/voxec.h +++ b/voxec.h @@ -1634,6 +1634,7 @@ class op_mesh : public voxel_operation { symbol_value invoke(const scope_map& scope) const { auto voxels = scope.get_value("input"); auto use_value = scope.get_value_or("use_value", -1); + auto with_components = scope.get_value_or("with_components", -1); auto filename = scope.get_value("filename"); auto groups = (regular_voxel_storage*) scope.get_value_or("groups", nullptr); @@ -1649,10 +1650,9 @@ class op_mesh : public voxel_operation { } else { if (voxels->value_bits() == 1) { - auto with_components = scope.get_value_or("with_components", -1); ((regular_voxel_storage*)voxels)->obj_export(ofs, with_components != 0); } else { - ((regular_voxel_storage*)voxels)->obj_export(ofs, use_value != 1, use_value == 1); + ((regular_voxel_storage*)voxels)->obj_export(ofs, with_components != 0 && use_value != 1, use_value == 1); } } symbol_value v; From 292b87069ae7c2664efb8bec4cbdbc5d1f16178b Mon Sep 17 00:00:00 2001 From: Thomas Krijnen Date: Sun, 25 Apr 2021 11:43:16 +0200 Subject: [PATCH 069/112] copy(type=) command --- voxec.cpp | 1 + voxec.h | 30 ++++++++++++++++++++++++++++++ 2 files changed, 31 insertions(+) diff --git a/voxec.cpp b/voxec.cpp index 4ac1c66..5d2ac62 100644 --- a/voxec.cpp +++ b/voxec.cpp @@ -24,6 +24,7 @@ voxel_operation_map::map_t& voxel_operation_map::map() { m.insert(std::make_pair(std::string("volume2"), &instantiate)); m.insert(std::make_pair(std::string("exterior"), &instantiate)); m.insert(std::make_pair(std::string("invert"), &instantiate)); + m.insert(std::make_pair(std::string("copy"), &instantiate)); m.insert(std::make_pair(std::string("fill_volume"), &instantiate)); m.insert(std::make_pair(std::string("union"), &instantiate)); m.insert(std::make_pair(std::string("subtract"), &instantiate)); diff --git a/voxec.h b/voxec.h index 0c0ca93..184a23e 100644 --- a/voxec.h +++ b/voxec.h @@ -1524,6 +1524,36 @@ class op_constant_like : public voxel_operation { } }; +class op_copy : public voxel_operation { +public: + const std::vector& arg_names() const { + static std::vector nm_ = { { true, "input", "voxels" }, { false, "type", "string"} }; + return nm_; + } + symbol_value invoke(const scope_map& scope) const { + regular_voxel_storage* voxels = (regular_voxel_storage*)scope.get_value("input"); + + abstract_chunked_voxel_storage* output; + + if (scope.has("type")) { + if (scope.get_value("type") == "uint") { + voxel_uint32_t fmt; + output = (abstract_chunked_voxel_storage*)voxels->copy_as(&fmt); + } else if (scope.get_value("type") == "bit") { + bit_t fmt; + output = (abstract_chunked_voxel_storage*)voxels->copy_as(&fmt); + } else { + throw std::runtime_error("not implemented"); + } + } else { + output = (abstract_chunked_voxel_storage*)voxels->copy(); + } + + return output; + } +}; + + #ifdef WITH_IFC namespace { From 914aefc917fe2966116cb76e588f5aa267a74739 Mon Sep 17 00:00:00 2001 From: Thomas Krijnen Date: Sun, 25 Apr 2021 14:55:59 +0200 Subject: [PATCH 070/112] dist: focal --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 37f7d3a..42abf09 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,3 +1,4 @@ +dist: focal language: cpp sudo: required compiler: From ba17a9d421b81756d8628da25089f455e8a17295 Mon Sep 17 00:00:00 2001 From: Thomas Krijnen Date: Sun, 25 Apr 2021 15:04:15 +0200 Subject: [PATCH 071/112] Update .travis.yml --- .travis.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 42abf09..2a61d66 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,13 +6,12 @@ compiler: - clang before_install: - - "sudo add-apt-repository -y ppa:freecad-community/ppa" - sudo apt-get update -qq install: - | sudo apt-get install -y \ - libocct-data-exchange-dev libocct-foundation-dev libocct-modeling-algorithms-dev libocct-modeling-data-dev \ + libocct-* libboost-all-dev script: From 3cda43d7ae8c73e06767a40daca151f42cb2355a Mon Sep 17 00:00:00 2001 From: Thomas Krijnen Date: Sun, 25 Apr 2021 15:12:19 +0200 Subject: [PATCH 072/112] Update .travis.yml --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 2a61d66..2974493 100644 --- a/.travis.yml +++ b/.travis.yml @@ -11,7 +11,7 @@ before_install: install: - | sudo apt-get install -y \ - libocct-* + libocct-* \ libboost-all-dev script: From 69a4f1cb07ceb80d867f43ccf58325f2c354c30c Mon Sep 17 00:00:00 2001 From: johltn Date: Mon, 26 Apr 2021 09:20:55 +0200 Subject: [PATCH 073/112] Store bit values into HDF file --- tests/test_hdf.cpp | 67 +++++++++---------------- writer.h | 120 ++++++++++++++++++++++++++++++++++----------- 2 files changed, 114 insertions(+), 73 deletions(-) diff --git a/tests/test_hdf.cpp b/tests/test_hdf.cpp index bed2e5c..3109e76 100644 --- a/tests/test_hdf.cpp +++ b/tests/test_hdf.cpp @@ -6,53 +6,30 @@ #include #include #include +#include +#include +#include +#include + + TEST(HdfFileName, HDF) { - auto storage = new chunked_voxel_storage(0., 0., 0., 0.1, 1000, 1000, 100, 32); - - for (int i = 0; i < 10; ++i) { - for (int j = 0; j < 10; ++j) { - BRepPrimAPI_MakeBox mb(gp_Pnt(10 * i, 10 * j, 0.), 8., 8., 8.); - - auto vox = voxelizer(mb.Solid(), storage); - vox.Convert(); - } - } - - /*HDF WRITER*/ - hdf_writer writer; - writer.SetVoxels(storage); - writer.Write("test_hdf_writer.vox"); - /*HDF WRITER*/ - - - const H5std_string FILE_NAME("SDS.h5"); - const H5std_string DATASET_NAME("IntArray"); - const int NX = 5; - const int NY = 6; - const int RANK = 2; - - int i, j; - int data[NX][NY]; - for (j = 0; j < NX; j++) - { - for (i = 0; i < NY; i++) - data[j][i] = i + j; - } - - H5::H5File file(FILE_NAME, H5F_ACC_TRUNC); - hsize_t dimsf[2]; - dimsf[0] = NX; - dimsf[1] = NY; - H5::DataSpace dataspace(RANK, dimsf); - H5::IntType datatype(H5::PredType::NATIVE_INT); - datatype.setOrder(H5T_ORDER_LE); - - H5::DataSet dataset = file.createDataSet(DATASET_NAME, datatype, dataspace); - dataset.write(data, H5::PredType::NATIVE_INT); - - std::string file_name = file.getFileName(); - EXPECT_TRUE(file_name == FILE_NAME); + auto storage2 = new chunked_voxel_storage(0., 0., 0., 1, 32, 32, 32, 32); + + BRepPrimAPI_MakeBox mb(gp_Pnt(1, 1, 1), gp_Pnt(10, 10, 10)); + auto x = mb.Solid(); + BRepMesh_IncrementalMesh(x, 0.001); + auto vox = voxelizer(x, storage2); + vox.Convert(); + + hdf_writer writer; + writer.SetVoxels(storage2); + writer.Write("voxels.h5"); + + std::ofstream fs("voxobj.obj"); + obj_export_helper oeh{ fs }; + storage2->obj_export(oeh, false, false); + } diff --git a/writer.h b/writer.h index 59c48e4..51ee20c 100644 --- a/writer.h +++ b/writer.h @@ -31,6 +31,7 @@ class voxel_writer :public abstract_writer { } return fs; } + public: void Write(const std::string& fnc) { { @@ -62,64 +63,127 @@ class hdf_writer :public abstract_writer { public: void Write(const std::string& fnc) { - const int RANK = 2; - const H5std_string FILE_NAME(fnc); - const H5std_string DATASET_NAME("Continuous"); - H5::H5File file(FILE_NAME, H5F_ACC_TRUNC); + chunked_voxel_storage* storage = (chunked_voxel_storage*)voxels_; + + int continuous_count = 0; + int planar_count = 0; + int constant_count = 0; + + for (int i = 0; i < storage->num_chunks().get(0); i++) { + auto chunk = storage->get_chunk(make_vec(i, 0, 0)); + + if (chunk && chunk->is_explicit()) { + continuous_count++; + + } + else { + if (chunk && chunk->is_constant()) { + constant_count++; + } + + else { + planar_count++; + } + } + } + //std::cout << continuous_count << std::endl; + //std::cout << planar_count << std::endl; + //std::cout << constant_count << std::endl; + + int chunk_stream = storage->chunk_size() * storage->voxel_size(); + + const H5std_string FILE_NAME(fnc); + const H5std_string DATASET_NAME("continuous_chunks"); + const int NX = 10; + const int NY = 5; + const int RANK = 2; hsize_t dims[2] = { 3, 3 }; // dataset dimensions at creation hsize_t maxdims[2] = { H5S_UNLIMITED, H5S_UNLIMITED }; H5::DataSpace mspace1(RANK, dims, maxdims); + + H5::H5File file(FILE_NAME, H5F_ACC_TRUNC); H5::DSetCreatPropList cparms; hsize_t chunk_dims[2] = { 2, 5 }; cparms.setChunk(RANK, chunk_dims); - + + int fill_val = 0; + cparms.setFillValue(H5::PredType::NATIVE_INT, &fill_val); + H5::DataSet dataset = file.createDataSet(DATASET_NAME, H5::PredType::NATIVE_INT, mspace1, cparms); - - chunked_voxel_storage* storage = (chunked_voxel_storage*)voxels_; + + hsize_t size[2]; + size[0] = continuous_count; + size[1] = 32 * 32 * 32; + dataset.extend(size); + + H5::DataSpace fspace1 = dataset.getSpace(); + hsize_t offset[2]; + offset[0] = 1; + offset[1] = 0; + hsize_t dims1[2] = { 3, 3 }; /* data1 dimensions */ + fspace1.selectHyperslab(H5S_SELECT_SET, dims1, offset); + + H5::DataSpace fspace2 = dataset.getSpace(); + offset[0] = 1; + offset[1] = 0; + hsize_t dims2[2] = { 1, 32*32*32 }; + + fspace2.selectHyperslab(H5S_SELECT_SET, dims2, offset); + + H5::DataSpace mspace2(RANK, dims2); + + int cont_count = 0; for (int i = 0; i < storage->num_chunks().get(0); i++) { + + std::vector bits_container; auto c = storage->get_chunk(make_vec(i, 0, 0)); if (c && c->is_explicit()) { + + offset[0] = cont_count; + cont_count++; + offset[1] = 0; + fspace2.selectHyperslab(H5S_SELECT_SET, dims2, offset); + continuous_voxel_storage* convox = (continuous_voxel_storage*)c; auto d = convox->data(); + //std::cout << convox->size(); for (int j = 0; j < convox->size(); j++) { auto vd = convox->data()[j]; + // consider using boost::dynamic_bitset std::bitset<8> b(convox->data()[j]); + + for (int k = 0; k < 8; k++) { + bits_container.push_back(b[k]); + + } + } - /*std::cout << b<is_constant()) { + std::cout << "Constant handling to implement." << std::endl; - std::cout << (int)convox->data()[j] << std::endl; - for (int k = 0; k < 8; k++) { - int bitmask = 1 << k; - int masked = vd & bitmask; - int voxbit = masked >> vd; - std::cout << voxbit; - } - - } + } + else { + std::cout << "Plane handling to implement." << std::endl; - if (c && c->is_constant()) { - std::cout << "Constant handling to implement." << std::endl; + } - } - else { - std::cout << "Plane handling to implement." << std::endl; - } } } - std:cout << std::endl; - } + + + + }; From 1343f77d536736ef7947c851d792c83a97b21085 Mon Sep 17 00:00:00 2001 From: johltn Date: Wed, 28 Apr 2021 16:29:53 +0200 Subject: [PATCH 074/112] Hyperslab writing on fixed dataset dimensions and add test case --- tests/test_hdf.cpp | 14 +++++++- writer.h | 82 +++++++++++++++++++--------------------------- 2 files changed, 47 insertions(+), 49 deletions(-) diff --git a/tests/test_hdf.cpp b/tests/test_hdf.cpp index 3109e76..38591c3 100644 --- a/tests/test_hdf.cpp +++ b/tests/test_hdf.cpp @@ -23,9 +23,21 @@ TEST(HdfFileName, HDF) { auto vox = voxelizer(x, storage2); vox.Convert(); + + auto storage = new chunked_voxel_storage(0., 0., 0., 0.1, 200, 150, 10, 32); + + { + BRepBuilderAPI_MakePolygon mp(gp_Pnt(1, 1, 0), gp_Pnt(16, 1, 0), gp_Pnt(16, 9.8, 0), gp_Pnt(1, 9.8, 0), true); + BRepBuilderAPI_MakeFace mf(mp.Wire()); + TopoDS_Face face = mf.Face(); + + auto vox = voxelizer(face, storage); + vox.Convert(); + } + hdf_writer writer; writer.SetVoxels(storage2); - writer.Write("voxels.h5"); + writer.Write("voxels2.h5"); std::ofstream fs("voxobj.obj"); obj_export_helper oeh{ fs }; diff --git a/writer.h b/writer.h index 51ee20c..2f3ef33 100644 --- a/writer.h +++ b/writer.h @@ -66,14 +66,21 @@ class hdf_writer :public abstract_writer { chunked_voxel_storage* storage = (chunked_voxel_storage*)voxels_; - int continuous_count = 0; - int planar_count = 0; - int constant_count = 0; + int continuous_count = 0; + int planar_count = 0; + int constant_count = 0; + + int col_n = 0; for (int i = 0; i < storage->num_chunks().get(0); i++) { auto chunk = storage->get_chunk(make_vec(i, 0, 0)); - + if (chunk && chunk->is_explicit()) { + if (col_n == 0) { + continuous_voxel_storage* casted = (continuous_voxel_storage*)chunk; + col_n = casted->size() * casted->value_bits() * 8; + } + continuous_count++; } @@ -88,52 +95,35 @@ class hdf_writer :public abstract_writer { } } - //std::cout << continuous_count << std::endl; - //std::cout << planar_count << std::endl; - //std::cout << constant_count << std::endl; - - int chunk_stream = storage->chunk_size() * storage->voxel_size(); + + int chunk_stream = storage->chunk_size() * storage->voxel_size(); + + std::vector bits_container; const H5std_string FILE_NAME(fnc); const H5std_string DATASET_NAME("continuous_chunks"); - const int NX = 10; - const int NY = 5; + const int NX = continuous_count; + const int NY = col_n; const int RANK = 2; - hsize_t dims[2] = { 3, 3 }; // dataset dimensions at creation - hsize_t maxdims[2] = { H5S_UNLIMITED, H5S_UNLIMITED }; - H5::DataSpace mspace1(RANK, dims, maxdims); - H5::H5File file(FILE_NAME, H5F_ACC_TRUNC); - H5::DSetCreatPropList cparms; - hsize_t chunk_dims[2] = { 2, 5 }; - cparms.setChunk(RANK, chunk_dims); - - int fill_val = 0; - cparms.setFillValue(H5::PredType::NATIVE_INT, &fill_val); - - H5::DataSet dataset = file.createDataSet(DATASET_NAME, H5::PredType::NATIVE_INT, mspace1, cparms); - - hsize_t size[2]; - size[0] = continuous_count; - size[1] = 32 * 32 * 32; - dataset.extend(size); - - H5::DataSpace fspace1 = dataset.getSpace(); - hsize_t offset[2]; - offset[0] = 1; - offset[1] = 0; - hsize_t dims1[2] = { 3, 3 }; /* data1 dimensions */ - fspace1.selectHyperslab(H5S_SELECT_SET, dims1, offset); + hsize_t dimsf[2]; + dimsf[0] = NX; + dimsf[1] = NY; + H5::DataSpace dataspace(RANK, dimsf); + H5::IntType datatype(H5::PredType::NATIVE_INT); + datatype.setOrder(H5T_ORDER_LE); + H5::DataSet dataset = file.createDataSet(DATASET_NAME, datatype, dataspace); - H5::DataSpace fspace2 = dataset.getSpace(); - offset[0] = 1; + int column_number = col_n; + hsize_t offset[2]; + offset[0] = 0; offset[1] = 0; - hsize_t dims2[2] = { 1, 32*32*32 }; - fspace2.selectHyperslab(H5S_SELECT_SET, dims2, offset); + hsize_t slab_dimsf[2] = { 1, col_n }; + dataspace.selectHyperslab(H5S_SELECT_SET, slab_dimsf, offset); + H5::DataSpace mspace2(RANK, slab_dimsf); - H5::DataSpace mspace2(RANK, dims2); int cont_count = 0; @@ -147,12 +137,11 @@ class hdf_writer :public abstract_writer { offset[0] = cont_count; cont_count++; offset[1] = 0; - fspace2.selectHyperslab(H5S_SELECT_SET, dims2, offset); + dataspace.selectHyperslab(H5S_SELECT_SET, slab_dimsf, offset); continuous_voxel_storage* convox = (continuous_voxel_storage*)c; auto d = convox->data(); - //std::cout << convox->size(); - + for (int j = 0; j < convox->size(); j++) { auto vd = convox->data()[j]; // consider using boost::dynamic_bitset @@ -164,8 +153,7 @@ class hdf_writer :public abstract_writer { } } - - dataset.write(bits_container.data(), H5::PredType::NATIVE_INT, mspace2, fspace2); + dataset.write(bits_container.data(), H5::PredType::NATIVE_INT, mspace2, dataspace); } if (c && c->is_constant()) { @@ -182,12 +170,10 @@ class hdf_writer :public abstract_writer { } - + }; -}; - From 7a15675c4b250860b642d5b163149be4080ae918 Mon Sep 17 00:00:00 2001 From: johltn Date: Wed, 28 Apr 2021 21:27:07 +0200 Subject: [PATCH 075/112] Write multidimensional dataset for continuous chunks --- tests/test_hdf.cpp | 43 +++++++++++++++++++++++++++++++++++++------ writer.h | 45 ++++++++++++++++++++++++++++++--------------- 2 files changed, 67 insertions(+), 21 deletions(-) diff --git a/tests/test_hdf.cpp b/tests/test_hdf.cpp index 38591c3..cbc6aef 100644 --- a/tests/test_hdf.cpp +++ b/tests/test_hdf.cpp @@ -23,7 +23,6 @@ TEST(HdfFileName, HDF) { auto vox = voxelizer(x, storage2); vox.Convert(); - auto storage = new chunked_voxel_storage(0., 0., 0., 0.1, 200, 150, 10, 32); { @@ -36,12 +35,44 @@ TEST(HdfFileName, HDF) { } hdf_writer writer; - writer.SetVoxels(storage2); - writer.Write("voxels2.h5"); + writer.SetVoxels(storage); + writer.Write("voxels.h5"); + + //std::ofstream fs("voxobj.obj"); + //obj_export_helper oeh{ fs }; + //storage2->obj_export(oeh, false, false); + + // Write a 4D dataset + + const H5std_string FILE_NAME("multidim.h5"); + const H5std_string DATASET_NAME("continuous_chunks"); + const int NX = 32; // dataset dimensions + const int NY = 32; + const int NZ = 32; + const int NC = 3; + + const int RANK = 4; + H5::H5File file(FILE_NAME, H5F_ACC_TRUNC); - std::ofstream fs("voxobj.obj"); - obj_export_helper oeh{ fs }; - storage2->obj_export(oeh, false, false); + hsize_t dimsf[4]; // dataset dimensions + dimsf[0] = NC; + dimsf[1] = NX; + dimsf[2] = NY; + dimsf[3] = NZ; + + H5::DataSpace dataspace(RANK, dimsf); + + H5::IntType datatype(H5::PredType::NATIVE_INT); + datatype.setOrder(H5T_ORDER_LE); + + H5::DataSet dataset = file.createDataSet(DATASET_NAME, datatype, dataspace); + + std::vector data; + + for (int i = 0; i < NX*NY*NZ*NC; i++) { + data.push_back(0); + } + dataset.write(data.data(), H5::PredType::NATIVE_INT); } diff --git a/writer.h b/writer.h index 2f3ef33..12e9552 100644 --- a/writer.h +++ b/writer.h @@ -66,12 +66,19 @@ class hdf_writer :public abstract_writer { chunked_voxel_storage* storage = (chunked_voxel_storage*)voxels_; + std::cout<num_chunks().get(0)<num_chunks().get(1) << std::endl; + std::cout << storage->num_chunks().get(2) << std::endl; + + std::cout << storage->chunk_size() << std::endl; + int continuous_count = 0; int planar_count = 0; int constant_count = 0; int col_n = 0; + for (int i = 0; i < storage->num_chunks().get(0); i++) { auto chunk = storage->get_chunk(make_vec(i, 0, 0)); @@ -95,30 +102,38 @@ class hdf_writer :public abstract_writer { } } - int chunk_stream = storage->chunk_size() * storage->voxel_size(); std::vector bits_container; const H5std_string FILE_NAME(fnc); const H5std_string DATASET_NAME("continuous_chunks"); - const int NX = continuous_count; - const int NY = col_n; - const int RANK = 2; + + const int NC = continuous_count; + const int NX = storage->num_chunks().get(0); + const int NY = storage->num_chunks().get(1); + const int NZ = storage->num_chunks().get(2); + + const int RANK = 4; H5::H5File file(FILE_NAME, H5F_ACC_TRUNC); - hsize_t dimsf[2]; - dimsf[0] = NX; - dimsf[1] = NY; + hsize_t dimsf[4]; + dimsf[0] = NC; + dimsf[1] = NX; + dimsf[2] = NY; + dimsf[3] = NZ; + H5::DataSpace dataspace(RANK, dimsf); H5::IntType datatype(H5::PredType::NATIVE_INT); datatype.setOrder(H5T_ORDER_LE); H5::DataSet dataset = file.createDataSet(DATASET_NAME, datatype, dataspace); int column_number = col_n; - hsize_t offset[2]; + hsize_t offset[4]; offset[0] = 0; offset[1] = 0; + offset[2] = 0; + offset[3] = 0; hsize_t slab_dimsf[2] = { 1, col_n }; dataspace.selectHyperslab(H5S_SELECT_SET, slab_dimsf, offset); @@ -134,21 +149,21 @@ class hdf_writer :public abstract_writer { if (c && c->is_explicit()) { - offset[0] = cont_count; + /*offset[0] = cont_count; cont_count++; offset[1] = 0; - dataspace.selectHyperslab(H5S_SELECT_SET, slab_dimsf, offset); + dataspace.selectHyperslab(H5S_SELECT_SET, slab_dimsf, offset);*/ continuous_voxel_storage* convox = (continuous_voxel_storage*)c; auto d = convox->data(); - for (int j = 0; j < convox->size(); j++) { - auto vd = convox->data()[j]; + for (int v = 0; v < convox->size(); v++) { + auto vd = convox->data()[v]; // consider using boost::dynamic_bitset - std::bitset<8> b(convox->data()[j]); + std::bitset<8> b(convox->data()[v]); - for (int k = 0; k < 8; k++) { - bits_container.push_back(b[k]); + for (int l = 0; l< 8; l++) { + bits_container.push_back(b[l]); } } From c76b91d9f3f62a594c9bb3fca6df680371baa141 Mon Sep 17 00:00:00 2001 From: Thomas Krijnen Date: Thu, 29 Apr 2021 08:57:08 +0200 Subject: [PATCH 076/112] don't attempt full chunk union on non-bit types --- storage.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/storage.h b/storage.h index 68e1365..74b357f 100644 --- a/storage.h +++ b/storage.h @@ -1150,7 +1150,7 @@ class abstract_chunked_voxel_storage : public regular_voxel_storage { // std::cout << "boolean operatoin of chunk " << ijk.format() << " " << css[Sa] << " " << css[Sb] << std::endl; if (mode == OP_UNION) { - if ((Sa == CK_FULL || Sb == CK_FULL) && get_chunk(n, ijk) == nullptr) { + if ((Sa == CK_FULL || Sb == CK_FULL) && get_chunk(n, ijk) == nullptr && b->value_bits() == 1) { n->create_constant((ijk - left).as(), 1U); continue; } else if (Sb == CK_EMPTY) { From 18957a431bd353baf5c683035a3b889a95da3575 Mon Sep 17 00:00:00 2001 From: Thomas Krijnen Date: Thu, 29 Apr 2021 08:57:40 +0200 Subject: [PATCH 077/112] print_values() debugging option --- voxec.cpp | 1 + voxec.h | 27 +++++++++++++++++++++++++++ 2 files changed, 28 insertions(+) diff --git a/voxec.cpp b/voxec.cpp index 5d2ac62..6114a78 100644 --- a/voxec.cpp +++ b/voxec.cpp @@ -37,6 +37,7 @@ voxel_operation_map::map_t& voxel_operation_map::map() { m.insert(std::make_pair(std::string("collapse"), &instantiate)); m.insert(std::make_pair(std::string("collapse_count"), &instantiate)); m.insert(std::make_pair(std::string("print_components"), &instantiate)); + m.insert(std::make_pair(std::string("print_values"), &instantiate)); m.insert(std::make_pair(std::string("describe_components"), &instantiate)); m.insert(std::make_pair(std::string("describe_group_by"), &instantiate)); m.insert(std::make_pair(std::string("keep_components"), &instantiate)); diff --git a/voxec.h b/voxec.h index 184a23e..c7eb356 100644 --- a/voxec.h +++ b/voxec.h @@ -601,6 +601,33 @@ class op_print_components : public voxel_operation { } }; +class op_print_values : public voxel_operation { +public: + const std::vector& arg_names() const { + static std::vector nm_ = { { true, "input", "voxels" } }; + return nm_; + } + symbol_value invoke(const scope_map& scope) const { + auto voxels = (regular_voxel_storage*) scope.get_value("input"); + uint32_t v; + std::set vs; + for (auto& ijk : *voxels) { + voxels->Get(ijk, &v); + vs.insert(v); + } + bool first = true; + for (auto& x : vs) { + if (!first) { + std::cout << " "; + } + first = false; + std::cout << x; + } + symbol_value vv; + return vv; + } +}; + class op_describe_components : public voxel_operation { public: const std::vector& arg_names() const { From e9df48ac833e98518a87c5871ecb58045d8eb3d1 Mon Sep 17 00:00:00 2001 From: Thomas Krijnen Date: Thu, 29 Apr 2021 12:30:34 +0200 Subject: [PATCH 078/112] more chunk union fixes --- storage.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/storage.h b/storage.h index 74b357f..6ae72d6 100644 --- a/storage.h +++ b/storage.h @@ -1150,7 +1150,7 @@ class abstract_chunked_voxel_storage : public regular_voxel_storage { // std::cout << "boolean operatoin of chunk " << ijk.format() << " " << css[Sa] << " " << css[Sb] << std::endl; if (mode == OP_UNION) { - if ((Sa == CK_FULL || Sb == CK_FULL) && get_chunk(n, ijk) == nullptr && b->value_bits() == 1) { + if (Sb == CK_FULL && get_chunk(n, ijk) == nullptr && b->value_bits() == 1) { n->create_constant((ijk - left).as(), 1U); continue; } else if (Sb == CK_EMPTY) { @@ -1158,7 +1158,7 @@ class abstract_chunked_voxel_storage : public regular_voxel_storage { set_chunk(n, ijk, a->copy(n->next_slot())); } continue; - } else if (Sa == CK_EMPTY && n->value_bits() == b->value_bits()) { + } else if ((Sa == CK_EMPTY || Sb == CK_FULL) && n->value_bits() == b->value_bits()) { // When copying from b we need to check whether datatypes match set_chunk(n, ijk, b->copy(n->next_slot())); continue; From f0cef42c3fb753db5014ff680f3a1fa93bdd857f Mon Sep 17 00:00:00 2001 From: johltn Date: Sat, 1 May 2021 11:37:22 +0200 Subject: [PATCH 079/112] Use extents to write multidimensional HDF --- writer.h | 119 +++++++++++++++++++++++++------------------------------ 1 file changed, 55 insertions(+), 64 deletions(-) diff --git a/writer.h b/writer.h index 12e9552..bd07fea 100644 --- a/writer.h +++ b/writer.h @@ -9,6 +9,27 @@ #include +#define BEGIN_LOOP(i0, i1, j0, j1, k0, k1) {\ + vec_n<3, typename std::remove_reference::type> ijk;\ + for (ijk.get(0) = i0; ijk.get(0) < i1; ++ijk.get(0)) {\ + for (ijk.get(1) = j0; ijk.get(1) < j1; ++ijk.get(1)) {\ + for (ijk.get(2) = k0; ijk.get(2) < k1; ++ijk.get(2)) { + +#define BEGIN_LOOP_I(i0, i1, j0, j1, k0, k1) {\ + vec_n<3, typename std::remove_reference::type> ijk;\ + for (ijk.get(0) = i0; ijk.get(0) <= i1; ++ijk.get(0)) {\ + for (ijk.get(1) = j0; ijk.get(1) <= j1; ++ijk.get(1)) {\ + for (ijk.get(2) = k0; ijk.get(2) <= k1; ++ijk.get(2)) { + +#define BEGIN_LOOP2(v0, v1) BEGIN_LOOP(v0.template get<0>(), v1.template get<0>(), v0.template get<1>(), v1.template get<1>(), v0.template get<2>(), v1.template get<2>()) + +#define BEGIN_LOOP_I2(v0, v1) BEGIN_LOOP_I(v0.template get<0>(), v1.template get<0>(), v0.template get<1>(), v1.template get<1>(), v0.template get<2>(), v1.template get<2>()) + +#define BEGIN_LOOP_ZERO_2(v1) BEGIN_LOOP2(make_vec((decltype(v1)::element_type)0, (decltype(v1)::element_type)0, (decltype(v1)::element_type)0), v1) + +#define END_LOOP }}}} + + class abstract_writer { public: @@ -58,7 +79,6 @@ class voxel_writer :public abstract_writer { }; - class hdf_writer :public abstract_writer { public: @@ -66,28 +86,21 @@ class hdf_writer :public abstract_writer { chunked_voxel_storage* storage = (chunked_voxel_storage*)voxels_; - std::cout<num_chunks().get(0)<num_chunks().get(1) << std::endl; - std::cout << storage->num_chunks().get(2) << std::endl; - - std::cout << storage->chunk_size() << std::endl; - int continuous_count = 0; int planar_count = 0; int constant_count = 0; int col_n = 0; - for (int i = 0; i < storage->num_chunks().get(0); i++) { auto chunk = storage->get_chunk(make_vec(i, 0, 0)); - + if (chunk && chunk->is_explicit()) { if (col_n == 0) { continuous_voxel_storage* casted = (continuous_voxel_storage*)chunk; col_n = casted->size() * casted->value_bits() * 8; } - + continuous_count++; } @@ -102,17 +115,14 @@ class hdf_writer :public abstract_writer { } } - int chunk_stream = storage->chunk_size() * storage->voxel_size(); - - std::vector bits_container; const H5std_string FILE_NAME(fnc); const H5std_string DATASET_NAME("continuous_chunks"); - + const int NC = continuous_count; - const int NX = storage->num_chunks().get(0); - const int NY = storage->num_chunks().get(1); - const int NZ = storage->num_chunks().get(2); + const int NX = storage->chunk_size(); + const int NY = storage->chunk_size(); + const int NZ = storage->chunk_size(); const int RANK = 4; H5::H5File file(FILE_NAME, H5F_ACC_TRUNC); @@ -128,65 +138,46 @@ class hdf_writer :public abstract_writer { datatype.setOrder(H5T_ORDER_LE); H5::DataSet dataset = file.createDataSet(DATASET_NAME, datatype, dataspace); - int column_number = col_n; - hsize_t offset[4]; - offset[0] = 0; - offset[1] = 0; - offset[2] = 0; - offset[3] = 0; - - hsize_t slab_dimsf[2] = { 1, col_n }; - dataspace.selectHyperslab(H5S_SELECT_SET, slab_dimsf, offset); - H5::DataSpace mspace2(RANK, slab_dimsf); - - int cont_count = 0; - for (int i = 0; i < storage->num_chunks().get(0); i++) { + std::vector bits_container; - std::vector bits_container; - auto c = storage->get_chunk(make_vec(i, 0, 0)); - - if (c && c->is_explicit()) { - - /*offset[0] = cont_count; - cont_count++; - offset[1] = 0; - dataspace.selectHyperslab(H5S_SELECT_SET, slab_dimsf, offset);*/ - - continuous_voxel_storage* convox = (continuous_voxel_storage*)c; - auto d = convox->data(); - - for (int v = 0; v < convox->size(); v++) { - auto vd = convox->data()[v]; - // consider using boost::dynamic_bitset - std::bitset<8> b(convox->data()[v]); - - for (int l = 0; l< 8; l++) { - bits_container.push_back(b[l]); - + for (int i = 0; i < storage->num_chunks().get(0); i++) { + for (int j = 0; j < storage->num_chunks().get(1); j++) { + for (int k = 0; k < storage->num_chunks().get(2); k++) { + auto c = storage->get_chunk(make_vec(i, j, k)); + if (c && c->is_explicit()) { + + continuous_voxel_storage* convox = (continuous_voxel_storage*)c; + size_t i0, j0, k0, i1, j1, k1; + i0 = 0; + j0 = 0; + k0 = 0; + convox->extents().tie(i1, j1, k1); + + BEGIN_LOOP_ZERO_2(make_vec(i1, j1, k1)) + bits_container.push_back(convox->Get(ijk)); + END_LOOP; } - } - dataset.write(bits_container.data(), H5::PredType::NATIVE_INT, mspace2, dataspace); - } - - if (c && c->is_constant()) { - std::cout << "Constant handling to implement." << std::endl; + if (c && c->is_constant()) { + std::cout << "Constant handling to implement." << std::endl; + } + else { + std::cout << "Plane handling to implement." << std::endl; + } + } } - else { - std::cout << "Plane handling to implement." << std::endl; + } - } + dataset.write(bits_container.data(), H5::PredType::NATIVE_INT); + } - } - } +}; - }; - From 12438d110e7df6c38a3a33a736ba66eabb010d1b Mon Sep 17 00:00:00 2001 From: johltn Date: Tue, 4 May 2021 09:15:52 +0200 Subject: [PATCH 080/112] Handle constant null storage chunks --- writer.h | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/writer.h b/writer.h index bd07fea..2e078ce 100644 --- a/writer.h +++ b/writer.h @@ -160,12 +160,14 @@ class hdf_writer :public abstract_writer { END_LOOP; } - if (c && c->is_constant()) { + if (c && c->is_constant() || c==nullptr ){ std::cout << "Constant handling to implement." << std::endl; } else { std::cout << "Plane handling to implement." << std::endl; - + planar_voxel_storage* planvox = (planar_voxel_storage*)c; + continuous_voxel_storage* continuoused = planvox->make_explicit(); + } } } From c2dea44e8128d222c95575696b9b46350d9a63b6 Mon Sep 17 00:00:00 2001 From: Thomas Krijnen Date: Tue, 11 May 2021 15:39:52 +0200 Subject: [PATCH 081/112] more optimal meshing in case of many chunks --- storage.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/storage.cpp b/storage.cpp index d31b409..ec30b4f 100644 --- a/storage.cpp +++ b/storage.cpp @@ -108,7 +108,8 @@ void regular_voxel_storage::obj_export(obj_export_helper& obj, bool with_compone std::map< std::array, std::vector< std::pair, size_t> > > triangles; const double d = voxel_size(); - for (auto it = begin(); it != end(); ++it) { + auto end_ = end(); + for (auto it = begin(); it != end_; ++it) { it.value(V1); for (size_t f = 0; f < 6; ++f) { vec_n<3, long> n; From cd33f5f8c1c812ca79138c0c258871826c2d37c3 Mon Sep 17 00:00:00 2001 From: Thomas Krijnen Date: Tue, 11 May 2021 15:40:54 +0200 Subject: [PATCH 082/112] Support for functions and small grammar changes --- tests/test_parser.cpp | 3 +- tests/test_parser_resample.cpp | 2 +- voxec.cpp | 191 ++++++++++++++++++++++++++------- voxec.h | 16 ++- voxec_main.cpp | 2 +- voxelfile.h | 30 +++++- 6 files changed, 193 insertions(+), 51 deletions(-) diff --git a/tests/test_parser.cpp b/tests/test_parser.cpp index 38e0e35..94840fb 100644 --- a/tests/test_parser.cpp +++ b/tests/test_parser.cpp @@ -29,7 +29,8 @@ TEST(Voxelfile, Parser) { auto f = first; voxelfile_parser parser; - std::vector tree; + + std::vector tree; phrase_parse(first, last, parser, blank, tree); ASSERT_EQ(std::distance(f, first), size); diff --git a/tests/test_parser_resample.cpp b/tests/test_parser_resample.cpp index 209338d..eba9829 100644 --- a/tests/test_parser_resample.cpp +++ b/tests/test_parser_resample.cpp @@ -27,7 +27,7 @@ TEST(Voxelfile, Parser) { auto f = first; voxelfile_parser parser; - std::vector tree; + std::vector tree; phrase_parse(first, last, parser, blank, tree); ASSERT_EQ(std::distance(f, first), size); diff --git a/voxec.cpp b/voxec.cpp index 6114a78..ca29151 100644 --- a/voxec.cpp +++ b/voxec.cpp @@ -113,7 +113,11 @@ std::ostream& operator<<(std::ostream& a, const function_call_type& b) { } std::ostream& operator<<(std::ostream& a, const statement_type& b) { - a << boost::fusion::at_c<0>(b) << " = " << boost::fusion::at_c<1>(b); + std::string assignee = boost::fusion::at_c<0>(b); + if (assignee.size()) { + a << assignee << " = "; + } + a << boost::fusion::at_c<1>(b); return a; } @@ -157,33 +161,72 @@ class log_visitor { void operator()(abstract_voxel_storage* const v) { } }; +template +class assigner { + Ts& ts_; + Us& us_; -scope_map run(const std::vector& statements, double size, size_t threads, size_t chunk_size, bool with_mesh, bool with_progress_on_cout) { - scope_map context; +public: + assigner(Ts& ts, Us& us) + : ts_(ts) + , us_(us) + {} - std::unique_ptr pb; - std::unique_ptr ap; - std::function pfn = [&pb](float f) { (*pb)(f); }; - std::function apfn = [&ap](float f) { (*ap)(f); }; - if (with_progress_on_cout) { - pb = std::make_unique(std::cout, progress_bar::DOTS); + void operator()(const T& t) { + ts_.push_back(t); } - else { - pb = std::make_unique(std::cout); + + void operator()(const U& u) { + // @nb overwritten on purpose + us_[u.name()] = u; } +}; - std::vector estimates(statements.size(), 1.f); - ap = std::make_unique(estimates, pfn); +void map_arguments(const std::vector parameters, scope_map& context, const std::vector& args) { - function_arg_value_type threads_ = (int)threads; - context["THREADS"] = threads_; + // @todo revisit the similarities with voxel_operation_runner - function_arg_value_type chunk_size_ = (int)chunk_size; - context["CHUNKSIZE"] = chunk_size_; + bool has_keyword = false; + std::map function_values; + size_t position = 0; - function_arg_value_type vsize_ = size; - context["VOXELSIZE"] = vsize_; + for (auto& arg : args) { + if (has_keyword && !arg.has_keyword()) { + throw std::runtime_error("Keyword argument supplied after positional argument"); + } + has_keyword = arg.has_keyword(); + if (has_keyword) { + if (function_values.find(*arg.keyword()) != function_values.end()) { + throw std::runtime_error("Value for " + *arg.keyword() + "supplied multiple times"); + } + function_values[*arg.keyword()] = arg.value(); + } + else { + if (position >= parameters.size()) { + throw std::runtime_error("Too many positional arguments"); + } + function_values[parameters[position]] = arg.value(); + } + position++; + } + + for (auto& pname : parameters) { + if (function_values.find(pname) == function_values.end()) { + throw std::runtime_error("No value supplied for " + pname); + } + + symbol_value_assignment_visitor v(&context[pname], "*", context); + function_values[pname].apply_visitor(v); + } + + for (auto& arg : args) { + if (arg.has_keyword() && context.find(*arg.keyword()) == context.end()) { + throw std::runtime_error("Keyword argument " + *arg.keyword() + " unused"); + } + } +} +void execute_statements(scope_map& context, const std::map& functions, const std::vector& statements, bool silent, std::unique_ptr& ap, std::function apfn, std::string prefix="") { size_t n = 0; long HAS_VOXELS = boost::mpl::distance< @@ -193,44 +236,63 @@ scope_map run(const std::vector& statements, double size, size_t for (auto it = statements.begin(); it != statements.end(); ++it) { auto& st = *it; - const bool is_last = it == --statements.end(); + // const bool is_last = it == --statements.end(); auto statement_string = boost::lexical_cast(st); - json_logger::message(json_logger::LOG_NOTICE, "executing {statement}", { + json_logger::message(json_logger::LOG_NOTICE, "executing {statement}", { {"statement", {{"text", statement_string}}} - }); - - voxel_operation_runner op(st); - op.application_progress_callback = apfn; - op.silent = with_progress_on_cout; - context[st.assignee()] = op.run(st, context); + }); + + auto fnit = functions.find(st.call().name()); + if (fnit != functions.end()) { + auto context_copy = context; + map_arguments( + fnit->second.args(), + context_copy, + st.call().args() + ); + std::unique_ptr ap_unused; + execute_statements( + context_copy, + functions, + fnit->second.statements(), + silent, + ap_unused, + [](float) {}, + boost::lexical_cast(n) + "." + ); + if (context_copy.find(fnit->second.result_identifier()) == context_copy.end()) { + throw scope_map::not_in_scope("Undefined variable " + fnit->second.result_identifier()); + } + context[st.assignee()] = context_copy[fnit->second.result_identifier()]; + } else { + voxel_operation_runner op(st); + op.application_progress_callback = apfn; + op.silent = silent; + context[st.assignee()] = op.run(st, context); + } if (context[st.assignee()].which() == HAS_VOXELS) { auto voxels = boost::get(context[st.assignee()]); voxel_writer w; w.SetVoxels(voxels); - w.Write(boost::lexical_cast(n) + ".vox"); + w.Write(prefix + boost::lexical_cast(n) + ".vox"); + /* + // @nb automatic meshing of last operand output no longer supported if (with_mesh && is_last) { std::string ofn = boost::lexical_cast(n) + ".obj"; std::ofstream ofs(ofn.c_str()); ((regular_voxel_storage*)voxels)->obj_export(ofs); } + */ if (dynamic_cast(voxels)) { - auto left = dynamic_cast(voxels)->grid_offset(); - auto nc = dynamic_cast(voxels)->num_chunks(); - auto right = (left + nc.as()) - (decltype(left)::element_type)1; - auto sz = dynamic_cast(voxels)->voxel_size(); - auto szl = (long)dynamic_cast(voxels)->chunk_size(); - auto left_world = ((voxels->bounds()[0].as() + left * szl).as() * sz); - auto right_world = ((voxels->bounds()[1].as() + left * szl).as() * sz); - json_logger::message(json_logger::LOG_NOTICE, "storing {value} in {variable}", { {"variable", {{"name", st.assignee()}}}, {"value", dump_info(voxels)}, - }); + }); } } @@ -245,12 +307,61 @@ scope_map run(const std::vector& statements, double size, size_t log_visitor v; context[statements.back().assignee()].apply_visitor(v); - json_logger::message(json_logger::LOG_NOTICE, "scripted finished with {variable}", { + std::string msg = prefix.empty() + ? std::string("script finished with {variable}") + : std::string("function finished with {variable}"); + + json_logger::message(json_logger::LOG_NOTICE, msg, { {"variable", {{"value", v.value()}}}, - }); - } else { + }); + } + else { json_logger::message(json_logger::LOG_WARNING, "no operations in input", {}); } +} + +scope_map run(const std::vector& statements_and_functions, double size, size_t threads, size_t chunk_size, bool with_mesh, bool with_progress_on_cout) { + scope_map context; + + std::unique_ptr pb; + std::unique_ptr ap; + std::function pfn = [&pb](float f) { (*pb)(f); }; + std::function apfn = [&ap](float f) { (*ap)(f); }; + if (with_progress_on_cout) { + pb = std::make_unique(std::cout, progress_bar::DOTS); + } + else { + pb = std::make_unique(std::cout); + } + + // split into statements and function calls + std::vector statements; + std::map functions; + + assigner< + statement_type, + function_def_type, + decltype(statements), + decltype(functions)> vis( + statements, + functions + ); + + std::for_each(statements_and_functions.begin(), statements_and_functions.end(), boost::apply_visitor(vis)); + + std::vector estimates(statements.size(), 1.f); + ap = std::make_unique(estimates, pfn); + + function_arg_value_type threads_ = (int)threads; + context["THREADS"] = threads_; + + function_arg_value_type chunk_size_ = (int)chunk_size; + context["CHUNKSIZE"] = chunk_size_; + + function_arg_value_type vsize_ = size; + context["VOXELSIZE"] = vsize_; + + execute_statements(context, functions, statements, with_progress_on_cout, ap, apfn); return context; } diff --git a/voxec.h b/voxec.h index c7eb356..509c516 100644 --- a/voxec.h +++ b/voxec.h @@ -88,6 +88,10 @@ class scope_map : public std::map { using std::runtime_error::runtime_error; }; + class value_error : public std::runtime_error { + using std::runtime_error::runtime_error; + }; + template const T& get_value_or(const std::string& symbol, const T& default_value) const { auto it = find(symbol); @@ -101,15 +105,19 @@ class scope_map : public std::map { const T& get_value(const std::string& symbol) const { auto it = find(symbol); if (it == end()) { - throw not_in_scope("Not in scope " + symbol); + throw not_in_scope("Undefined variable " + symbol); + } + try { + return get_value_(it->second); + } catch (boost::bad_get&) { + throw value_error(std::string("Expected ") + typeid(T).name() + " got type index " + std::to_string(it->second.which())); } - return get_value_(it->second); } const int get_length(const std::string& symbol) const { auto it = find(symbol); if (it == end()) { - throw not_in_scope("Not in scope " + symbol); + throw not_in_scope("Undefined variable " + symbol); } try { return get_value_(it->second); @@ -1943,6 +1951,6 @@ class voxel_operation_runner { } }; -scope_map run(const std::vector& statements, double size, size_t threads = 0, size_t chunk_size = 128, bool with_mesh = false, bool with_progress_on_cout = false); +scope_map run(const std::vector& statements, double size, size_t threads = 0, size_t chunk_size = 128, bool with_mesh = false, bool with_progress_on_cout = false); #endif diff --git a/voxec_main.cpp b/voxec_main.cpp index c63071e..0b6c8b0 100644 --- a/voxec_main.cpp +++ b/voxec_main.cpp @@ -98,7 +98,7 @@ int main(int argc, char** argv) { auto f = first; voxelfile_parser parser; - std::vector tree; + std::vector tree; phrase_parse(first, last, parser, blank, tree); if (std::distance(f, first) != size) { diff --git a/voxelfile.h b/voxelfile.h index 65fc6c8..1c6bbc3 100644 --- a/voxelfile.h +++ b/voxelfile.h @@ -47,16 +47,34 @@ class statement_type : public boost::fusion::vector, std::vector, std::string> { +public: + const std::string& name() const { + return boost::fusion::at_c<0>(*this); + } + const std::vector& args() const { + return boost::fusion::at_c<1>(*this); + } + const std::vector& statements() const { + return boost::fusion::at_c<2>(*this); + } + const std::string& result_identifier() const { + return boost::fusion::at_c<3>(*this); + } +}; + +typedef boost::variant statement_or_function_def; + std::ostream& operator<<(std::ostream& a, const function_call_type& b); std::ostream& operator<<(std::ostream& a, const statement_type& b); template -struct voxelfile_parser : grammar(), blank_type> { +struct voxelfile_parser : grammar(), blank_type> { - voxelfile_parser() : grammar(), blank_type>(start) { + voxelfile_parser() : grammar(), blank_type>(start) { iden = lexeme[(alpha >> *(alnum | char_('_')))]; - statement = iden >> '=' >> function_call >> eol; + statement = -hold[iden >> '='] >> function_call >> eol >> *eol; quoted_string = char_('"') >> *(char_ - '"') >> char_('"'); // @todo only sequence of strings for now sequence = '{' >> quoted_string % ',' >> '}'; @@ -64,7 +82,9 @@ struct voxelfile_parser : grammar(), blank_type> function_arg = -hold[iden >> '='] >> value; function_args = function_arg % ','; function_call = iden >> '(' >> function_args >> ')'; - start = *statement; + argument_list = iden % ','; + function_def = "function" >> iden >> '(' >> argument_list >> ')' >> eol >> *eol >> *statement >> "return" >> -hold[iden] >> eol >> *eol; + start = *(statement | function_def); } real_parser> strict_double; @@ -75,6 +95,8 @@ struct voxelfile_parser : grammar(), blank_type> rule function_call; rule statement; rule iden, quoted_string; + rule(), blank_type> argument_list; + rule function_def; typename voxelfile_parser::start_type start; }; From 2640dc33b7b315dd6201d05fb3adcff3a86448bf Mon Sep 17 00:00:00 2001 From: johltn Date: Sat, 15 May 2021 21:51:09 +0200 Subject: [PATCH 083/112] Better control flow --- tests/test_hdf.cpp | 57 +++++++++++++++++------ writer.h | 111 ++++++++++++++++++++++++++++++++------------- 2 files changed, 123 insertions(+), 45 deletions(-) diff --git a/tests/test_hdf.cpp b/tests/test_hdf.cpp index cbc6aef..92f7e8a 100644 --- a/tests/test_hdf.cpp +++ b/tests/test_hdf.cpp @@ -1,24 +1,51 @@ #include "../voxelizer.h" #include "../writer.h" +#include "../processor.h" + #include "H5Cpp.h" #include #include #include #include +#include #include #include #include #include +#include + +#ifdef WITH_IFC +#include +#ifdef IFCOPENSHELL_05 +#include +using namespace Ifc2x3; +#else +#include +#endif +#endif + +#ifdef WIN32 +#define DIRSEP "\\" +#else +#define DIRSEP "/" +#endif TEST(HdfFileName, HDF) { - auto storage2 = new chunked_voxel_storage(0., 0., 0., 1, 32, 32, 32, 32); - BRepPrimAPI_MakeBox mb(gp_Pnt(1, 1, 1), gp_Pnt(10, 10, 10)); - auto x = mb.Solid(); + BRepPrimAPI_MakeBox mb(gp_Pnt(0, 0, 0), gp_Pnt(10, 10, 10)); + BRepPrimAPI_MakeSphere s(gp_Pnt(10, 10, 10), 7); + + auto x = s.Solid(); + + auto storage2 = new chunked_voxel_storage( 0, 0, 0, 1, 32, 32, 32, 32); + BRepTools breptools; + + breptools.Write(x, "sphere.brep"); + BRepMesh_IncrementalMesh(x, 0.001); auto vox = voxelizer(x, storage2); vox.Convert(); @@ -26,7 +53,7 @@ TEST(HdfFileName, HDF) { auto storage = new chunked_voxel_storage(0., 0., 0., 0.1, 200, 150, 10, 32); { - BRepBuilderAPI_MakePolygon mp(gp_Pnt(1, 1, 0), gp_Pnt(16, 1, 0), gp_Pnt(16, 9.8, 0), gp_Pnt(1, 9.8, 0), true); + BRepBuilderAPI_MakePolygon mp(gp_Pnt(1, 1, 2), gp_Pnt(16, 1, 2), gp_Pnt(16, 9.8, 2), gp_Pnt(1, 9.8, 2), true); BRepBuilderAPI_MakeFace mf(mp.Wire()); TopoDS_Face face = mf.Face(); @@ -34,9 +61,11 @@ TEST(HdfFileName, HDF) { vox.Convert(); } - hdf_writer writer; - writer.SetVoxels(storage); - writer.Write("voxels.h5"); + + + hdf_writer writer; + writer.SetVoxels(storage2); + writer.Write("multi_dim_vox.h5"); //std::ofstream fs("voxobj.obj"); //obj_export_helper oeh{ fs }; @@ -49,7 +78,7 @@ TEST(HdfFileName, HDF) { const int NX = 32; // dataset dimensions const int NY = 32; const int NZ = 32; - const int NC = 3; + const int NC = 3; const int RANK = 4; H5::H5File file(FILE_NAME, H5F_ACC_TRUNC); @@ -61,18 +90,18 @@ TEST(HdfFileName, HDF) { dimsf[3] = NZ; H5::DataSpace dataspace(RANK, dimsf); - + H5::IntType datatype(H5::PredType::NATIVE_INT); datatype.setOrder(H5T_ORDER_LE); H5::DataSet dataset = file.createDataSet(DATASET_NAME, datatype, dataspace); - std::vector data; + //std::vector data; - for (int i = 0; i < NX*NY*NZ*NC; i++) { - data.push_back(0); - } + //for (int i = 0; i < NX*NY*NZ*NC; i++) { + // data.push_back(0); + //} - dataset.write(data.data(), H5::PredType::NATIVE_INT); + //dataset.write(data.data(), H5::PredType::NATIVE_INT); } diff --git a/writer.h b/writer.h index 2e078ce..7e7e3bd 100644 --- a/writer.h +++ b/writer.h @@ -93,23 +93,28 @@ class hdf_writer :public abstract_writer { int col_n = 0; for (int i = 0; i < storage->num_chunks().get(0); i++) { - auto chunk = storage->get_chunk(make_vec(i, 0, 0)); - - if (chunk && chunk->is_explicit()) { - if (col_n == 0) { - continuous_voxel_storage* casted = (continuous_voxel_storage*)chunk; - col_n = casted->size() * casted->value_bits() * 8; - } - - continuous_count++; + auto c = storage->get_chunk(make_vec(i, 0, 0)); + if (c == nullptr) { + std::cout << "Null pointer" << std::endl; } + else { - if (chunk && chunk->is_constant()) { - constant_count++; + if (c->is_explicit() || c->is_constant()) { + if (c->is_constant()) { + std::cout << "Constant chunk" << std::endl; + constant_count++; + } + else { + std::cout << "Continuous chunk" << std::endl; + continuous_count++; + + } } else { + + std::cout << "Planar chunk" << std::endl; planar_count++; } } @@ -138,36 +143,80 @@ class hdf_writer :public abstract_writer { datatype.setOrder(H5T_ORDER_LE); H5::DataSet dataset = file.createDataSet(DATASET_NAME, datatype, dataspace); - int cont_count = 0; + + //const H5std_string DATASET_NAME2("planar_chunks"); + + //const int NP = planar_count; + //const int NXX = storage->chunk_size(); + //const int NYY = storage->chunk_size(); + + //const int RANK = 3; + //H5::H5File file(FILE_NAME, H5F_ACC_TRUNC); + + //hsize_t dimsf2[3]; + //dimsf2[0] = NP; + //dimsf2[1] = NXX; + //dimsf2[2] = NYY; + // + //H5::DataSpace dataspace2(RANK, dimsf2); + //H5::IntType datatype2(H5::PredType::NATIVE_INT); + //datatype.setOrder(H5T_ORDER_LE); + //H5::DataSet dataset2 = file.createDataSet(DATASET_NAME2, datatype2, dataspace2); + + + std::vector planar_container; std::vector bits_container; for (int i = 0; i < storage->num_chunks().get(0); i++) { for (int j = 0; j < storage->num_chunks().get(1); j++) { for (int k = 0; k < storage->num_chunks().get(2); k++) { auto c = storage->get_chunk(make_vec(i, j, k)); - if (c && c->is_explicit()) { - - continuous_voxel_storage* convox = (continuous_voxel_storage*)c; - size_t i0, j0, k0, i1, j1, k1; - i0 = 0; - j0 = 0; - k0 = 0; - convox->extents().tie(i1, j1, k1); - - BEGIN_LOOP_ZERO_2(make_vec(i1, j1, k1)) - bits_container.push_back(convox->Get(ijk)); - END_LOOP; + if (c == nullptr) { + std::cout << "Null pointer" << std::endl; } - if (c && c->is_constant() || c==nullptr ){ - std::cout << "Constant handling to implement." << std::endl; - } else { - std::cout << "Plane handling to implement." << std::endl; - planar_voxel_storage* planvox = (planar_voxel_storage*)c; - continuous_voxel_storage* continuoused = planvox->make_explicit(); - + if (c->is_explicit() || c->is_constant()) { + if (c->is_constant()) { + std::cout << "Constant chunk" << std::endl; + } + else { + std::cout << "Continuous chunk" << std::endl; + continuous_voxel_storage* convox = (continuous_voxel_storage*)c; + size_t i0, j0, k0, i1, j1, k1; + i0 = 0; + j0 = 0; + k0 = 0; + convox->extents().tie(i1, j1, k1); + + BEGIN_LOOP_ZERO_2(make_vec(i1, j1, k1)) + bits_container.push_back(convox->Get(ijk)); + END_LOOP; + } + } + + else { + std::cout << "Planar chunk" << std::endl; + planar_voxel_storage* planvox = (planar_voxel_storage*)c; + auto off = planvox->offsets(); + auto axis = planvox->axis(); + + bool make_explicit = 0; + if (make_explicit) { + continuous_voxel_storage* convox = planvox->make_explicit(); + size_t i0, j0, k0, i1, j1, k1; + i0 = 0; + j0 = 0; + k0 = 0; + convox->extents().tie(i1, j1, k1); + + BEGIN_LOOP_ZERO_2(make_vec(i1, j1, k1)) + bits_container.push_back(convox->Get(ijk)); + END_LOOP; + + } + } } } } From 8120ab4e8f3149f896e792c9672d3395ffe6ad23 Mon Sep 17 00:00:00 2001 From: johltn Date: Sun, 16 May 2021 13:20:41 +0200 Subject: [PATCH 084/112] Count all chunks and make planar explicit --- writer.h | 43 +++++++++++++++++++++++++------------------ 1 file changed, 25 insertions(+), 18 deletions(-) diff --git a/writer.h b/writer.h index 7e7e3bd..cf3005f 100644 --- a/writer.h +++ b/writer.h @@ -93,32 +93,39 @@ class hdf_writer :public abstract_writer { int col_n = 0; for (int i = 0; i < storage->num_chunks().get(0); i++) { - auto c = storage->get_chunk(make_vec(i, 0, 0)); - - if (c == nullptr) { - std::cout << "Null pointer" << std::endl; - } + for (int j = 0; j < storage->num_chunks().get(1); j++) { + for (int k = 0; k < storage->num_chunks().get(2); k++) { + auto c = storage->get_chunk(make_vec(i, j, k)); - else { - if (c->is_explicit() || c->is_constant()) { - if (c->is_constant()) { - std::cout << "Constant chunk" << std::endl; + if (c == nullptr) { + std::cout << "Null pointer" << std::endl; constant_count++; } + else { - std::cout << "Continuous chunk" << std::endl; - continuous_count++; - - } - } + if (c->is_explicit() || c->is_constant()) { + if (c->is_constant()) { + std::cout << "Count Constant chunk" << std::endl; + constant_count++; + } + else { + std::cout << "Count Continuous chunk" << std::endl; + continuous_count++; - else { + } + } - std::cout << "Planar chunk" << std::endl; - planar_count++; + else { + + std::cout << " Count Planar chunk" << std::endl; + planar_count++; + } + } } } } + + const H5std_string FILE_NAME(fnc); @@ -202,7 +209,7 @@ class hdf_writer :public abstract_writer { auto off = planvox->offsets(); auto axis = planvox->axis(); - bool make_explicit = 0; + bool make_explicit = 1; if (make_explicit) { continuous_voxel_storage* convox = planvox->make_explicit(); size_t i0, j0, k0, i1, j1, k1; From fae86ecf7b7f9e6cc9a6290441585d0ac867919c Mon Sep 17 00:00:00 2001 From: johltn Date: Tue, 1 Jun 2021 08:09:39 +0200 Subject: [PATCH 085/112] Write bits incrementally --- writer.h | 76 +++++++++++++++++++++----------------------------------- 1 file changed, 29 insertions(+), 47 deletions(-) diff --git a/writer.h b/writer.h index cf3005f..ba793e4 100644 --- a/writer.h +++ b/writer.h @@ -90,8 +90,6 @@ class hdf_writer :public abstract_writer { int planar_count = 0; int constant_count = 0; - int col_n = 0; - for (int i = 0; i < storage->num_chunks().get(0); i++) { for (int j = 0; j < storage->num_chunks().get(1); j++) { for (int k = 0; k < storage->num_chunks().get(2); k++) { @@ -111,7 +109,6 @@ class hdf_writer :public abstract_writer { else { std::cout << "Count Continuous chunk" << std::endl; continuous_count++; - } } @@ -127,11 +124,13 @@ class hdf_writer :public abstract_writer { - const H5std_string FILE_NAME(fnc); - const H5std_string DATASET_NAME("continuous_chunks"); + const H5std_string CONTINUOUS_DATASET_NAME("continuous_chunks"); + const H5std_string PLANAR_DATASET_NAME("continuous_chunks"); const int NC = continuous_count; + const int NP = planar_count; + const int NX = storage->chunk_size(); const int NY = storage->chunk_size(); const int NZ = storage->chunk_size(); @@ -148,48 +147,42 @@ class hdf_writer :public abstract_writer { H5::DataSpace dataspace(RANK, dimsf); H5::IntType datatype(H5::PredType::NATIVE_INT); datatype.setOrder(H5T_ORDER_LE); - H5::DataSet dataset = file.createDataSet(DATASET_NAME, datatype, dataspace); + H5::DataSet dataset = file.createDataSet(CONTINUOUS_DATASET_NAME, datatype, dataspace); + //Hyperslab prep + hsize_t offset[4]; + offset[0] = -1; + offset[1] = 0; + offset[2] = 0; + offset[3] = 0; - //const H5std_string DATASET_NAME2("planar_chunks"); + hsize_t slab_dimsf[4] = { 1, 1, 1, 1 }; + H5::DataSpace mspace2(RANK, slab_dimsf); - //const int NP = planar_count; - //const int NXX = storage->chunk_size(); - //const int NYY = storage->chunk_size(); - - //const int RANK = 3; - //H5::H5File file(FILE_NAME, H5F_ACC_TRUNC); - - //hsize_t dimsf2[3]; - //dimsf2[0] = NP; - //dimsf2[1] = NXX; - //dimsf2[2] = NYY; - // - //H5::DataSpace dataspace2(RANK, dimsf2); - //H5::IntType datatype2(H5::PredType::NATIVE_INT); - //datatype.setOrder(H5T_ORDER_LE); - //H5::DataSet dataset2 = file.createDataSet(DATASET_NAME2, datatype2, dataspace2); + for (int x = 0; x < storage->num_chunks().get(0); x++) { + for (int y = 0; y < storage->num_chunks().get(1); y++) { + for (int z = 0; z < storage->num_chunks().get(2); z++) { + auto c = storage->get_chunk(make_vec(x, y, z)); - std::vector planar_container; - std::vector bits_container; - for (int i = 0; i < storage->num_chunks().get(0); i++) { - for (int j = 0; j < storage->num_chunks().get(1); j++) { - for (int k = 0; k < storage->num_chunks().get(2); k++) { - auto c = storage->get_chunk(make_vec(i, j, k)); if (c == nullptr) { std::cout << "Null pointer" << std::endl; + } else { if (c->is_explicit() || c->is_constant()) { if (c->is_constant()) { std::cout << "Constant chunk" << std::endl; + } else { std::cout << "Continuous chunk" << std::endl; + + offset[0]++; + continuous_voxel_storage* convox = (continuous_voxel_storage*)c; size_t i0, j0, k0, i1, j1, k1; i0 = 0; @@ -198,7 +191,13 @@ class hdf_writer :public abstract_writer { convox->extents().tie(i1, j1, k1); BEGIN_LOOP_ZERO_2(make_vec(i1, j1, k1)) - bits_container.push_back(convox->Get(ijk)); + offset[1] = ijk.get(0); + offset[2] = ijk.get(1); + offset[3] = ijk.get(2); + dataspace.selectHyperslab(H5S_SELECT_SET, slab_dimsf, offset); + + std::vector hslab = { convox->Get(ijk) }; + dataset.write(hslab.data(), H5::PredType::NATIVE_INT, mspace2, dataspace); END_LOOP; } } @@ -209,28 +208,11 @@ class hdf_writer :public abstract_writer { auto off = planvox->offsets(); auto axis = planvox->axis(); - bool make_explicit = 1; - if (make_explicit) { - continuous_voxel_storage* convox = planvox->make_explicit(); - size_t i0, j0, k0, i1, j1, k1; - i0 = 0; - j0 = 0; - k0 = 0; - convox->extents().tie(i1, j1, k1); - - BEGIN_LOOP_ZERO_2(make_vec(i1, j1, k1)) - bits_container.push_back(convox->Get(ijk)); - END_LOOP; - - } } } } } } - - dataset.write(bits_container.data(), H5::PredType::NATIVE_INT); - } From b34b14a91d3b5bdcc044d92a72359dfa408c258c Mon Sep 17 00:00:00 2001 From: johltn Date: Tue, 1 Jun 2021 20:47:34 +0200 Subject: [PATCH 086/112] Incrementally write dataset regions in Region dataset --- tests/test_hdf.cpp | 8 ++++++-- writer.h | 46 ++++++++++++++++++++++++++++++++++++++++------ 2 files changed, 46 insertions(+), 8 deletions(-) diff --git a/tests/test_hdf.cpp b/tests/test_hdf.cpp index 92f7e8a..7e51745 100644 --- a/tests/test_hdf.cpp +++ b/tests/test_hdf.cpp @@ -61,10 +61,14 @@ TEST(HdfFileName, HDF) { vox.Convert(); } + const double dim = 0.1; + auto storage3 = new chunked_voxel_storage(-dim, -dim, -dim, dim, 100, 100, 100, 16); + BRepPrimAPI_MakeBox make_box(8., 8., 8.); + auto vox3 = voxelizer(make_box.Solid(), storage3); + vox3.Convert(); - hdf_writer writer; - writer.SetVoxels(storage2); + writer.SetVoxels(storage3); writer.Write("multi_dim_vox.h5"); //std::ofstream fs("voxobj.obj"); diff --git a/writer.h b/writer.h index ba793e4..d0c1ba8 100644 --- a/writer.h +++ b/writer.h @@ -123,10 +123,9 @@ class hdf_writer :public abstract_writer { } - const H5std_string FILE_NAME(fnc); const H5std_string CONTINUOUS_DATASET_NAME("continuous_chunks"); - const H5std_string PLANAR_DATASET_NAME("continuous_chunks"); + const H5std_string PLANAR_DATASET_NAME("planar_chunks"); const int NC = continuous_count; const int NP = planar_count; @@ -149,7 +148,6 @@ class hdf_writer :public abstract_writer { datatype.setOrder(H5T_ORDER_LE); H5::DataSet dataset = file.createDataSet(CONTINUOUS_DATASET_NAME, datatype, dataspace); - //Hyperslab prep hsize_t offset[4]; offset[0] = -1; @@ -160,12 +158,39 @@ class hdf_writer :public abstract_writer { hsize_t slab_dimsf[4] = { 1, 1, 1, 1 }; H5::DataSpace mspace2(RANK, slab_dimsf); + //Regions dataset + const H5std_string REGIONS_DATASET_NAME("regions"); + const int REGION_RANK = 2; + hsize_t regions_dimsf[2]; + regions_dimsf[0] = NC; + regions_dimsf[1] = 1; + H5::DataSpace regions_dataspace(REGION_RANK, regions_dimsf); + H5::PredType regions_datatype(H5::PredType::STD_REF_DSETREG); + regions_datatype.setOrder(H5T_ORDER_LE); + + H5::DataSet regions_dataset = file.createDataSet(REGIONS_DATASET_NAME, regions_datatype, regions_dataspace); + // Regions hyperlab + hsize_t region_offset[2]; + region_offset[0] = -1; + region_offset[1] = 0; + hsize_t region_slab_dimsf[2] = { 1, 1 }; + H5::DataSpace mspace3(REGION_RANK, region_slab_dimsf); + + //Chunk hyperslab + hsize_t chunk_offset[4]; + const int CHUNK_RANK = 4; + chunk_offset[0] = -1; + chunk_offset[1] = 0; + chunk_offset[2] = 0; + chunk_offset[3] = 0; + hsize_t chunk_dimsf[4] = { 1, NX, NY, NZ }; + H5::DataSpace chunk_space(CHUNK_RANK, chunk_dimsf); for (int x = 0; x < storage->num_chunks().get(0); x++) { for (int y = 0; y < storage->num_chunks().get(1); y++) { for (int z = 0; z < storage->num_chunks().get(2); z++) { auto c = storage->get_chunk(make_vec(x, y, z)); - + if (c == nullptr) { std::cout << "Null pointer" << std::endl; @@ -199,6 +224,17 @@ class hdf_writer :public abstract_writer { std::vector hslab = { convox->Get(ijk) }; dataset.write(hslab.data(), H5::PredType::NATIVE_INT, mspace2, dataspace); END_LOOP; + + region_offset[0]++; + regions_dataspace.selectHyperslab(H5S_SELECT_SET, region_slab_dimsf, region_offset); + + chunk_offset[0]++; + + dataspace.selectHyperslab(H5S_SELECT_SET, chunk_dimsf, chunk_offset); + + hobj_ref_t inter[1]; + file.reference(&inter[0], "/continuous_chunks", dataspace, H5R_DATASET_REGION); + regions_dataset.write(inter, H5::PredType::STD_REF_DSETREG, mspace3, regions_dataspace); } } @@ -214,8 +250,6 @@ class hdf_writer :public abstract_writer { } } } - - }; From d4623162577ffe4a77d88ec15b6ee97990ef473b Mon Sep 17 00:00:00 2001 From: johltn Date: Wed, 2 Jun 2021 09:35:59 +0200 Subject: [PATCH 087/112] New test case --- tests/test_hdf_2.cpp | 102 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 102 insertions(+) create mode 100644 tests/test_hdf_2.cpp diff --git a/tests/test_hdf_2.cpp b/tests/test_hdf_2.cpp new file mode 100644 index 0000000..4583968 --- /dev/null +++ b/tests/test_hdf_2.cpp @@ -0,0 +1,102 @@ +#include "../voxelizer.h" +#include "../writer.h" +#include "../processor.h" + +#ifdef WITH_IFC +#include +#ifdef IFCOPENSHELL_05 +#include +using namespace Ifc2x3; +#else +#include +#endif +#endif + +#include +#include +#include + +#include + +#ifdef WIN32 +#define DIRSEP "\\" +#else +#define DIRSEP "/" +#endif + + +#if defined(WITH_IFC) && !defined(IFCOPENSHELL_05) +TEST(Voxelization, IfcWallIds) { + const std::string input_filename = ".." DIRSEP "tests" DIRSEP "fixtures" DIRSEP "duplex.ifc"; + + IfcParse::IfcFile ifc_file(input_filename); + + const double d = 0.5; + + ASSERT_TRUE(ifc_file.good()); + + IfcGeom::IteratorSettings settings_surface; + settings_surface.set(IfcGeom::IteratorSettings::DISABLE_TRIANGULATION, true); + settings_surface.set(IfcGeom::IteratorSettings::USE_WORLD_COORDS, true); + + IfcGeom::entity_filter ef; + ef.include = true; + ef.entity_names = { "IfcWall" }; + ef.traverse = false; + + auto filters = std::vector({ ef }); + + IfcGeom::Iterator it(settings_surface, &ifc_file, filters, 1); + + ASSERT_TRUE(it.initialize()); + + geometry_collection_t geoms; + + while (true) { + TopoDS_Compound C = it.get_native()->geometry().as_compound(); + BRepMesh_IncrementalMesh(C, 0.001); + geoms.push_back({ it.get_native()->id(), C }); + if (!it.next()) { + break; + } + } + + Bnd_Box global_bounds; + for (auto& P : geoms) { + BRepBndLib::Add(P.second, global_bounds); + } + + double x1, y1, z1, x2, y2, z2; + global_bounds.Get(x1, y1, z1, x2, y2, z2); + int nx = (int)ceil((x2 - x1) / d) + 10; + int ny = (int)ceil((y2 - y1) / d) + 10; + int nz = (int)ceil((z2 - z1) / d) + 10; + + x1 -= d * 5; + y1 -= d * 5; + z1 -= d * 5; + + chunked_voxel_storage* storage = new chunked_voxel_storage(x1, y1, z1, d, nx, ny, nz, 64); + + { + progress_writer progress_1("test_wall_ids"); + processor pr(storage, progress_1); + pr.use_scanline() = false; + pr.process(geoms.begin(), geoms.end(), VOLUME_PRODUCT_ID(), output(MERGED(), "test_wall_ids.vox")); + } + + // PRINT WORLD BOUNDS OF STORAGE...e b + + //std::ofstream fs("boundaries.obj"); + //obj_export_helper oeh{ fs }; + //storage->obj_export(oeh, false, true); + + + hdf_writer writer; + writer.SetVoxels(storage); + writer.Write("test_walls.h5"); + +} +#else +TEST(Voxelization, DISABLED_IfcSpaceIds) {} +#endif \ No newline at end of file From 7ae6a5f486fc812684cb3851c93655491ce99de1 Mon Sep 17 00:00:00 2001 From: johltn Date: Wed, 2 Jun 2021 09:59:53 +0200 Subject: [PATCH 088/112] Install hdf5 --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 2974493..899d3c5 100644 --- a/.travis.yml +++ b/.travis.yml @@ -12,6 +12,7 @@ install: - | sudo apt-get install -y \ libocct-* \ + libhdf5-dev \ libboost-all-dev script: From 05793b7549bedc0698bc7a0063c2a4823f4d7e68 Mon Sep 17 00:00:00 2001 From: johltn Date: Wed, 2 Jun 2021 10:17:04 +0200 Subject: [PATCH 089/112] Add pathes to include and lib dirs --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 899d3c5..9740af7 100644 --- a/.travis.yml +++ b/.travis.yml @@ -18,7 +18,7 @@ install: script: - mkdir build && cd build - | - cmake .. \ + cmake -DHDF5_INCLUDE_DIR=/usr/include/hdf5 -DHDF5_LIBRARY_DIR=/usr/lib/x86_64-linux-gnu/libhdf5_cpp.a;/usr/lib/x86_64-linux-gnu/libhdf5.a;/usr/lib/x86_64-linux-gnu/libzlib.a;/usr/lib/x86_64-linux-gnu/libsz.a;/usr/lib/x86_64-linux-gnu/libaec.a .. \ -DIFC_SUPPORT=Off \ -DOCC_INCLUDE_DIR=/usr/include/opencascade \ -DOCC_LIBRARY_DIR=/usr/lib/x86_64-linux-gnu \ From e8d4f672d545e6cd19e3419990b151ed5986f33c Mon Sep 17 00:00:00 2001 From: johltn Date: Wed, 2 Jun 2021 10:33:33 +0200 Subject: [PATCH 090/112] Add quotes --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 9740af7..68b12fa 100644 --- a/.travis.yml +++ b/.travis.yml @@ -18,7 +18,7 @@ install: script: - mkdir build && cd build - | - cmake -DHDF5_INCLUDE_DIR=/usr/include/hdf5 -DHDF5_LIBRARY_DIR=/usr/lib/x86_64-linux-gnu/libhdf5_cpp.a;/usr/lib/x86_64-linux-gnu/libhdf5.a;/usr/lib/x86_64-linux-gnu/libzlib.a;/usr/lib/x86_64-linux-gnu/libsz.a;/usr/lib/x86_64-linux-gnu/libaec.a .. \ + cmake -DHDF5_INCLUDE_DIR=/usr/include/hdf5 -DHDF5_LIBRARY_DIR="/usr/lib/x86_64-linux-gnu/libhdf5_cpp.a;/usr/lib/x86_64-linux-gnu/libhdf5.a;/usr/lib/x86_64-linux-gnu/libzlib.a;/usr/lib/x86_64-linux-gnu/libsz.a;/usr/lib/x86_64-linux-gnu/libaec.a" .. \ -DIFC_SUPPORT=Off \ -DOCC_INCLUDE_DIR=/usr/include/opencascade \ -DOCC_LIBRARY_DIR=/usr/lib/x86_64-linux-gnu \ From ac94ad1febfe7f345bf5465a909db9fe8cf66c54 Mon Sep 17 00:00:00 2001 From: johltn Date: Wed, 2 Jun 2021 11:02:04 +0200 Subject: [PATCH 091/112] Add subdir for include path --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 68b12fa..1c71903 100644 --- a/.travis.yml +++ b/.travis.yml @@ -18,7 +18,7 @@ install: script: - mkdir build && cd build - | - cmake -DHDF5_INCLUDE_DIR=/usr/include/hdf5 -DHDF5_LIBRARY_DIR="/usr/lib/x86_64-linux-gnu/libhdf5_cpp.a;/usr/lib/x86_64-linux-gnu/libhdf5.a;/usr/lib/x86_64-linux-gnu/libzlib.a;/usr/lib/x86_64-linux-gnu/libsz.a;/usr/lib/x86_64-linux-gnu/libaec.a" .. \ + cmake -DHDF5_INCLUDE_DIR=/usr/include/hdf5/serial -DHDF5_LIBRARY_DIR="/usr/lib/x86_64-linux-gnu/libhdf5_cpp.a;/usr/lib/x86_64-linux-gnu/libhdf5.a;/usr/lib/x86_64-linux-gnu/libzlib.a;/usr/lib/x86_64-linux-gnu/libsz.a;/usr/lib/x86_64-linux-gnu/libaec.a" .. \ -DIFC_SUPPORT=Off \ -DOCC_INCLUDE_DIR=/usr/include/opencascade \ -DOCC_LIBRARY_DIR=/usr/lib/x86_64-linux-gnu \ From d58742be7252a02bba809a93d539fd92e2b0d2b5 Mon Sep 17 00:00:00 2001 From: johltn Date: Wed, 2 Jun 2021 14:17:37 +0200 Subject: [PATCH 092/112] Move HDF installation variables --- CMakeLists.txt | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 2fea951..12a0244 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -112,18 +112,6 @@ endif() set(BOOST_COMPONENTS regex program_options iostreams system) -SET(HDF5_INCLUDE_DIR ${IFCOPENSHELL_ROOT}/_deps-vs2017-x64-installed/HDF5-1.8.22-win64/include) -SET(HDF5_LIBRARY_DIR ${IFCOPENSHELL_ROOT}/_deps-vs2017-x64-installed/HDF5-1.8.22-win64/lib) - - -SET(HDF5_LIBRARIES - "${HDF5_LIBRARY_DIR}/libhdf5_cpp.lib" - "${HDF5_LIBRARY_DIR}/libhdf5.lib" - "${HDF5_LIBRARY_DIR}/libzlib.lib" - "${HDF5_LIBRARY_DIR}/libsz.lib" - "${HDF5_LIBRARY_DIR}/libaec.lib") - - if (IFC_SUPPORT) # Find IfcOpenShell version, v0.6.0 does not have the IfcSchema namespace anymore file(READ "${IFCOPENSHELL_ROOT}/src/ifcparse/IfcParse.h" header) @@ -161,6 +149,17 @@ if(UNIX) string(REGEX REPLACE "([^;]+)" "${ICU_LIBRARY_DIR}/lib\\1.a" ICU_LIBRARIES "${ICU_LIBRARY_NAMES}") endif() else() + + SET(HDF5_INCLUDE_DIR ${IFCOPENSHELL_ROOT}/_deps-vs2017-x64-installed/HDF5-1.8.22-win64/include) + SET(HDF5_LIBRARY_DIR ${IFCOPENSHELL_ROOT}/_deps-vs2017-x64-installed/HDF5-1.8.22-win64/lib) + + SET(HDF5_LIBRARIES + "${HDF5_LIBRARY_DIR}/libhdf5_cpp.lib" + "${HDF5_LIBRARY_DIR}/libhdf5.lib" + "${HDF5_LIBRARY_DIR}/libzlib.lib" + "${HDF5_LIBRARY_DIR}/libsz.lib" + "${HDF5_LIBRARY_DIR}/libaec.lib") + set(BOOST_ROOT ${IFCOPENSHELL_ROOT}/_deps/boost_1_74_0) set(BOOST_LIBRARYDIR ${IFCOPENSHELL_ROOT}/_deps/boost_1_74_0/stage/vs${MSVC_YEAR}-${WIN_ARCH}/lib) From 05dd0576381d0dd919a813385c11527c86a1b0aa Mon Sep 17 00:00:00 2001 From: johltn Date: Wed, 2 Jun 2021 15:04:49 +0200 Subject: [PATCH 093/112] Libraries variable --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 1c71903..b383928 100644 --- a/.travis.yml +++ b/.travis.yml @@ -18,7 +18,7 @@ install: script: - mkdir build && cd build - | - cmake -DHDF5_INCLUDE_DIR=/usr/include/hdf5/serial -DHDF5_LIBRARY_DIR="/usr/lib/x86_64-linux-gnu/libhdf5_cpp.a;/usr/lib/x86_64-linux-gnu/libhdf5.a;/usr/lib/x86_64-linux-gnu/libzlib.a;/usr/lib/x86_64-linux-gnu/libsz.a;/usr/lib/x86_64-linux-gnu/libaec.a" .. \ + cmake -DHDF5_INCLUDE_DIR=/usr/include/hdf5/serial -DHDF5_LIBRARIES="/usr/lib/x86_64-linux-gnu/libhdf5_cpp.a;/usr/lib/x86_64-linux-gnu/libhdf5.a;/usr/lib/x86_64-linux-gnu/libzlib.a;/usr/lib/x86_64-linux-gnu/libsz.a;/usr/lib/x86_64-linux-gnu/libaec.a" .. \ -DIFC_SUPPORT=Off \ -DOCC_INCLUDE_DIR=/usr/include/opencascade \ -DOCC_LIBRARY_DIR=/usr/lib/x86_64-linux-gnu \ From d7612354897862fe3dab85e758e14f02e2240754 Mon Sep 17 00:00:00 2001 From: johltn Date: Wed, 2 Jun 2021 15:17:58 +0200 Subject: [PATCH 094/112] Update Travis file --- .travis.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index b383928..38dd2fd 100644 --- a/.travis.yml +++ b/.travis.yml @@ -18,7 +18,9 @@ install: script: - mkdir build && cd build - | - cmake -DHDF5_INCLUDE_DIR=/usr/include/hdf5/serial -DHDF5_LIBRARIES="/usr/lib/x86_64-linux-gnu/libhdf5_cpp.a;/usr/lib/x86_64-linux-gnu/libhdf5.a;/usr/lib/x86_64-linux-gnu/libzlib.a;/usr/lib/x86_64-linux-gnu/libsz.a;/usr/lib/x86_64-linux-gnu/libaec.a" .. \ + cmake .. \ + -DHDF5_INCLUDE_DIR=/usr/include/hdf5/serial + -DHDF5_LIBRARIES="/usr/lib/x86_64-linux-gnu/libhdf5_cpp.a;/usr/lib/x86_64-linux-gnu/libhdf5.a;/usr/lib/x86_64-linux-gnu/libzlib.a;/usr/lib/x86_64-linux-gnu/libsz.a;/usr/lib/x86_64-linux-gnu/libaec.a" -DIFC_SUPPORT=Off \ -DOCC_INCLUDE_DIR=/usr/include/opencascade \ -DOCC_LIBRARY_DIR=/usr/lib/x86_64-linux-gnu \ From 94d7c0a0ef9c350080379f8b70059f64cf71085a Mon Sep 17 00:00:00 2001 From: johltn Date: Wed, 2 Jun 2021 15:22:35 +0200 Subject: [PATCH 095/112] Update Travis file --- .travis.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 38dd2fd..b383928 100644 --- a/.travis.yml +++ b/.travis.yml @@ -18,9 +18,7 @@ install: script: - mkdir build && cd build - | - cmake .. \ - -DHDF5_INCLUDE_DIR=/usr/include/hdf5/serial - -DHDF5_LIBRARIES="/usr/lib/x86_64-linux-gnu/libhdf5_cpp.a;/usr/lib/x86_64-linux-gnu/libhdf5.a;/usr/lib/x86_64-linux-gnu/libzlib.a;/usr/lib/x86_64-linux-gnu/libsz.a;/usr/lib/x86_64-linux-gnu/libaec.a" + cmake -DHDF5_INCLUDE_DIR=/usr/include/hdf5/serial -DHDF5_LIBRARIES="/usr/lib/x86_64-linux-gnu/libhdf5_cpp.a;/usr/lib/x86_64-linux-gnu/libhdf5.a;/usr/lib/x86_64-linux-gnu/libzlib.a;/usr/lib/x86_64-linux-gnu/libsz.a;/usr/lib/x86_64-linux-gnu/libaec.a" .. \ -DIFC_SUPPORT=Off \ -DOCC_INCLUDE_DIR=/usr/include/opencascade \ -DOCC_LIBRARY_DIR=/usr/lib/x86_64-linux-gnu \ From 80cc06c0bf180950df42ad1340bb501151631e5e Mon Sep 17 00:00:00 2001 From: johltn Date: Wed, 2 Jun 2021 15:40:08 +0200 Subject: [PATCH 096/112] serial --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index b383928..b134f0b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -18,7 +18,7 @@ install: script: - mkdir build && cd build - | - cmake -DHDF5_INCLUDE_DIR=/usr/include/hdf5/serial -DHDF5_LIBRARIES="/usr/lib/x86_64-linux-gnu/libhdf5_cpp.a;/usr/lib/x86_64-linux-gnu/libhdf5.a;/usr/lib/x86_64-linux-gnu/libzlib.a;/usr/lib/x86_64-linux-gnu/libsz.a;/usr/lib/x86_64-linux-gnu/libaec.a" .. \ + cmake -DHDF5_INCLUDE_DIR=/usr/include/hdf5/serial -DHDF5_LIBRARIES="/usr/lib/x86_64-linux-gnu/hdf5/serial/libhdf5_cpp.a;/usr/lib/x86_64-linux-gnu/hdf5/serial/libhdf5.a;/usr/lib/x86_64-linux-gnu/hdf5/serial/libzlib.a;/usr/lib/x86_64-linux-gnu/hdf5/serial/libsz.a;/usr/lib/x86_64-linux-gnu/hdf5/serial/libaec.a" .. \ -DIFC_SUPPORT=Off \ -DOCC_INCLUDE_DIR=/usr/include/opencascade \ -DOCC_LIBRARY_DIR=/usr/lib/x86_64-linux-gnu \ From 866d71611b56f65400612bcf1f2729b6a8d98093 Mon Sep 17 00:00:00 2001 From: johltn Date: Wed, 2 Jun 2021 16:11:44 +0200 Subject: [PATCH 097/112] Replace lib name --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index b134f0b..8ba56df 100644 --- a/.travis.yml +++ b/.travis.yml @@ -18,7 +18,7 @@ install: script: - mkdir build && cd build - | - cmake -DHDF5_INCLUDE_DIR=/usr/include/hdf5/serial -DHDF5_LIBRARIES="/usr/lib/x86_64-linux-gnu/hdf5/serial/libhdf5_cpp.a;/usr/lib/x86_64-linux-gnu/hdf5/serial/libhdf5.a;/usr/lib/x86_64-linux-gnu/hdf5/serial/libzlib.a;/usr/lib/x86_64-linux-gnu/hdf5/serial/libsz.a;/usr/lib/x86_64-linux-gnu/hdf5/serial/libaec.a" .. \ + cmake -DHDF5_INCLUDE_DIR=/usr/include/hdf5/serial -DHDF5_LIBRARIES="/usr/lib/x86_64-linux-gnu/hdf5/serial/libhdf5_cpp.a;/usr/lib/x86_64-linux-gnu/hdf5/serial/libhdf5.a;/usr/lib/x86_64-linux-gnu/libz.a;/usr/lib/x86_64-linux-gnu/hdf5/serial/libsz.a;/usr/lib/x86_64-linux-gnu/hdf5/serial/libaec.a" .. \ -DIFC_SUPPORT=Off \ -DOCC_INCLUDE_DIR=/usr/include/opencascade \ -DOCC_LIBRARY_DIR=/usr/lib/x86_64-linux-gnu \ From c04b2e4a9706094998bf5a9e6a29b45adda2622b Mon Sep 17 00:00:00 2001 From: johltn Date: Wed, 2 Jun 2021 16:25:28 +0200 Subject: [PATCH 098/112] serial --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 8ba56df..3f294f6 100644 --- a/.travis.yml +++ b/.travis.yml @@ -18,7 +18,7 @@ install: script: - mkdir build && cd build - | - cmake -DHDF5_INCLUDE_DIR=/usr/include/hdf5/serial -DHDF5_LIBRARIES="/usr/lib/x86_64-linux-gnu/hdf5/serial/libhdf5_cpp.a;/usr/lib/x86_64-linux-gnu/hdf5/serial/libhdf5.a;/usr/lib/x86_64-linux-gnu/libz.a;/usr/lib/x86_64-linux-gnu/hdf5/serial/libsz.a;/usr/lib/x86_64-linux-gnu/hdf5/serial/libaec.a" .. \ + cmake -DHDF5_INCLUDE_DIR=/usr/include/hdf5/serial -DHDF5_LIBRARIES="/usr/lib/x86_64-linux-gnu/hdf5/serial/libhdf5_cpp.a;/usr/lib/x86_64-linux-gnu/hdf5/serial/libhdf5.a;/usr/lib/x86_64-linux-gnu/libz.a;/usr/lib/x86_64-linux-gnu/libsz.a;/usr/lib/x86_64-linux-gnu/hdf5/serial/libaec.a" .. \ -DIFC_SUPPORT=Off \ -DOCC_INCLUDE_DIR=/usr/include/opencascade \ -DOCC_LIBRARY_DIR=/usr/lib/x86_64-linux-gnu \ From 1f7a652aab8329422a7ff9d7a51e7c6dacdd5113 Mon Sep 17 00:00:00 2001 From: johltn Date: Wed, 2 Jun 2021 19:24:04 +0200 Subject: [PATCH 099/112] Update Travis file --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 3f294f6..4a50ad4 100644 --- a/.travis.yml +++ b/.travis.yml @@ -18,7 +18,7 @@ install: script: - mkdir build && cd build - | - cmake -DHDF5_INCLUDE_DIR=/usr/include/hdf5/serial -DHDF5_LIBRARIES="/usr/lib/x86_64-linux-gnu/hdf5/serial/libhdf5_cpp.a;/usr/lib/x86_64-linux-gnu/hdf5/serial/libhdf5.a;/usr/lib/x86_64-linux-gnu/libz.a;/usr/lib/x86_64-linux-gnu/libsz.a;/usr/lib/x86_64-linux-gnu/hdf5/serial/libaec.a" .. \ + cmake -DHDF5_INCLUDE_DIR=/usr/include/hdf5/serial -DHDF5_LIBRARIES="/usr/lib/x86_64-linux-gnu/hdf5/serial/libhdf5_cpp.a;/usr/lib/x86_64-linux-gnu/hdf5/serial/libhdf5.a;/usr/lib/x86_64-linux-gnu/libz.a;/usr/lib/x86_64-linux-gnu/libsz.a;/usr/lib/x86_64-linux-gnu/libaec.a" .. \ -DIFC_SUPPORT=Off \ -DOCC_INCLUDE_DIR=/usr/include/opencascade \ -DOCC_LIBRARY_DIR=/usr/lib/x86_64-linux-gnu \ From 59281bdc577703992950a10330f622279d27a893 Mon Sep 17 00:00:00 2001 From: johltn Date: Thu, 3 Jun 2021 10:17:51 +0200 Subject: [PATCH 100/112] Add dl --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 4a50ad4..b52625d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -18,7 +18,7 @@ install: script: - mkdir build && cd build - | - cmake -DHDF5_INCLUDE_DIR=/usr/include/hdf5/serial -DHDF5_LIBRARIES="/usr/lib/x86_64-linux-gnu/hdf5/serial/libhdf5_cpp.a;/usr/lib/x86_64-linux-gnu/hdf5/serial/libhdf5.a;/usr/lib/x86_64-linux-gnu/libz.a;/usr/lib/x86_64-linux-gnu/libsz.a;/usr/lib/x86_64-linux-gnu/libaec.a" .. \ + cmake -DHDF5_INCLUDE_DIR=/usr/include/hdf5/serial -DHDF5_LIBRARIES="/usr/lib/x86_64-linux-gnu/hdf5/serial/libhdf5_cpp.a;/usr/lib/x86_64-linux-gnu/hdf5/serial/libhdf5.a;/usr/lib/x86_64-linux-gnu/libz.a;/usr/lib/x86_64-linux-gnu/libsz.a;/usr/lib/x86_64-linux-gnu/libaec.a;dl" .. \ -DIFC_SUPPORT=Off \ -DOCC_INCLUDE_DIR=/usr/include/opencascade \ -DOCC_LIBRARY_DIR=/usr/lib/x86_64-linux-gnu \ From 92f1eec00639a333d44bb63294a95328ac9c32b3 Mon Sep 17 00:00:00 2001 From: johltn Date: Sun, 6 Jun 2021 13:12:53 +0200 Subject: [PATCH 101/112] Fix iterations over chunks --- writer.h | 159 +++++++++++++++++++++++++++---------------------------- 1 file changed, 77 insertions(+), 82 deletions(-) diff --git a/writer.h b/writer.h index d0c1ba8..699b8bd 100644 --- a/writer.h +++ b/writer.h @@ -90,38 +90,37 @@ class hdf_writer :public abstract_writer { int planar_count = 0; int constant_count = 0; - for (int i = 0; i < storage->num_chunks().get(0); i++) { - for (int j = 0; j < storage->num_chunks().get(1); j++) { - for (int k = 0; k < storage->num_chunks().get(2); k++) { - auto c = storage->get_chunk(make_vec(i, j, k)); + hsize_t nchunks_x = storage->num_chunks().get(0); + hsize_t nchunks_y = storage->num_chunks().get(1); + hsize_t nchunks_z = storage->num_chunks().get(2); - if (c == nullptr) { - std::cout << "Null pointer" << std::endl; + BEGIN_LOOP(size_t(0), nchunks_x, 0U, nchunks_y, 0U, nchunks_z) + auto c = storage->get_chunk(ijk); + + if (c == nullptr) { + std::cout << "Null pointer" << std::endl; + constant_count++; + } + + else { + if (c->is_explicit() || c->is_constant()) { + if (c->is_constant()) { + std::cout << "Count Constant chunk" << std::endl; constant_count++; } - else { - if (c->is_explicit() || c->is_constant()) { - if (c->is_constant()) { - std::cout << "Count Constant chunk" << std::endl; - constant_count++; - } - else { - std::cout << "Count Continuous chunk" << std::endl; - continuous_count++; - } - } - - else { - - std::cout << " Count Planar chunk" << std::endl; - planar_count++; - } + std::cout << "Count Continuous chunk" << std::endl; + continuous_count++; } } + + else { + + std::cout << " Count Planar chunk" << std::endl; + planar_count++; + } } - } - + END_LOOP; const H5std_string FILE_NAME(fnc); const H5std_string CONTINUOUS_DATASET_NAME("continuous_chunks"); @@ -129,7 +128,7 @@ class hdf_writer :public abstract_writer { const int NC = continuous_count; const int NP = planar_count; - + const int NX = storage->chunk_size(); const int NY = storage->chunk_size(); const int NZ = storage->chunk_size(); @@ -186,69 +185,65 @@ class hdf_writer :public abstract_writer { hsize_t chunk_dimsf[4] = { 1, NX, NY, NZ }; H5::DataSpace chunk_space(CHUNK_RANK, chunk_dimsf); - for (int x = 0; x < storage->num_chunks().get(0); x++) { - for (int y = 0; y < storage->num_chunks().get(1); y++) { - for (int z = 0; z < storage->num_chunks().get(2); z++) { - auto c = storage->get_chunk(make_vec(x, y, z)); - - if (c == nullptr) { - std::cout << "Null pointer" << std::endl; + BEGIN_LOOP(size_t(0), nchunks_x, 0U, nchunks_y, 0U, nchunks_z) + auto c = storage->get_chunk(ijk); - } + if (c == nullptr) { + std::cout << "Null pointer" << std::endl; + + } + + else { + if (c->is_explicit() || c->is_constant()) { + if (c->is_constant()) { + std::cout << "Constant chunk" << std::endl; + } else { - if (c->is_explicit() || c->is_constant()) { - if (c->is_constant()) { - std::cout << "Constant chunk" << std::endl; - - } - else { - std::cout << "Continuous chunk" << std::endl; - - offset[0]++; - - continuous_voxel_storage* convox = (continuous_voxel_storage*)c; - size_t i0, j0, k0, i1, j1, k1; - i0 = 0; - j0 = 0; - k0 = 0; - convox->extents().tie(i1, j1, k1); - - BEGIN_LOOP_ZERO_2(make_vec(i1, j1, k1)) - offset[1] = ijk.get(0); - offset[2] = ijk.get(1); - offset[3] = ijk.get(2); - dataspace.selectHyperslab(H5S_SELECT_SET, slab_dimsf, offset); - - std::vector hslab = { convox->Get(ijk) }; - dataset.write(hslab.data(), H5::PredType::NATIVE_INT, mspace2, dataspace); - END_LOOP; - - region_offset[0]++; - regions_dataspace.selectHyperslab(H5S_SELECT_SET, region_slab_dimsf, region_offset); - - chunk_offset[0]++; - - dataspace.selectHyperslab(H5S_SELECT_SET, chunk_dimsf, chunk_offset); - - hobj_ref_t inter[1]; - file.reference(&inter[0], "/continuous_chunks", dataspace, H5R_DATASET_REGION); - regions_dataset.write(inter, H5::PredType::STD_REF_DSETREG, mspace3, regions_dataspace); - } - } - - else { - std::cout << "Planar chunk" << std::endl; - planar_voxel_storage* planvox = (planar_voxel_storage*)c; - auto off = planvox->offsets(); - auto axis = planvox->axis(); - - } + std::cout << "Continuous chunk" << std::endl; + + offset[0]++; + + continuous_voxel_storage* convox = (continuous_voxel_storage*)c; + size_t i0, j0, k0, i1, j1, k1; + i0 = 0; + j0 = 0; + k0 = 0; + convox->extents().tie(i1, j1, k1); + + BEGIN_LOOP_ZERO_2(make_vec(i1, j1, k1)) + offset[1] = ijk.get(0); + offset[2] = ijk.get(1); + offset[3] = ijk.get(2); + dataspace.selectHyperslab(H5S_SELECT_SET, slab_dimsf, offset); + + std::vector hslab = { convox->Get(ijk) }; + dataset.write(hslab.data(), H5::PredType::NATIVE_INT, mspace2, dataspace); + END_LOOP; + + region_offset[0]++; + regions_dataspace.selectHyperslab(H5S_SELECT_SET, region_slab_dimsf, region_offset); + + chunk_offset[0]++; + + dataspace.selectHyperslab(H5S_SELECT_SET, chunk_dimsf, chunk_offset); + + hobj_ref_t inter[1]; + file.reference(&inter[0], "/continuous_chunks", dataspace, H5R_DATASET_REGION); + //regions_dataset.write(inter, H5::PredType::STD_REF_DSETREG, mspace3, regions_dataspace); } } + + else { + std::cout << "Planar chunk" << std::endl; + planar_voxel_storage* planvox = (planar_voxel_storage*)c; + auto off = planvox->offsets(); + auto axis = planvox->axis(); + + } } - } + END_LOOP } }; From 835a0608084cc76afbfbff80735b9c672e65e45b Mon Sep 17 00:00:00 2001 From: johltn Date: Sun, 6 Jun 2021 13:17:57 +0200 Subject: [PATCH 102/112] Fix region slabs dimensions --- writer.h | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/writer.h b/writer.h index 699b8bd..ec6d5cd 100644 --- a/writer.h +++ b/writer.h @@ -159,20 +159,20 @@ class hdf_writer :public abstract_writer { //Regions dataset const H5std_string REGIONS_DATASET_NAME("regions"); - const int REGION_RANK = 2; - hsize_t regions_dimsf[2]; + const int REGION_RANK = 1; + hsize_t regions_dimsf[1]; regions_dimsf[0] = NC; - regions_dimsf[1] = 1; + //regions_dimsf[1] = 1; H5::DataSpace regions_dataspace(REGION_RANK, regions_dimsf); H5::PredType regions_datatype(H5::PredType::STD_REF_DSETREG); regions_datatype.setOrder(H5T_ORDER_LE); H5::DataSet regions_dataset = file.createDataSet(REGIONS_DATASET_NAME, regions_datatype, regions_dataspace); // Regions hyperlab - hsize_t region_offset[2]; + hsize_t region_offset[1]; region_offset[0] = -1; - region_offset[1] = 0; - hsize_t region_slab_dimsf[2] = { 1, 1 }; + //region_offset[1] = 0; + hsize_t region_slab_dimsf[1] = { 1}; H5::DataSpace mspace3(REGION_RANK, region_slab_dimsf); //Chunk hyperslab @@ -231,7 +231,7 @@ class hdf_writer :public abstract_writer { hobj_ref_t inter[1]; file.reference(&inter[0], "/continuous_chunks", dataspace, H5R_DATASET_REGION); - //regions_dataset.write(inter, H5::PredType::STD_REF_DSETREG, mspace3, regions_dataspace); + regions_dataset.write(inter, H5::PredType::STD_REF_DSETREG, mspace3, regions_dataspace); } } From d4012d0e5162fd81100ff302560dbbc6246bb056 Mon Sep 17 00:00:00 2001 From: johltn Date: Mon, 7 Jun 2021 12:00:52 +0200 Subject: [PATCH 103/112] Tentative to incrementally write planar chunks to compound typed dataset --- writer.h | 189 ++++++++++++++++++++++++++++++++++++------------------- 1 file changed, 126 insertions(+), 63 deletions(-) diff --git a/writer.h b/writer.h index ec6d5cd..811c8fe 100644 --- a/writer.h +++ b/writer.h @@ -4,6 +4,7 @@ #include "storage.h" #include "H5Cpp.h" + #include #include #include @@ -84,6 +85,10 @@ class hdf_writer :public abstract_writer { public: void Write(const std::string& fnc) { + + + + chunked_voxel_storage* storage = (chunked_voxel_storage*)voxels_; int continuous_count = 0; @@ -97,34 +102,34 @@ class hdf_writer :public abstract_writer { BEGIN_LOOP(size_t(0), nchunks_x, 0U, nchunks_y, 0U, nchunks_z) auto c = storage->get_chunk(ijk); - if (c == nullptr) { - std::cout << "Null pointer" << std::endl; - constant_count++; - } + if (c == nullptr) { + std::cout << "Null pointer" << std::endl; + constant_count++; + } - else { - if (c->is_explicit() || c->is_constant()) { - if (c->is_constant()) { - std::cout << "Count Constant chunk" << std::endl; - constant_count++; - } - else { - std::cout << "Count Continuous chunk" << std::endl; - continuous_count++; - } + else { + if (c->is_explicit() || c->is_constant()) { + if (c->is_constant()) { + std::cout << "Count Constant chunk" << std::endl; + constant_count++; } - else { - - std::cout << " Count Planar chunk" << std::endl; - planar_count++; + std::cout << "Count Continuous chunk" << std::endl; + continuous_count++; } } + + else { + + std::cout << " Count Planar chunk" << std::endl; + planar_count++; + } + } END_LOOP; const H5std_string FILE_NAME(fnc); const H5std_string CONTINUOUS_DATASET_NAME("continuous_chunks"); - const H5std_string PLANAR_DATASET_NAME("planar_chunks"); + const int NC = continuous_count; const int NP = planar_count; @@ -147,7 +152,7 @@ class hdf_writer :public abstract_writer { datatype.setOrder(H5T_ORDER_LE); H5::DataSet dataset = file.createDataSet(CONTINUOUS_DATASET_NAME, datatype, dataspace); - //Hyperslab prep + // Hyperslab prep hsize_t offset[4]; offset[0] = -1; offset[1] = 0; @@ -157,11 +162,11 @@ class hdf_writer :public abstract_writer { hsize_t slab_dimsf[4] = { 1, 1, 1, 1 }; H5::DataSpace mspace2(RANK, slab_dimsf); - //Regions dataset + // Regions dataset const H5std_string REGIONS_DATASET_NAME("regions"); const int REGION_RANK = 1; hsize_t regions_dimsf[1]; - regions_dimsf[0] = NC; + regions_dimsf[0] = continuous_count + constant_count + planar_count; //regions_dimsf[1] = 1; H5::DataSpace regions_dataspace(REGION_RANK, regions_dimsf); H5::PredType regions_datatype(H5::PredType::STD_REF_DSETREG); @@ -172,10 +177,10 @@ class hdf_writer :public abstract_writer { hsize_t region_offset[1]; region_offset[0] = -1; //region_offset[1] = 0; - hsize_t region_slab_dimsf[1] = { 1}; + hsize_t region_slab_dimsf[1] = { 1 }; H5::DataSpace mspace3(REGION_RANK, region_slab_dimsf); - //Chunk hyperslab + // Chunk hyperslab hsize_t chunk_offset[4]; const int CHUNK_RANK = 4; chunk_offset[0] = -1; @@ -186,63 +191,121 @@ class hdf_writer :public abstract_writer { H5::DataSpace chunk_space(CHUNK_RANK, chunk_dimsf); + typedef struct s2_t { + double a; + hvl_t b; + } s2_t; + + const H5std_string AXIS("axis"); + const H5std_string OFFSETS("offsets"); + + H5::VarLenType varlen_type(&H5::PredType::NATIVE_INT); + H5::CompType mtype2(sizeof(s2_t)); + mtype2.insertMember(AXIS, HOFFSET(s2_t, a), H5::PredType::NATIVE_INT); + mtype2.insertMember(OFFSETS, HOFFSET(s2_t, b), varlen_type); + const H5std_string PLANAR_DATASET_NAME("planar_chunks"); + + + const int PLANAR_RANK = 2; + hsize_t planar_dimsf[1]; + planar_dimsf[0] = planar_count; + planar_dimsf[1] = 1; + + + H5::DataSpace planar_dataspace(PLANAR_RANK, planar_dimsf); + + H5::DataSet planar_dataset = file.createDataSet(PLANAR_DATASET_NAME, mtype2, planar_dataspace); + + + + hsize_t planar_offset[2]; + planar_offset[0] = 1; + planar_offset[1] = 0; + + hsize_t planar_dims[2] = { 1,1 }; + H5::DataSpace planar_space(PLANAR_RANK, planar_dims); + + planar_dataspace.selectHyperslab(H5S_SELECT_SET, planar_dims, planar_offset); + + + + std::vector input_vec; + + hvl_t varlen_offsets; + std::vector tt = { 1,2,4,4,4,4 }; + varlen_offsets.p = &tt; + + + // No bug but nothing written when inspecting output file + s2_t ss; + ss.a = 4; + ss.b = varlen_offsets; + input_vec.push_back(ss); + planar_dataset.write(input_vec.data(), mtype2, planar_space, planar_dataspace); + + + BEGIN_LOOP(size_t(0), nchunks_x, 0U, nchunks_y, 0U, nchunks_z) auto c = storage->get_chunk(ijk); - if (c == nullptr) { - std::cout << "Null pointer" << std::endl; + if (c == nullptr) { + std::cout << "Null pointer" << std::endl; - } + } - else { - if (c->is_explicit() || c->is_constant()) { - if (c->is_constant()) { - std::cout << "Constant chunk" << std::endl; + else { + if (c->is_explicit() || c->is_constant()) { + if (c->is_constant()) { + std::cout << "Constant chunk" << std::endl; - } - else { - std::cout << "Continuous chunk" << std::endl; + } + else { + std::cout << "Continuous chunk" << std::endl; - offset[0]++; + offset[0]++; - continuous_voxel_storage* convox = (continuous_voxel_storage*)c; - size_t i0, j0, k0, i1, j1, k1; - i0 = 0; - j0 = 0; - k0 = 0; - convox->extents().tie(i1, j1, k1); + continuous_voxel_storage* convox = (continuous_voxel_storage*)c; + size_t i0, j0, k0, i1, j1, k1; + i0 = 0; + j0 = 0; + k0 = 0; + convox->extents().tie(i1, j1, k1); - BEGIN_LOOP_ZERO_2(make_vec(i1, j1, k1)) - offset[1] = ijk.get(0); - offset[2] = ijk.get(1); - offset[3] = ijk.get(2); - dataspace.selectHyperslab(H5S_SELECT_SET, slab_dimsf, offset); + BEGIN_LOOP_ZERO_2(make_vec(i1, j1, k1)) + offset[1] = ijk.get(0); + offset[2] = ijk.get(1); + offset[3] = ijk.get(2); + dataspace.selectHyperslab(H5S_SELECT_SET, slab_dimsf, offset); - std::vector hslab = { convox->Get(ijk) }; - dataset.write(hslab.data(), H5::PredType::NATIVE_INT, mspace2, dataspace); - END_LOOP; + std::vector hslab = { convox->Get(ijk) }; + dataset.write(hslab.data(), H5::PredType::NATIVE_INT, mspace2, dataspace); + END_LOOP; - region_offset[0]++; - regions_dataspace.selectHyperslab(H5S_SELECT_SET, region_slab_dimsf, region_offset); + region_offset[0]++; + regions_dataspace.selectHyperslab(H5S_SELECT_SET, region_slab_dimsf, region_offset); - chunk_offset[0]++; + chunk_offset[0]++; - dataspace.selectHyperslab(H5S_SELECT_SET, chunk_dimsf, chunk_offset); + dataspace.selectHyperslab(H5S_SELECT_SET, chunk_dimsf, chunk_offset); - hobj_ref_t inter[1]; - file.reference(&inter[0], "/continuous_chunks", dataspace, H5R_DATASET_REGION); - regions_dataset.write(inter, H5::PredType::STD_REF_DSETREG, mspace3, regions_dataspace); - } + hobj_ref_t inter[1]; + file.reference(&inter[0], "/continuous_chunks", dataspace, H5R_DATASET_REGION); + regions_dataset.write(inter, H5::PredType::STD_REF_DSETREG, mspace3, regions_dataspace); } + } + + else { + std::cout << "Planar chunk" << std::endl; + planar_voxel_storage* planvox = (planar_voxel_storage*)c; + + auto off = planvox->offsets(); + auto axis = planvox->axis(); + + - else { - std::cout << "Planar chunk" << std::endl; - planar_voxel_storage* planvox = (planar_voxel_storage*)c; - auto off = planvox->offsets(); - auto axis = planvox->axis(); - } } + } END_LOOP } }; From d196c31f695e97dbe48169cc68699e8e0d5f341d Mon Sep 17 00:00:00 2001 From: johltn Date: Mon, 7 Jun 2021 13:16:48 +0200 Subject: [PATCH 104/112] Planar dataset writing ok for simple example --- writer.h | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/writer.h b/writer.h index 811c8fe..60df017 100644 --- a/writer.h +++ b/writer.h @@ -192,7 +192,7 @@ class hdf_writer :public abstract_writer { typedef struct s2_t { - double a; + int a; hvl_t b; } s2_t; @@ -206,10 +206,10 @@ class hdf_writer :public abstract_writer { const H5std_string PLANAR_DATASET_NAME("planar_chunks"); - const int PLANAR_RANK = 2; + const int PLANAR_RANK = 1; hsize_t planar_dimsf[1]; planar_dimsf[0] = planar_count; - planar_dimsf[1] = 1; + //planar_dimsf[1] = 1; H5::DataSpace planar_dataspace(PLANAR_RANK, planar_dimsf); @@ -218,11 +218,11 @@ class hdf_writer :public abstract_writer { - hsize_t planar_offset[2]; + hsize_t planar_offset[1]; planar_offset[0] = 1; - planar_offset[1] = 0; + - hsize_t planar_dims[2] = { 1,1 }; + hsize_t planar_dims[2] = { 1,1}; H5::DataSpace planar_space(PLANAR_RANK, planar_dims); planar_dataspace.selectHyperslab(H5S_SELECT_SET, planar_dims, planar_offset); @@ -233,7 +233,7 @@ class hdf_writer :public abstract_writer { hvl_t varlen_offsets; std::vector tt = { 1,2,4,4,4,4 }; - varlen_offsets.p = &tt; + varlen_offsets.p = tt.data(); // No bug but nothing written when inspecting output file @@ -241,7 +241,7 @@ class hdf_writer :public abstract_writer { ss.a = 4; ss.b = varlen_offsets; input_vec.push_back(ss); - planar_dataset.write(input_vec.data(), mtype2, planar_space, planar_dataspace); + planar_dataset.write(&ss, mtype2, planar_space, planar_dataspace); From db468403a755920295931cfc1764cc1347fa19e4 Mon Sep 17 00:00:00 2001 From: johltn Date: Mon, 7 Jun 2021 21:44:17 +0200 Subject: [PATCH 105/112] Start cleaning before cosntant chunks handling --- writer.h | 159 +++++++++++++++++++++++++++++-------------------------- 1 file changed, 83 insertions(+), 76 deletions(-) diff --git a/writer.h b/writer.h index 60df017..7b08e71 100644 --- a/writer.h +++ b/writer.h @@ -85,10 +85,6 @@ class hdf_writer :public abstract_writer { public: void Write(const std::string& fnc) { - - - - chunked_voxel_storage* storage = (chunked_voxel_storage*)voxels_; int continuous_count = 0; @@ -127,20 +123,20 @@ class hdf_writer :public abstract_writer { } END_LOOP; + const H5std_string FILE_NAME(fnc); - const H5std_string CONTINUOUS_DATASET_NAME("continuous_chunks"); + H5::H5File file(FILE_NAME, H5F_ACC_TRUNC); + // Continuous chunks dataset + const H5std_string CONTINUOUS_DATASET_NAME("continuous_chunks"); const int NC = continuous_count; - const int NP = planar_count; - const int NX = storage->chunk_size(); const int NY = storage->chunk_size(); const int NZ = storage->chunk_size(); const int RANK = 4; - H5::H5File file(FILE_NAME, H5F_ACC_TRUNC); - + hsize_t dimsf[4]; dimsf[0] = NC; dimsf[1] = NX; @@ -152,7 +148,7 @@ class hdf_writer :public abstract_writer { datatype.setOrder(H5T_ORDER_LE); H5::DataSet dataset = file.createDataSet(CONTINUOUS_DATASET_NAME, datatype, dataspace); - // Hyperslab prep + // Continuous dataset hyperslab preparation hsize_t offset[4]; offset[0] = -1; offset[1] = 0; @@ -162,38 +158,12 @@ class hdf_writer :public abstract_writer { hsize_t slab_dimsf[4] = { 1, 1, 1, 1 }; H5::DataSpace mspace2(RANK, slab_dimsf); - // Regions dataset - const H5std_string REGIONS_DATASET_NAME("regions"); - const int REGION_RANK = 1; - hsize_t regions_dimsf[1]; - regions_dimsf[0] = continuous_count + constant_count + planar_count; - //regions_dimsf[1] = 1; - H5::DataSpace regions_dataspace(REGION_RANK, regions_dimsf); - H5::PredType regions_datatype(H5::PredType::STD_REF_DSETREG); - regions_datatype.setOrder(H5T_ORDER_LE); - - H5::DataSet regions_dataset = file.createDataSet(REGIONS_DATASET_NAME, regions_datatype, regions_dataspace); - // Regions hyperlab - hsize_t region_offset[1]; - region_offset[0] = -1; - //region_offset[1] = 0; - hsize_t region_slab_dimsf[1] = { 1 }; - H5::DataSpace mspace3(REGION_RANK, region_slab_dimsf); - - // Chunk hyperslab - hsize_t chunk_offset[4]; - const int CHUNK_RANK = 4; - chunk_offset[0] = -1; - chunk_offset[1] = 0; - chunk_offset[2] = 0; - chunk_offset[3] = 0; - hsize_t chunk_dimsf[4] = { 1, NX, NY, NZ }; - H5::DataSpace chunk_space(CHUNK_RANK, chunk_dimsf); + //Planar chunks dataset typedef struct s2_t { - int a; - hvl_t b; + int axis; + hvl_t offsets; } s2_t; const H5std_string AXIS("axis"); @@ -201,64 +171,74 @@ class hdf_writer :public abstract_writer { H5::VarLenType varlen_type(&H5::PredType::NATIVE_INT); H5::CompType mtype2(sizeof(s2_t)); - mtype2.insertMember(AXIS, HOFFSET(s2_t, a), H5::PredType::NATIVE_INT); - mtype2.insertMember(OFFSETS, HOFFSET(s2_t, b), varlen_type); + mtype2.insertMember(AXIS, HOFFSET(s2_t, axis), H5::PredType::NATIVE_INT); + mtype2.insertMember(OFFSETS, HOFFSET(s2_t, offsets), varlen_type); const H5std_string PLANAR_DATASET_NAME("planar_chunks"); - const int PLANAR_RANK = 1; hsize_t planar_dimsf[1]; planar_dimsf[0] = planar_count; - //planar_dimsf[1] = 1; - - H5::DataSpace planar_dataspace(PLANAR_RANK, planar_dimsf); - H5::DataSet planar_dataset = file.createDataSet(PLANAR_DATASET_NAME, mtype2, planar_dataspace); - - - hsize_t planar_offset[1]; - planar_offset[0] = 1; - - - hsize_t planar_dims[2] = { 1,1}; + planar_offset[0] = -1; + hsize_t planar_dims[1] = { 1 }; H5::DataSpace planar_space(PLANAR_RANK, planar_dims); - planar_dataspace.selectHyperslab(H5S_SELECT_SET, planar_dims, planar_offset); + // Regions dataset + const H5std_string REGIONS_DATASET_NAME("regions"); - std::vector input_vec; + hsize_t regions_dimsf[1]; + regions_dimsf[0] = continuous_count + constant_count + planar_count; - hvl_t varlen_offsets; - std::vector tt = { 1,2,4,4,4,4 }; - varlen_offsets.p = tt.data(); + const int REGION_RANK = 1; + H5::DataSpace regions_dataspace(REGION_RANK, regions_dimsf); + H5::PredType regions_datatype(H5::PredType::STD_REF_DSETREG); + regions_datatype.setOrder(H5T_ORDER_LE); + H5::DataSet regions_dataset = file.createDataSet(REGIONS_DATASET_NAME, regions_datatype, regions_dataspace); + + // Regions dataset hyperslab preparation + hsize_t region_offset[1]; + region_offset[0] = -1; + hsize_t region_slab_dimsf[1] = { 1 }; + H5::DataSpace mspace3(REGION_RANK, region_slab_dimsf); - // No bug but nothing written when inspecting output file - s2_t ss; - ss.a = 4; - ss.b = varlen_offsets; - input_vec.push_back(ss); - planar_dataset.write(&ss, mtype2, planar_space, planar_dataspace); + // Continuous chunks region hyperslab preparation + hsize_t chunk_offset[4]; + const int CHUNK_RANK = 4; + chunk_offset[0] = -1; + chunk_offset[1] = 0; + chunk_offset[2] = 0; + chunk_offset[3] = 0; + hsize_t chunk_dimsf[4] = { 1, NX, NY, NZ }; + H5::DataSpace chunk_space(CHUNK_RANK, chunk_dimsf); + // Planar chunks region hyperslab preparation + hsize_t planar_chunk_offset[1]; + const int PLANAR_CHUNK_RANK = 1; + planar_chunk_offset[0] = 1; + hsize_t planar_chunk_dimsf[1] = { 1 }; + H5::DataSpace planar_chunk_space(PLANAR_CHUNK_RANK, planar_chunk_dimsf); BEGIN_LOOP(size_t(0), nchunks_x, 0U, nchunks_y, 0U, nchunks_z) auto c = storage->get_chunk(ijk); - if (c == nullptr) { - std::cout << "Null pointer" << std::endl; + if (c == nullptr) { + std::cout << "Null pointer" << std::endl; - } + } - else { - if (c->is_explicit() || c->is_constant()) { - if (c->is_constant()) { + else { + if (c->is_explicit() || c->is_constant()) { + if (c->is_constant()) { std::cout << "Constant chunk" << std::endl; } + else { std::cout << "Continuous chunk" << std::endl; @@ -294,18 +274,45 @@ class hdf_writer :public abstract_writer { } } - else { - std::cout << "Planar chunk" << std::endl; - planar_voxel_storage* planvox = (planar_voxel_storage*)c; + else { + std::cout << "Planar chunk" << std::endl; + planar_voxel_storage* planvox = (planar_voxel_storage*)c; + + auto axis = planvox->axis(); + auto off = planvox->offsets(); + + std::vector offsets(off.begin(), off.end()); - auto off = planvox->offsets(); - auto axis = planvox->axis(); + planar_offset[0]++; + + planar_dataspace.selectHyperslab(H5S_SELECT_SET, planar_dims, planar_offset); + std::vector input_vec; + hvl_t varlen_offsets; + + varlen_offsets.len = offsets.size(); + varlen_offsets.p = offsets.data(); + s2_t compound_data; + compound_data.axis = axis; + compound_data.offsets = varlen_offsets; + input_vec.push_back(compound_data); + planar_dataset.write(&compound_data, mtype2, planar_space, planar_dataspace); + region_offset[0]++; + regions_dataspace.selectHyperslab(H5S_SELECT_SET, region_slab_dimsf, region_offset); + + planar_chunk_offset[0]++; + + dataspace.selectHyperslab(H5S_SELECT_SET, planar_chunk_dimsf, planar_chunk_offset); + + hobj_ref_t inter[1]; + file.reference(&inter[0], "/planar_chunks", planar_dataspace, H5R_DATASET_REGION); + regions_dataset.write(inter, H5::PredType::STD_REF_DSETREG, mspace3, regions_dataspace); + + } } - } END_LOOP } }; From b387a4c3d7c4ebfe72ebd01a2b668644e5e0140b Mon Sep 17 00:00:00 2001 From: johltn Date: Mon, 7 Jun 2021 22:12:06 +0200 Subject: [PATCH 106/112] Writing preparation to constant chunks dataset --- writer.h | 96 +++++++++++++++++++++++++++++++++++--------------------- 1 file changed, 61 insertions(+), 35 deletions(-) diff --git a/writer.h b/writer.h index 7b08e71..275de10 100644 --- a/writer.h +++ b/writer.h @@ -136,7 +136,7 @@ class hdf_writer :public abstract_writer { const int NZ = storage->chunk_size(); const int RANK = 4; - + hsize_t dimsf[4]; dimsf[0] = NC; dimsf[1] = NX; @@ -187,6 +187,21 @@ class hdf_writer :public abstract_writer { + //Constant chunks dataset + const H5std_string CONSTANT_DATASET_NAME("constant_chunks"); + + const int CONSTANT_RANK = 1; + hsize_t constant_dimsf[1]; + constant_dimsf[0] = constant_count; + H5::DataSpace constant_dataspace(CONSTANT_RANK, constant_dimsf); + H5::DataSet constant_dataset = file.createDataSet(CONSTANT_DATASET_NAME, H5::PredType::NATIVE_INT, constant_dataspace); + hsize_t constant_offset[1]; + constant_offset[0] = -1; + hsize_t constant_dims[1] = { 1 }; + H5::DataSpace constant_space(CONSTANT_RANK, constant_dims); + + + // Regions dataset const H5std_string REGIONS_DATASET_NAME("regions"); @@ -223,20 +238,31 @@ class hdf_writer :public abstract_writer { hsize_t planar_chunk_dimsf[1] = { 1 }; H5::DataSpace planar_chunk_space(PLANAR_CHUNK_RANK, planar_chunk_dimsf); + // Constant chunks region hyperslab preparation + hsize_t constant_chunk_offset[1]; + const int CONSTANT_CHUNK_RANK = 1; + constant_chunk_offset[0] = 1; + hsize_t constant_chunk_dimsf[1] = { 1 }; + H5::DataSpace constant_chunk_space(CONSTANT_CHUNK_RANK, constant_chunk_dimsf); + BEGIN_LOOP(size_t(0), nchunks_x, 0U, nchunks_y, 0U, nchunks_z) auto c = storage->get_chunk(ijk); - if (c == nullptr) { - std::cout << "Null pointer" << std::endl; - - } + if (c == nullptr) { + std::cout << "Null pointer" << std::endl; + } - else { - if (c->is_explicit() || c->is_constant()) { - if (c->is_constant()) { + else { + if (c->is_explicit() || c->is_constant()) { + if (c->is_constant()) { std::cout << "Constant chunk" << std::endl; + constant_offset[0]++; + constant_dataspace.selectHyperslab(H5S_SELECT_SET, constant_dims, constant_offset); + int inter[1] = { 1 }; + dataset.write(inter, H5::PredType::NATIVE_INT, constant_space, constant_dataspace); + } else { @@ -274,45 +300,45 @@ class hdf_writer :public abstract_writer { } } - else { - std::cout << "Planar chunk" << std::endl; - planar_voxel_storage* planvox = (planar_voxel_storage*)c; + else { + std::cout << "Planar chunk" << std::endl; + planar_voxel_storage* planvox = (planar_voxel_storage*)c; - auto axis = planvox->axis(); - auto off = planvox->offsets(); + auto axis = planvox->axis(); + auto off = planvox->offsets(); - std::vector offsets(off.begin(), off.end()); + std::vector offsets(off.begin(), off.end()); - planar_offset[0]++; - - planar_dataspace.selectHyperslab(H5S_SELECT_SET, planar_dims, planar_offset); - std::vector input_vec; - hvl_t varlen_offsets; + planar_offset[0]++; - - varlen_offsets.len = offsets.size(); + planar_dataspace.selectHyperslab(H5S_SELECT_SET, planar_dims, planar_offset); + std::vector input_vec; + hvl_t varlen_offsets; - varlen_offsets.p = offsets.data(); - s2_t compound_data; - compound_data.axis = axis; - compound_data.offsets = varlen_offsets; - input_vec.push_back(compound_data); - planar_dataset.write(&compound_data, mtype2, planar_space, planar_dataspace); + varlen_offsets.len = offsets.size(); - region_offset[0]++; - regions_dataspace.selectHyperslab(H5S_SELECT_SET, region_slab_dimsf, region_offset); + varlen_offsets.p = offsets.data(); + s2_t compound_data; + compound_data.axis = axis; + compound_data.offsets = varlen_offsets; + input_vec.push_back(compound_data); + planar_dataset.write(&compound_data, mtype2, planar_space, planar_dataspace); - planar_chunk_offset[0]++; - dataspace.selectHyperslab(H5S_SELECT_SET, planar_chunk_dimsf, planar_chunk_offset); + region_offset[0]++; + regions_dataspace.selectHyperslab(H5S_SELECT_SET, region_slab_dimsf, region_offset); - hobj_ref_t inter[1]; - file.reference(&inter[0], "/planar_chunks", planar_dataspace, H5R_DATASET_REGION); - regions_dataset.write(inter, H5::PredType::STD_REF_DSETREG, mspace3, regions_dataspace); + planar_chunk_offset[0]++; + + dataspace.selectHyperslab(H5S_SELECT_SET, planar_chunk_dimsf, planar_chunk_offset); + + hobj_ref_t inter[1]; + file.reference(&inter[0], "/planar_chunks", planar_dataspace, H5R_DATASET_REGION); + regions_dataset.write(inter, H5::PredType::STD_REF_DSETREG, mspace3, regions_dataspace); - } } + } END_LOOP } }; From 075e55dc29bda181f6f983fb33453ef111dc4bb6 Mon Sep 17 00:00:00 2001 From: johltn Date: Mon, 7 Jun 2021 22:19:41 +0200 Subject: [PATCH 107/112] Write constant chunks + region references --- writer.h | 37 ++++++++++++++++++++++++++++++++++--- 1 file changed, 34 insertions(+), 3 deletions(-) diff --git a/writer.h b/writer.h index 275de10..efb2761 100644 --- a/writer.h +++ b/writer.h @@ -251,6 +251,23 @@ class hdf_writer :public abstract_writer { if (c == nullptr) { std::cout << "Null pointer" << std::endl; + + constant_offset[0]++; + constant_dataspace.selectHyperslab(H5S_SELECT_SET, constant_dims, constant_offset); + int constant_value[1] = { 0 }; + dataset.write(constant_value, H5::PredType::NATIVE_INT, constant_space, constant_dataspace); + + + region_offset[0]++; + regions_dataspace.selectHyperslab(H5S_SELECT_SET, region_slab_dimsf, region_offset); + + constant_chunk_offset[0]++; + + //dataspace.selectHyperslab(H5S_SELECT_SET, planar_chunk_dimsf, planar_chunk_offset); + + hobj_ref_t inter[1]; + file.reference(&inter[0], "/constant_chunks", constant_dataspace, H5R_DATASET_REGION); + regions_dataset.write(inter, H5::PredType::STD_REF_DSETREG, mspace3, regions_dataspace); } else { @@ -260,8 +277,22 @@ class hdf_writer :public abstract_writer { constant_offset[0]++; constant_dataspace.selectHyperslab(H5S_SELECT_SET, constant_dims, constant_offset); - int inter[1] = { 1 }; - dataset.write(inter, H5::PredType::NATIVE_INT, constant_space, constant_dataspace); + int constant_value[1] = { 1 }; + dataset.write(constant_value , H5::PredType::NATIVE_INT, constant_space, constant_dataspace); + + + region_offset[0]++; + regions_dataspace.selectHyperslab(H5S_SELECT_SET, region_slab_dimsf, region_offset); + + constant_chunk_offset[0]++; + + //dataspace.selectHyperslab(H5S_SELECT_SET, planar_chunk_dimsf, planar_chunk_offset); + + hobj_ref_t inter[1]; + file.reference(&inter[0], "/constant_chunks", constant_dataspace, H5R_DATASET_REGION); + regions_dataset.write(inter, H5::PredType::STD_REF_DSETREG, mspace3, regions_dataspace); + + } @@ -331,7 +362,7 @@ class hdf_writer :public abstract_writer { planar_chunk_offset[0]++; - dataspace.selectHyperslab(H5S_SELECT_SET, planar_chunk_dimsf, planar_chunk_offset); + //dataspace.selectHyperslab(H5S_SELECT_SET, planar_chunk_dimsf, planar_chunk_offset); hobj_ref_t inter[1]; file.reference(&inter[0], "/planar_chunks", planar_dataspace, H5R_DATASET_REGION); From 6e818543e2edcb126547962926d1b2538d304e15 Mon Sep 17 00:00:00 2001 From: johltn Date: Mon, 7 Jun 2021 22:29:41 +0200 Subject: [PATCH 108/112] Change dimensions types to comply with clang compilator --- writer.h | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/writer.h b/writer.h index efb2761..71ad373 100644 --- a/writer.h +++ b/writer.h @@ -130,10 +130,10 @@ class hdf_writer :public abstract_writer { // Continuous chunks dataset const H5std_string CONTINUOUS_DATASET_NAME("continuous_chunks"); - const int NC = continuous_count; - const int NX = storage->chunk_size(); - const int NY = storage->chunk_size(); - const int NZ = storage->chunk_size(); + const hsize_t NC = continuous_count; + const hsize_t NX = storage->chunk_size(); + const hsize_t NY = storage->chunk_size(); + const hsize_t NZ = storage->chunk_size(); const int RANK = 4; From 3d62d8b697dd65aedbb92408d72ccd109410427d Mon Sep 17 00:00:00 2001 From: johltn Date: Tue, 8 Jun 2021 10:14:57 +0200 Subject: [PATCH 109/112] Modify control flow --- writer.h | 146 ++++++++++++++++++++++++++++++------------------------- 1 file changed, 79 insertions(+), 67 deletions(-) diff --git a/writer.h b/writer.h index 71ad373..aa95a68 100644 --- a/writer.h +++ b/writer.h @@ -99,28 +99,35 @@ class hdf_writer :public abstract_writer { auto c = storage->get_chunk(ijk); if (c == nullptr) { - std::cout << "Null pointer" << std::endl; + std::cout << "Null pointer count" << std::endl; constant_count++; + } else { - if (c->is_explicit() || c->is_constant()) { + + if (c->is_explicit()) { + std::cout << "Continuous chunk count" << std::endl; + continuous_count++; + } + + else { if (c->is_constant()) { - std::cout << "Count Constant chunk" << std::endl; + std::cout << "Constant chunk count" << std::endl; constant_count++; } + + else { - std::cout << "Count Continuous chunk" << std::endl; - continuous_count++; + std::cout << "Planar chunk count" << std::endl; + planar_count++; + } } - else { - std::cout << " Count Planar chunk" << std::endl; - planar_count++; - } } + END_LOOP; @@ -271,14 +278,50 @@ class hdf_writer :public abstract_writer { } else { - if (c->is_explicit() || c->is_constant()) { + + if (c->is_explicit()) { + std::cout << "Continuous chunk" << std::endl; + + offset[0]++; + + continuous_voxel_storage* convox = (continuous_voxel_storage*)c; + size_t i0, j0, k0, i1, j1, k1; + i0 = 0; + j0 = 0; + k0 = 0; + convox->extents().tie(i1, j1, k1); + + BEGIN_LOOP_ZERO_2(make_vec(i1, j1, k1)) + offset[1] = ijk.get(0); + offset[2] = ijk.get(1); + offset[3] = ijk.get(2); + dataspace.selectHyperslab(H5S_SELECT_SET, slab_dimsf, offset); + + std::vector hslab = { convox->Get(ijk) }; + dataset.write(hslab.data(), H5::PredType::NATIVE_INT, mspace2, dataspace); + END_LOOP; + + region_offset[0]++; + regions_dataspace.selectHyperslab(H5S_SELECT_SET, region_slab_dimsf, region_offset); + + chunk_offset[0]++; + + dataspace.selectHyperslab(H5S_SELECT_SET, chunk_dimsf, chunk_offset); + + hobj_ref_t inter[1]; + file.reference(&inter[0], "/continuous_chunks", dataspace, H5R_DATASET_REGION); + regions_dataset.write(inter, H5::PredType::STD_REF_DSETREG, mspace3, regions_dataspace); + + } + + else { if (c->is_constant()) { std::cout << "Constant chunk" << std::endl; constant_offset[0]++; constant_dataspace.selectHyperslab(H5S_SELECT_SET, constant_dims, constant_offset); int constant_value[1] = { 1 }; - dataset.write(constant_value , H5::PredType::NATIVE_INT, constant_space, constant_dataspace); + dataset.write(constant_value, H5::PredType::NATIVE_INT, constant_space, constant_dataspace); region_offset[0]++; @@ -296,79 +339,48 @@ class hdf_writer :public abstract_writer { } + else { - std::cout << "Continuous chunk" << std::endl; + std::cout << "Planar chunk" << std::endl; + planar_voxel_storage* planvox = (planar_voxel_storage*)c; + + auto axis = planvox->axis(); + auto off = planvox->offsets(); + + std::vector offsets(off.begin(), off.end()); - offset[0]++; + planar_offset[0]++; - continuous_voxel_storage* convox = (continuous_voxel_storage*)c; - size_t i0, j0, k0, i1, j1, k1; - i0 = 0; - j0 = 0; - k0 = 0; - convox->extents().tie(i1, j1, k1); + planar_dataspace.selectHyperslab(H5S_SELECT_SET, planar_dims, planar_offset); + std::vector input_vec; + hvl_t varlen_offsets; - BEGIN_LOOP_ZERO_2(make_vec(i1, j1, k1)) - offset[1] = ijk.get(0); - offset[2] = ijk.get(1); - offset[3] = ijk.get(2); - dataspace.selectHyperslab(H5S_SELECT_SET, slab_dimsf, offset); - std::vector hslab = { convox->Get(ijk) }; - dataset.write(hslab.data(), H5::PredType::NATIVE_INT, mspace2, dataspace); - END_LOOP; + varlen_offsets.len = offsets.size(); + + varlen_offsets.p = offsets.data(); + s2_t compound_data; + compound_data.axis = axis; + compound_data.offsets = varlen_offsets; + input_vec.push_back(compound_data); + planar_dataset.write(&compound_data, mtype2, planar_space, planar_dataspace); + region_offset[0]++; regions_dataspace.selectHyperslab(H5S_SELECT_SET, region_slab_dimsf, region_offset); - chunk_offset[0]++; + planar_chunk_offset[0]++; - dataspace.selectHyperslab(H5S_SELECT_SET, chunk_dimsf, chunk_offset); + //dataspace.selectHyperslab(H5S_SELECT_SET, planar_chunk_dimsf, planar_chunk_offset); hobj_ref_t inter[1]; - file.reference(&inter[0], "/continuous_chunks", dataspace, H5R_DATASET_REGION); + file.reference(&inter[0], "/planar_chunks", planar_dataspace, H5R_DATASET_REGION); regions_dataset.write(inter, H5::PredType::STD_REF_DSETREG, mspace3, regions_dataspace); + } } - else { - std::cout << "Planar chunk" << std::endl; - planar_voxel_storage* planvox = (planar_voxel_storage*)c; - - auto axis = planvox->axis(); - auto off = planvox->offsets(); - - std::vector offsets(off.begin(), off.end()); - - planar_offset[0]++; - - planar_dataspace.selectHyperslab(H5S_SELECT_SET, planar_dims, planar_offset); - std::vector input_vec; - hvl_t varlen_offsets; - - varlen_offsets.len = offsets.size(); - - varlen_offsets.p = offsets.data(); - s2_t compound_data; - compound_data.axis = axis; - compound_data.offsets = varlen_offsets; - input_vec.push_back(compound_data); - planar_dataset.write(&compound_data, mtype2, planar_space, planar_dataspace); - - - region_offset[0]++; - regions_dataspace.selectHyperslab(H5S_SELECT_SET, region_slab_dimsf, region_offset); - - planar_chunk_offset[0]++; - - //dataspace.selectHyperslab(H5S_SELECT_SET, planar_chunk_dimsf, planar_chunk_offset); - - hobj_ref_t inter[1]; - file.reference(&inter[0], "/planar_chunks", planar_dataspace, H5R_DATASET_REGION); - regions_dataset.write(inter, H5::PredType::STD_REF_DSETREG, mspace3, regions_dataspace); - - } } END_LOOP } From 612ced3a1d948308891c890c760842f2412685a0 Mon Sep 17 00:00:00 2001 From: johltn Date: Thu, 17 Jun 2021 13:50:14 +0200 Subject: [PATCH 110/112] Start handling voxels stored as integers --- writer.h | 37 ++++++++++++++++++++++++++----------- 1 file changed, 26 insertions(+), 11 deletions(-) diff --git a/writer.h b/writer.h index aa95a68..c70f0a0 100644 --- a/writer.h +++ b/writer.h @@ -85,8 +85,8 @@ class hdf_writer :public abstract_writer { public: void Write(const std::string& fnc) { - chunked_voxel_storage* storage = (chunked_voxel_storage*)voxels_; - + abstract_chunked_voxel_storage* storage = (abstract_chunked_voxel_storage*)voxels_; + int continuous_count = 0; int planar_count = 0; int constant_count = 0; @@ -255,6 +255,8 @@ class hdf_writer :public abstract_writer { BEGIN_LOOP(size_t(0), nchunks_x, 0U, nchunks_y, 0U, nchunks_z) auto c = storage->get_chunk(ijk); + + if (c == nullptr) { std::cout << "Null pointer" << std::endl; @@ -283,22 +285,35 @@ class hdf_writer :public abstract_writer { std::cout << "Continuous chunk" << std::endl; offset[0]++; - - continuous_voxel_storage* convox = (continuous_voxel_storage*)c; + size_t i0, j0, k0, i1, j1, k1; i0 = 0; j0 = 0; k0 = 0; - convox->extents().tie(i1, j1, k1); + c->extents().tie(i1, j1, k1); BEGIN_LOOP_ZERO_2(make_vec(i1, j1, k1)) offset[1] = ijk.get(0); - offset[2] = ijk.get(1); - offset[3] = ijk.get(2); - dataspace.selectHyperslab(H5S_SELECT_SET, slab_dimsf, offset); - - std::vector hslab = { convox->Get(ijk) }; - dataset.write(hslab.data(), H5::PredType::NATIVE_INT, mspace2, dataspace); + offset[2] = ijk.get(1); + offset[3] = ijk.get(2); + dataspace.selectHyperslab(H5S_SELECT_SET, slab_dimsf, offset); + + if (c->value_bits() == 32) { + + uint32_t v; + std::vector hslab; + c->Get(ijk, &v); + hslab.push_back(v); + dataset.write(hslab.data(), H5::PredType::NATIVE_INT, mspace2, dataspace); + + } + + else { + std::vector hslab = { c->Get(ijk) }; + dataset.write(hslab.data(), H5::PredType::NATIVE_INT, mspace2, dataspace); + } + + END_LOOP; region_offset[0]++; From b6da6f3a590eaad41b3fb064002c12dfcc2ed0e9 Mon Sep 17 00:00:00 2001 From: johltn Date: Mon, 21 Jun 2021 09:37:10 +0200 Subject: [PATCH 111/112] Store operations as groups containing chunk datasets --- writer.h | 52 ++++++++++++++++++++++------------------------------ 1 file changed, 22 insertions(+), 30 deletions(-) diff --git a/writer.h b/writer.h index c70f0a0..af67b39 100644 --- a/writer.h +++ b/writer.h @@ -39,7 +39,7 @@ class abstract_writer { voxels_ = voxels; } - virtual void Write(const std::string& fnc) = 0; + virtual void Write(const std::string& fnc, std::string statement="default") = 0; }; @@ -55,7 +55,7 @@ class voxel_writer :public abstract_writer { } public: - void Write(const std::string& fnc) { + void Write(const std::string& fnc, std::string statement = "default") { { std::string fn = fnc + std::string(".index"); std::ofstream fs(fn.c_str()); @@ -83,10 +83,10 @@ class voxel_writer :public abstract_writer { class hdf_writer :public abstract_writer { public: - void Write(const std::string& fnc) { + void Write(const std::string& fnc, std::string statement="default") { abstract_chunked_voxel_storage* storage = (abstract_chunked_voxel_storage*)voxels_; - + int continuous_count = 0; int planar_count = 0; int constant_count = 0; @@ -99,7 +99,6 @@ class hdf_writer :public abstract_writer { auto c = storage->get_chunk(ijk); if (c == nullptr) { - std::cout << "Null pointer count" << std::endl; constant_count++; } @@ -107,19 +106,16 @@ class hdf_writer :public abstract_writer { else { if (c->is_explicit()) { - std::cout << "Continuous chunk count" << std::endl; continuous_count++; } else { if (c->is_constant()) { - std::cout << "Constant chunk count" << std::endl; constant_count++; } else { - std::cout << "Planar chunk count" << std::endl; planar_count++; } @@ -131,9 +127,14 @@ class hdf_writer :public abstract_writer { END_LOOP; + + //TODO: Handle existing file (with group) case const H5std_string FILE_NAME(fnc); - H5::H5File file(FILE_NAME, H5F_ACC_TRUNC); + H5::H5File file(FILE_NAME, H5F_ACC_RDWR); + + H5::Group operation_group = H5::Group(file.createGroup(statement)); + // Continuous chunks dataset const H5std_string CONTINUOUS_DATASET_NAME("continuous_chunks"); @@ -153,7 +154,7 @@ class hdf_writer :public abstract_writer { H5::DataSpace dataspace(RANK, dimsf); H5::IntType datatype(H5::PredType::NATIVE_INT); datatype.setOrder(H5T_ORDER_LE); - H5::DataSet dataset = file.createDataSet(CONTINUOUS_DATASET_NAME, datatype, dataspace); + H5::DataSet dataset = operation_group.createDataSet(CONTINUOUS_DATASET_NAME, datatype, dataspace); // Continuous dataset hyperslab preparation hsize_t offset[4]; @@ -186,7 +187,7 @@ class hdf_writer :public abstract_writer { hsize_t planar_dimsf[1]; planar_dimsf[0] = planar_count; H5::DataSpace planar_dataspace(PLANAR_RANK, planar_dimsf); - H5::DataSet planar_dataset = file.createDataSet(PLANAR_DATASET_NAME, mtype2, planar_dataspace); + H5::DataSet planar_dataset = operation_group.createDataSet(PLANAR_DATASET_NAME, mtype2, planar_dataspace); hsize_t planar_offset[1]; planar_offset[0] = -1; hsize_t planar_dims[1] = { 1 }; @@ -201,7 +202,7 @@ class hdf_writer :public abstract_writer { hsize_t constant_dimsf[1]; constant_dimsf[0] = constant_count; H5::DataSpace constant_dataspace(CONSTANT_RANK, constant_dimsf); - H5::DataSet constant_dataset = file.createDataSet(CONSTANT_DATASET_NAME, H5::PredType::NATIVE_INT, constant_dataspace); + H5::DataSet constant_dataset = operation_group.createDataSet(CONSTANT_DATASET_NAME, H5::PredType::NATIVE_INT, constant_dataspace); hsize_t constant_offset[1]; constant_offset[0] = -1; hsize_t constant_dims[1] = { 1 }; @@ -220,7 +221,7 @@ class hdf_writer :public abstract_writer { H5::DataSpace regions_dataspace(REGION_RANK, regions_dimsf); H5::PredType regions_datatype(H5::PredType::STD_REF_DSETREG); regions_datatype.setOrder(H5T_ORDER_LE); - H5::DataSet regions_dataset = file.createDataSet(REGIONS_DATASET_NAME, regions_datatype, regions_dataspace); + H5::DataSet regions_dataset = operation_group.createDataSet(REGIONS_DATASET_NAME, regions_datatype, regions_dataspace); // Regions dataset hyperslab preparation hsize_t region_offset[1]; @@ -259,8 +260,7 @@ class hdf_writer :public abstract_writer { if (c == nullptr) { - std::cout << "Null pointer" << std::endl; - + constant_offset[0]++; constant_dataspace.selectHyperslab(H5S_SELECT_SET, constant_dims, constant_offset); int constant_value[1] = { 0 }; @@ -272,18 +272,15 @@ class hdf_writer :public abstract_writer { constant_chunk_offset[0]++; - //dataspace.selectHyperslab(H5S_SELECT_SET, planar_chunk_dimsf, planar_chunk_offset); - hobj_ref_t inter[1]; - file.reference(&inter[0], "/constant_chunks", constant_dataspace, H5R_DATASET_REGION); + file.reference(&inter[0], "/" + statement + "/constant_chunks", constant_dataspace, H5R_DATASET_REGION); regions_dataset.write(inter, H5::PredType::STD_REF_DSETREG, mspace3, regions_dataspace); } else { if (c->is_explicit()) { - std::cout << "Continuous chunk" << std::endl; - + offset[0]++; size_t i0, j0, k0, i1, j1, k1; @@ -324,15 +321,14 @@ class hdf_writer :public abstract_writer { dataspace.selectHyperslab(H5S_SELECT_SET, chunk_dimsf, chunk_offset); hobj_ref_t inter[1]; - file.reference(&inter[0], "/continuous_chunks", dataspace, H5R_DATASET_REGION); + file.reference(&inter[0], "/" + statement + "/continuous_chunks", dataspace, H5R_DATASET_REGION); regions_dataset.write(inter, H5::PredType::STD_REF_DSETREG, mspace3, regions_dataspace); } else { if (c->is_constant()) { - std::cout << "Constant chunk" << std::endl; - + constant_offset[0]++; constant_dataspace.selectHyperslab(H5S_SELECT_SET, constant_dims, constant_offset); int constant_value[1] = { 1 }; @@ -344,19 +340,16 @@ class hdf_writer :public abstract_writer { constant_chunk_offset[0]++; - //dataspace.selectHyperslab(H5S_SELECT_SET, planar_chunk_dimsf, planar_chunk_offset); hobj_ref_t inter[1]; - file.reference(&inter[0], "/constant_chunks", constant_dataspace, H5R_DATASET_REGION); + file.reference(&inter[0], "/" + statement + "/constant_chunks", constant_dataspace, H5R_DATASET_REGION); regions_dataset.write(inter, H5::PredType::STD_REF_DSETREG, mspace3, regions_dataspace); - - } else { - std::cout << "Planar chunk" << std::endl; + planar_voxel_storage* planvox = (planar_voxel_storage*)c; auto axis = planvox->axis(); @@ -386,10 +379,9 @@ class hdf_writer :public abstract_writer { planar_chunk_offset[0]++; - //dataspace.selectHyperslab(H5S_SELECT_SET, planar_chunk_dimsf, planar_chunk_offset); hobj_ref_t inter[1]; - file.reference(&inter[0], "/planar_chunks", planar_dataspace, H5R_DATASET_REGION); + file.reference(&inter[0], "/" + statement + "/planar_chunks", planar_dataspace, H5R_DATASET_REGION); regions_dataset.write(inter, H5::PredType::STD_REF_DSETREG, mspace3, regions_dataspace); } From 163d991cf25a1d049875a1b89806173925a752c8 Mon Sep 17 00:00:00 2001 From: johltn Date: Thu, 3 Feb 2022 11:39:33 -0500 Subject: [PATCH 112/112] Voxelization toolkit works with IfcOpenShell v0.7.0 including HDF support --- CMakeLists.txt | 47 +++++++++++++++++++++++++++------------- processor.h | 5 +++++ tests/test_hdf_2.cpp | 5 ++++- tests/test_space_ids.cpp | 4 ++++ voxec.h | 22 +++++++++++++++++-- 5 files changed, 65 insertions(+), 18 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 12a0244..585ab4e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -69,16 +69,27 @@ else() elseif(MSVC_VERSION GREATER 1919) set(MSVC_YEAR 2019) endif() - + + file(READ "${IFCOPENSHELL_ROOT}/src/ifcparse/IfcParse.h" header) + string(REGEX MATCH "#define IFCOPENSHELL_VERSION \"[0-9a-zA-Z\\.\\-]+\"" ifcopenshell_version "${header}") + string(REGEX MATCH "[0-9]\\.[0-9]" ifcopenshell_major_version "${ifcopenshell_version}") file(TO_CMAKE_PATH "${IFCOPENSHELL_ROOT}" IFCOPENSHELL_ROOT) if (EXISTS ${IFCOPENSHELL_ROOT}/_deps-vs${MSVC_YEAR}-${WIN_ARCH}-installed/oce/${WIN_ARCH}/lib) set(OCC_LIBRARY_DIR ${IFCOPENSHELL_ROOT}/_deps-vs${MSVC_YEAR}-${WIN_ARCH}-installed/oce/${WIN_ARCH}/lib) else() - set(OCC_LIBRARY_DIR ${IFCOPENSHELL_ROOT}/_deps-vs${MSVC_YEAR}-${WIN_ARCH}-installed/opencascade-7.3.0p3/win${BITS}/lib) + if (${ifcopenshell_major_version} STREQUAL "0.7") + set(OCC_LIBRARY_DIR ${IFCOPENSHELL_ROOT}/_deps-vs${MSVC_YEAR}-${WIN_ARCH}-installed/opencascade-7.5.3/win${BITS}/lib) + else() + set(OCC_LIBRARY_DIR ${IFCOPENSHELL_ROOT}/_deps-vs${MSVC_YEAR}-${WIN_ARCH}-installed/opencascade-7.3.0p3/win${BITS}/lib) + endif() endif() - - set(OCC_INCLUDE_DIR ${IFCOPENSHELL_ROOT}/_deps-vs${MSVC_YEAR}-${WIN_ARCH}-installed/oce/include/oce - ${IFCOPENSHELL_ROOT}/_deps-vs${MSVC_YEAR}-${WIN_ARCH}-installed/opencascade-7.3.0p3/inc) + if (${ifcopenshell_major_version} STREQUAL "0.7") + set(OCC_INCLUDE_DIR ${IFCOPENSHELL_ROOT}/_deps-vs${MSVC_YEAR}-${WIN_ARCH}-installed/opencascade-7.5.3/inc) + else() + set(OCC_INCLUDE_DIR ${IFCOPENSHELL_ROOT}/_deps-vs${MSVC_YEAR}-${WIN_ARCH}-installed/oce/include/oce + ${IFCOPENSHELL_ROOT}/_deps-vs${MSVC_YEAR}-${WIN_ARCH}-installed/opencascade-7.3.0p3/inc) + endif() + else() file(TO_CMAKE_PATH "${OCC_LIBRARY_DIR}" OCC_LIBRARY_DIR) endif() @@ -119,6 +130,11 @@ string(REGEX MATCH "#define IFCOPENSHELL_VERSION \"[0-9a-zA-Z\\.\\-]+\"" ifcopen string(REGEX MATCH "[0-9]\\.[0-9]" ifcopenshell_major_version "${ifcopenshell_version}") message(STATUS "IfcOpenShell version ${ifcopenshell_major_version}") +if (${ifcopenshell_major_version} STREQUAL "0.7") + add_definitions("-DIFCOPENSHELL_07") +endif() + + if (${ifcopenshell_major_version} STREQUAL "0.5") set(IFC_LIBRARY_NAMES IfcGeom IfcParse) add_definitions("-DIFCOPENSHELL_05") @@ -149,17 +165,18 @@ if(UNIX) string(REGEX REPLACE "([^;]+)" "${ICU_LIBRARY_DIR}/lib\\1.a" ICU_LIBRARIES "${ICU_LIBRARY_NAMES}") endif() else() + if (${ifcopenshell_major_version} STREQUAL "0.7") + SET(HDF5_INCLUDE_DIR ${IFCOPENSHELL_ROOT}/_deps-vs${MSVC_YEAR}-${WIN_ARCH}-installed/HDF5-1.8.22-win64/include) + SET(HDF5_LIBRARY_DIR ${IFCOPENSHELL_ROOT}/_deps-vs${MSVC_YEAR}-${WIN_ARCH}-installed/HDF5-1.8.22-win64/lib) + + SET(HDF5_LIBRARIES + "${HDF5_LIBRARY_DIR}/libhdf5_cpp.lib" + "${HDF5_LIBRARY_DIR}/libhdf5.lib" + "${HDF5_LIBRARY_DIR}/libzlib.lib" + "${HDF5_LIBRARY_DIR}/libsz.lib" + "${HDF5_LIBRARY_DIR}/libaec.lib") + endif() - SET(HDF5_INCLUDE_DIR ${IFCOPENSHELL_ROOT}/_deps-vs2017-x64-installed/HDF5-1.8.22-win64/include) - SET(HDF5_LIBRARY_DIR ${IFCOPENSHELL_ROOT}/_deps-vs2017-x64-installed/HDF5-1.8.22-win64/lib) - - SET(HDF5_LIBRARIES - "${HDF5_LIBRARY_DIR}/libhdf5_cpp.lib" - "${HDF5_LIBRARY_DIR}/libhdf5.lib" - "${HDF5_LIBRARY_DIR}/libzlib.lib" - "${HDF5_LIBRARY_DIR}/libsz.lib" - "${HDF5_LIBRARY_DIR}/libaec.lib") - set(BOOST_ROOT ${IFCOPENSHELL_ROOT}/_deps/boost_1_74_0) set(BOOST_LIBRARYDIR ${IFCOPENSHELL_ROOT}/_deps/boost_1_74_0/stage/vs${MSVC_YEAR}-${WIN_ARCH}/lib) diff --git a/processor.h b/processor.h index d3d68ad..04d7cc5 100644 --- a/processor.h +++ b/processor.h @@ -19,9 +19,14 @@ #include // @todo < commit in IfcopenShell #include +#ifdef IFCOPENSHELL_07 +typedef IfcGeom::BRepElement elem_t; +#else typedef IfcGeom::BRepElement elem_t; #endif +#endif + typedef TopoDS_Shape geometry_t; typedef std::vector > geometry_collection_t; diff --git a/tests/test_hdf_2.cpp b/tests/test_hdf_2.cpp index 4583968..34c53c8 100644 --- a/tests/test_hdf_2.cpp +++ b/tests/test_hdf_2.cpp @@ -46,8 +46,11 @@ TEST(Voxelization, IfcWallIds) { auto filters = std::vector({ ef }); +#ifdef IFCOPENSHELL_07 + IfcGeom::Iterator it(settings_surface, &ifc_file, filters, 1); +#else IfcGeom::Iterator it(settings_surface, &ifc_file, filters, 1); - +#endif ASSERT_TRUE(it.initialize()); geometry_collection_t geoms; diff --git a/tests/test_space_ids.cpp b/tests/test_space_ids.cpp index 5dd6358..9f6d726 100644 --- a/tests/test_space_ids.cpp +++ b/tests/test_space_ids.cpp @@ -45,7 +45,11 @@ TEST(Voxelization, IfcSpaceIds) { auto filters = std::vector({ ef }); +#ifdef IFCOPENSHELL_07 + IfcGeom::Iterator it(settings_surface, &ifc_file, filters, 1); +#else IfcGeom::Iterator it(settings_surface, &ifc_file, filters, 1); +#endif ASSERT_TRUE(it.initialize()); diff --git a/voxec.h b/voxec.h index 509c516..59528a4 100644 --- a/voxec.h +++ b/voxec.h @@ -309,8 +309,11 @@ class op_create_geometry : public voxel_operation { settings_surface.set(IfcGeom::IteratorSettings::DISABLE_TRIANGULATION, true); settings_surface.set(IfcGeom::IteratorSettings::USE_WORLD_COORDS, true); // Only to determine whether building element parts decompositions of slabs should be processed as roofs +#ifdef IFCOPENSHELL_07 + settings_surface.set(IfcGeom::IteratorSettings::ELEMENT_HIERARCHY, true); +#else settings_surface.set(IfcGeom::IteratorSettings::SEARCH_FLOOR, true); - +#endif boost::optional include, roof_slabs; std::vector entities; @@ -376,10 +379,25 @@ class op_create_geometry : public voxel_operation { for (auto ifc_file : ifc_files.files) { - std::unique_ptr> iterator; +#ifdef IFCOPENSHELL_07 + std::unique_ptr iterator; +#else + std::unique_ptr> iterator; +#endif + #ifdef IFCOPENSHELL_05 iterator.reset(IfcGeom::Iterator(settings_surface, ifc_file, filters_surface)); + + +#elif IFCOPENSHELL_07 + + if (threads) { + iterator.reset(new IfcGeom::Iterator(settings_surface, ifc_file, filters_surface, *threads)); + } + else { + iterator.reset(new IfcGeom::Iterator(settings_surface, ifc_file, filters_surface)); + } #else if (threads) { iterator.reset(new IfcGeom::Iterator(settings_surface, ifc_file, filters_surface, *threads));