diff --git a/.github/workflows/trx-cpp-tests.yml b/.github/workflows/trx-cpp-tests.yml new file mode 100644 index 0000000..04de69b --- /dev/null +++ b/.github/workflows/trx-cpp-tests.yml @@ -0,0 +1,86 @@ +name: trx-cpp tests + +on: + push: + paths: + - "**" + pull_request: + paths: + - "**" + +jobs: + build-and-test: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Install system dependencies + run: | + sudo apt-get update + sudo apt-get install -y \ + cmake \ + g++ \ + zlib1g-dev \ + libeigen3-dev \ + nlohmann-json3-dev \ + libspdlog-dev + + - name: Fetch mio + run: | + git clone --depth 1 https://github.com/mandreyel/mio.git deps/mio + + - name: Build and install libzip (with tools) + run: | + git clone --depth 1 https://github.com/nih-at/libzip.git deps/libzip + cmake -S deps/libzip -B deps/libzip/build \ + -DCMAKE_BUILD_TYPE=Release \ + -DBUILD_TOOLS=ON \ + -DBUILD_SHARED_LIBS=ON \ + -DCMAKE_INSTALL_PREFIX=${GITHUB_WORKSPACE}/deps/libzip/install + cmake --build deps/libzip/build --target install + + - name: Build and install GoogleTest + run: | + git clone --depth 1 https://github.com/google/googletest.git deps/googletest + cmake -S deps/googletest -B deps/googletest/build \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_INSTALL_PREFIX=${GITHUB_WORKSPACE}/deps/googletest/install + cmake --build deps/googletest/build --target install + + - name: Configure trx-cpp + run: | + cmake -S . -B build \ + -DTRX_BUILD_TESTS=ON \ + -DMIO_INCLUDE_DIR=${GITHUB_WORKSPACE}/deps/mio/include \ + -DGTest_DIR=${GITHUB_WORKSPACE}/deps/googletest/install/lib/cmake/GTest \ + -DCMAKE_PREFIX_PATH=${GITHUB_WORKSPACE}/deps/libzip/install + + - name: Build + run: cmake --build build + + - name: Test + run: ctest --test-dir build --output-on-failure + + conan-create: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Install build tooling + run: | + sudo apt-get update + sudo apt-get install -y cmake ninja-build g++ + + - name: Install Conan + run: | + python3 -m pip install --upgrade pip + python3 -m pip install "conan>=2.0,<3.0" + + - name: Conan create (with tests) + env: + CONAN_HOME: ${{ runner.temp }}/.conan2 + run: | + conan profile detect --force + conan create . --build=missing -o with_tests=True -s build_type=Release \ No newline at end of file diff --git a/.gitignore b/.gitignore index 94a144d..c1d7cb4 100644 --- a/.gitignore +++ b/.gitignore @@ -9,3 +9,8 @@ libtrx.a tests/data .vscode +test_package/build/ +test_package/CMakeUserPresets.json + +test_package/build +test_package/CMakeUserPresets.json \ No newline at end of file diff --git a/CMakeLists.txt b/CMakeLists.txt index b3a8300..9a6766b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,28 +1,145 @@ -cmake_minimum_required(VERSION 3.0.0) +cmake_minimum_required(VERSION 3.10) cmake_policy(SET CMP0074 NEW) cmake_policy(SET CMP0079 NEW) + project(trx VERSION 0.1.0) -set (CMAKE_CXX_STANDARD 11) -#set(CMAKE_BUILD_TYPE RelWithDebInfo) -set(CMAKE_BUILD_TYPE Debug) +include(GNUInstallDirs) +include(CMakePackageConfigHelpers) + +set(CMAKE_CXX_STANDARD 17) +if(NOT CMAKE_BUILD_TYPE AND NOT CMAKE_CONFIGURATION_TYPES) + set(CMAKE_BUILD_TYPE Debug) +endif() + +option(TRX_USE_CONAN "Should Conan package manager be used?" ON) +option(TRX_BUILD_TESTS "Build trx tests" ON) find_package(libzip REQUIRED) -find_package (Eigen3 CONFIG REQUIRED) +set(TRX_LIBZIP_TARGET "") +if(TARGET libzip::libzip) + set(TRX_LIBZIP_TARGET libzip::libzip) +elseif(TARGET libzip::zip) + set(TRX_LIBZIP_TARGET libzip::zip) +elseif(TARGET zip::zip) + set(TRX_LIBZIP_TARGET zip::zip) +else() + message(FATAL_ERROR "No suitable libzip target (expected libzip::libzip or zip::zip)") +endif() +find_package(Eigen3 CONFIG QUIET) +if (NOT Eigen3_FOUND) + find_package(Eigen3 REQUIRED) # try module mode +endif() +# Create an imported target if the package did not provide one (module mode) +if (NOT TARGET Eigen3::Eigen AND EXISTS "${EIGEN3_INCLUDE_DIR}") + add_library(Eigen3::Eigen INTERFACE IMPORTED) + set_target_properties(Eigen3::Eigen PROPERTIES + INTERFACE_INCLUDE_DIRECTORIES "${EIGEN3_INCLUDE_DIR}" + ) +endif() find_package(nlohmann_json CONFIG REQUIRED) find_package(spdlog CONFIG REQUIRED) +find_package(mio CONFIG QUIET) +if(TARGET mio::mio) + set(TRX_HAVE_MIO_TARGET ON) +else() + find_path(MIO_INCLUDE_DIR mio/mmap.hpp PATH_SUFFIXES include) + if (NOT MIO_INCLUDE_DIR) + message(FATAL_ERROR "mio headers not found. Set MIO_INCLUDE_DIR to the folder containing mio/mmap.hpp.") + endif() +endif() + +add_library(trx src/trx.cpp include/trx/trx.h include/trx/trx.tpp) +add_library(trx-cpp::trx ALIAS trx) + +if(TRX_USE_CONAN AND EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/cmake/ConanSetup.cmake") + include(cmake/ConanSetup.cmake) +elseif(TRX_USE_CONAN) + message(STATUS "TRX_USE_CONAN enabled but cmake/ConanSetup.cmake not found; skipping Conan.") +endif() -add_library(trx src/trx.cpp src/trx.tpp src/trx.h) +# Fallback for libzip packages that don't expose include dirs via CMake targets. +set(TRX_LIBZIP_INCLUDE_DIR "") +get_target_property(_trx_libzip_includes ${TRX_LIBZIP_TARGET} INTERFACE_INCLUDE_DIRECTORIES) +if(NOT _trx_libzip_includes) + find_path(TRX_LIBZIP_INCLUDE_DIR zip.h) + if(NOT TRX_LIBZIP_INCLUDE_DIR) + message(FATAL_ERROR "libzip headers not found. Set TRX_LIBZIP_INCLUDE_DIR or fix libzip CMake targets.") + endif() +endif() TARGET_LINK_LIBRARIES(trx - PRIVATE - nlohmann_json::nlohmann_json - libzip::zip - Eigen3::Eigen - spdlog::spdlog - spdlog::spdlog_header_only + PUBLIC + nlohmann_json::nlohmann_json + ${TRX_LIBZIP_TARGET} + Eigen3::Eigen + spdlog::spdlog + $<$:mio::mio> +) +target_include_directories(trx + PUBLIC + $ + $ + $<$>:$> + PRIVATE + ${CMAKE_CURRENT_SOURCE_DIR}/src +) +if(TRX_LIBZIP_INCLUDE_DIR) + target_include_directories(trx PUBLIC + $ + $ + ) +endif() + +if(TRX_BUILD_TESTS) + find_package(GTest CONFIG QUIET) + if(NOT GTest_FOUND) + find_package(GTest QUIET) + endif() + if(GTest_FOUND) + enable_testing() + add_subdirectory(tests) + else() + message(STATUS "GTest not found; skipping tests. Set GTest_DIR to a config path to enable.") + endif() +endif() + +# Installation and package config +set(TRX_INSTALL_CONFIGDIR "${CMAKE_INSTALL_LIBDIR}/cmake/trx-cpp") + +install(TARGETS trx + EXPORT trx-cppTargets + RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} + LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} + ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} + INCLUDES DESTINATION ${CMAKE_INSTALL_INCLUDEDIR} ) +install(DIRECTORY include/ DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}) + +install(EXPORT trx-cppTargets + FILE trx-cppTargets.cmake + NAMESPACE trx-cpp:: + DESTINATION ${TRX_INSTALL_CONFIGDIR} +) + +configure_package_config_file( + cmake/trx-cppConfig.cmake.in + ${CMAKE_CURRENT_BINARY_DIR}/trx-cppConfig.cmake + INSTALL_DESTINATION ${TRX_INSTALL_CONFIGDIR} +) + +write_basic_package_version_file( + ${CMAKE_CURRENT_BINARY_DIR}/trx-cppConfigVersion.cmake + VERSION ${PROJECT_VERSION} + COMPATIBILITY SameMinorVersion +) + +install(FILES + ${CMAKE_CURRENT_BINARY_DIR}/trx-cppConfig.cmake + ${CMAKE_CURRENT_BINARY_DIR}/trx-cppConfigVersion.cmake + DESTINATION ${TRX_INSTALL_CONFIGDIR} +) set(CPACK_PROJECT_NAME ${PROJECT_NAME}) set(CPACK_PROJECT_VERSION ${PROJECT_VERSION}) diff --git a/cmake/trx-cppConfig.cmake.in b/cmake/trx-cppConfig.cmake.in new file mode 100644 index 0000000..1b46b66 --- /dev/null +++ b/cmake/trx-cppConfig.cmake.in @@ -0,0 +1,13 @@ +@PACKAGE_INIT@ + +include(CMakeFindDependencyMacro) +find_dependency(libzip) +find_dependency(Eigen3) +find_dependency(nlohmann_json) +find_dependency(spdlog) +# mio is header-only; try to locate if packaged +find_dependency(mio QUIET) + +include("${CMAKE_CURRENT_LIST_DIR}/trx-cppTargets.cmake") + +check_required_components("trx-cpp") diff --git a/conanfile.py b/conanfile.py new file mode 100644 index 0000000..3cf80bb --- /dev/null +++ b/conanfile.py @@ -0,0 +1,132 @@ +from conan import ConanFile +from conan.tools.cmake import CMake, CMakeToolchain, cmake_layout +from conan.tools.files import get, copy +import os + + +class TrxCppConan(ConanFile): + name = "trx-cpp" + version = "0.1.0" + package_type = "library" + license = "MIT" # Update if the project adopts a different license. + url = "https://github.com/tractdata/trx-cpp" + homepage = "https://github.com/tractdata/trx-cpp" + description = "C++ library for reading and writing the TRX tractography format." + topics = ("tractography", "mmap", "neuroimaging") + settings = "os", "arch", "compiler", "build_type" + options = { + "shared": [True, False], + "fPIC": [True, False], + "with_tests": [True, False], + } + default_options = { + "shared": False, + "fPIC": True, + "with_tests": False, + } + generators = ("CMakeDeps",) + exports_sources = ( + "CMakeLists.txt", + "src/*", + "include/*", + "cmake/*", + "tests/*", + ) + + def config_options(self): + if self.settings.os == "Windows": + self.options.rm_safe("fPIC") + + def requirements(self): + self.requires("libzip/1.10.1") + self.requires("nlohmann_json/3.11.3") + self.requires("eigen/3.4.0") + self.requires("spdlog/1.12.0") + + def build_requirements(self): + if self.options.with_tests: + # Only needed for building/running tests, not for consumers. + self.test_requires("gtest/1.14.0") + + def layout(self): + cmake_layout(self) + + def source(self): + # Fetch header-only mio directly to avoid external package availability issues. + # Using the upstream main branch archive to ensure availability. + get( + self, + url="https://github.com/mandreyel/mio/archive/refs/heads/master.zip", + strip_root=True, + destination="mio", + ) + + def generate(self): + tc = CMakeToolchain(self) + mio_include = os.path.join(self.source_folder, "mio", "include") + tc.variables["MIO_INCLUDE_DIR"] = mio_include + tc.generate() + + def build(self): + cmake = CMake(self) + cmake.configure(variables={"TRX_BUILD_TESTS": self.options.with_tests}) + cmake.build() + if self.options.with_tests: + cmake.ctest() + + def package(self): + cmake = CMake(self) + cmake.install() + # Ensure libzip headers are available to consumers even when the + # libzip Conan package does not expose include dirs via CMakeDeps. + libzip_dep = self.dependencies.get("libzip") + if libzip_dep and libzip_dep.package_folder: + libzip_include = os.path.join(libzip_dep.package_folder, "include") + copy(self, "zip.h", src=libzip_include, dst=os.path.join(self.package_folder, "include")) + copy(self, "zipconf.h", src=libzip_include, dst=os.path.join(self.package_folder, "include")) + + nlohmann_dep = self.dependencies.get("nlohmann_json") + if nlohmann_dep and nlohmann_dep.package_folder: + nlohmann_include = os.path.join(nlohmann_dep.package_folder, "include") + copy(self, "nlohmann/*", src=nlohmann_include, dst=os.path.join(self.package_folder, "include")) + eigen_dep = self.dependencies.get("eigen") + if eigen_dep and eigen_dep.package_folder: + eigen_include = os.path.join(eigen_dep.package_folder, "include", "eigen3") + copy(self, "Eigen/*", src=eigen_include, dst=os.path.join(self.package_folder, "include")) + + mio_include = os.path.join(self.source_folder, "mio", "include") + copy(self, "mio/*", src=mio_include, dst=os.path.join(self.package_folder, "include")) + + spdlog_dep = self.dependencies.get("spdlog") + if spdlog_dep and spdlog_dep.package_folder: + spdlog_include = os.path.join(spdlog_dep.package_folder, "include") + copy(self, "spdlog/*", src=spdlog_include, dst=os.path.join(self.package_folder, "include")) + fmt_dep = self.dependencies.get("fmt") + if fmt_dep and fmt_dep.package_folder: + fmt_include = os.path.join(fmt_dep.package_folder, "include") + copy(self, "fmt/*", src=fmt_include, dst=os.path.join(self.package_folder, "include")) + + def package_info(self): + self.cpp_info.set_property("cmake_file_name", "trx-cpp") + self.cpp_info.set_property("cmake_target_name", "trx-cpp::trx") + self.cpp_info.set_property("pkg_config_name", "trx-cpp") + self.cpp_info.components["trx"].set_property("cmake_target_name", "trx-cpp::trx") + self.cpp_info.components["trx"].set_property("pkg_config_name", "trx-cpp") + self.cpp_info.components["trx"].libs = ["trx"] + self.cpp_info.components["trx"].requires = [ + "libzip::libzip", + "spdlog::spdlog", + "nlohmann_json::nlohmann_json", + "eigen::Eigen3::Eigen", + ] + extra_includes = [] + for dep_name in ("nlohmann_json", "eigen"): + dep = self.dependencies.get(dep_name) + if dep and dep.package_folder: + dep_include = os.path.join(dep.package_folder, "include") + if os.path.isdir(dep_include): + extra_includes.append(dep_include) + if extra_includes: + self.cpp_info.components["trx"].includedirs.extend(extra_includes) + if not self.options.shared and self.settings.os in ("Linux", "FreeBSD"): + self.cpp_info.system_libs.append("pthread") diff --git a/src/trx.h b/include/trx/trx.h similarity index 94% rename from src/trx.h rename to include/trx/trx.h index 4cf112a..f4d0c79 100644 --- a/src/trx.h +++ b/include/trx/trx.h @@ -13,10 +13,16 @@ #include #include #include +#include +#include +#include #include #include +#ifndef SPDLOG_FMT_EXTERNAL +#define SPDLOG_FMT_EXTERNAL +#endif #include "spdlog/spdlog.h" using namespace Eigen; @@ -66,10 +72,12 @@ namespace trxmmap std::map *>> data_per_group; std::string _uncompressed_folder_handle; bool _copy_safe; + bool _owns_uncompressed_folder = false; // Member Functions() // TrxFile(int nb_vertices = 0, int nb_streamlines = 0); TrxFile(int nb_vertices = 0, int nb_streamlines = 0, const TrxFile
*init_as = NULL, std::string reference = ""); + ~TrxFile(); /** * @brief After reading the structure of a zip/folder, create a TrxFile @@ -103,6 +111,7 @@ namespace trxmmap * */ void close(); + void _cleanup_temporary_directory(); private: /** @@ -219,7 +228,7 @@ namespace trxmmap // TODO: ADD order?? // TODO: change tuple to vector to support ND arrays? // TODO: remove data type as that's done outside of this function - mio::shared_mmap_sink _create_memmap(std::string &filename, std::tuple &shape, std::string mode = "r", std::string dtype = "float32", long long offset = 0); + mio::shared_mmap_sink _create_memmap(std::string filename, std::tuple &shape, std::string mode = "r", std::string dtype = "float32", long long offset = 0); template std::string _generate_filename_from_data(const MatrixBase
&arr, const std::string filename); @@ -289,9 +298,11 @@ namespace trxmmap void copy_dir(const char *src, const char *dst); void copy_file(const char *src, const char *dst); int rm_dir(const char *d); + std::string make_temp_dir(const std::string &prefix); + std::string extract_zip_to_directory(zip_t *zfolder); std::string rm_root(std::string root, const std::string path); -#include "trx.tpp" +#include } diff --git a/src/trx.tpp b/include/trx/trx.tpp similarity index 87% rename from src/trx.tpp rename to include/trx/trx.tpp index 92eebdc..730a8f0 100644 --- a/src/trx.tpp +++ b/include/trx/trx.tpp @@ -60,17 +60,11 @@ std::string _generate_filename_from_data(const MatrixBase
&arr, std::string std::string new_filename; if (n_cols == 1) { - int buffsize = filename.size() + dt.size() + 2; - char buff[buffsize]; - snprintf(buff, sizeof(buff), "%s.%s", base.c_str(), dt.c_str()); - new_filename = buff; + new_filename = base + "." + dt; } else { - int buffsize = filename.size() + dt.size() + n_cols + 3; - char buff[buffsize]; - snprintf(buff, sizeof(buff), "%s.%i.%s", base.c_str(), n_cols, dt.c_str()); - new_filename = buff; + new_filename = base + "." + std::to_string(n_cols) + "." + dt; } return new_filename; @@ -81,31 +75,17 @@ Matrix _compute_lengths(const MatrixBase
&offsets, int { if (offsets.size() > 1) { - int last_elem_pos = _dichotomic_search(offsets); - Matrix lengths; - - if (last_elem_pos == offsets.size() - 1) + const auto casted = offsets.template cast(); + const Eigen::Index len = offsets.size() - 1; + Matrix lengths(len); + for (Eigen::Index i = 0; i < len; ++i) { - Matrix tmp(offsets.template cast()); - ediff1d(lengths, tmp, uint32_t(nb_vertices - offsets(last))); + lengths(i) = static_cast(casted(i + 1) - casted(i)); } - else - { - Matrix tmp(offsets.template cast()); - tmp(last_elem_pos + 1) = uint32_t(nb_vertices); - ediff1d(lengths, tmp, 0); - lengths(last_elem_pos + 1) = uint32_t(0); - } - return lengths; - } - if (offsets.size() == 1) - { - Matrix lengths(nb_vertices); return lengths; } - - Matrix lengths(0); - return lengths; + // If offsets are empty or only contain the sentinel, there are zero streamlines. + return Matrix(0); } template @@ -203,7 +183,11 @@ TrxFile
::TrxFile(int nb_vertices, int nb_streamlines, const TrxFile
*ini this->data_per_vertex = trx->data_per_vertex; this->data_per_group = trx->data_per_group; this->_uncompressed_folder_handle = trx->_uncompressed_folder_handle; + this->_owns_uncompressed_folder = trx->_owns_uncompressed_folder; this->_copy_safe = trx->_copy_safe; + trx->_owns_uncompressed_folder = false; + trx->_uncompressed_folder_handle.clear(); + delete trx; } else { @@ -223,11 +207,7 @@ TrxFile
*_initialize_empty_trx(int nb_streamlines, int nb_vertices, const Tr { TrxFile
*trx = new TrxFile
(); - char *dirname; - char t[] = "/tmp/trx_XXXXXX"; - dirname = mkdtemp(t); - - std::string tmp_dir(dirname); + std::string tmp_dir = make_temp_dir("trx"); spdlog::info("Temporary folder for memmaps: {}", tmp_dir); @@ -282,12 +262,13 @@ TrxFile
*_initialize_empty_trx(int nb_streamlines, int nb_vertices, const Tr std::string offsets_filename(tmp_dir); offsets_filename += "/offsets." + offsets_dtype; - std::tuple shape_off = std::make_tuple(nb_streamlines, 1); + std::tuple shape_off = std::make_tuple(nb_streamlines + 1, 1); trx->streamlines->mmap_off = trxmmap::_create_memmap(offsets_filename, shape_off, "w+", offsets_dtype); new (&(trx->streamlines->_offsets)) Map>(reinterpret_cast(trx->streamlines->mmap_off.data()), std::get<0>(shape_off), std::get<1>(shape_off)); trx->streamlines->_lengths.resize(nb_streamlines); + trx->streamlines->_lengths.setZero(); if (init_as != NULL) { @@ -390,6 +371,7 @@ TrxFile
*_initialize_empty_trx(int nb_streamlines, int nb_vertices, const Tr } trx->_uncompressed_folder_handle = tmp_dir; + trx->_owns_uncompressed_folder = true; return trx; } @@ -481,20 +463,21 @@ TrxFile
*TrxFile
::_create_trx_from_pointer(json header, std::mapheader["NB_STREAMLINES"]) || dim != 1) + if (size != int(trx->header["NB_STREAMLINES"]) + 1 || dim != 1) { - - throw std::invalid_argument("Wrong offsets size/dimensionality"); + throw std::invalid_argument("Wrong offsets size/dimensionality: size=" + + std::to_string(size) + " nb_streamlines=" + + std::to_string(int(trx->header["NB_STREAMLINES"])) + + " dim=" + std::to_string(dim) + " filename=" + elem_filename); } - std::tuple shape = std::make_tuple(trx->header["NB_STREAMLINES"], 1); + const int nb_str = int(trx->header["NB_STREAMLINES"]); + std::tuple shape = std::make_tuple(nb_str + 1, 1); trx->streamlines->mmap_off = trxmmap::_create_memmap(filename, shape, "r+", ext, mem_adress); new (&(trx->streamlines->_offsets)) Map>(reinterpret_cast(trx->streamlines->mmap_off.data()), std::get<0>(shape), std::get<1>(shape)); - // TODO : adapt compute_lengths to accept a map - Matrix offsets; - offsets = trx->streamlines->_offsets; + Matrix offsets = trx->streamlines->_offsets; trx->streamlines->_lengths = _compute_lengths(offsets, int(trx->header["NB_VERTICES"])); } @@ -630,11 +613,7 @@ TrxFile
*TrxFile
::_create_trx_from_pointer(json header, std::map TrxFile
*TrxFile
::deepcopy() { - char *dirname; - char t[] = "/tmp/trx_XXXXXX"; - dirname = mkdtemp(t); - - std::string tmp_dir(dirname); + std::string tmp_dir = make_temp_dir("trx"); std::string header = tmp_dir + SEPARATOR + "header.json"; std::ofstream out_json(header); @@ -650,9 +629,14 @@ TrxFile
*TrxFile
::deepcopy() if (!this->_copy_safe) { - tmp_header["NB_STREAMLINES"] = to_dump->_offsets.size(); + tmp_header["NB_STREAMLINES"] = to_dump->_offsets.size() > 0 ? to_dump->_offsets.size() - 1 : 0; tmp_header["NB_VERTICES"] = to_dump->_data.size() / 3; } + // Ensure sentinel is correct before persisting + if (to_dump->_offsets.size() > 0) + { + to_dump->_offsets(to_dump->_offsets.size() - 1) = tmp_header["NB_VERTICES"]; + } if (out_json.is_open()) { out_json << std::setw(4) << tmp_header << std::endl; @@ -756,6 +740,7 @@ TrxFile
*TrxFile
::deepcopy() TrxFile
*copy_trx = load_from_directory
(tmp_dir); copy_trx->_uncompressed_folder_handle = tmp_dir; + copy_trx->_owns_uncompressed_folder = true; return copy_trx; } @@ -812,20 +797,27 @@ std::tuple TrxFile
::_copy_fixed_arrays_from(TrxFile
*trx, int if (curr_pts_len == 0) return std::make_tuple(strs_start, pts_start); - this->streamlines->_data(seq(pts_start, pts_end - 1), all) = trx->streamlines->_data(seq(0, curr_pts_len - 1), all); - this->streamlines->_offsets(seq(strs_start, strs_end - 1), all) = (trx->streamlines->_offsets(seq(0, curr_strs_len - 1), all).array() + pts_start).matrix(); - this->streamlines->_lengths(seq(strs_start, strs_end - 1), all) = trx->streamlines->_lengths(seq(0, curr_strs_len - 1), all); + this->streamlines->_data.block(pts_start, 0, curr_pts_len, this->streamlines->_data.cols()) = + trx->streamlines->_data.block(0, 0, curr_pts_len, trx->streamlines->_data.cols()); + + this->streamlines->_offsets.block(strs_start, 0, curr_strs_len + 1, 1) = + (trx->streamlines->_offsets.block(0, 0, curr_strs_len + 1, 1).array() + pts_start).matrix(); + + this->streamlines->_lengths.block(strs_start, 0, curr_strs_len, 1) = + trx->streamlines->_lengths.block(0, 0, curr_strs_len, 1); for (auto const &x : this->data_per_vertex) { - this->data_per_vertex[x.first]->_data(seq(pts_start, pts_end - 1), all) = trx->data_per_vertex[x.first]->_data(seq(0, curr_pts_len - 1), all); + this->data_per_vertex[x.first]->_data.block(pts_start, 0, curr_pts_len, this->data_per_vertex[x.first]->_data.cols()) = + trx->data_per_vertex[x.first]->_data.block(0, 0, curr_pts_len, trx->data_per_vertex[x.first]->_data.cols()); new (&(this->data_per_vertex[x.first]->_offsets)) Map>(trx->data_per_vertex[x.first]->_offsets.data(), trx->data_per_vertex[x.first]->_offsets.rows(), trx->data_per_vertex[x.first]->_offsets.cols()); this->data_per_vertex[x.first]->_lengths = trx->data_per_vertex[x.first]->_lengths; } for (auto const &x : this->data_per_streamline) { - this->data_per_streamline[x.first]->_matrix(seq(strs_start, strs_end - 1), all) = trx->data_per_streamline[x.first]->_matrix(seq(0, curr_strs_len - 1), all); + this->data_per_streamline[x.first]->_matrix.block(strs_start, 0, curr_strs_len, this->data_per_streamline[x.first]->_matrix.cols()) = + trx->data_per_streamline[x.first]->_matrix.block(0, 0, curr_strs_len, trx->data_per_streamline[x.first]->_matrix.cols()); } return std::make_tuple(strs_end, pts_end); @@ -834,15 +826,31 @@ std::tuple TrxFile
::_copy_fixed_arrays_from(TrxFile
*trx, int template void TrxFile
::close() { - if (this->_uncompressed_folder_handle != "") - { - this->_uncompressed_folder_handle = ""; - } - + this->_cleanup_temporary_directory(); *this = TrxFile
(); // probably dangerous to do spdlog::debug("Deleted memmaps and initialized empty TrxFile."); } +template +TrxFile
::~TrxFile() +{ + this->_cleanup_temporary_directory(); +} + +template +void TrxFile
::_cleanup_temporary_directory() +{ + if (this->_owns_uncompressed_folder && !this->_uncompressed_folder_handle.empty()) + { + if (rm_dir(this->_uncompressed_folder_handle.c_str()) != 0) + { + spdlog::warn("Could not remove temporary folder {}", this->_uncompressed_folder_handle); + } + this->_uncompressed_folder_handle.clear(); + this->_owns_uncompressed_folder = false; + } +} + template void TrxFile
::resize(int nb_streamlines, int nb_vertices, bool delete_dpg) { @@ -863,7 +871,7 @@ void TrxFile
::resize(int nb_streamlines, int nb_vertices, bool delete_dpg) if (nb_vertices == -1) { - ptrs_end = this->streamlines->_lengths(all, 0).sum(); + ptrs_end = this->streamlines->_lengths.sum(); nb_vertices = ptrs_end; } else if (nb_vertices < ptrs_end) @@ -886,7 +894,7 @@ void TrxFile
::resize(int nb_streamlines, int nb_vertices, bool delete_dpg) TrxFile
*trx = _initialize_empty_trx(nb_streamlines, nb_vertices, this); spdlog::info("Resizing streamlines from size {} to {}", this->streamlines->_lengths.size(), nb_streamlines); - spdlog::info("Resizing vertices from size {} to {}", this->streamlines->_data(all, 0).size(), nb_vertices); + spdlog::info("Resizing vertices from size {} to {}", this->streamlines->_data.rows(), nb_vertices); if (nb_streamlines < this->header["NB_STREAMLINES"]) trx->_copy_fixed_arrays_from(this, -1, -1, nb_streamlines); @@ -920,7 +928,7 @@ void TrxFile
::resize(int nb_streamlines, int nb_vertices, bool delete_dpg) { for (int j = 0; j < x.second->_matrix.cols(); ++j) { - if (x.second->_matrix(i, j) < strs_end) + if (static_cast(x.second->_matrix(i, j)) < strs_end) { keep_rows.push_back(i); } @@ -1014,87 +1022,31 @@ void TrxFile
::resize(int nb_streamlines, int nb_vertices, bool delete_dpg) template TrxFile
*load_from_zip(std::string filename) { - // TODO: check error values - int *errorp; - zip_t *zf = zip_open(filename.c_str(), 0, errorp); - json header = load_header(zf); - - std::map> file_pointer_size; - long long global_pos = 0; - long long mem_address = 0; - - int num_entries = zip_get_num_entries(zf, ZIP_FL_UNCHANGED); - - for (int i = 0; i < num_entries; ++i) + int errorp = 0; + zip_t *zf = zip_open(filename.c_str(), 0, &errorp); + if (zf == nullptr) { - std::string elem_filename = zip_get_name(zf, i, ZIP_FL_UNCHANGED); - - zip_stat_t sb; - zip_file_t *zft; - - if (zip_stat(zf, elem_filename.c_str(), ZIP_FL_UNCHANGED, &sb) != 0) - { - return NULL; - } - - global_pos += 30 + elem_filename.size(); - - size_t lastdot = elem_filename.find_last_of("."); - - if (lastdot == std::string::npos) - { - global_pos += sb.comp_size; - continue; - } - std::string ext = elem_filename.substr(lastdot + 1, std::string::npos); - - // apparently all zip directory names end with a slash. may be a better way - if (ext.compare("json") == 0 || elem_filename.rfind("/") == elem_filename.size() - 1) - { - global_pos += sb.comp_size; - continue; - } - - if (!_is_dtype_valid(ext)) - { - global_pos += sb.comp_size; - continue; - // maybe throw error here instead? - // throw std::invalid_argument("The dtype is not supported"); - } - - if (ext.compare("bit") == 0) - { - ext = "bool"; - } - - // get file stats - - // std::ifstream file(filename, std::ios::binary); - // file.seekg(global_pos); - - // unsigned char signature[4] = {0}; - // const unsigned char local_sig[4] = {0x50, 0x4b, 0x03, 0x04}; - // file.read((char *)signature, sizeof(signature)); + throw std::runtime_error("Could not open zip file: " + filename); + } - // if (memcmp(signature, local_sig, sizeof(signature)) == 0) - // { - // global_pos += 30; - // // global_pos += sb.comp_size + elem_filename.size(); - // } + std::string temp_dir = extract_zip_to_directory(zf); + zip_close(zf); - long long size = sb.size / _sizeof_dtype(ext); - mem_address = global_pos; - file_pointer_size[elem_filename] = {mem_address, size}; - global_pos += sb.comp_size; - } - return TrxFile
::_create_trx_from_pointer(header, file_pointer_size, filename); + TrxFile
*trx = load_from_directory
(temp_dir); + trx->_uncompressed_folder_handle = temp_dir; + trx->_owns_uncompressed_folder = true; + return trx; } template TrxFile
*load_from_directory(std::string path) { - std::string directory = (std::string)canonicalize_file_name(path.c_str()); + std::string directory = path; + char resolved[PATH_MAX]; + if (realpath(path.c_str(), resolved) != nullptr) + { + directory = resolved; + } std::string header_name = directory + SEPARATOR + "header.json"; // TODO: add check to verify that it's open @@ -1144,6 +1096,12 @@ void save(TrxFile
&trx, const std::string filename, zip_uint32_t compression { struct stat sb; + struct stat tmp_sb; + if (stat(tmp_dir_name.c_str(), &tmp_sb) != 0 || !S_ISDIR(tmp_sb.st_mode)) + { + throw std::runtime_error("Temporary TRX directory does not exist: " + tmp_dir_name); + } + if (stat(filename.c_str(), &sb) == 0 && S_ISDIR(sb.st_mode)) { if (rm_dir(filename.c_str()) != 0) @@ -1151,7 +1109,27 @@ void save(TrxFile
&trx, const std::string filename, zip_uint32_t compression spdlog::error("Could not remove existing directory {}", filename); } } + std::filesystem::path dest_path(filename); + if (dest_path.has_parent_path()) + { + std::error_code ec; + std::filesystem::create_directories(dest_path.parent_path(), ec); + if (ec) + { + throw std::runtime_error("Could not create output parent directory: " + + dest_path.parent_path().string()); + } + } copy_dir(tmp_dir_name.c_str(), filename.c_str()); + if (stat(filename.c_str(), &sb) != 0 || !S_ISDIR(sb.st_mode)) + { + throw std::runtime_error("Failed to create output directory: " + filename); + } + const std::filesystem::path header_path = dest_path / "header.json"; + if (!std::filesystem::exists(header_path)) + { + throw std::runtime_error("Missing header.json in output directory: " + header_path.string()); + } copy_trx->close(); } } diff --git a/src/trx.cpp b/src/trx.cpp index e65ff2d..127cc07 100644 --- a/src/trx.cpp +++ b/src/trx.cpp @@ -1,8 +1,10 @@ -#include "trx.h" +#include #include #include #include #include +#include +#include #define SYSERROR() errno //#define ZIP_DD_SIG 0x08074b50 @@ -104,6 +106,21 @@ namespace trxmmap } return ext; } + +bool _is_path_within(const std::filesystem::path &child, const std::filesystem::path &parent) +{ + auto parent_it = parent.begin(); + auto child_it = child.begin(); + + for (; parent_it != parent.end(); ++parent_it, ++child_it) + { + if (child_it == child.end() || *parent_it != *child_it) + { + return false; + } + } + return true; +} // TODO: check if there's a better way int _sizeof_dtype(std::string dtype) { @@ -159,6 +176,8 @@ namespace trxmmap return "uint32"; case 'm': return "uint64"; + case 'y': // unsigned long long (Itanium ABI) + return "uint64"; case 'a': return "int8"; case 's': @@ -167,6 +186,8 @@ namespace trxmmap return "int32"; case 'l': return "int64"; + case 'x': // long long (Itanium ABI) + return "int64"; case 'f': return "float32"; case 'd': @@ -228,8 +249,16 @@ namespace trxmmap json load_header(zip_t *zfolder) { + if (zfolder == nullptr) + { + throw std::invalid_argument("Zip archive pointer is null"); + } // load file zip_file_t *zh = zip_fopen(zfolder, "header.json", ZIP_FL_UNCHANGED); + if (zh == nullptr) + { + throw std::runtime_error("Failed to open header.json in zip archive"); + } // read data from file in chunks of 255 characters until data is fully loaded int buff_len = 255 * sizeof(char); @@ -245,7 +274,7 @@ namespace trxmmap } } - free(zh); + zip_fclose(zh); free(buffer); // convert jstream data into Json. @@ -269,7 +298,7 @@ namespace trxmmap } } - mio::shared_mmap_sink _create_memmap(std::string &filename, std::tuple &shape, std::string mode, std::string dtype, long long offset) +mio::shared_mmap_sink _create_memmap(std::string filename, std::tuple &shape, std::string mode, std::string dtype, long long offset) { if (dtype.compare("bool") == 0) { @@ -282,14 +311,19 @@ namespace trxmmap // if file does not exist, create and allocate it struct stat buffer; - if (stat(filename.c_str(), &buffer) != 0) - { - allocate_file(filename, filesize); - } + if (stat(filename.c_str(), &buffer) != 0) + { + allocate_file(filename, filesize); + } + + if (filesize == 0) + { + return mio::shared_mmap_sink(); + } // std::error_code error; - mio::shared_mmap_sink rw_mmap(filename, offset, filesize); + mio::shared_mmap_sink rw_mmap(filename, offset, filesize); return rw_mmap; } @@ -446,6 +480,172 @@ namespace trxmmap return rmdir(d); } + std::string make_temp_dir(const std::string &prefix) + { + const char *env_tmp = std::getenv("TRX_TMPDIR"); + std::string base_dir; + + if (env_tmp != nullptr) + { + std::string val(env_tmp); + if (val == "use_working_dir") + { + base_dir = "."; + } + else + { + std::filesystem::path env_path(val); + std::error_code ec; + if (std::filesystem::exists(env_path, ec) && std::filesystem::is_directory(env_path, ec)) + { + base_dir = env_path.string(); + } + } + } + + if (base_dir.empty()) + { + const char *candidates[] = {std::getenv("TMPDIR"), std::getenv("TEMP"), std::getenv("TMP")}; + for (const char *candidate : candidates) + { + if (candidate == nullptr || std::string(candidate).empty()) + { + continue; + } + std::filesystem::path path(candidate); + std::error_code ec; + if (std::filesystem::exists(path, ec) && std::filesystem::is_directory(path, ec)) + { + base_dir = path.string(); + break; + } + } + } + if (base_dir.empty()) + { + std::error_code ec; + auto sys_tmp = std::filesystem::temp_directory_path(ec); + if (!ec) + { + base_dir = sys_tmp.string(); + } + } + if (base_dir.empty()) + { + base_dir = "/tmp"; + } + + std::filesystem::path tmpl = std::filesystem::path(base_dir) / (prefix + "_XXXXXX"); + std::string tmpl_str = tmpl.string(); + std::vector buf(tmpl_str.begin(), tmpl_str.end()); + buf.push_back('\0'); + char *dirname = mkdtemp(buf.data()); + if (dirname == nullptr) + { + throw std::runtime_error("Failed to create temporary directory"); + } + return std::string(dirname); + } + + std::string extract_zip_to_directory(zip_t *zfolder) + { + if (zfolder == nullptr) + { + throw std::invalid_argument("Zip archive pointer is null"); + } + std::string root_dir = make_temp_dir("trx_zip"); + std::filesystem::path normalized_root = std::filesystem::path(root_dir).lexically_normal(); + + zip_int64_t num_entries = zip_get_num_entries(zfolder, ZIP_FL_UNCHANGED); + for (zip_int64_t i = 0; i < num_entries; ++i) + { + const char *entry_name = zip_get_name(zfolder, i, ZIP_FL_UNCHANGED); + if (entry_name == nullptr) + { + continue; + } + std::string entry(entry_name); + + std::filesystem::path entry_path(entry); + if (entry_path.is_absolute()) + { + throw std::runtime_error("Zip entry has absolute path: " + entry); + } + + std::filesystem::path normalized_entry = entry_path.lexically_normal(); + std::filesystem::path out_path = normalized_root / normalized_entry; + std::filesystem::path normalized_out = out_path.lexically_normal(); + + if (!_is_path_within(normalized_out, normalized_root)) + { + throw std::runtime_error("Zip entry escapes temporary directory: " + entry); + } + + if (!entry.empty() && entry.back() == '/') + { + std::error_code ec; + std::filesystem::create_directories(normalized_out, ec); + if (ec) + { + throw std::runtime_error("Failed to create directory: " + normalized_out.string()); + } + continue; + } + + std::error_code ec; + std::filesystem::create_directories(normalized_out.parent_path(), ec); + if (ec) + { + throw std::runtime_error("Failed to create parent directory: " + normalized_out.parent_path().string()); + } + + zip_file_t *zf = zip_fopen_index(zfolder, i, ZIP_FL_UNCHANGED); + if (zf == nullptr) + { + throw std::runtime_error("Failed to open zip entry: " + entry); + } + + std::ofstream out(normalized_out, std::ios::binary); + if (!out.is_open()) + { + zip_fclose(zf); + throw std::runtime_error("Failed to open output file: " + normalized_out.string()); + } + + char buffer[4096]; + zip_int64_t nbytes = 0; + while ((nbytes = zip_fread(zf, buffer, sizeof(buffer))) > 0) + { + out.write(buffer, nbytes); + if (!out) + { + out.close(); + zip_fclose(zf); + throw std::runtime_error("Failed to write to output file: " + normalized_out.string()); + } + } + if (nbytes < 0) + { + out.close(); + zip_fclose(zf); + throw std::runtime_error("Failed to read data from zip entry: " + entry); + } + + out.flush(); + if (!out) + { + out.close(); + zip_fclose(zf); + throw std::runtime_error("Failed to flush output file: " + normalized_out.string()); + } + + out.close(); + zip_fclose(zf); + } + + return root_dir; + } + void zip_from_folder(zip_t *zf, const std::string root, const std::string directory, zip_uint32_t compression_standard) { DIR *dir; @@ -478,12 +678,17 @@ namespace trxmmap zip_source_t *s; + zip_int64_t file_idx = -1; if ((s = zip_source_file(zf, fullpath, 0, 0)) == NULL || - zip_file_add(zf, fn.c_str(), s, ZIP_FL_ENC_UTF_8) < 0) + (file_idx = zip_file_add(zf, fn.c_str(), s, ZIP_FL_ENC_UTF_8)) < 0) { zip_source_free(s); spdlog::error("error adding file {}: {}", fn, zip_strerror(zf)); } + else if (zip_set_file_compression(zf, file_idx, compression_standard, 0) < 0) + { + spdlog::error("error setting compression for {}: {}", fn, zip_strerror(zf)); + } } } closedir(dir); diff --git a/test_package/CMakeLists.txt b/test_package/CMakeLists.txt new file mode 100644 index 0000000..d463faa --- /dev/null +++ b/test_package/CMakeLists.txt @@ -0,0 +1,10 @@ +cmake_minimum_required(VERSION 3.15) +project(trx_cpp_test_package LANGUAGES CXX) + +set(CMAKE_CXX_STANDARD 17) +set(CMAKE_CXX_STANDARD_REQUIRED ON) + +find_package(trx-cpp CONFIG REQUIRED) + +add_executable(example src/example.cpp) +target_link_libraries(example PRIVATE trx-cpp::trx) diff --git a/test_package/conanfile.py b/test_package/conanfile.py new file mode 100644 index 0000000..d9d2a8a --- /dev/null +++ b/test_package/conanfile.py @@ -0,0 +1,25 @@ +import os + +from conan import ConanFile +from conan.tools.cmake import CMake, cmake_layout + + +class TrxCppTestPackage(ConanFile): + settings = "os", "arch", "compiler", "build_type" + generators = "CMakeDeps", "CMakeToolchain" + test_type = "explicit" + + def requirements(self): + self.requires(self.tested_reference_str) + + def layout(self): + cmake_layout(self) + + def build(self): + cmake = CMake(self) + cmake.configure() + cmake.build() + + def test(self): + exe_name = "example.exe" if self.settings.os == "Windows" else "example" + self.run(os.path.join(self.build_folder, exe_name), env="conanrun") diff --git a/test_package/src/example.cpp b/test_package/src/example.cpp new file mode 100644 index 0000000..7c52822 --- /dev/null +++ b/test_package/src/example.cpp @@ -0,0 +1,9 @@ +#include + +int main() +{ + // Basic construction and cleanup exercises the public API and linkage. + trxmmap::TrxFile file; + file.close(); + return 0; +} diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 847dc83..56fb1af 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -1,41 +1,12 @@ -cmake_minimum_required(VERSION 3.0.0) -cmake_policy(SET CMP0074 NEW) -cmake_policy(SET CMP0079 NEW) -project(trx) -set (CMAKE_CXX_STANDARD 11) - -set(PROJECT_BINARY_DIR ../../builds) -set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/build/tests) -set(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/build/tests) -set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin) - -set(CMAKE_BUILD_TYPE Debug) - -find_package(libzip REQUIRED) -find_package (Eigen3 CONFIG REQUIRED) -find_package(nlohmann_json CONFIG REQUIRED) -find_package(spdlog CONFIG REQUIRED) -find_package(GTest CONFIG REQUIRED) - enable_testing() -include_directories(../src) -add_executable(test_mmap test_trx_mmap.cpp ../src/trx.h ../src/trx.tpp ../src/trx.cpp) - - -TARGET_LINK_LIBRARIES(test_mmap - nlohmann_json::nlohmann_json - Eigen3::Eigen - libzip::zip - GTest::gtest - GTest::gtest_main - spdlog::spdlog - spdlog::spdlog_header_only -) +find_package(GTest CONFIG QUIET) +if(NOT GTest_FOUND) + find_package(GTest REQUIRED) +endif() +add_executable(test_mmap test_trx_mmap.cpp) +target_link_libraries(test_mmap PRIVATE trx GTest::gtest GTest::gtest_main) -set(CPACK_PROJECT_NAME ${PROJECT_NAME}) -set(CPACK_PROJECT_VERSION ${PROJECT_VERSION}) -include(CPack) include(GoogleTest) gtest_discover_tests(test_mmap) diff --git a/tests/test_trx_mmap.cpp b/tests/test_trx_mmap.cpp index d381731..30b13da 100644 --- a/tests/test_trx_mmap.cpp +++ b/tests/test_trx_mmap.cpp @@ -1,10 +1,188 @@ #include -#include "../src/trx.h" +#include #include +#include +#include +#include +#include +#include +#include using namespace Eigen; using namespace trxmmap; +namespace +{ + struct TestTrxFixture + { + std::filesystem::path root_dir; + std::string path; + std::string dir_path; + json expected_header; + int nb_vertices; + int nb_streamlines; + + ~TestTrxFixture() + { + std::error_code ec; + if (!root_dir.empty()) + { + std::filesystem::remove_all(root_dir, ec); + if (ec) + { + std::cerr << "Failed to clean up test directory " << root_dir.string() + << ": " << ec.message() << std::endl; + } + root_dir.clear(); + } + } + }; + + std::filesystem::path make_temp_test_dir(const std::string &prefix) + { + std::error_code ec; + auto base = std::filesystem::temp_directory_path(ec); + if (ec) + { + throw std::runtime_error("Failed to get temp directory: " + ec.message()); + } + + static std::mt19937_64 rng(std::random_device{}()); + std::uniform_int_distribution dist; + + for (int attempt = 0; attempt < 100; ++attempt) + { + std::filesystem::path candidate = base / (prefix + "_" + std::to_string(dist(rng))); + std::error_code dir_ec; + if (std::filesystem::create_directory(candidate, dir_ec)) + { + return candidate; + } + if (dir_ec && dir_ec != std::errc::file_exists) + { + throw std::runtime_error("Failed to create temporary directory: " + dir_ec.message()); + } + } + throw std::runtime_error("Unable to create unique temporary directory"); + } + + TestTrxFixture create_fixture() + { + + TestTrxFixture fixture; + std::filesystem::path root_dir = make_temp_test_dir("trx_test"); + std::filesystem::path trx_dir = root_dir / "trx_data"; + std::error_code ec; + if (!std::filesystem::create_directory(trx_dir, ec) && ec) + { + throw std::runtime_error("Failed to create trx data directory: " + ec.message()); + } + + fixture.root_dir = root_dir; + fixture.path = (root_dir / "small.trx").string(); + fixture.dir_path = trx_dir.string(); + fixture.nb_vertices = 12; + fixture.nb_streamlines = 4; + + fixture.expected_header["DIMENSIONS"] = {117, 151, 115}; + fixture.expected_header["NB_STREAMLINES"] = fixture.nb_streamlines; + fixture.expected_header["NB_VERTICES"] = fixture.nb_vertices; + fixture.expected_header["VOXEL_TO_RASMM"] = {{-1.25, 0.0, 0.0, 72.5}, + {0.0, 1.25, 0.0, -109.75}, + {0.0, 0.0, 1.25, -64.5}, + {0.0, 0.0, 0.0, 1.0}}; + + // Write header.json + std::filesystem::path header_path = trx_dir / "header.json"; + std::ofstream header_out(header_path); + if (!header_out.is_open()) + { + throw std::runtime_error("Failed to write header.json"); + } + header_out << std::setw(4) << fixture.expected_header << std::endl; + header_out.close(); + + // Write positions (float16) + Matrix positions(fixture.nb_vertices, 3); + positions.setZero(); + std::filesystem::path positions_path = trx_dir / "positions.3.float16"; + trxmmap::write_binary(positions_path.c_str(), positions); + struct stat sb; + if (stat(positions_path.c_str(), &sb) != 0) + { + throw std::runtime_error("Failed to stat positions file"); + } + const size_t expected_positions_bytes = fixture.nb_vertices * 3 * sizeof(half); + if (static_cast(sb.st_size) != expected_positions_bytes) + { + throw std::runtime_error("Positions file size mismatch"); + } + + // Write offsets (uint64) with sentinel (NB_STREAMLINES + 1) + Matrix offsets(fixture.nb_streamlines + 1, 1); + for (int i = 0; i < fixture.nb_streamlines; ++i) + { + offsets(i, 0) = static_cast(i * (fixture.nb_vertices / fixture.nb_streamlines)); + } + offsets(fixture.nb_streamlines, 0) = static_cast(fixture.nb_vertices); + + std::filesystem::path offsets_path = trx_dir / "offsets.uint64"; + trxmmap::write_binary(offsets_path.c_str(), offsets); + if (stat(offsets_path.c_str(), &sb) != 0) + { + throw std::runtime_error("Failed to stat offsets file"); + } + const size_t expected_offsets_bytes = (fixture.nb_streamlines + 1) * sizeof(uint64_t); + if (static_cast(sb.st_size) != expected_offsets_bytes) + { + throw std::runtime_error("Offsets file size mismatch"); + } + + // Zip the directory into a trx file without compression + int errorp = 0; + zip_t *zf = zip_open(fixture.path.c_str(), ZIP_CREATE | ZIP_TRUNCATE, &errorp); + if (zf == nullptr) + { + throw std::runtime_error("Failed to create trx zip file"); + } + trxmmap::zip_from_folder(zf, trx_dir.string(), trx_dir.string(), ZIP_CM_STORE); + if (zip_close(zf) != 0) + { + throw std::runtime_error("Failed to close trx zip file"); + } + + // Validate zip entry sizes + int zip_err = 0; + zip_t *verify_zip = zip_open(fixture.path.c_str(), 0, &zip_err); + if (verify_zip == nullptr) + { + throw std::runtime_error("Failed to reopen trx zip file"); + } + zip_stat_t stat_buf; + if (zip_stat(verify_zip, "offsets.uint64", ZIP_FL_UNCHANGED, &stat_buf) != 0 || + static_cast(stat_buf.size) != expected_offsets_bytes) + { + zip_close(verify_zip); + throw std::runtime_error("Zip offsets entry size mismatch"); + } + if (zip_stat(verify_zip, "positions.3.float16", ZIP_FL_UNCHANGED, &stat_buf) != 0 || + static_cast(stat_buf.size) != expected_positions_bytes) + { + zip_close(verify_zip); + throw std::runtime_error("Zip positions entry size mismatch"); + } + zip_close(verify_zip); + + return fixture; + } + + const TestTrxFixture &get_fixture() + { + static TestTrxFixture fixture = create_fixture(); + return fixture; + } +} + // TODO: Test null filenames. Maybe use MatrixBase instead of ArrayBase // TODO: try to update test case to use GTest parameterization TEST(TrxFileMemmap, __generate_filename_from_data) @@ -94,34 +272,33 @@ TEST(TrxFileMemmap, __split_ext_with_dimensionality) TEST(TrxFileMemmap, __compute_lengths) { Matrix offsets{uint64_t(0), uint64_t(1), uint64_t(2), uint64_t(3), uint64_t(4)}; - Matrix lengths(trxmmap::_compute_lengths(offsets, 4)); - Matrix result{uint32_t(1), uint32_t(1), uint32_t(1), uint32_t(1), uint32_t(0)}; + Matrix lengths(trxmmap::_compute_lengths(offsets, 4)); + Matrix result{uint32_t(1), uint32_t(1), uint32_t(1), uint32_t(1)}; EXPECT_EQ(lengths, result); - Matrix offsets2{uint64_t(0), uint64_t(1), uint64_t(0), uint64_t(3), uint64_t(4)}; - Matrix lengths2(trxmmap::_compute_lengths(offsets2, 4)); - Matrix result2{uint32_t(1), uint32_t(3), uint32_t(0), uint32_t(1), uint32_t(0)}; + Matrix offsets2{uint64_t(0), uint64_t(1), uint64_t(1), uint64_t(3), uint64_t(4)}; + Matrix lengths2(trxmmap::_compute_lengths(offsets2, 4)); + Matrix result2{uint32_t(1), uint32_t(0), uint32_t(2), uint32_t(1)}; EXPECT_EQ(lengths2, result2); - Matrix offsets3{uint64_t(0), uint64_t(1), uint64_t(2), uint64_t(3)}; - Matrix lengths3(trxmmap::_compute_lengths(offsets3, 4)); - Matrix result3{uint32_t(1), uint32_t(1), uint32_t(1), uint32_t(1)}; + Matrix offsets3{uint64_t(0), uint64_t(1), uint64_t(2), uint64_t(4)}; + Matrix lengths3(trxmmap::_compute_lengths(offsets3, 4)); + Matrix result3{uint32_t(1), uint32_t(1), uint32_t(2)}; EXPECT_EQ(lengths3, result3); - Matrix offsets4(uint64_t(4)); + Matrix offsets4; + offsets4 << uint64_t(0), uint64_t(2); Matrix lengths4(trxmmap::_compute_lengths(offsets4, 2)); Matrix result4(uint32_t(2)); EXPECT_EQ(lengths4, result4); Matrix offsets5; - Matrix lengths5(trxmmap::_compute_lengths(offsets5, 2)); - Matrix result5(uint32_t(0)); - - EXPECT_EQ(lengths5, result5); + Matrix lengths5(trxmmap::_compute_lengths(offsets5, 2)); + EXPECT_EQ(lengths5.size(), 0); } TEST(TrxFileMemmap, __is_dtype_valid) @@ -172,17 +349,13 @@ TEST(TrxFileMemmap, __dichotomic_search) TEST(TrxFileMemmap, __create_memmap) { - char *dirname; - char t[] = "/tmp/trx_XXXXXX"; - dirname = mkdtemp(t); - - std::string path(dirname); - path += "/offsets.int16"; + std::filesystem::path dir = make_temp_test_dir("trx_memmap"); + std::filesystem::path path = dir / "offsets.int16"; std::tuple shape = std::make_tuple(3, 4); // Test 1: create file and allocate space assert that correct data is filled - mio::shared_mmap_sink empty_mmap = trxmmap::_create_memmap(path, shape); + mio::shared_mmap_sink empty_mmap = trxmmap::_create_memmap(path.string(), shape); Map> expected_m(reinterpret_cast(empty_mmap.data())); Matrix zero_filled{{half(0), half(0), half(0), half(0)}, {half(0), half(0), half(0), half(0)}, @@ -196,37 +369,43 @@ TEST(TrxFileMemmap, __create_memmap) expected_m(i) = half(i); } - mio::shared_mmap_sink filled_mmap = trxmmap::_create_memmap(path, shape); + mio::shared_mmap_sink filled_mmap = trxmmap::_create_memmap(path.string(), shape); Map> real_m(reinterpret_cast(filled_mmap.data()), std::get<0>(shape), std::get<1>(shape)); EXPECT_EQ(expected_m, real_m); + + std::error_code ec; + std::filesystem::remove_all(dir, ec); } -TEST(TrxFileMemmap, load_header) +TEST(TrxFileMemmap, __create_memmap_empty) { - std::string path = "data/small.trx"; - int *errorp; - zip_t *zf = zip_open(path.c_str(), 0, errorp); - json root = trxmmap::load_header(zf); + std::filesystem::path dir = make_temp_test_dir("trx_memmap_empty"); + std::filesystem::path path = dir / "empty.float32"; - // expected output - json expected; + std::tuple shape = std::make_tuple(0, 1); + mio::shared_mmap_sink empty_mmap = trxmmap::_create_memmap(path.string(), shape); - expected["DIMENSIONS"] = {117, 151, 115}; - expected["NB_STREAMLINES"] = 1000; - expected["NB_VERTICES"] = 33886; - expected["VOXEL_TO_RASMM"] = {{-1.25, 0.0, 0.0, 72.5}, - {0.0, 1.25, 0.0, -109.75}, - {0.0, 0.0, 1.25, -64.5}, - {0.0, 0.0, 0.0, 1.0}}; + struct stat sb; + ASSERT_EQ(stat(path.c_str(), &sb), 0); + EXPECT_EQ(sb.st_size, 0); + EXPECT_EQ(empty_mmap.size(), 0u); - EXPECT_EQ(root, expected); + std::error_code ec; + std::filesystem::remove_all(dir, ec); +} - std::string expected_str = "{\"DIMENSIONS\":[117,151,115],\"NB_STREAMLINES\":1000,\"NB_VERTICES\":33886,\"VOXEL_TO_RASMM\":[[-1.25,0.0,0.0,72.5],[0.0,1.25,0.0,-109.75],[0.0,0.0,1.25,-64.5],[0.0,0.0,0.0,1.0]]}"; +TEST(TrxFileMemmap, load_header) +{ + const auto &fixture = get_fixture(); + int errorp = 0; + zip_t *zf = zip_open(fixture.path.c_str(), 0, &errorp); + json root = trxmmap::load_header(zf); - EXPECT_EQ(root.dump(), expected_str); + EXPECT_EQ(root, fixture.expected_header); + EXPECT_EQ(root.dump(), fixture.expected_header.dump()); - free(zf); + zip_close(zf); } // TEST(TrxFileMemmap, _load) @@ -247,8 +426,18 @@ TEST(TrxFileMemmap, load_header) TEST(TrxFileMemmap, load_zip) { - trxmmap::TrxFile *trx = trxmmap::load_from_zip("data/small.trx"); + const auto &fixture = get_fixture(); + trxmmap::TrxFile *trx = trxmmap::load_from_zip(fixture.path); + EXPECT_GT(trx->streamlines->_data.size(), 0); + delete trx; +} + +TEST(TrxFileMemmap, load_directory) +{ + const auto &fixture = get_fixture(); + trxmmap::TrxFile *trx = trxmmap::load_from_directory(fixture.dir_path); EXPECT_GT(trx->streamlines->_data.size(), 0); + delete trx; } TEST(TrxFileMemmap, TrxFile) @@ -268,55 +457,67 @@ TEST(TrxFileMemmap, TrxFile) EXPECT_EQ(trx->header, expected); - std::string path = "data/small.trx"; - int *errorp; - zip_t *zf = zip_open(path.c_str(), 0, errorp); + const auto &fixture = get_fixture(); + int errorp = 0; + zip_t *zf = zip_open(fixture.path.c_str(), 0, &errorp); json root = trxmmap::load_header(zf); TrxFile *root_init = new TrxFile(); root_init->header = root; + zip_close(zf); // TODO: test for now.. - trxmmap::TrxFile *trx_init = new TrxFile(33886, 1000, root_init); + trxmmap::TrxFile *trx_init = new TrxFile(fixture.nb_vertices, fixture.nb_streamlines, root_init); json init_as; init_as["DIMENSIONS"] = {117, 151, 115}; - init_as["NB_STREAMLINES"] = 1000; - init_as["NB_VERTICES"] = 33886; + init_as["NB_STREAMLINES"] = fixture.nb_streamlines; + init_as["NB_VERTICES"] = fixture.nb_vertices; init_as["VOXEL_TO_RASMM"] = {{-1.25, 0.0, 0.0, 72.5}, {0.0, 1.25, 0.0, -109.75}, {0.0, 0.0, 1.25, -64.5}, {0.0, 0.0, 0.0, 1.0}}; EXPECT_EQ(root_init->header, init_as); - EXPECT_EQ(trx_init->streamlines->_data.size(), 33886 * 3); - EXPECT_EQ(trx_init->streamlines->_offsets.size(), 1000); - EXPECT_EQ(trx_init->streamlines->_lengths.size(), 1000); + EXPECT_EQ(trx_init->streamlines->_data.size(), fixture.nb_vertices * 3); + EXPECT_EQ(trx_init->streamlines->_offsets.size(), fixture.nb_streamlines + 1); + EXPECT_EQ(trx_init->streamlines->_lengths.size(), fixture.nb_streamlines); + delete trx; + delete root_init; + delete trx_init; } TEST(TrxFileMemmap, deepcopy) { - trxmmap::TrxFile *trx = trxmmap::load_from_zip("data/small.trx"); + const auto &fixture = get_fixture(); + trxmmap::TrxFile *trx = trxmmap::load_from_zip(fixture.path); trxmmap::TrxFile *copy = trx->deepcopy(); EXPECT_EQ(trx->header, copy->header); EXPECT_EQ(trx->streamlines->_data, trx->streamlines->_data); EXPECT_EQ(trx->streamlines->_offsets, trx->streamlines->_offsets); EXPECT_EQ(trx->streamlines->_lengths, trx->streamlines->_lengths); + delete trx; + delete copy; } TEST(TrxFileMemmap, resize) { - trxmmap::TrxFile *trx = trxmmap::load_from_zip("data/small.trx"); + const auto &fixture = get_fixture(); + trxmmap::TrxFile *trx = trxmmap::load_from_zip(fixture.path); trx->resize(); trx->resize(10); + delete trx; } TEST(TrxFileMemmap, save) { - trxmmap::TrxFile *trx = trxmmap::load_from_zip("data/small.trx"); + const auto &fixture = get_fixture(); + trxmmap::TrxFile *trx = trxmmap::load_from_zip(fixture.path); trxmmap::save(*trx, (std::string) "testsave"); trxmmap::save(*trx, (std::string) "testsave.trx"); + delete trx; + // trxmmap::TrxFile *saved = trxmmap::load_from_zip("testsave.trx"); // EXPECT_EQ(saved->data_per_vertex["color_x.float16"]->_data, trx->data_per_vertex["color_x.float16"]->_data); }