Block API minor improvements

clickhouse/block.cpp

@@ -81,8 +81,7 @@ size_t Block::GetRowCount() const {
     return rows_;
 }
 
-size_t Block::RefreshRowCount()
-{
+size_t Block::RefreshRowCount() {
     size_t rows = 0UL;
 
     for (size_t idx = 0UL; idx < columns_.size(); ++idx)
@@ -100,6 +99,22 @@ size_t Block::RefreshRowCount()
     return rows_;
 }
 
+void Block::Clear() {
+    for (auto & c : columns_) {
+        c.column->Clear();
+    }
+
+    RefreshRowCount();
+}
+
+void Block::Reserve(size_t new_cap) {
+    for (auto & c : columns_) {
+        c.column->Reserve(new_cap);
+    }
+}
+
+
+
 ColumnRef Block::operator [] (size_t idx) const {
     if (idx < columns_.size()) {
         return columns_[idx].column;

clickhouse/block.h

@@ -85,6 +85,12 @@ class Block {
         return columns_.at(idx).name;
     }
 
+    /// Convinience method to wipe out all rows from all columns
+    void Clear();
+
+    /// Convinience method to do Reserve() on all columns
+    void Reserve(size_t new_cap);
+
     /// Reference to column by index in the block.
     ColumnRef operator [] (size_t idx) const;
 

clickhouse/columns/itemview.h

@@ -65,7 +65,7 @@ struct ItemView {
         using ValueType = std::remove_cv_t<std::decay_t<T>>;
         if constexpr (std::is_same_v<std::string_view, ValueType> || std::is_same_v<std::string, ValueType>) {
             return data;
-        } else if constexpr (std::is_fundamental_v<ValueType> || std::is_same_v<Int128, ValueType>) {
+        } else if constexpr (std::is_fundamental_v<ValueType> || std::is_same_v<Int128, ValueType> || std::is_same_v<UInt128, ValueType>) {
             if (sizeof(ValueType) == data.size()) {
                 return *reinterpret_cast<const T*>(data.data());
             } else {

clickhouse/columns/lowcardinality.cpp

@@ -9,6 +9,7 @@
 #include <functional>
 #include <string_view>
 #include <type_traits>
+#include <cmath>
 
 #include <cassert>
 
@@ -175,8 +176,11 @@ ColumnLowCardinality::~ColumnLowCardinality()
 {}
 
 void ColumnLowCardinality::Reserve(size_t new_cap) {
-    dictionary_column_->Reserve(new_cap);
-    index_column_->Reserve(new_cap);
+    // Assumption is that dictionary must be smaller than index.
+    // NOTE(vnemkov): Formula below (`ceil(sqrt(x))`) is a gut-feeling-good-enough estimation,
+    // feel free to replace/adjust if you have better one suported by actual data.
+    dictionary_column_->Reserve(static_cast<size_t>(ceil(sqrt(static_cast<double>(new_cap)))));    
+    index_column_->Reserve(new_cap + 2); // + 1 for null item (at pos 0), + 1 for default item (at pos 1)
 }
 
 void ColumnLowCardinality::Setup(ColumnRef dictionary_column) {

ut/block_ut.cpp

@@ -1,21 +1,37 @@
 #include <clickhouse/client.h>
-#include "readonly_client_test.h"
-#include "connection_failed_client_test.h"
+#include <clickhouse/columns/tuple.h>
+#include <clickhouse/types/types.h>
+
+#include "clickhouse/columns/column.h"
+#include "clickhouse/columns/lowcardinality.h"
+#include "gtest/gtest-message.h"
+
+#include "ut/utils_comparison.h"
 #include "utils.h"
 
 #include <gtest/gtest.h>
+#include <memory>
 
 namespace {
 using namespace clickhouse;
 
 Block MakeBlock(std::vector<std::pair<std::string, ColumnRef>> columns) {
     Block result;
 
+    const size_t number_of_rows = columns.size() ? columns[0].second->Size() : 0;
+    size_t i = 0;
     for (const auto & name_and_col : columns) {
+        EXPECT_EQ(number_of_rows, name_and_col.second->Size())
+        << "Column #" << i <<  " " << name_and_col.first << " has incorrect number of rows";
+
         result.AppendColumn(name_and_col.first, name_and_col.second);
+
+        ++i;
     }
 
     result.RefreshRowCount();
+    EXPECT_EQ(number_of_rows, result.GetRowCount());
+
     return result;
 }
 
@@ -83,3 +99,83 @@ TEST(BlockTest, Iterators) {
     ASSERT_NE(block.cbegin(), block.cend());
 }
 
+TEST(BlockTest, Clear) {
+    // Test that Block::Clear removes all rows from all of the columns,
+    // without changing column instances, types, names, etc.
+
+    auto block = MakeBlock({
+        {"foo", std::make_shared<ColumnUInt8>(std::vector<uint8_t>{1, 2, 3, 4, 5})},
+        {"bar", std::make_shared<ColumnString>(std::vector<std::string>{"1", "2", "3", "4", "5"})},
+    });
+
+    std::vector<std::tuple<std::string, Column*>> expected_columns_description;
+    for (const auto & c : block) {
+        expected_columns_description.emplace_back(c.Name(), c.Column().get());
+    }
+
+    block.Clear();
+
+    // Block must report empty after being cleared
+    EXPECT_EQ(0u, block.GetRowCount());
+
+    size_t i = 0;
+    for (const auto & c : block) {
+        const auto & [expected_name, expected_column] = expected_columns_description[i];
+        SCOPED_TRACE(testing::Message("col #") << c.ColumnIndex() << " \"" << c.Name() << "\"");
+
+        // MUST be same column object
+        EXPECT_EQ(expected_column, c.Column().get());
+
+        // MUST have same column name
+        EXPECT_EQ(expected_name, c.Name());
+
+        // column MUST be empty
+        EXPECT_EQ(0u, c.Column()->Size());
+
+        ++i;
+    }
+}
+
+TEST(BlockTest, Reserve) {
+    // Test that Block::Reserve reserves space in all columns (uncheckable now),
+    // without changing column instances, names, and previously stored rows.
+
+    auto block = MakeBlock({
+        {"foo", std::make_shared<ColumnUInt8>(std::vector<uint8_t>{1, 2, 3, 4, 5})},
+        {"bar", std::make_shared<ColumnString>(std::vector<std::string>{"1", "2", "3", "4", "5"})},
+        {"quix", std::make_shared<ColumnLowCardinalityT<ColumnString>>(std::vector<std::string>{"1", "2", "3", "4", "5"})},
+    });
+
+    const size_t initial_rows_count = block.GetRowCount();
+
+    std::vector<std::tuple<std::string, Column*, ColumnRef>> expected_columns_description;
+    for (const auto & c : block) {
+        expected_columns_description.emplace_back(
+            c.Name(),
+            c.Column().get(),                        // reference to the actual object
+            c.Column()->Slice(0, c.Column()->Size()) // reference to the values
+        );
+    }
+
+    block.Reserve(1000); // 1000 is arbitrary value
+
+    // Block must same number of rows as before Reserve
+    EXPECT_EQ(initial_rows_count, block.GetRowCount());
+
+    size_t i = 0;
+    for (const auto & c : block) {
+        const auto & [expected_name, expected_column, expected_values] = expected_columns_description[i];
+        SCOPED_TRACE(testing::Message("col #") << c.ColumnIndex() << " \"" << c.Name() << "\"");
+
+        // MUST have same column name
+        EXPECT_EQ(expected_name, c.Name());
+
+        // MUST be same column object
+        EXPECT_EQ(expected_column, c.Column().get());
+
+        // column MUST have the same values
+        EXPECT_TRUE(CompareRecursive(*expected_values, *c.Column()));
+
+        ++i;
+    }
+}

ut/utils.cpp

@@ -19,9 +19,15 @@
 #include <clickhouse/base/socket.h> // for ipv4-ipv6 platform-specific stuff
 
 #include <cinttypes>
+#include <cstdint>
+#include <ctime>
 #include <iomanip>
 #include <sstream>
+#include <stdexcept>
 #include <type_traits>
+#include "clickhouse/types/types.h"
+#include "absl/numeric/int128.h"
+
 
 
 namespace {
@@ -42,7 +48,7 @@ struct DateTimeValue {
 };
 
 std::ostream& operator<<(std::ostream & ostr, const DateTimeValue & time) {
-    const auto t = std::localtime(&time.value);
+    const auto t = std::gmtime(&time.value);
     char buffer[] = "2015-05-18 07:40:12\0\0";
     std::strftime(buffer, sizeof(buffer), "%Y-%m-%d %H:%M:%S", t);
 
@@ -173,6 +179,7 @@ std::ostream & printColumnValue(const ColumnRef& c, const size_t row, std::ostre
         || doPrintValue<ColumnEnum8>(c, row, ostr)
         || doPrintValue<ColumnEnum16>(c, row, ostr)
         || doPrintValue<ColumnDate, DateTimeValue>(c, row, ostr)
+        || doPrintValue<ColumnDate32, DateTimeValue>(c, row, ostr)
         || doPrintValue<ColumnDateTime, DateTimeValue>(c, row, ostr)
         || doPrintValue<ColumnDateTime64, DateTimeValue>(c, row, ostr)
         || doPrintValue<ColumnDecimal>(c, row, ostr)
@@ -332,6 +339,141 @@ std::ostream & operator<<(std::ostream & ostr, const Progress & progress) {
         << " written_bytes : " << progress.written_bytes;
 }
 
+std::ostream& operator<<(std::ostream& ostr, const ItemView& item_view) {
+    ostr << "ItemView {" << clickhouse::Type::TypeName(item_view.type) << " : ";
+
+    switch (item_view.type) {
+        case Type::Void:
+            ostr << "--void--";
+            break;
+        case Type::Int8:
+            ostr << static_cast<int>(item_view.get<int8_t>());
+            break;
+        case Type::Int16:
+            ostr << static_cast<int>(item_view.get<int16_t>());
+            break;
+        case Type::Int32:
+            ostr << static_cast<int>(item_view.get<int32_t>());
+            break;
+        case Type::Int64:
+            ostr << item_view.get<int64_t>();
+            break;
+        case Type::UInt8:
+            ostr << static_cast<unsigned int>(item_view.get<uint8_t>());
+            break;
+        case Type::UInt16:
+            ostr << static_cast<unsigned int>(item_view.get<uint16_t>());
+            break;
+        case Type::UInt32:
+            ostr << static_cast<unsigned int>(item_view.get<uint32_t>());
+            break;
+        case Type::UInt64:
+            ostr << item_view.get<uint64_t>();
+            break;
+        case Type::Float32:
+            ostr << static_cast<float>(item_view.get<float>());
+            break;
+        case Type::Float64:
+            ostr << static_cast<double>(item_view.get<double>());
+            break;
+        case Type::String:
+        case Type::FixedString:
+            ostr << "\"" << item_view.data << "\" (" << item_view.data.size() << " bytes)";
+            break;
+        case Type::Date:
+            ostr << DateTimeValue(item_view.get<uint16_t>() * 86400);
+            break;
+        case Type::Date32:
+            ostr << DateTimeValue(item_view.get<int32_t>() * 86400);
+            break;
+        case Type::DateTime:
+            ostr << DateTimeValue(item_view.get<uint32_t>());
+            break;
+        case Type::DateTime64: {
+            if (item_view.data.size() == 4) {
+                ostr << DateTimeValue(item_view.get<int32_t>());
+            }
+            else if (item_view.data.size() == 8) {
+                ostr << DateTimeValue(item_view.get<int64_t>());
+            }
+            else if (item_view.data.size() == 16) {
+                ostr << DateTimeValue(item_view.get<Int128>());
+            }
+            else {
+                throw std::runtime_error("Invalid data size of ItemView of type DateTime64");
+            }
+            break;
+        }
+        case Type::Enum8:
+            ostr << static_cast<unsigned int>(item_view.get<uint8_t>());
+            break;
+        case Type::Enum16:
+            ostr << static_cast<unsigned int>(item_view.get<uint16_t>());
+            break;
+        case Type::UUID: {
+            const auto & uuid_vals = reinterpret_cast<const uint64_t*>(item_view.data.data());
+            ostr << ToString(clickhouse::UUID{uuid_vals[0], uuid_vals[1]});
+            break;
+        }
+        case Type::IPv4: {
+            in_addr addr;
+            addr.s_addr = ntohl(item_view.get<uint32_t>());
+            ostr << addr;
+            break;
+        }
+        case Type::IPv6:
+            ostr << *reinterpret_cast<const in6_addr*>(item_view.AsBinaryData().data());
+            break;
+        case Type::Int128:
+            ostr << item_view.get<Int128>();
+            break;
+        case Type::UInt128:
+            ostr << item_view.get<UInt128>();
+            break;
+        case Type::Decimal: {
+            if (item_view.data.size() == 4) {
+                ostr << item_view.get<int32_t>();
+            }
+            else if (item_view.data.size() == 8) {
+                ostr << item_view.get<int64_t>();
+            }
+            else if (item_view.data.size() == 16) {
+                ostr << item_view.get<Int128>();
+            }
+            else {
+                throw std::runtime_error("Invalid data size of ItemView of type Decimal");
+            }
+        }
+        break;
+        case Type::Decimal32:
+            ostr << DateTimeValue(item_view.get<int32_t>());
+            break;
+        case Type::Decimal64:
+            ostr << DateTimeValue(item_view.get<int64_t>());
+            break;
+        case Type::Decimal128:
+            ostr << DateTimeValue(item_view.get<Int128>());
+            break;
+        // Unsupported types. i.e. there shouldn't be `ItemView`s of those types in practice.
+        // either because GetItem() is not implemented for corresponding column type
+        // OR this type code is never used, for `ItemView`s (but type code of wrapped column is).
+        case Type::LowCardinality:
+        case Type::Array:
+        case Type::Nullable:
+        case Type::Tuple:
+        case Type::Map:
+        case Type::Point:
+        case Type::Ring:
+        case Type::Polygon:
+        case Type::MultiPolygon: {
+            throw std::runtime_error("Invalid data size of ItemView of type " + std::string(Type::TypeName(item_view.type)));
+        }
+    };
+
+    return ostr << "}";
+}
+
+
 }
 
 uint64_t versionNumber(const ServerInfo & server_info) {

ut/utils.h

@@ -3,6 +3,7 @@
 #include <clickhouse/base/platform.h>
 #include <clickhouse/base/uuid.h>
 
+#include "clickhouse/columns/itemview.h"
 #include "clickhouse/query.h"
 #include "utils_meta.h"
 #include "utils_comparison.h"
@@ -144,6 +145,7 @@ std::ostream& operator<<(std::ostream & ostr, const Type & type);
 std::ostream & operator<<(std::ostream & ostr, const ServerInfo & server_info);
 std::ostream & operator<<(std::ostream & ostr, const Profile & profile);
 std::ostream & operator<<(std::ostream & ostr, const Progress & progress);
+std::ostream & operator<<(std::ostream& ostr, const ItemView& item_view);
 }
 
 std::ostream& operator<<(std::ostream & ostr, const PrettyPrintBlock & block);