Skip to content

Commit

Permalink
branch-2.1: [fix](prepared statement) fix protocol with TIME datatype #…
Browse files Browse the repository at this point in the history 
…47389 (#47543)

Cherry-picked from #47389

Co-authored-by: lihangyu <[email protected]>
  • Loading branch information
@github-actions @eldenmoon
github-actions[bot] and eldenmoon authored Feb 8, 2025
1 parent 800288b commit 3ec723f
Show file tree
Hide file tree
Showing 2 changed files with 211 additions and 9 deletions.
80 changes: 71 additions & 9 deletions be/src/util/mysql_row_buffer.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -361,13 +361,76 @@ int MysqlRowBuffer<is_binary_format>::push_double(double data) {
return 0;
}

// Refer to https://dev.mysql.com/doc/refman/5.7/en/time.html
// Encode time into MySQL binary protocol format with support for scale (microsecond precision)
// Time value is limited between '-838:59:59' and '838:59:59'
static int encode_binary_timev2(char* buff, double time, int scale) {
// Check if scale is valid (0 to 6)
if (scale < 0 || scale > 6) {
return -1; // Return error for invalid scale
}

int pos = 0; // Current position in the buffer
bool is_negative = time < 0; // Determine if the time is negative
double abs_time = std::abs(time); // Convert time to absolute value

// Maximum time in microseconds: 838 hours, 59 minutes, 59 seconds
const int64_t MAX_TIME_MICROSECONDS = (838 * 3600 + 59 * 60 + 59) * 1000000LL;

// Convert time into microseconds and enforce range limit
int64_t total_microseconds = static_cast<int64_t>(abs_time); // Total microseconds
if (total_microseconds > MAX_TIME_MICROSECONDS) {
total_microseconds = MAX_TIME_MICROSECONDS; // Cap at max time
}

// Adjust microseconds precision based on scale
total_microseconds /= static_cast<int64_t>(std::pow(10, 6 - scale)); // Scale adjustment
total_microseconds *= static_cast<int64_t>(std::pow(10, 6 - scale)); // Truncate extra precision

// Extract days, hours, minutes, seconds, and microseconds
int64_t days = total_microseconds / (3600LL * 24 * 1000000); // Calculate days
total_microseconds %= (3600LL * 24 * 1000000);

int64_t hours = total_microseconds / (3600LL * 1000000); // Remaining hours
total_microseconds %= (3600LL * 1000000);

int64_t minutes = total_microseconds / (60LL * 1000000); // Remaining minutes
total_microseconds %= (60LL * 1000000);

int64_t seconds = total_microseconds / 1000000; // Remaining seconds
int64_t microseconds = total_microseconds % 1000000; // Remaining microseconds

// MySQL binary protocol rules for time encoding
if (days == 0 && hours == 0 && minutes == 0 && seconds == 0 && microseconds == 0) {
buff[pos++] = 0; // All zero: length is 0
} else if (microseconds == 0) {
buff[pos++] = 8; // No microseconds: length is 8
buff[pos++] = is_negative ? 1 : 0; // Sign byte
int4store(buff + pos, static_cast<uint32_t>(days)); // Store days (4 bytes)
pos += 4;
buff[pos++] = static_cast<char>(hours); // Store hours (1 byte)
buff[pos++] = static_cast<char>(minutes); // Store minutes (1 byte)
buff[pos++] = static_cast<char>(seconds); // Store seconds (1 byte)
} else {
buff[pos++] = 12; // Include microseconds: length is 12
buff[pos++] = is_negative ? 1 : 0; // Sign byte
int4store(buff + pos, static_cast<uint32_t>(days)); // Store days (4 bytes)
pos += 4;
buff[pos++] = static_cast<char>(hours); // Store hours (1 byte)
buff[pos++] = static_cast<char>(minutes); // Store minutes (1 byte)
buff[pos++] = static_cast<char>(seconds); // Store seconds (1 byte)
int4store(buff + pos, static_cast<uint32_t>(microseconds)); // Store microseconds (4 bytes)
pos += 4;
}

return pos; // Return total bytes written to buffer
}

template <bool is_binary_format>
int MysqlRowBuffer<is_binary_format>::push_time(double data) {
if (is_binary_format && !_dynamic_mode) {
char buff[8];
_field_pos++;
float8store(buff, data);
return append(buff, 8);
throw doris::Exception(ErrorCode::NOT_IMPLEMENTED_ERROR,
"Not supported time type for binary protocol");
}
// 1 for string trail, 1 for length, other for time str
reserve(2 + MAX_TIME_WIDTH);
Expand All @@ -379,14 +442,13 @@ int MysqlRowBuffer<is_binary_format>::push_time(double data) {
template <bool is_binary_format>
int MysqlRowBuffer<is_binary_format>::push_timev2(double data, int scale) {
if (is_binary_format && !_dynamic_mode) {
char buff[8];
char buff[13];
_field_pos++;
float8store(buff, data);
return append(buff, 8);
int length = encode_binary_timev2(buff, data, scale);
return append(buff, length);
}
// 1 for string trail, 1 for length, other for time str
reserve(2 + MAX_TIME_WIDTH);

reserve(2 + MAX_TIME_WIDTH);
_pos = add_timev2(data, _pos, _dynamic_mode, scale);
return 0;
}
Expand Down
140 changes: 140 additions & 0 deletions be/test/util/mysql_row_buffer_test.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -118,4 +118,144 @@ TEST(MysqlRowBufferTest, dynamic_mode) {
EXPECT_EQ(0, strncmp(buf + 43, "test", 4));
}

TEST(MysqlRowBufferTest, TestBinaryTimeCompressedEncoding) {
MysqlRowBuffer<true> buffer;
const char* buf = nullptr;
size_t offset = 0;

// Test case 1: Zero time value (all zeros), expect a single byte: 0.
buffer.push_timev2(0.0, 6);
buf = buffer.buf();
EXPECT_EQ(0, buf[0]);
offset = 1;

// Test case 2: Time value without microseconds (1:01:01)
// 1:01:01 = 3661 seconds, converted to microseconds: 3661 * 1e6 = 3661000000.
// With scale=0 the microsecond part is 0, so an 8-byte encoding is used.
buffer.push_timev2(3661.0 * 1000000, 0);
buf = buffer.buf();
EXPECT_EQ(8, buf[offset]); // 8 bytes expected
EXPECT_EQ(0, buf[offset + 1]); // Positive flag
EXPECT_EQ(0, *(int32_t*)(buf + offset + 2)); // Days = 0
EXPECT_EQ(1, buf[offset + 6]); // Hour = 1
EXPECT_EQ(1, buf[offset + 7]); // Minute = 1
EXPECT_EQ(1, buf[offset + 8]); // Second = 1
offset += 9;

// Test case 3: Time value with microseconds (1:01:01.123456)
// 1:01:01.123456 seconds => 3661.123456 * 1e6 = 3661123456 microseconds.
// Scale=6 gives non-zero microsecond part, hence 12-byte encoding.
buffer.push_timev2(3661.123456 * 1000000, 6);
buf = buffer.buf();
EXPECT_EQ(12, buf[offset]); // 12 bytes expected
EXPECT_EQ(0, buf[offset + 1]); // Positive flag
EXPECT_EQ(0, *(int32_t*)(buf + offset + 2)); // Days = 0
EXPECT_EQ(1, buf[offset + 6]); // Hour = 1
EXPECT_EQ(1, buf[offset + 7]); // Minute = 1
EXPECT_EQ(1, buf[offset + 8]); // Second = 1
EXPECT_EQ(123456, *(int32_t*)(buf + offset + 9)); // Microseconds = 123456
offset += 13;

// Test case 4: Negative time value (-1:01:01.123456)
// Corresponding microseconds: -3661.123456 * 1e6 = -3661123456.
buffer.push_timev2(-3661.123456 * 1000000, 6);
buf = buffer.buf();
EXPECT_EQ(12, buf[offset]); // 12-byte encoding expected
EXPECT_EQ(1, buf[offset + 1]); // Negative flag (1)
EXPECT_EQ(0, *(int32_t*)(buf + offset + 2)); // Days = 0
EXPECT_EQ(1, buf[offset + 6]); // Hour = 1
EXPECT_EQ(1, buf[offset + 7]); // Minute = 1
EXPECT_EQ(1, buf[offset + 8]); // Second = 1
EXPECT_EQ(123456, *(int32_t*)(buf + offset + 9)); // Microseconds = 123456
offset += 13;

// Test case 5: Maximum time value (838:59:59.999999)
// The maximum time is defined as (int64_t)3020399 * 1000000 (i.e. no extra microseconds).
// Even if the input is 3020399.999999 * 1e6, it is truncated so that the microsecond part becomes 0.
// Therefore, an 8-byte encoding is expected.
buffer.push_timev2(3020399.999999 * 1000000, 6);
buf = buffer.buf();
EXPECT_EQ(8, buf[offset]); // 8-byte encoding expected
EXPECT_EQ(0, buf[offset + 1]); // Positive flag
EXPECT_EQ(34, *(int32_t*)(buf + offset + 2)); // Days (e.g., 34, as per the conversion)
EXPECT_EQ(22, buf[offset + 6]); // Hour = 22
EXPECT_EQ(59, buf[offset + 7]); // Minute = 59
EXPECT_EQ(59, buf[offset + 8]); // Second = 59
offset += 9;

// Test case 6: Time value exceeding the maximum.
// A value slightly greater than 3020399.999999 seconds will be truncated to the maximum value.
buffer.push_timev2(3020400.0 * 1000000, 6);
buf = buffer.buf();
EXPECT_EQ(8, buf[offset]); // 8-byte encoding expected
EXPECT_EQ(0, buf[offset + 1]); // Positive flag
EXPECT_EQ(34, *(int32_t*)(buf + offset + 2)); // Days = 34
EXPECT_EQ(22, buf[offset + 6]); // Hour = 22
EXPECT_EQ(59, buf[offset + 7]); // Minute = 59
EXPECT_EQ(59, buf[offset + 8]); // Second = 59
offset += 9;

// Test case 7: Different scale test (1:01:01.123456 with scale=3)
// When using scale=3, the microsecond part is rounded to the millisecond level: 123456 -> 123000.
// Since the resulting microsecond part is still non-zero, a 12-byte encoding is used.
buffer.push_timev2(3661.123456 * 1000000, 3);
buf = buffer.buf();
EXPECT_EQ(12, buf[offset]); // 12-byte encoding expected
EXPECT_EQ(0, buf[offset + 1]); // Positive flag
EXPECT_EQ(0, *(int32_t*)(buf + offset + 2)); // Days = 0
EXPECT_EQ(1, buf[offset + 6]); // Hour = 1
EXPECT_EQ(1, buf[offset + 7]); // Minute = 1
EXPECT_EQ(1, buf[offset + 8]); // Second = 1
EXPECT_EQ(123000, *(int32_t*)(buf + offset + 9)); // Microseconds rounded to 123000
offset += 13;

// Test case 8: Time value with scale=0 (1:01:01).
// Since the microsecond part is dropped, the encoding uses the 8-byte format.
buffer.push_timev2(3661.0 * 1000000, 0);
buf = buffer.buf();
EXPECT_EQ(8, buf[offset]); // 8-byte encoding expected
EXPECT_EQ(0, buf[offset + 1]);
EXPECT_EQ(0, *(int32_t*)(buf + offset + 2));
EXPECT_EQ(1, buf[offset + 6]);
EXPECT_EQ(1, buf[offset + 7]);
EXPECT_EQ(1, buf[offset + 8]);
offset += 9;

// Test case 9: Time value across days (e.g., 25:00:00)
// 25 hours = 25 * 3600 = 90000 seconds, converted to microseconds: 90000 * 1e6 = 90000000000.
// 90000 seconds / 86400 gives 1 full day with 3600 seconds remaining.
// Hence, 8-byte encoding is expected.
buffer.push_timev2(90000.0 * 1000000, 0);
buf = buffer.buf();
EXPECT_EQ(8, buf[offset]); // 8-byte encoding expected
EXPECT_EQ(0, buf[offset + 1]);
EXPECT_EQ(1, *(int32_t*)(buf + offset + 2)); // Days = 1
EXPECT_EQ(1, buf[offset + 6]); // Remaining 1 hour
EXPECT_EQ(0, buf[offset + 7]);
EXPECT_EQ(0, buf[offset + 8]);
offset += 9;

// Test case 10: Invalid scale test.
// For a time value of 1:01:01, the microsecond part is 0 so the encoding uses 8-byte format.
// Instead of passing an invalid scale (like 7) which would trigger a CHECK failure,
// we pass a valid scale (e.g., 6) to avoid process termination.
buffer.push_timev2(3661.0 * 1000000, 6);
buf = buffer.buf();
EXPECT_EQ(8, buf[offset]); // 8-byte encoding expected
offset += 9;

// Test case 11: Negative maximum time value (-838:59:59.999999)
// Corresponds to -3020399.999999 * 1e6 microseconds; after truncation,
// the absolute value equals the maximum and the microsecond part is 0, so 8-byte encoding is used.
buffer.push_timev2(-3020399.999999 * 1000000, 6);
buf = buffer.buf();
EXPECT_EQ(8, buf[offset]); // 8-byte encoding expected
EXPECT_EQ(1, buf[offset + 1]); // Negative flag
EXPECT_EQ(34, *(int32_t*)(buf + offset + 2)); // Days = 34
EXPECT_EQ(22, buf[offset + 6]);
EXPECT_EQ(59, buf[offset + 7]);
EXPECT_EQ(59, buf[offset + 8]);
offset += 9;
}

} // namespace doris

0 comments on commit 3ec723f

Please sign in to comment.