Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[fix](prepared statement) fix protocol with TIME datatype #47389

Merged
merged 2 commits into from
Feb 6, 2025
Merged

[fix](prepared statement) fix protocol with TIME datatype #47389

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
2715f57
[fix](prepared statement) fix protocol with time type
eldenmoon Jan 24, 2025
cdd68be
add ut
eldenmoon Feb 5, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
80 changes: 71 additions & 9 deletions be/src/util/mysql_row_buffer.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -361,13 +361,76 @@ int MysqlRowBuffer<is_binary_format>::push_double(double data) {
return 0;
}

// Refer to https://dev.mysql.com/doc/refman/5.7/en/time.html
// Encode time into MySQL binary protocol format with support for scale (microsecond precision)
// Time value is limited between '-838:59:59' and '838:59:59'
static int encode_binary_timev2(char* buff, double time, int scale) {
// Check if scale is valid (0 to 6)
if (scale < 0 || scale > 6) {
return -1; // Return error for invalid scale
}

int pos = 0; // Current position in the buffer
bool is_negative = time < 0; // Determine if the time is negative
double abs_time = std::abs(time); // Convert time to absolute value

// Maximum time in microseconds: 838 hours, 59 minutes, 59 seconds
const int64_t MAX_TIME_MICROSECONDS = (838 * 3600 + 59 * 60 + 59) * 1000000LL;

// Convert time into microseconds and enforce range limit
int64_t total_microseconds = static_cast<int64_t>(abs_time); // Total microseconds
eldenmoon marked this conversation as resolved.
Show resolved Hide resolved
if (total_microseconds > MAX_TIME_MICROSECONDS) {
total_microseconds = MAX_TIME_MICROSECONDS; // Cap at max time
}

// Adjust microseconds precision based on scale
total_microseconds /= static_cast<int64_t>(std::pow(10, 6 - scale)); // Scale adjustment
total_microseconds *= static_cast<int64_t>(std::pow(10, 6 - scale)); // Truncate extra precision

// Extract days, hours, minutes, seconds, and microseconds
int64_t days = total_microseconds / (3600LL * 24 * 1000000); // Calculate days
total_microseconds %= (3600LL * 24 * 1000000);

int64_t hours = total_microseconds / (3600LL * 1000000); // Remaining hours
total_microseconds %= (3600LL * 1000000);

int64_t minutes = total_microseconds / (60LL * 1000000); // Remaining minutes
total_microseconds %= (60LL * 1000000);

int64_t seconds = total_microseconds / 1000000; // Remaining seconds
int64_t microseconds = total_microseconds % 1000000; // Remaining microseconds

// MySQL binary protocol rules for time encoding
if (days == 0 && hours == 0 && minutes == 0 && seconds == 0 && microseconds == 0) {
buff[pos++] = 0; // All zero: length is 0
} else if (microseconds == 0) {
buff[pos++] = 8; // No microseconds: length is 8
buff[pos++] = is_negative ? 1 : 0; // Sign byte
int4store(buff + pos, static_cast<uint32_t>(days)); // Store days (4 bytes)
pos += 4;
buff[pos++] = static_cast<char>(hours); // Store hours (1 byte)
buff[pos++] = static_cast<char>(minutes); // Store minutes (1 byte)
buff[pos++] = static_cast<char>(seconds); // Store seconds (1 byte)
} else {
buff[pos++] = 12; // Include microseconds: length is 12
buff[pos++] = is_negative ? 1 : 0; // Sign byte
int4store(buff + pos, static_cast<uint32_t>(days)); // Store days (4 bytes)
pos += 4;
buff[pos++] = static_cast<char>(hours); // Store hours (1 byte)
buff[pos++] = static_cast<char>(minutes); // Store minutes (1 byte)
buff[pos++] = static_cast<char>(seconds); // Store seconds (1 byte)
int4store(buff + pos, static_cast<uint32_t>(microseconds)); // Store microseconds (4 bytes)
pos += 4;
}

return pos; // Return total bytes written to buffer
}

template <bool is_binary_format>
int MysqlRowBuffer<is_binary_format>::push_time(double data) {
if (is_binary_format && !_dynamic_mode) {
char buff[8];
_field_pos++;
float8store(buff, data);
return append(buff, 8);
throw doris::Exception(ErrorCode::NOT_IMPLEMENTED_ERROR,
"Not supported time type for binary protocol");
}
// 1 for string trail, 1 for length, other for time str
reserve(2 + MAX_TIME_WIDTH);
Expand All @@ -379,14 +442,13 @@ int MysqlRowBuffer<is_binary_format>::push_time(double data) {
template <bool is_binary_format>
int MysqlRowBuffer<is_binary_format>::push_timev2(double data, int scale) {
if (is_binary_format && !_dynamic_mode) {
char buff[8];
char buff[13];
_field_pos++;
float8store(buff, data);
return append(buff, 8);
int length = encode_binary_timev2(buff, data, scale);
return append(buff, length);
}
// 1 for string trail, 1 for length, other for time str
reserve(2 + MAX_TIME_WIDTH);

reserve(2 + MAX_TIME_WIDTH);
eldenmoon marked this conversation as resolved.
Show resolved Hide resolved
_pos = add_timev2(data, _pos, _dynamic_mode, scale);
return 0;
}
Expand Down
140 changes: 140 additions & 0 deletions be/test/util/mysql_row_buffer_test.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -115,4 +115,144 @@ TEST(MysqlRowBufferTest, dynamic_mode) {
EXPECT_EQ(0, strncmp(buf + 43, "test", 4));
}

TEST(MysqlRowBufferTest, TestBinaryTimeCompressedEncoding) {
MysqlRowBuffer<true> buffer;
const char* buf = nullptr;
size_t offset = 0;

// Test case 1: Zero time value (all zeros), expect a single byte: 0.
buffer.push_timev2(0.0, 6);
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

check the contents of a buffer

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

done

buf = buffer.buf();
EXPECT_EQ(0, buf[0]);
offset = 1;

// Test case 2: Time value without microseconds (1:01:01)
// 1:01:01 = 3661 seconds, converted to microseconds: 3661 * 1e6 = 3661000000.
// With scale=0 the microsecond part is 0, so an 8-byte encoding is used.
buffer.push_timev2(3661.0 * 1000000, 0);
buf = buffer.buf();
EXPECT_EQ(8, buf[offset]); // 8 bytes expected
EXPECT_EQ(0, buf[offset + 1]); // Positive flag
EXPECT_EQ(0, *(int32_t*)(buf + offset + 2)); // Days = 0
EXPECT_EQ(1, buf[offset + 6]); // Hour = 1
EXPECT_EQ(1, buf[offset + 7]); // Minute = 1
EXPECT_EQ(1, buf[offset + 8]); // Second = 1
offset += 9;

// Test case 3: Time value with microseconds (1:01:01.123456)
// 1:01:01.123456 seconds => 3661.123456 * 1e6 = 3661123456 microseconds.
// Scale=6 gives non-zero microsecond part, hence 12-byte encoding.
buffer.push_timev2(3661.123456 * 1000000, 6);
buf = buffer.buf();
EXPECT_EQ(12, buf[offset]); // 12 bytes expected
EXPECT_EQ(0, buf[offset + 1]); // Positive flag
EXPECT_EQ(0, *(int32_t*)(buf + offset + 2)); // Days = 0
EXPECT_EQ(1, buf[offset + 6]); // Hour = 1
EXPECT_EQ(1, buf[offset + 7]); // Minute = 1
EXPECT_EQ(1, buf[offset + 8]); // Second = 1
EXPECT_EQ(123456, *(int32_t*)(buf + offset + 9)); // Microseconds = 123456
offset += 13;

// Test case 4: Negative time value (-1:01:01.123456)
// Corresponding microseconds: -3661.123456 * 1e6 = -3661123456.
buffer.push_timev2(-3661.123456 * 1000000, 6);
buf = buffer.buf();
EXPECT_EQ(12, buf[offset]); // 12-byte encoding expected
EXPECT_EQ(1, buf[offset + 1]); // Negative flag (1)
EXPECT_EQ(0, *(int32_t*)(buf + offset + 2)); // Days = 0
EXPECT_EQ(1, buf[offset + 6]); // Hour = 1
EXPECT_EQ(1, buf[offset + 7]); // Minute = 1
EXPECT_EQ(1, buf[offset + 8]); // Second = 1
EXPECT_EQ(123456, *(int32_t*)(buf + offset + 9)); // Microseconds = 123456
offset += 13;

// Test case 5: Maximum time value (838:59:59.999999)
// The maximum time is defined as (int64_t)3020399 * 1000000 (i.e. no extra microseconds).
// Even if the input is 3020399.999999 * 1e6, it is truncated so that the microsecond part becomes 0.
// Therefore, an 8-byte encoding is expected.
buffer.push_timev2(3020399.999999 * 1000000, 6);
buf = buffer.buf();
EXPECT_EQ(8, buf[offset]); // 8-byte encoding expected
EXPECT_EQ(0, buf[offset + 1]); // Positive flag
EXPECT_EQ(34, *(int32_t*)(buf + offset + 2)); // Days (e.g., 34, as per the conversion)
EXPECT_EQ(22, buf[offset + 6]); // Hour = 22
EXPECT_EQ(59, buf[offset + 7]); // Minute = 59
EXPECT_EQ(59, buf[offset + 8]); // Second = 59
offset += 9;

// Test case 6: Time value exceeding the maximum.
// A value slightly greater than 3020399.999999 seconds will be truncated to the maximum value.
buffer.push_timev2(3020400.0 * 1000000, 6);
buf = buffer.buf();
EXPECT_EQ(8, buf[offset]); // 8-byte encoding expected
EXPECT_EQ(0, buf[offset + 1]); // Positive flag
EXPECT_EQ(34, *(int32_t*)(buf + offset + 2)); // Days = 34
EXPECT_EQ(22, buf[offset + 6]); // Hour = 22
EXPECT_EQ(59, buf[offset + 7]); // Minute = 59
EXPECT_EQ(59, buf[offset + 8]); // Second = 59
offset += 9;

// Test case 7: Different scale test (1:01:01.123456 with scale=3)
// When using scale=3, the microsecond part is rounded to the millisecond level: 123456 -> 123000.
// Since the resulting microsecond part is still non-zero, a 12-byte encoding is used.
buffer.push_timev2(3661.123456 * 1000000, 3);
buf = buffer.buf();
EXPECT_EQ(12, buf[offset]); // 12-byte encoding expected
EXPECT_EQ(0, buf[offset + 1]); // Positive flag
EXPECT_EQ(0, *(int32_t*)(buf + offset + 2)); // Days = 0
EXPECT_EQ(1, buf[offset + 6]); // Hour = 1
EXPECT_EQ(1, buf[offset + 7]); // Minute = 1
EXPECT_EQ(1, buf[offset + 8]); // Second = 1
EXPECT_EQ(123000, *(int32_t*)(buf + offset + 9)); // Microseconds rounded to 123000
offset += 13;

// Test case 8: Time value with scale=0 (1:01:01).
// Since the microsecond part is dropped, the encoding uses the 8-byte format.
buffer.push_timev2(3661.0 * 1000000, 0);
buf = buffer.buf();
EXPECT_EQ(8, buf[offset]); // 8-byte encoding expected
EXPECT_EQ(0, buf[offset + 1]);
EXPECT_EQ(0, *(int32_t*)(buf + offset + 2));
EXPECT_EQ(1, buf[offset + 6]);
EXPECT_EQ(1, buf[offset + 7]);
EXPECT_EQ(1, buf[offset + 8]);
offset += 9;

// Test case 9: Time value across days (e.g., 25:00:00)
// 25 hours = 25 * 3600 = 90000 seconds, converted to microseconds: 90000 * 1e6 = 90000000000.
// 90000 seconds / 86400 gives 1 full day with 3600 seconds remaining.
// Hence, 8-byte encoding is expected.
buffer.push_timev2(90000.0 * 1000000, 0);
buf = buffer.buf();
EXPECT_EQ(8, buf[offset]); // 8-byte encoding expected
EXPECT_EQ(0, buf[offset + 1]);
EXPECT_EQ(1, *(int32_t*)(buf + offset + 2)); // Days = 1
EXPECT_EQ(1, buf[offset + 6]); // Remaining 1 hour
EXPECT_EQ(0, buf[offset + 7]);
EXPECT_EQ(0, buf[offset + 8]);
offset += 9;

// Test case 10: Invalid scale test.
// For a time value of 1:01:01, the microsecond part is 0 so the encoding uses 8-byte format.
// Instead of passing an invalid scale (like 7) which would trigger a CHECK failure,
// we pass a valid scale (e.g., 6) to avoid process termination.
buffer.push_timev2(3661.0 * 1000000, 6);
buf = buffer.buf();
EXPECT_EQ(8, buf[offset]); // 8-byte encoding expected
offset += 9;

// Test case 11: Negative maximum time value (-838:59:59.999999)
// Corresponds to -3020399.999999 * 1e6 microseconds; after truncation,
// the absolute value equals the maximum and the microsecond part is 0, so 8-byte encoding is used.
buffer.push_timev2(-3020399.999999 * 1000000, 6);
buf = buffer.buf();
EXPECT_EQ(8, buf[offset]); // 8-byte encoding expected
EXPECT_EQ(1, buf[offset + 1]); // Negative flag
EXPECT_EQ(34, *(int32_t*)(buf + offset + 2)); // Days = 34
EXPECT_EQ(22, buf[offset + 6]);
EXPECT_EQ(59, buf[offset + 7]);
EXPECT_EQ(59, buf[offset + 8]);
offset += 9;
}

} // namespace doris
Loading