Skip to content

Commit

Permalink
[fix](group commit) fix wal reader handle empty block (#48290)
Browse files Browse the repository at this point in the history
### What problem does this PR solve?

Problem Summary:

If kill BE, wal writer may write block length and does not write block
data. Then when replay wal, will get:
```
failed to replay wal...st=[INTERNAL_ERROR]cur path: . failed to deserialize row
```
  • Loading branch information
mymeiyi authored and Your Name committed Feb 25, 2025
1 parent d9926da commit d6cab45
Show file tree
Hide file tree
Showing 2 changed files with 14 additions and 3 deletions.
16 changes: 13 additions & 3 deletions be/src/olap/wal/wal_reader.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -31,9 +31,13 @@ WalReader::WalReader(const std::string& file_name) : _file_name(file_name), _off

WalReader::~WalReader() = default;

static Status _deserialize(PBlock& block, const std::string& buf) {
Status WalReader::_deserialize(PBlock& block, const std::string& buf, size_t block_len,
size_t bytes_read) {
if (UNLIKELY(!block.ParseFromString(buf))) {
return Status::InternalError("failed to deserialize row");
return Status::InternalError(
"failed to deserialize row, file_size=" + std::to_string(file_reader->size()) +
", read_offset=" + std::to_string(_offset) + +", block_bytes=" +
std::to_string(block_len) + ", read_block_bytes=" + std::to_string(bytes_read));
}
return Status::OK();
}
Expand Down Expand Up @@ -69,12 +73,18 @@ Status WalReader::read_block(PBlock& block) {
if (block_len == 0) {
return Status::DataQualityError("fail to read wal {} ,block is empty", _file_name);
}
if (_offset == file_reader->size()) {
LOG(WARNING) << "need read block with length=" << block_len << ", but offset=" << _offset
<< " reached end of WAL (path=" << _file_name
<< ", size=" << file_reader->size() << ")";
return Status::EndOfFile("end of wal file");
}
// read block
std::string block_buf;
block_buf.resize(block_len);
RETURN_IF_ERROR(file_reader->read_at(_offset, {block_buf.c_str(), block_len}, &bytes_read));
RETURN_IF_ERROR(_deserialize(block, block_buf, block_len, bytes_read));
_offset += block_len;
RETURN_IF_ERROR(_deserialize(block, block_buf));
// checksum
uint8_t checksum_len_buf[WalWriter::CHECKSUM_SIZE];
RETURN_IF_ERROR(file_reader->read_at(_offset, {checksum_len_buf, WalWriter::CHECKSUM_SIZE},
Expand Down
1 change: 1 addition & 0 deletions be/src/olap/wal/wal_reader.h
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@ class WalReader {
Status read_header(uint32_t& version, std::string& col_ids);

private:
Status _deserialize(PBlock& block, const std::string& buf, size_t block_len, size_t bytes_read);
Status _check_checksum(const char* binary, size_t size, uint32_t checksum);

std::string _file_name;
Expand Down

0 comments on commit d6cab45

Please sign in to comment.