Skip to content

Commit 95bb9fa

Browse files
nieklinnenbanknieklinnenbank
committed
lib/libexec: add LZ4 decompression algorithm support
1 parent f84f704 commit 95bb9fa

File tree

6 files changed

+493
-11
lines changed

6 files changed

+493
-11
lines changed

‎README.md

+2-2
Original file line numberDiff line numberDiff line change
@@ -49,7 +49,7 @@ the build dependencies on your host OS.
4949
Update your system repository cache and install the required development tools using:
5050

5151
$ sudo apt-get update
52-
$ sudo apt-get install build-essential scons genisoimage xorriso qemu-system binutils-multiarch u-boot-tools
52+
$ sudo apt-get install build-essential scons genisoimage xorriso qemu-system binutils-multiarch u-boot-tools liblz4-tool
5353

5454
If your Ubuntu host is 64-bit, you need to install the GCC multilib package
5555
to cross compile for the 32-bit architecture:
@@ -66,7 +66,7 @@ Update your system repository cache and install the required development tools u
6666

6767
% su -
6868
# pkg update
69-
# pkg install qemu scons cdrkit-genisoimage xorriso gcc u-boot-tools
69+
# pkg install qemu scons cdrkit-genisoimage xorriso gcc u-boot-tools liblz4
7070

7171
On FreeBSD, make sure that the latest version of the GNU linker (from pkg) is used:
7272

‎SConstruct

+10-7
Original file line numberDiff line numberDiff line change
@@ -81,18 +81,21 @@ else:
8181
SConscript(host['BUILDROOT'] + '/server/SConscript')
8282
SConscript(host['BUILDROOT'] + '/test/SConscript')
8383

84-
#
85-
# Boot Image
86-
#
87-
target.BootImage('#${BUILDROOT}/boot.img', '#config/' + target['ARCH'] + '/' + target['SYSTEM'] + '/boot.imgdesc')
88-
target.Lz4Compress('#${BUILDROOT}/boot.img.lz4', '#${BUILDROOT}/boot.img')
89-
9084
#
9185
# RootFS
9286
#
9387
Import('rootfs_files')
9488
target.LinnImage('#${BUILDROOT}/rootfs.linn', rootfs_files)
95-
target.Depends('#${BUILDROOT}/rootfs.linn', '#build/host')
89+
target.Depends('#${BUILDROOT}/rootfs.linn', '#build/host/server/filesystem/linn/create')
90+
target.Depends('#${BUILDROOT}/rootfs.linn', '#build/host/bin/img/img')
91+
92+
#
93+
# Boot Image
94+
#
95+
target.BootImage('#${BUILDROOT}/boot.img', '#config/' + target['ARCH'] + '/' + target['SYSTEM'] + '/boot.imgdesc')
96+
target.Lz4Compress('#${BUILDROOT}/boot.img.lz4', '#${BUILDROOT}/boot.img')
97+
host.Install('#${BUILDROOT}', target['BUILDROOT'] + '/boot.img')
98+
host.Install('#${BUILDROOT}', target['BUILDROOT'] + '/boot.img.lz4')
9699

97100
#
98101
# Source Release

‎lib/libexec/Lz4Decompressor.cpp

+280
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,280 @@
1+
/*
2+
* Copyright (C) 2020 Niek Linnenbank
3+
*
4+
* This program is free software: you can redistribute it and/or modify
5+
* it under the terms of the GNU General Public License as published by
6+
* the Free Software Foundation, either version 3 of the License, or
7+
* (at your option) any later version.
8+
*
9+
* This program is distributed in the hope that it will be useful,
10+
* but WITHOUT ANY WARRANTY; without even the implied warranty of
11+
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12+
* GNU General Public License for more details.
13+
*
14+
* You should have received a copy of the GNU General Public License
15+
* along with this program. If not, see <http://www.gnu.org/licenses/>.
16+
*/
17+
18+
#include <FreeNOS/User.h>
19+
#include <Log.h>
20+
#include <Endian.h>
21+
#include <MemoryBlock.h>
22+
#include "Lz4Decompressor.h"
23+
24+
Lz4Decompressor::Lz4Decompressor(const void *data, const Size size)
25+
: m_inputData(static_cast<const u8 *>(data))
26+
, m_inputSize(size)
27+
, m_frameDescSize(3)
28+
, m_blockChecksums(false)
29+
, m_contentChecksum(false)
30+
, m_contentSize(0)
31+
, m_blockMaximumSize(0)
32+
{
33+
}
34+
35+
Lz4Decompressor::Result Lz4Decompressor::initialize()
36+
{
37+
// Reset state
38+
m_frameDescSize = 3;
39+
40+
// Verify minimum input size
41+
if (m_inputSize < 27)
42+
{
43+
ERROR("invalid size of input data: " << m_inputSize);
44+
return InvalidArgument;
45+
}
46+
47+
// Verify the input is an actual LZ4 frame
48+
if (readU32(m_inputData) != FrameMagic)
49+
{
50+
ERROR("invalid magic value " << readU32(m_inputData) << " != " << FrameMagic);
51+
return InvalidArgument;
52+
}
53+
54+
// Read the FLG byte
55+
const u8 flg = *(m_inputData + sizeof(u32));
56+
57+
// Verify the version bits
58+
const u8 version = flg >> FrameVersionShift;
59+
if (version != FrameVersion)
60+
{
61+
ERROR("invalid version value " << version << " != " << FrameVersion);
62+
return InvalidArgument;
63+
}
64+
65+
// This code only supports independent blocks
66+
const bool independent = (flg >> FrameBlockIndShift) & 0x1;
67+
if (!independent)
68+
{
69+
ERROR("inter-dependent blocks not supported");
70+
return NotSupported;
71+
}
72+
73+
// Check for block checksum flag
74+
m_blockChecksums = (flg >> FrameBlockChkShift) & 0x1;
75+
76+
// Check for content size flag
77+
if ((flg >> FrameContentSzShift) & 0x1)
78+
{
79+
m_contentSize = readU64(m_inputData + sizeof(u32) + (sizeof(u8) * 2));
80+
m_frameDescSize += 8;
81+
}
82+
83+
// Content size must be non-zero
84+
if (m_contentSize == 0)
85+
{
86+
ERROR("content size must not be zero");
87+
return NotSupported;
88+
}
89+
90+
// Check for the content checksum flag
91+
m_contentChecksum = (flg >> FrameContentChkShift) & 0x1 ? true : false;
92+
93+
// Check for the DictID flag
94+
if ((flg >> FrameDictIdShift) & 0x1)
95+
{
96+
m_frameDescSize += 4;
97+
}
98+
99+
// Read the BD byte which contains the maximum block size
100+
const u8 bd = *(m_inputData + sizeof(u32) + sizeof(u8));
101+
switch (bd >> 4)
102+
{
103+
case 4:
104+
m_blockMaximumSize = KiloByte(64);
105+
break;
106+
case 5:
107+
m_blockMaximumSize = KiloByte(256);
108+
break;
109+
case 6:
110+
m_blockMaximumSize = MegaByte(1);
111+
break;
112+
case 7:
113+
m_blockMaximumSize = MegaByte(4);
114+
break;
115+
default:
116+
{
117+
ERROR("invalid maximum block size value: " << (bd >> 4));
118+
return InvalidArgument;
119+
}
120+
}
121+
122+
return Success;
123+
}
124+
125+
u64 Lz4Decompressor::getUncompressedSize() const
126+
{
127+
return m_contentSize;
128+
}
129+
130+
Lz4Decompressor::Result Lz4Decompressor::read(void *buffer,
131+
const Size size) const
132+
{
133+
const u8 *input = m_inputData + m_frameDescSize + sizeof(u32);
134+
const u8 *inputEnd = m_inputData + m_inputSize;
135+
u8 *output = static_cast<u8 *>(buffer);
136+
Size copied = 0;
137+
138+
while (copied < size && input < inputEnd)
139+
{
140+
// Fetch the next block
141+
const u32 blockSizeByte = readU32(input);
142+
const u32 blockSize = blockSizeByte & ~(1 << 31);
143+
const bool isCompressed = blockSizeByte & (1 << 31) ? false : true;
144+
Size uncompSize;
145+
146+
// Last block has the EndMark as size value
147+
if (blockSize == EndMark)
148+
{
149+
break;
150+
}
151+
assert(blockSize <= m_blockMaximumSize);
152+
input += sizeof(u32);
153+
154+
// Decompress the block
155+
if (isCompressed)
156+
{
157+
uncompSize = decompress(input, blockSize, output, size - copied);
158+
}
159+
// Return data as-is when the block is not compressed
160+
else
161+
{
162+
MemoryBlock::copy(output, input, blockSize);
163+
uncompSize = blockSize;
164+
}
165+
166+
// Move to the next block
167+
copied += uncompSize;
168+
output += uncompSize;
169+
input += blockSize;
170+
if (m_blockChecksums)
171+
{
172+
input += sizeof(u32);
173+
}
174+
}
175+
176+
return Success;
177+
}
178+
179+
inline const u64 Lz4Decompressor::readU64(const void *data) const
180+
{
181+
u64 value;
182+
MemoryBlock::copy(&value, data, sizeof(value));
183+
return le64_to_cpu(value);
184+
}
185+
186+
inline const u32 Lz4Decompressor::readU32(const void *data) const
187+
{
188+
u32 value;
189+
MemoryBlock::copy(&value, data, sizeof(value));
190+
return le32_to_cpu(value);
191+
}
192+
193+
inline const u16 Lz4Decompressor::readU16(const void *data) const
194+
{
195+
u16 value;
196+
MemoryBlock::copy(&value, data, sizeof(value));
197+
return le16_to_cpu(value);
198+
}
199+
200+
inline const u32 Lz4Decompressor::integerDecode(const u32 initial,
201+
const u8 *next,
202+
Size &byteCount) const
203+
{
204+
u32 value = initial;
205+
206+
if (initial < 0xf)
207+
{
208+
return initial;
209+
}
210+
211+
for (byteCount = 1; ; byteCount++)
212+
{
213+
const u8 byte = *next++;
214+
value += byte;
215+
216+
if (byte < 0xff)
217+
{
218+
break;
219+
}
220+
}
221+
222+
return value;
223+
}
224+
225+
const u32 Lz4Decompressor::decompress(const u8 *input,
226+
const Size inputSize,
227+
u8 *output,
228+
const Size outputSize) const
229+
{
230+
const u8 *inputEnd = input + inputSize;
231+
Size outputOffset = 0;
232+
233+
// Decompress the whole block
234+
while (input < inputEnd && outputOffset < outputSize)
235+
{
236+
u32 literalBytes = 0;
237+
u32 matchBytes = 0;
238+
239+
// Read the token
240+
const u8 token = *input;
241+
input++;
242+
243+
// Read literals count
244+
const u32 literalsCount = integerDecode(token >> 4, input, literalBytes);
245+
input += literalBytes;
246+
DEBUG("token = " << token << " literalsCount = " << literalsCount << " literalBytes = " << literalBytes);
247+
248+
// Copy literals
249+
if (literalsCount > 0)
250+
{
251+
MemoryBlock::copy(output + outputOffset, input, literalsCount);
252+
input += literalsCount;
253+
outputOffset += literalsCount;
254+
}
255+
256+
// End of block reached? Last 5 bytes are only literals
257+
if (input >= inputEnd)
258+
{
259+
break;
260+
}
261+
262+
// Read match offset
263+
const u16 off = readU16(input);
264+
assert(off <= outputOffset);
265+
const u32 matchOffset = outputOffset - off;
266+
input += sizeof(u16);
267+
268+
// Read match length
269+
const u32 matchCount = integerDecode(token & 0xf, input, matchBytes) + 4u;
270+
input += matchBytes;
271+
272+
// Copy the match from previous decoded bytes
273+
DEBUG("matchOffset = " << matchOffset << " matchCount = " << matchCount);
274+
MemoryBlock::copy(output + outputOffset, output + matchOffset, matchCount);
275+
outputOffset += matchCount;
276+
}
277+
278+
assert(outputOffset <= m_blockMaximumSize);
279+
return outputOffset;
280+
}

0 commit comments

Comments
 (0)