Skip to content

Commit 5b43fbf

Browse files
lpawelczmtdudekrw1nklerrdob-ant
committed
modules: Add ZSTD Decoder module
This commit adds an initial implementation of the ZSTD Decoder. It is capable of decoding simple ZSTD frames containing raw and rle blocks. This is a squashed commit that was created from the following changes: modules/zstd: Add buffer library modules/zstd: Add Buffer use-case example modules/zstd: Add library for parsing magic number modules/zstd: Add library for parsing frame header dependency_support/libzstd: Make zstd_errors.h public dependency_support: Add decodecorpus binary modules/zstd: Add data generator library modules/zstd: Add zstd frame header tests modules/zstd: Add common zstd definitions modules/zstd: Add raw block decoder modules/zstd: Add rle block decoder modules/zstd: Add block header parsing library modules/zstd: Add SequenceExecutorPacket to common definitions modules/zstd: Add block data muxer library modules/zstd: Add block demuxer library modules/zstd: Add block decoder module modules/zstd/common: Specify decoder output format examples/ram: Export internal RAM API to other modules modules/zstd: Add Offset type to common zstd definitions modules/zstd: Add RamPrinter Proc modules/zstd: Add SequenceExecutor Proc modules/zstd: Add repacketizer modules/zstd: Add ZSTD decoder modules/zstd: Add ZSTD Decoder documentation CI: Add custom ZSTD module workflow Co-authored-by: Maciej Dudek <[email protected]> Co-authored-by: Pawel Czarnecki <[email protected]> Co-authored-by: Robert Winkler <[email protected]> Co-authored-by: Roman Dobrodii <[email protected]> Internal-tag: [#52186] Signed-off-by: Maciej Dudek <[email protected]> Signed-off-by: Pawel Czarnecki <[email protected]> Signed-off-by: Robert Winkler <[email protected]> Signed-off-by: Roman Dobrodii <[email protected]>
1 parent 8a9c4bb commit 5b43fbf

30 files changed

+8287
-8
lines changed

‎.github/workflows/modules-zstd.yml

Lines changed: 73 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,73 @@
1+
name: Modules (ZSTD)
2+
on:
3+
# Avoid triggering on pushes to /all/ open PR branches.
4+
push:
5+
branches:
6+
- main
7+
paths:
8+
- 'xls/modules/zstd/**'
9+
pull_request:
10+
branches:
11+
- main
12+
paths:
13+
- 'xls/modules/zstd/**'
14+
# This lets us trigger manually from the UI.
15+
workflow_dispatch:
16+
17+
jobs:
18+
test:
19+
name: Test ZSTD module (opt)
20+
runs-on:
21+
labels: ubuntu-22.04-64core
22+
timeout-minutes: 600
23+
continue-on-error: true
24+
steps:
25+
- uses: actions/checkout@v2
26+
27+
- name: Restore Nightly Bazel Cache
28+
uses: actions/cache/restore@v4
29+
with:
30+
path: "~/.cache/bazel"
31+
key: bazel-cache-nightly-${{ runner.os }}-${{ github.sha }}
32+
restore-keys: bazel-cache-nightly-${{ runner.os }}-
33+
34+
- name: Install dependencies via apt
35+
run: sudo apt-get install python3-distutils python3-dev python-is-python3 libtinfo5 build-essential liblapack-dev libblas-dev gfortran
36+
37+
- name: Bazel Build Tools (opt)
38+
run: |
39+
bazel build -c opt --test_output=errors -- //xls/dslx:interpreter_main //xls/dslx/ir_convert:ir_converter_main //xls/tools:opt_main //xls/tools:codegen_main
40+
41+
- name: Build ZSTD Module (opt)
42+
run: |
43+
bazel build -c opt --test_output=errors -- //xls/modules/zstd:all
44+
45+
- name: Test ZSTD Module - DSLX Tests (opt)
46+
if: ${{ !cancelled() }}
47+
run: |
48+
bazel test -c opt --test_output=errors -- $(bazel query 'filter(".*_dslx_test", kind(rule, //xls/modules/zstd/...))')
49+
50+
- name: Test ZSTD Module - CC Tests (opt)
51+
if: ${{ !cancelled() }}
52+
run: |
53+
bazel test -c opt --test_output=errors -- $(bazel query 'filter(".*_cc_test", kind(rule, //xls/modules/zstd/...))')
54+
55+
- name: Build ZSTD verilog targets (opt)
56+
if: ${{ !cancelled() }}
57+
run: |
58+
bazel build -c opt -- $(bazel query 'filter(".*_verilog", kind(rule, //xls/modules/zstd/...))')
59+
60+
- name: Build and run ZSTD IR benchmark rules (opt)
61+
if: ${{ !cancelled() }}
62+
run: |
63+
bazel run -c opt -- $(bazel query 'filter(".*_ir_benchmark", kind(rule, //xls/modules/zstd/...))')
64+
65+
- name: Build and run synthesis benchmarks of the ZSTD module (opt)
66+
if: ${{ !cancelled() }}
67+
run: |
68+
bazel run -c opt -- $(bazel query 'filter(".*_benchmark_synth", kind(rule, //xls/modules/zstd/...))')
69+
70+
- name: Build ZSTD place and route targets (opt)
71+
if: ${{ !cancelled() }}
72+
run: |
73+
bazel build -c opt -- $(bazel query 'filter(".*_place_and_route", kind(rule, //xls/modules/zstd/...))')

‎dependency_support/com_github_facebook_zstd/bundled.BUILD.bazel

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -174,3 +174,42 @@ cc_test(
174174
":zstd",
175175
],
176176
)
177+
178+
# NOTE: Required because of direct zstd_compress.c include in decodecorpus sources
179+
cc_library(
180+
name = "decodecorpus_includes",
181+
hdrs = [
182+
"lib/compress/zstd_compress.c",
183+
],
184+
)
185+
186+
cc_binary(
187+
name = "decodecorpus",
188+
srcs = [
189+
"tests/decodecorpus.c",
190+
] + glob(
191+
[
192+
"programs/*.c",
193+
"programs/*.h",
194+
],
195+
exclude = [
196+
"programs/zstdcli.c",
197+
],
198+
),
199+
deps = [
200+
":zstd",
201+
":decodecorpus_includes",
202+
],
203+
includes = [
204+
"lib/",
205+
"lib/common/",
206+
"lib/compress/",
207+
"lib/dictBuilder/",
208+
"lib/deprecated/",
209+
"programs/",
210+
],
211+
local_defines = [
212+
"XXH_NAMESPACE=ZSTD_",
213+
],
214+
visibility = ["//visibility:public"],
215+
)

‎xls/examples/ram.x

Lines changed: 5 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -55,7 +55,7 @@ pub fn ReadWordReq<NUM_PARTITIONS:u32, ADDR_WIDTH:u32>(addr:uN[ADDR_WIDTH]) ->
5555
}
5656

5757
// Behavior of reads and writes to the same address in the same "tick".
58-
enum SimultaneousReadWriteBehavior : u2 {
58+
pub enum SimultaneousReadWriteBehavior : u2 {
5959
// The read shows the contents at the address before the write.
6060
READ_BEFORE_WRITE = 0,
6161
// The read shows the contents at the address after the write.
@@ -160,7 +160,7 @@ fn write_word_test() {
160160

161161
// Function to compute num partitions (e.g. mask width) for a data_width-wide
162162
// word divided into word_partition_size-chunks.
163-
fn num_partitions(word_partition_size: u32, data_width: u32) -> u32 {
163+
pub fn num_partitions(word_partition_size: u32, data_width: u32) -> u32 {
164164
match word_partition_size {
165165
u32:0 => u32:0,
166166
_ => (word_partition_size + data_width - u32:1) / word_partition_size,
@@ -251,7 +251,7 @@ proc RamModel<DATA_WIDTH:u32, SIZE:u32, WORD_PARTITION_SIZE:u32={u32:0},
251251
if read_req_valid && ASSERT_VALID_READ {
252252
let mem_initialized_as_bits =
253253
std::convert_to_bits_msb0(array_rev(mem_initialized[read_req.addr]));
254-
assert_eq(read_req.mask & !mem_initialized_as_bits, uN[NUM_PARTITIONS]:0)
254+
assert!(read_req.mask & !mem_initialized_as_bits == uN[NUM_PARTITIONS]:0, "memory_not_initialized")
255255
} else { () };
256256

257257
let (value_to_write, written_mem_initialized) = write_word(
@@ -265,11 +265,8 @@ proc RamModel<DATA_WIDTH:u32, SIZE:u32, WORD_PARTITION_SIZE:u32={u32:0},
265265
match SIMULTANEOUS_READ_WRITE_BEHAVIOR {
266266
SimultaneousReadWriteBehavior::READ_BEFORE_WRITE => mem[read_req.addr],
267267
SimultaneousReadWriteBehavior::WRITE_BEFORE_READ => value_to_write,
268-
SimultaneousReadWriteBehavior::ASSERT_NO_CONFLICT => {
269-
// Assertion failure, we have a conflicting read and write.
270-
assert_eq(true, false);
271-
mem[read_req.addr] // Need to return something.
272-
},
268+
SimultaneousReadWriteBehavior::ASSERT_NO_CONFLICT => fail!("conflicting_read_and_write", mem[read_req.addr]),
269+
_ => fail!("impossible_case", uN[DATA_WIDTH]:0),
273270
}
274271
} else { mem[read_req.addr] };
275272
let read_resp_value = ReadResp<DATA_WIDTH> {

0 commit comments

Comments
 (0)