Skip to content
This repository was archived by the owner on Nov 26, 2024. It is now read-only.

Commit 79a7390

Browse files
apoelstraapoelstra
committed
Merge #19: Import jsonrpc crate
7238857 Add fuzzing for jsonrpc (Tobin C. Harding) 49b72c1 Import jsonrpc crate (Tobin C. Harding) 45addbd justfile: Add a docs build command (Tobin C. Harding) Pull request description: Import the `rust-jsonrpc` crate from https://github.com/apoelstra/rust-jsonrpc using current tip of master `59646e6 Merge apoelstra/rust-jsonrpc#119: Use rust-bitcoin-maintainer-tools and re-write CI` Full commit hash: 59646e6e6ac95f07998133b1709e4a1fa2dbc7bd Do so using the following commands: mkdir jsonrpc mkdir jsonrpc/contrib rsync -avz ../../rust-jsonrpc/master/README.md jsonrpc rsync -avz ../../rust-jsonrpc/master/src jsonrpc rsync -avz ../../rust-jsonrpc/master/contrib/test_vars.sh jsonrpc/contrib Then: - Update `contrib/crates.sh` to include `jsonrpc`. - Remove workspaces from `jsonrpc/Cargo.toml`. - Add `jsonrpc` to repository workspace. Finally import fuzzing, and fix up to mimic current `rust-bitcoin` setup. ACKs for top commit: apoelstra: ACK 7238857 successfully ran local tests Tree-SHA512: 309c214d50e78bcd67b49b8f68df91792100d95fda65a6ec700c422d28d64f41498e52f4eabeebcfae46685ee997ffb8c444180863d8f8003ed542a5ed80d174
2 parents 73be7c9 + 7238857 commit 79a7390

26 files changed

+2762
-6
lines changed

‎.github/workflows/cron-daily-fuzz.yml‎

Lines changed: 62 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,62 @@
1+
# Automatically generated by fuzz/generate-files.sh
2+
name: Fuzz
3+
on:
4+
schedule:
5+
# 6am every day UTC, this correlates to:
6+
# - 11pm PDT
7+
# - 7am CET
8+
# - 5pm AEDT
9+
- cron: '00 06 * * *'
10+
11+
jobs:
12+
fuzz:
13+
if: ${{ !github.event.act }}
14+
runs-on: ubuntu-latest
15+
strategy:
16+
fail-fast: false
17+
matrix:
18+
# We only get 20 jobs at a time, we probably don't want to go
19+
# over that limit with fuzzing because of the hour run time.
20+
fuzz_target: [
21+
minreq_http,
22+
simple_http,
23+
]
24+
steps:
25+
- name: Install test dependencies
26+
run: sudo apt-get update -y && sudo apt-get install -y binutils-dev libunwind8-dev libcurl4-openssl-dev libelf-dev libdw-dev cmake gcc libiberty-dev
27+
- uses: actions/checkout@v4
28+
- uses: actions/cache@v4
29+
id: cache-fuzz
30+
with:
31+
path: |
32+
~/.cargo/bin
33+
fuzz/target
34+
target
35+
key: cache-${{ matrix.target }}-${{ hashFiles('**/Cargo.toml','**/Cargo.lock') }}
36+
- uses: dtolnay/rust-toolchain@stable
37+
with:
38+
toolchain: '1.65.0'
39+
- name: fuzz
40+
run: |
41+
if [[ "${{ matrix.fuzz_target }}" =~ ^bitcoin ]]; then
42+
export RUSTFLAGS='--cfg=hashes_fuzz --cfg=secp256k1_fuzz'
43+
fi
44+
echo "Using RUSTFLAGS $RUSTFLAGS"
45+
cd fuzz && ./fuzz.sh "${{ matrix.fuzz_target }}"
46+
- run: echo "${{ matrix.fuzz_target }}" >executed_${{ matrix.fuzz_target }}
47+
- uses: actions/upload-artifact@v3
48+
with:
49+
name: executed_${{ matrix.fuzz_target }}
50+
path: executed_${{ matrix.fuzz_target }}
51+
52+
verify-execution:
53+
if: ${{ !github.event.act }}
54+
needs: fuzz
55+
runs-on: ubuntu-latest
56+
steps:
57+
- uses: actions/checkout@v4
58+
- uses: actions/download-artifact@v3
59+
- name: Display structure of downloaded files
60+
run: ls -R
61+
- run: find executed_* -type f -exec cat {} + | sort > executed
62+
- run: source ./fuzz/fuzz-util.sh && listTargetNames | sort | diff - executed

‎Cargo-minimal.lock‎

Lines changed: 46 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -106,6 +106,12 @@ dependencies = [
106106
"serde_json",
107107
]
108108

109+
[[package]]
110+
name = "byteorder"
111+
version = "1.5.0"
112+
source = "registry+https://github.com/rust-lang/crates.io-index"
113+
checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b"
114+
109115
[[package]]
110116
name = "cc"
111117
version = "1.0.28"
@@ -136,15 +142,20 @@ checksum = "49f1f14873335454500d59611f1cf4a4b0f786f9ac11f4312a78e4cf2566695b"
136142
[[package]]
137143
name = "jsonrpc"
138144
version = "0.18.0"
139-
source = "registry+https://github.com/rust-lang/crates.io-index"
140-
checksum = "3662a38d341d77efecb73caf01420cfa5aa63c0253fd7bc05289ef9f6616e1bf"
141145
dependencies = [
142146
"base64",
143147
"minreq",
144148
"serde",
145149
"serde_json",
150+
"socks",
146151
]
147152

153+
[[package]]
154+
name = "libc"
155+
version = "0.2.158"
156+
source = "registry+https://github.com/rust-lang/crates.io-index"
157+
checksum = "d8adc4bb1803a324070e64a98ae98f38934d91957a99cfb3a43dcbc01bc56439"
158+
148159
[[package]]
149160
name = "log"
150161
version = "0.4.18"
@@ -237,6 +248,17 @@ dependencies = [
237248
"serde",
238249
]
239250

251+
[[package]]
252+
name = "socks"
253+
version = "0.3.4"
254+
source = "registry+https://github.com/rust-lang/crates.io-index"
255+
checksum = "f0c3dbbd9ae980613c6dd8e28a9407b50509d3803b57624d5dfe8315218cd58b"
256+
dependencies = [
257+
"byteorder",
258+
"libc",
259+
"winapi",
260+
]
261+
240262
[[package]]
241263
name = "syn"
242264
version = "2.0.56"
@@ -253,3 +275,25 @@ name = "unicode-ident"
253275
version = "1.0.8"
254276
source = "registry+https://github.com/rust-lang/crates.io-index"
255277
checksum = "e5464a87b239f13a63a501f2701565754bae92d243d4bb7eb12f6d57d2269bf4"
278+
279+
[[package]]
280+
name = "winapi"
281+
version = "0.3.9"
282+
source = "registry+https://github.com/rust-lang/crates.io-index"
283+
checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419"
284+
dependencies = [
285+
"winapi-i686-pc-windows-gnu",
286+
"winapi-x86_64-pc-windows-gnu",
287+
]
288+
289+
[[package]]
290+
name = "winapi-i686-pc-windows-gnu"
291+
version = "0.4.0"
292+
source = "registry+https://github.com/rust-lang/crates.io-index"
293+
checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6"
294+
295+
[[package]]
296+
name = "winapi-x86_64-pc-windows-gnu"
297+
version = "0.4.0"
298+
source = "registry+https://github.com/rust-lang/crates.io-index"
299+
checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"

‎Cargo-recent.lock‎

Lines changed: 46 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -106,6 +106,12 @@ dependencies = [
106106
"serde_json",
107107
]
108108

109+
[[package]]
110+
name = "byteorder"
111+
version = "1.5.0"
112+
source = "registry+https://github.com/rust-lang/crates.io-index"
113+
checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b"
114+
109115
[[package]]
110116
name = "cc"
111117
version = "1.0.28"
@@ -136,15 +142,20 @@ checksum = "49f1f14873335454500d59611f1cf4a4b0f786f9ac11f4312a78e4cf2566695b"
136142
[[package]]
137143
name = "jsonrpc"
138144
version = "0.18.0"
139-
source = "registry+https://github.com/rust-lang/crates.io-index"
140-
checksum = "3662a38d341d77efecb73caf01420cfa5aa63c0253fd7bc05289ef9f6616e1bf"
141145
dependencies = [
142146
"base64",
143147
"minreq",
144148
"serde",
145149
"serde_json",
150+
"socks",
146151
]
147152

153+
[[package]]
154+
name = "libc"
155+
version = "0.2.158"
156+
source = "registry+https://github.com/rust-lang/crates.io-index"
157+
checksum = "d8adc4bb1803a324070e64a98ae98f38934d91957a99cfb3a43dcbc01bc56439"
158+
148159
[[package]]
149160
name = "log"
150161
version = "0.4.18"
@@ -237,6 +248,17 @@ dependencies = [
237248
"serde",
238249
]
239250

251+
[[package]]
252+
name = "socks"
253+
version = "0.3.4"
254+
source = "registry+https://github.com/rust-lang/crates.io-index"
255+
checksum = "f0c3dbbd9ae980613c6dd8e28a9407b50509d3803b57624d5dfe8315218cd58b"
256+
dependencies = [
257+
"byteorder",
258+
"libc",
259+
"winapi",
260+
]
261+
240262
[[package]]
241263
name = "syn"
242264
version = "2.0.56"
@@ -253,3 +275,25 @@ name = "unicode-ident"
253275
version = "1.0.8"
254276
source = "registry+https://github.com/rust-lang/crates.io-index"
255277
checksum = "e5464a87b239f13a63a501f2701565754bae92d243d4bb7eb12f6d57d2269bf4"
278+
279+
[[package]]
280+
name = "winapi"
281+
version = "0.3.9"
282+
source = "registry+https://github.com/rust-lang/crates.io-index"
283+
checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419"
284+
dependencies = [
285+
"winapi-i686-pc-windows-gnu",
286+
"winapi-x86_64-pc-windows-gnu",
287+
]
288+
289+
[[package]]
290+
name = "winapi-i686-pc-windows-gnu"
291+
version = "0.4.0"
292+
source = "registry+https://github.com/rust-lang/crates.io-index"
293+
checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6"
294+
295+
[[package]]
296+
name = "winapi-x86_64-pc-windows-gnu"
297+
version = "0.4.0"
298+
source = "registry+https://github.com/rust-lang/crates.io-index"
299+
checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"

‎Cargo.toml‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
[workspace]
2-
members = ["client", "json"]
2+
members = ["client", "json", "jsonrpc"]
33
exclude = ["integration_test", "regtest"]
44
resolver = "2"
55

@@ -8,3 +8,6 @@ path = "client"
88

99
[patch.crates-io.bitcoind-json-rpc-types]
1010
path = "json"
11+
12+
[patch.crates-io.jsonrpc]
13+
path = "jsonrpc"

‎contrib/crates.sh‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,4 +2,4 @@
22
# shellcheck disable=SC2148
33

44
# Crates in this workspace to test.
5-
CRATES=("json" "client")
5+
CRATES=("json" "client" "jsonrpc")

‎fuzz/.gitignore‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
2+
target
3+
corpus
4+
artifacts

‎fuzz/Cargo.toml‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
[package]
2+
name = "jsonrpc-fuzz"
3+
edition = "2021"
4+
rust-version = "1.63.0"
5+
version = "0.0.1"
6+
authors = ["Generated by fuzz/generate-files.sh"]
7+
publish = false
8+
9+
[package.metadata]
10+
cargo-fuzz = true
11+
12+
[dependencies]
13+
honggfuzz = { version = "0.5.55", default-features = false }
14+
jsonrpc = { path = "..", features = ["minreq_http"] }
15+
16+
serde = { version = "1.0.103", features = [ "derive" ] }
17+
serde_json = "1.0"
18+
19+
[lints.rust]
20+
unexpected_cfgs = { level = "deny", check-cfg = ['cfg(fuzzing)'] }
21+
22+
[[bin]]
23+
name = "minreq_http"
24+
path = "fuzz_targets/minreq_http.rs"
25+
26+
[[bin]]
27+
name = "simple_http"
28+
path = "fuzz_targets/simple_http.rs"

‎fuzz/README.md‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
# Note to devs
2+
3+
If you are considering adding fuzzing for the other crates take a look
4+
at how we set up `fuzz_target` in `rust-bitcoin/fuzz`.

‎fuzz/cycle.sh‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
#!/usr/bin/env bash
2+
3+
# Continuosly cycle over fuzz targets running each for 1 hour.
4+
# It uses chrt SCHED_IDLE so that other process takes priority.
5+
#
6+
# For hfuzz options see https://github.com/google/honggfuzz/blob/master/docs/USAGE.md
7+
8+
set -e
9+
REPO_DIR=$(git rev-parse --show-toplevel)
10+
# shellcheck source=./fuzz-util.sh
11+
source "$REPO_DIR/fuzz/fuzz-util.sh"
12+
13+
while :
14+
do
15+
for targetFile in $(listTargetFiles); do
16+
targetName=$(targetFileToName "$targetFile")
17+
echo "Fuzzing target $targetName ($targetFile)"
18+
19+
# fuzz for one hour
20+
HFUZZ_RUN_ARGS='--run_time 3600' chrt -i 0 cargo hfuzz run "$targetName"
21+
# minimize the corpus
22+
HFUZZ_RUN_ARGS="-i hfuzz_workspace/$targetName/input/ -P -M" chrt -i 0 cargo hfuzz run "$targetName"
23+
done
24+
done

‎fuzz/fuzz-util.sh‎

Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,55 @@
1+
#!/usr/bin/env bash
2+
3+
REPO_DIR=$(git rev-parse --show-toplevel)
4+
5+
# Sort order is effected by locale. See `man sort`.
6+
# > Set LC_ALL=C to get the traditional sort order that uses native byte values.
7+
export LC_ALL=C
8+
9+
listTargetFiles() {
10+
pushd "$REPO_DIR/fuzz" > /dev/null || exit 1
11+
find fuzz_targets/ -type f -name "*.rs" | sort
12+
popd > /dev/null || exit 1
13+
}
14+
15+
targetFileToName() {
16+
echo "$1" \
17+
| sed 's/^fuzz_targets\///' \
18+
| sed 's/\.rs$//' \
19+
| sed 's/\//_/g'
20+
}
21+
22+
targetFileToHFuzzInputArg() {
23+
baseName=$(basename "$1")
24+
dirName="${baseName%.*}"
25+
if [ -d "hfuzz_input/$dirName" ]; then
26+
echo "HFUZZ_INPUT_ARGS=\"-f hfuzz_input/$FILE/input\""
27+
fi
28+
}
29+
30+
listTargetNames() {
31+
for target in $(listTargetFiles); do
32+
targetFileToName "$target"
33+
done
34+
}
35+
36+
# Utility function to avoid CI failures on Windows
37+
checkWindowsFiles() {
38+
incorrectFilenames=$(find . -type f -name "*,*" -o -name "*:*" -o -name "*<*" -o -name "*>*" -o -name "*|*" -o -name "*\?*" -o -name "*\**" -o -name "*\"*" | wc -l)
39+
if [ "$incorrectFilenames" -gt 0 ]; then
40+
echo "Bailing early because there is a Windows-incompatible filename in the tree."
41+
exit 2
42+
fi
43+
}
44+
45+
# Checks whether a fuzz case output some report, and dumps it in hex
46+
checkReport() {
47+
reportFile="hfuzz_workspace/$1/HONGGFUZZ.REPORT.TXT"
48+
if [ -f "$reportFile" ]; then
49+
cat "$reportFile"
50+
for CASE in "hfuzz_workspace/$1/SIG"*; do
51+
xxd -p -c10000 < "$CASE"
52+
done
53+
exit 1
54+
fi
55+
}

0 commit comments

Comments
 (0)