Merge #19: Import jsonrpc crate

7238857 Add fuzzing for jsonrpc (Tobin C. Harding)
49b72c1 Import jsonrpc crate (Tobin C. Harding)
45addbd justfile: Add a docs build command (Tobin C. Harding)

Pull request description:

  Import the `rust-jsonrpc` crate from
  https://github.com/apoelstra/rust-jsonrpc using current tip of master

  `59646e6 Merge apoelstra/rust-jsonrpc#119: Use rust-bitcoin-maintainer-tools and re-write CI`

  Full commit hash: 59646e6e6ac95f07998133b1709e4a1fa2dbc7bd

  Do so using the following commands:

  mkdir jsonrpc
  mkdir jsonrpc/contrib
  rsync -avz ../../rust-jsonrpc/master/README.md jsonrpc rsync -avz ../../rust-jsonrpc/master/src jsonrpc
  rsync -avz ../../rust-jsonrpc/master/contrib/test_vars.sh jsonrpc/contrib

  Then:

  - Update `contrib/crates.sh` to include `jsonrpc`.
  - Remove workspaces from `jsonrpc/Cargo.toml`.
  - Add `jsonrpc` to repository workspace.

  Finally import fuzzing, and fix up to mimic current `rust-bitcoin` setup.

ACKs for top commit:
  apoelstra:
    ACK 7238857 successfully ran local tests

Tree-SHA512: 309c214d50e78bcd67b49b8f68df91792100d95fda65a6ec700c422d28d64f41498e52f4eabeebcfae46685ee997ffb8c444180863d8f8003ed542a5ed80d174

Author: apoelstra

.github/workflows/cron-daily-fuzz.yml

@@ -0,0 +1,62 @@
+# Automatically generated by fuzz/generate-files.sh
+name: Fuzz
+on:
+  schedule:
+    # 6am every day UTC, this correlates to:
+    # - 11pm PDT
+    # - 7am CET
+    # - 5pm AEDT
+    - cron: '00 06 * * *'
+
+jobs:
+  fuzz:
+    if: ${{ !github.event.act }}
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        # We only get 20 jobs at a time, we probably don't want to go
+        # over that limit with fuzzing because of the hour run time.
+        fuzz_target: [
+          minreq_http,
+          simple_http,
+        ]
+    steps:
+      - name: Install test dependencies
+        run: sudo apt-get update -y && sudo apt-get install -y binutils-dev libunwind8-dev libcurl4-openssl-dev libelf-dev libdw-dev cmake gcc libiberty-dev
+      - uses: actions/checkout@v4
+      - uses: actions/cache@v4
+        id: cache-fuzz
+        with:
+          path: |
+            ~/.cargo/bin
+            fuzz/target
+            target
+          key: cache-${{ matrix.target }}-${{ hashFiles('**/Cargo.toml','**/Cargo.lock') }}
+      - uses: dtolnay/rust-toolchain@stable
+        with:
+          toolchain: '1.65.0'
+      - name: fuzz
+        run: |
+          if [[ "${{ matrix.fuzz_target }}" =~ ^bitcoin ]]; then
+              export RUSTFLAGS='--cfg=hashes_fuzz --cfg=secp256k1_fuzz'
+          fi
+          echo "Using RUSTFLAGS $RUSTFLAGS"
+          cd fuzz && ./fuzz.sh "${{ matrix.fuzz_target }}"
+      - run: echo "${{ matrix.fuzz_target }}" >executed_${{ matrix.fuzz_target }}
+      - uses: actions/upload-artifact@v3
+        with:
+          name: executed_${{ matrix.fuzz_target }}
+          path: executed_${{ matrix.fuzz_target }}
+
+  verify-execution:
+    if: ${{ !github.event.act }}
+    needs: fuzz
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/download-artifact@v3
+      - name: Display structure of downloaded files
+        run: ls -R
+      - run: find executed_* -type f -exec cat {} + | sort > executed
+      - run: source ./fuzz/fuzz-util.sh && listTargetNames | sort | diff - executed

Cargo-minimal.lock

@@ -106,6 +106,12 @@ dependencies = [
  "serde_json",
 ]
 
+[[package]]
+name = "byteorder"
+version = "1.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b"
+
 [[package]]
 name = "cc"
 version = "1.0.28"
@@ -136,15 +142,20 @@ checksum = "49f1f14873335454500d59611f1cf4a4b0f786f9ac11f4312a78e4cf2566695b"
 [[package]]
 name = "jsonrpc"
 version = "0.18.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3662a38d341d77efecb73caf01420cfa5aa63c0253fd7bc05289ef9f6616e1bf"
 dependencies = [
  "base64",
  "minreq",
  "serde",
  "serde_json",
+ "socks",
 ]
 
+[[package]]
+name = "libc"
+version = "0.2.158"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d8adc4bb1803a324070e64a98ae98f38934d91957a99cfb3a43dcbc01bc56439"
+
 [[package]]
 name = "log"
 version = "0.4.18"
@@ -237,6 +248,17 @@ dependencies = [
  "serde",
 ]
 
+[[package]]
+name = "socks"
+version = "0.3.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f0c3dbbd9ae980613c6dd8e28a9407b50509d3803b57624d5dfe8315218cd58b"
+dependencies = [
+ "byteorder",
+ "libc",
+ "winapi",
+]
+
 [[package]]
 name = "syn"
 version = "2.0.56"
@@ -253,3 +275,25 @@ name = "unicode-ident"
 version = "1.0.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e5464a87b239f13a63a501f2701565754bae92d243d4bb7eb12f6d57d2269bf4"
+
+[[package]]
+name = "winapi"
+version = "0.3.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419"
+dependencies = [
+ "winapi-i686-pc-windows-gnu",
+ "winapi-x86_64-pc-windows-gnu",
+]
+
+[[package]]
+name = "winapi-i686-pc-windows-gnu"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6"
+
+[[package]]
+name = "winapi-x86_64-pc-windows-gnu"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"

Cargo-recent.lock

@@ -106,6 +106,12 @@ dependencies = [
  "serde_json",
 ]
 
+[[package]]
+name = "byteorder"
+version = "1.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b"
+
 [[package]]
 name = "cc"
 version = "1.0.28"
@@ -136,15 +142,20 @@ checksum = "49f1f14873335454500d59611f1cf4a4b0f786f9ac11f4312a78e4cf2566695b"
 [[package]]
 name = "jsonrpc"
 version = "0.18.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3662a38d341d77efecb73caf01420cfa5aa63c0253fd7bc05289ef9f6616e1bf"
 dependencies = [
  "base64",
  "minreq",
  "serde",
  "serde_json",
+ "socks",
 ]
 
+[[package]]
+name = "libc"
+version = "0.2.158"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d8adc4bb1803a324070e64a98ae98f38934d91957a99cfb3a43dcbc01bc56439"
+
 [[package]]
 name = "log"
 version = "0.4.18"
@@ -237,6 +248,17 @@ dependencies = [
  "serde",
 ]
 
+[[package]]
+name = "socks"
+version = "0.3.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f0c3dbbd9ae980613c6dd8e28a9407b50509d3803b57624d5dfe8315218cd58b"
+dependencies = [
+ "byteorder",
+ "libc",
+ "winapi",
+]
+
 [[package]]
 name = "syn"
 version = "2.0.56"
@@ -253,3 +275,25 @@ name = "unicode-ident"
 version = "1.0.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e5464a87b239f13a63a501f2701565754bae92d243d4bb7eb12f6d57d2269bf4"
+
+[[package]]
+name = "winapi"
+version = "0.3.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419"
+dependencies = [
+ "winapi-i686-pc-windows-gnu",
+ "winapi-x86_64-pc-windows-gnu",
+]
+
+[[package]]
+name = "winapi-i686-pc-windows-gnu"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6"
+
+[[package]]
+name = "winapi-x86_64-pc-windows-gnu"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"

Cargo.toml

@@ -1,5 +1,5 @@
 [workspace]
-members = ["client", "json"]
+members = ["client", "json", "jsonrpc"]
 exclude = ["integration_test", "regtest"]
 resolver = "2"
 
@@ -8,3 +8,6 @@ path = "client"
 
 [patch.crates-io.bitcoind-json-rpc-types]
 path = "json"
+
+[patch.crates-io.jsonrpc]
+path = "jsonrpc"

contrib/crates.sh

@@ -2,4 +2,4 @@
 # shellcheck disable=SC2148
 
 # Crates in this workspace to test.
-CRATES=("json" "client")
+CRATES=("json" "client" "jsonrpc")

fuzz/.gitignore

@@ -0,0 +1,4 @@
+
+target
+corpus
+artifacts

fuzz/Cargo.toml

@@ -0,0 +1,28 @@
+[package]
+name = "jsonrpc-fuzz"
+edition = "2021"
+rust-version = "1.63.0"
+version = "0.0.1"
+authors = ["Generated by fuzz/generate-files.sh"]
+publish = false
+
+[package.metadata]
+cargo-fuzz = true
+
+[dependencies]
+honggfuzz = { version = "0.5.55", default-features = false }
+jsonrpc = { path = "..", features = ["minreq_http"] }
+
+serde = { version = "1.0.103", features = [ "derive" ] }
+serde_json = "1.0"
+
+[lints.rust]
+unexpected_cfgs = { level = "deny", check-cfg = ['cfg(fuzzing)'] }
+
+[[bin]]
+name = "minreq_http"
+path = "fuzz_targets/minreq_http.rs"
+
+[[bin]]
+name = "simple_http"
+path = "fuzz_targets/simple_http.rs"

fuzz/README.md

@@ -0,0 +1,4 @@
+# Note to devs
+
+If you are considering adding fuzzing for the other crates take a look
+at how we set up `fuzz_target` in `rust-bitcoin/fuzz`.

fuzz/cycle.sh

@@ -0,0 +1,24 @@
+#!/usr/bin/env bash
+
+# Continuosly cycle over fuzz targets running each for 1 hour.
+# It uses chrt SCHED_IDLE so that other process takes priority.
+#
+# For hfuzz options see https://github.com/google/honggfuzz/blob/master/docs/USAGE.md
+
+set -e
+REPO_DIR=$(git rev-parse --show-toplevel)
+# shellcheck source=./fuzz-util.sh
+source "$REPO_DIR/fuzz/fuzz-util.sh"
+
+while :
+do
+  for targetFile in $(listTargetFiles); do
+    targetName=$(targetFileToName "$targetFile")
+    echo "Fuzzing target $targetName ($targetFile)"
+
+    # fuzz for one hour
+    HFUZZ_RUN_ARGS='--run_time 3600' chrt -i 0 cargo hfuzz run "$targetName"
+    # minimize the corpus
+    HFUZZ_RUN_ARGS="-i hfuzz_workspace/$targetName/input/ -P -M" chrt -i 0 cargo hfuzz run "$targetName"
+  done
+done

fuzz/fuzz-util.sh

@@ -0,0 +1,55 @@
+#!/usr/bin/env bash
+
+REPO_DIR=$(git rev-parse --show-toplevel)
+
+# Sort order is effected by locale. See `man sort`.
+# > Set LC_ALL=C to get the traditional sort order that uses native byte values.
+export LC_ALL=C
+
+listTargetFiles() {
+  pushd "$REPO_DIR/fuzz" > /dev/null || exit 1
+  find fuzz_targets/ -type f -name "*.rs" | sort
+  popd > /dev/null || exit 1
+}
+
+targetFileToName() {
+  echo "$1" \
+    | sed 's/^fuzz_targets\///' \
+    | sed 's/\.rs$//' \
+    | sed 's/\//_/g'
+}
+
+targetFileToHFuzzInputArg() {
+  baseName=$(basename "$1")
+  dirName="${baseName%.*}"
+  if [ -d "hfuzz_input/$dirName" ]; then
+    echo "HFUZZ_INPUT_ARGS=\"-f hfuzz_input/$FILE/input\""
+  fi
+}
+
+listTargetNames() {
+  for target in $(listTargetFiles); do
+    targetFileToName "$target"
+  done
+}
+
+# Utility function to avoid CI failures on Windows
+checkWindowsFiles() {
+  incorrectFilenames=$(find . -type f -name "*,*" -o -name "*:*" -o -name "*<*" -o -name "*>*" -o -name "*|*" -o -name "*\?*" -o -name "*\**" -o -name "*\"*" | wc -l)
+  if [ "$incorrectFilenames" -gt 0 ]; then
+    echo "Bailing early because there is a Windows-incompatible filename in the tree."
+    exit 2
+  fi
+}
+
+# Checks whether a fuzz case output some report, and dumps it in hex
+checkReport() {
+  reportFile="hfuzz_workspace/$1/HONGGFUZZ.REPORT.TXT"
+  if [ -f "$reportFile" ]; then
+    cat "$reportFile"
+    for CASE in "hfuzz_workspace/$1/SIG"*; do
+      xxd -p -c10000 < "$CASE"
+    done
+    exit 1
+  fi
+}

fuzz/fuzz.sh

@@ -0,0 +1,34 @@
+#!/usr/bin/env bash
+set -ex
+
+REPO_DIR=$(git rev-parse --show-toplevel)
+
+# shellcheck source=./fuzz-util.sh
+source "$REPO_DIR/fuzz/fuzz-util.sh"
+
+# Check that input files are correct Windows file names
+checkWindowsFiles
+
+if [ "$1" == "" ]; then
+  targetFiles="$(listTargetFiles)"
+else
+  targetFiles=fuzz_targets/"$1".rs
+fi
+
+cargo --version
+rustc --version
+
+# Testing
+cargo install --force honggfuzz --no-default-features
+for targetFile in $targetFiles; do
+  targetName=$(targetFileToName "$targetFile")
+  echo "Fuzzing target $targetName ($targetFile)"
+  if [ -d "hfuzz_input/$targetName" ]; then
+    HFUZZ_INPUT_ARGS="-f hfuzz_input/$targetName/input\""
+  else
+    HFUZZ_INPUT_ARGS=""
+  fi
+  RUSTFLAGS="--cfg=jsonrpc_fuzz" HFUZZ_RUN_ARGS="--run_time 30 --exit_upon_crash -v $HFUZZ_INPUT_ARGS" cargo hfuzz run "$targetName"
+
+  checkReport "$targetName"
+done