diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 69040d0afe8..801f85b7a4b 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -2,6 +2,9 @@ name: ci/gh-actions/cli on: push: + paths-ignore: + - 'docs/**' + - '**/README.md' pull_request: paths-ignore: - 'docs/**' @@ -27,16 +30,16 @@ jobs: env: CCACHE_TEMPDIR: /tmp/.ccache-temp steps: - - uses: actions/checkout@v1 + - uses: actions/checkout@v3 with: submodules: recursive - - uses: actions/cache@v2 + - uses: actions/cache@v3 with: path: /Users/runner/Library/Caches/ccache key: ccache-${{ runner.os }}-build-${{ github.sha }} restore-keys: ccache-${{ runner.os }}-build- - name: install dependencies - run: HOMEBREW_NO_AUTO_UPDATE=1 brew install boost hidapi openssl zmq libpgm miniupnpc ldns expat libunwind-headers protobuf ccache + run: HOMEBREW_NO_AUTO_UPDATE=1 brew install boost hidapi openssl zmq libpgm miniupnpc expat libunwind-headers protobuf ccache - name: build run: | ${{env.CCACHE_SETTINGS}} @@ -51,15 +54,15 @@ jobs: run: shell: msys2 {0} steps: - - uses: actions/checkout@v1 + - uses: actions/checkout@v3 with: submodules: recursive - - uses: actions/cache@v2 + - uses: actions/cache@v3 with: path: C:\Users\runneradmin\.ccache key: ccache-${{ runner.os }}-build-${{ github.sha }} restore-keys: ccache-${{ runner.os }}-build- - - uses: eine/setup-msys2@v2 + - uses: msys2/setup-msys2@v2 with: update: true install: mingw-w64-x86_64-toolchain make mingw-w64-x86_64-cmake mingw-w64-x86_64-ccache mingw-w64-x86_64-boost mingw-w64-x86_64-openssl mingw-w64-x86_64-zeromq mingw-w64-x86_64-libsodium mingw-w64-x86_64-hidapi mingw-w64-x86_64-protobuf-c mingw-w64-x86_64-libusb mingw-w64-x86_64-unbound git @@ -77,12 +80,12 @@ jobs: CCACHE_TEMPDIR: /tmp/.ccache-temp strategy: matrix: - os: [ubuntu-latest, ubuntu-18.04] + os: [ubuntu-22.04, ubuntu-20.04] steps: - - uses: actions/checkout@v1 + - uses: actions/checkout@v3 with: submodules: recursive - - uses: actions/cache@v2 + - uses: actions/cache@v3 with: path: ~/.ccache key: ccache-${{ runner.os }}-build-${{ matrix.os }}-${{ github.sha }} @@ -101,14 +104,14 @@ jobs: ${{env.BUILD_DEFAULT_LINUX}} libwallet-ubuntu: - runs-on: ubuntu-latest + runs-on: ubuntu-20.04 env: CCACHE_TEMPDIR: /tmp/.ccache-temp steps: - - uses: actions/checkout@v1 + - uses: actions/checkout@v3 with: submodules: recursive - - uses: actions/cache@v2 + - uses: actions/cache@v3 with: path: ~/.ccache key: ccache-${{ runner.os }}-libwallet-${{ github.sha }} @@ -129,15 +132,15 @@ jobs: test-ubuntu: needs: build-ubuntu - runs-on: ubuntu-latest + runs-on: ubuntu-20.04 env: CCACHE_TEMPDIR: /tmp/.ccache-temp steps: - - uses: actions/checkout@v1 + - uses: actions/checkout@v3 with: submodules: recursive - name: ccache - uses: actions/cache@v2 + uses: actions/cache@v3 with: path: ~/.ccache key: ccache-${{ runner.os }}-build-ubuntu-latest-${{ github.sha }} @@ -151,10 +154,11 @@ jobs: - name: install monero dependencies run: ${{env.APT_INSTALL_LINUX}} - name: install Python dependencies - run: pip install requests psutil monotonic + run: pip install requests psutil monotonic zmq - name: tests env: CTEST_OUTPUT_ON_FAILURE: ON + DNS_PUBLIC: tcp://9.9.9.9 run: | ${{env.CCACHE_SETTINGS}} ${{env.BUILD_DEFAULT_LINUX}} @@ -166,8 +170,9 @@ jobs: source-archive: runs-on: ubuntu-20.04 steps: - - uses: actions/checkout@v1 + - uses: actions/checkout@v3 with: + fetch-depth: 0 submodules: recursive - name: archive run: | @@ -176,7 +181,7 @@ jobs: export OUTPUT="$VERSION.tar" echo "OUTPUT=$OUTPUT" >> $GITHUB_ENV /home/runner/.local/bin/git-archive-all --prefix "$VERSION/" --force-submodules "$OUTPUT" - - uses: actions/upload-artifact@v2 + - uses: actions/upload-artifact@v3 with: name: ${{ env.OUTPUT }} path: /home/runner/work/monero/monero/${{ env.OUTPUT }} diff --git a/.github/workflows/depends.yml b/.github/workflows/depends.yml index f20bf949fd5..27b29450300 100644 --- a/.github/workflows/depends.yml +++ b/.github/workflows/depends.yml @@ -2,6 +2,9 @@ name: ci/gh-actions/depends on: push: + paths-ignore: + - 'docs/**' + - '**/README.md' pull_request: paths-ignore: - 'docs/**' @@ -17,8 +20,8 @@ env: ccache --set-config=compression=true jobs: - build-macos: - runs-on: ubuntu-18.04 + build-cross: + runs-on: ubuntu-20.04 env: CCACHE_TEMPDIR: /tmp/.ccache-temp strategy: @@ -36,13 +39,13 @@ jobs: packages: "python3 gperf g++-aarch64-linux-gnu" - name: "i686 Win" host: "i686-w64-mingw32" - packages: "python3 g++-mingw-w64-i686 qttools5-dev-tools" + packages: "python3 g++-mingw-w64-i686" - name: "i686 Linux" host: "i686-pc-linux-gnu" packages: "gperf cmake g++-multilib python3-zmq" - name: "Win64" host: "x86_64-w64-mingw32" - packages: "cmake python3 g++-mingw-w64-x86-64 qttools5-dev-tools" + packages: "cmake python3 g++-mingw-w64-x86-64" - name: "x86_64 Linux" host: "x86_64-unknown-linux-gnu" packages: "gperf cmake python3-zmq libdbus-1-dev libharfbuzz-dev" @@ -55,21 +58,25 @@ jobs: - name: "x86_64 Freebsd" host: "x86_64-unknown-freebsd" packages: "clang-8 gperf cmake python3-zmq libdbus-1-dev libharfbuzz-dev" + - name: "ARMv8 Android" + host: "aarch64-linux-android" + packages: "gperf cmake python3" name: ${{ matrix.toolchain.name }} steps: - - uses: actions/checkout@v1 + - uses: actions/checkout@v3 with: + fetch-depth: 0 submodules: recursive # Most volatile cache - name: ccache - uses: actions/cache@v2 + uses: actions/cache@v3 with: path: ~/.ccache key: ccache-${{ matrix.toolchain.host }}-${{ github.sha }} restore-keys: ccache-${{ matrix.toolchain.host }}- # Less volatile cache - name: depends cache - uses: actions/cache@v2 + uses: actions/cache@v3 with: path: contrib/depends/built key: depends-${{ matrix.toolchain.host }}-${{ hashFiles('contrib/depends/packages/*') }} @@ -78,7 +85,7 @@ jobs: depends-${{ matrix.toolchain.host }}- # Static cache - name: OSX SDK cache - uses: actions/cache@v2 + uses: actions/cache@v3 with: path: contrib/depends/sdk-sources key: sdk-${{ matrix.toolchain.host }}-${{ matrix.toolchain.osx_sdk }} @@ -96,7 +103,7 @@ jobs: run: | ${{env.CCACHE_SETTINGS}} make depends target=${{ matrix.toolchain.host }} -j2 - - uses: actions/upload-artifact@v2 + - uses: actions/upload-artifact@v3 if: ${{ matrix.toolchain.host == 'x86_64-w64-mingw32' || matrix.toolchain.host == 'x86_64-apple-darwin11' || matrix.toolchain.host == 'x86_64-unknown-linux-gnu' }} with: name: ${{ matrix.toolchain.name }} diff --git a/.github/workflows/gitian.yml b/.github/workflows/gitian.yml new file mode 100644 index 00000000000..91e60a88f5b --- /dev/null +++ b/.github/workflows/gitian.yml @@ -0,0 +1,49 @@ +name: ci/gh-actions/gitian + +on: + push: + tags: + - '*' + +jobs: + build-gitian: + runs-on: ubuntu-20.04 + strategy: + fail-fast: false + matrix: + operating-system: + - name: "Linux" + option: "l" + - name: "Windows" + option: "w" + - name: "Android" + option: "a" + - name: "FreeBSD" + option: "f" + - name: "macOS" + option: "m" + name: ${{ matrix.operating-system.name }} + steps: + - name: prepare + run: | + sudo apt update + curl -O https://raw.githubusercontent.com/monero-project/monero/${{ github.ref_name }}/contrib/gitian/gitian-build.py + chmod +x gitian-build.py + - name: setup + run: | + ./gitian-build.py --setup --docker github-actions ${{ github.ref_name }} + - name: build + run: | + ./gitian-build.py --docker --detach-sign --no-commit --build -j 3 -o ${{ matrix.operating-system.option }} github-actions ${{ github.ref_name }} + - name: post build + run: | + cd out/${{ github.ref_name }} + shasum -a256 * + echo \`\`\` >> $GITHUB_STEP_SUMMARY + shasum -a256 * >> $GITHUB_STEP_SUMMARY + echo \`\`\` >> $GITHUB_STEP_SUMMARY + - uses: actions/upload-artifact@v3 + with: + name: ${{ matrix.operating-system.name }} + path: | + out/${{ github.ref_name }}/* diff --git a/.gitignore b/.gitignore index a39168ac563..9f62575e5ab 100644 --- a/.gitignore +++ b/.gitignore @@ -24,6 +24,12 @@ external/miniupnpc/Makefile miniupnpcstrings.h version/ ClangBuildAnalyzerSession.txt + +# gitian +contrib/gitian/builder/ +contrib/gitian/docker/ +contrib/gitian/sigs/ + # Created by https://www.gitignore.io ### C++ ### diff --git a/CMakeLists.txt b/CMakeLists.txt index 3abd0722a1b..d036f745691 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,4 +1,4 @@ -# Copyright (c) 2014-2022, The Monero Project +# Copyright (c) 2014-2023, The Monero Project # # All rights reserved. # @@ -412,7 +412,7 @@ if (CMAKE_BUILD_TYPE STREQUAL "Debug") else() set(DEFAULT_BUILD_DEBUG_UTILITIES OFF) endif() -option(BUILD_DEBUG_UTILITIES "Build debug utilities." DEFAULT_BUILD_DEBUG_UTILITIES) +option(BUILD_DEBUG_UTILITIES "Build debug utilities." ${DEFAULT_BUILD_DEBUG_UTILITIES}) if(OSSFUZZ) message(STATUS "Using OSS-Fuzz fuzzing system") @@ -449,15 +449,12 @@ endif() # Check if we're on OpenBSD. See the README.md for build instructions. if(CMAKE_SYSTEM_NAME MATCHES "kOpenBSD.*|OpenBSD.*") set(OPENBSD TRUE) +elseif(CMAKE_SYSTEM_NAME MATCHES "kNetBSD.*|NetBSD.*") + set(NETBSD TRUE) +elseif(CMAKE_SYSTEM_NAME MATCHES ".*BSDI.*") + set(BSDI TRUE) endif() -# TODO: check bsdi, NetBSD, to see if they need the same FreeBSD changes -# -# elseif(CMAKE_SYSTEM_NAME MATCHES "kNetBSD.*|NetBSD.*") -# set(NETBSD TRUE) -# elseif(CMAKE_SYSTEM_NAME MATCHES ".*BSDI.*") -# set(BSDI TRUE) - include_directories(external/rapidjson/include external/easylogging++ src contrib/epee/include external external/supercop/include) if(APPLE) @@ -531,7 +528,7 @@ add_definitions("-DBLOCKCHAIN_DB=${BLOCKCHAIN_DB}") # Can't install hook in static build on OSX, because OSX linker does not support --wrap # On ARM, having libunwind package (with .so's only) installed breaks static link. # When possible, avoid stack tracing using libunwind in favor of using easylogging++. -if (APPLE) +if (APPLE OR NETBSD) set(DEFAULT_STACK_TRACE OFF) set(LIBUNWIND_LIBRARIES "") elseif (DEPENDS AND NOT LINUX) @@ -794,7 +791,7 @@ else() set(USE_LTO_DEFAULT false) set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,--stack,10485760") if(NOT BUILD_64) - add_definitions(-DWINVER=0x0501 -D_WIN32_WINNT=0x0501) + add_definitions(-DWINVER=0x0600 -D_WIN32_WINNT=0x0600) endif() endif() set(C_WARNINGS "-Waggregate-return -Wnested-externs -Wold-style-definition -Wstrict-prototypes") @@ -1076,6 +1073,7 @@ if(STATIC) set(Boost_USE_STATIC_RUNTIME ON) endif() find_package(Boost 1.58 QUIET REQUIRED COMPONENTS system filesystem thread date_time chrono regex serialization program_options locale) +add_definitions(-DBOOST_ASIO_ENABLE_SEQUENTIAL_STRAND_ALLOCATION) set(CMAKE_FIND_LIBRARY_SUFFIXES ${OLD_LIB_SUFFIXES}) if(NOT Boost_FOUND) @@ -1138,7 +1136,9 @@ if (HIDAPI_FOUND OR LibUSB_COMPILE_TEST_PASSED) endif() endif() -option(USE_READLINE "Build with GNU readline support." ON) +if(NOT OPENBSD) + option(USE_READLINE "Build with GNU readline support." ON) +endif() if(USE_READLINE AND NOT DEPENDS) find_package(Readline) if(READLINE_FOUND AND GNU_READLINE_FOUND) @@ -1177,6 +1177,9 @@ find_library(NORM_LIBRARY norm) find_library(GSSAPI_LIBRARY gssapi_krb5) find_library(PROTOLIB_LIBRARY protolib) find_library(SODIUM_LIBRARY sodium) +find_library(BSD_LIBRARY bsd) +find_library(MD_LIBRARY md) +find_library(PROTOKIT_LIBRARY protokit) if(NOT ZMQ_INCLUDE_PATH) message(FATAL_ERROR "Could not find required header zmq.h") @@ -1199,6 +1202,15 @@ endif() if(SODIUM_LIBRARY) set(ZMQ_LIB "${ZMQ_LIB};${SODIUM_LIBRARY}") endif() +if(BSD_LIBRARY) + set(ZMQ_LIB "${ZMQ_LIB};${BSD_LIBRARY}") +endif() +if(MD_LIBRARY) + set(ZMQ_LIB "${ZMQ_LIB};${MD_LIBRARY}") +endif() +if(PROTOKIT_LIBRARY) + set(ZMQ_LIB "${ZMQ_LIB};${PROTOKIT_LIBRARY}") +endif() include(external/supercop/functions.cmake) # place after setting flags and before src directory inclusion add_subdirectory(contrib) diff --git a/CMakeLists_IOS.txt b/CMakeLists_IOS.txt index 21aa92094bd..f2d44ac8810 100644 --- a/CMakeLists_IOS.txt +++ b/CMakeLists_IOS.txt @@ -1,4 +1,4 @@ -# Portions Copyright (c) 2017-2022, The Monero Project +# Portions Copyright (c) 2017-2023, The Monero Project # This file is based off of the https://code.google.com/archive/p/ios-cmake/ # It has been altered for Monero iOS development # diff --git a/Doxyfile b/Doxyfile index 81b2009865b..9ce0d2247f1 100644 --- a/Doxyfile +++ b/Doxyfile @@ -754,7 +754,7 @@ WARN_LOGFILE = # spaces. # Note: If this tag is empty the current directory is searched. -INPUT = . +INPUT = contrib/epee external/easylogging++ src # This tag can be used to specify the character encoding of the source files # that doxygen parses. Internally doxygen uses the UTF-8 encoding. Doxygen uses @@ -805,7 +805,7 @@ EXCLUDE_SYMLINKS = NO # Note that the wildcards are matched against the file with absolute path, so to # exclude all test directories for example use the pattern */test/* -EXCLUDE_PATTERNS = */build/* */contrib/depends/* +EXCLUDE_PATTERNS = */src/crypto/crypto_ops_builder/ref10* # The EXCLUDE_SYMBOLS tag can be used to specify one or more symbol names # (namespaces, classes, functions, etc.) that should be excluded from the diff --git a/LICENSE b/LICENSE index c55c48771b7..74b9a8118a9 100644 --- a/LICENSE +++ b/LICENSE @@ -1,4 +1,4 @@ -Copyright (c) 2014-2022, The Monero Project +Copyright (c) 2014-2023, The Monero Project All rights reserved. diff --git a/Makefile b/Makefile index a07ac77a108..cb6a24aab7a 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -# Copyright (c) 2014-2022, The Monero Project +# Copyright (c) 2014-2023, The Monero Project # # All rights reserved. # diff --git a/README.md b/README.md index b3422280d61..7689779c06b 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Monero -Copyright (c) 2014-2022 The Monero Project. +Copyright (c) 2014-2023, The Monero Project Portions Copyright (c) 2012-2013 The Cryptonote developers. ## Table of Contents @@ -20,6 +20,7 @@ Portions Copyright (c) 2012-2013 The Cryptonote developers. - [Release staging schedule and protocol](#release-staging-schedule-and-protocol) - [Compiling Monero from source](#compiling-monero-from-source) - [Dependencies](#dependencies) + - [Gitian builds](#gitian-builds) - [Internationalization](#Internationalization) - [Using Tor](#using-tor) - [Pruning](#Pruning) @@ -117,10 +118,11 @@ See [LICENSE](LICENSE). If you want to help out, see [CONTRIBUTING](docs/CONTRIBUTING.md) for a set of guidelines. -## Scheduled software upgrades +## Scheduled software/network upgrades -Monero uses a fixed-schedule software upgrade (hard fork) mechanism to implement new features. This means that users of Monero (end users and service providers) should run current versions and upgrade their software on a regular schedule. Software upgrades occur during the months of April and October. The required software for these upgrades will be available prior to the scheduled date. Please check the repository prior to this date for the proper Monero software version. Below is the historical schedule and the projected schedule for the next upgrade. -Dates are provided in the format YYYY-MM-DD. +Monero uses a scheduled software/network upgrade (hard fork) mechanism to implement new features into the Monero software and network. This means that users of Monero (end users and service providers) should run current versions and upgrade their software when new releases are available. Software upgrades occur when new features are developed and implemented in the codebase. Network upgrades occur in tandem with software upgrades that modify the consensus rules of the Monero network. The required software for network upgrades will be available prior to the scheduled network upgrade date. Please check the repository prior to this date for the proper Monero software version. Below is the historical schedule and the projected schedule for the next upgrade. + +Dates are provided in the format YYYY-MM-DD. The "Minimum" is the software version that follows the new consensus rules. The "Recommended" version may include bug fixes and other new features that do not affect the consensus rules. | Software upgrade block height | Date | Fork version | Minimum Monero version | Recommended Monero version | Details | @@ -138,6 +140,8 @@ Dates are provided in the format YYYY-MM-DD. | 1978433 | 2019-11-30 | v12 | v0.15.0.0 | v0.16.0.0 | New PoW based on RandomX, only allow >= 2 outputs, change to the block median used to calculate penalty, v1 coinbases are forbidden, rct sigs in coinbase forbidden, 10 block lock time for incoming outputs | 2210000 | 2020-10-17 | v13 | v0.17.0.0 | v0.17.3.2 | New CLSAG transaction format | 2210720 | 2020-10-18 | v14 | v0.17.1.1 | v0.17.3.2 | forbid old MLSAG transaction format +| 2688888 | 2022-08-13 | v15 | v0.18.0.0 | v0.18.1.2 | ringsize = 16, bulletproofs+, view tags, adjusted dynamic block weight algorithm +| 2689608 | 2022-08-14 | v16 | v0.18.0.0 | v0.18.1.2 | forbid old v14 transaction format | XXXXXXX | XXX-XX-XX | XXX | vX.XX.X.X | vX.XX.X.X | XXX | X's indicate that these details have not been determined as of commit date. @@ -167,16 +171,15 @@ library archives (`.a`). | CMake | 3.5 | NO | `cmake` | `cmake` | `cmake` | `cmake` | NO | | | pkg-config | any | NO | `pkg-config` | `base-devel` | `base-devel` | `pkgconf` | NO | | | Boost | 1.58 | NO | `libboost-all-dev` | `boost` | `boost-devel` | `boost-devel` | NO | C++ libraries | -| OpenSSL | basically any | NO | `libssl-dev` | `openssl` | `libressl-devel` | `openssl-devel` | NO | sha256 sum | +| OpenSSL | basically any | NO | `libssl-dev` | `openssl` | `openssl-devel` | `openssl-devel` | NO | sha256 sum | | libzmq | 4.2.0 | NO | `libzmq3-dev` | `zeromq` | `zeromq-devel` | `zeromq-devel` | NO | ZeroMQ library | | OpenPGM | ? | NO | `libpgm-dev` | `libpgm` | | `openpgm-devel` | NO | For ZeroMQ | | libnorm[2] | ? | NO | `libnorm-dev` | | | | YES | For ZeroMQ | -| libunbound | 1.4.16 | YES | `libunbound-dev` | `unbound` | `unbound-devel` | `unbound-devel` | NO | DNS resolver | +| libunbound | 1.4.16 | NO | `libunbound-dev` | `unbound` | `unbound-devel` | `unbound-devel` | NO | DNS resolver | | libsodium | ? | NO | `libsodium-dev` | `libsodium` | `libsodium-devel` | `libsodium-devel` | NO | cryptography | | libunwind | any | NO | `libunwind8-dev` | `libunwind` | `libunwind-devel` | `libunwind-devel` | YES | Stack traces | | liblzma | any | NO | `liblzma-dev` | `xz` | `liblzma-devel` | `xz-devel` | YES | For libunwind | | libreadline | 6.3.0 | NO | `libreadline6-dev` | `readline` | `readline-devel` | `readline-devel` | YES | Input editing | -| ldns | 1.6.17 | NO | `libldns-dev` | `ldns` | `libldns-devel` | `ldns-devel` | YES | SSL toolkit | | expat | 1.1 | NO | `libexpat1-dev` | `expat` | `expat-devel` | `expat-devel` | YES | XML parsing | | GTest | 1.5 | YES | `libgtest-dev`[1] | `gtest` | `gtest-devel` | `gtest-devel` | YES | Test suite | | ccache | any | NO | `ccache` | `ccache` | `ccache` | `ccache` | YES | Compil. cache | @@ -203,23 +206,23 @@ then: Install all dependencies at once on Debian/Ubuntu: ``` -sudo apt update && sudo apt install build-essential cmake pkg-config libssl-dev libzmq3-dev libunbound-dev libsodium-dev libunwind8-dev liblzma-dev libreadline6-dev libldns-dev libexpat1-dev libpgm-dev qttools5-dev-tools libhidapi-dev libusb-1.0-0-dev libprotobuf-dev protobuf-compiler libudev-dev libboost-chrono-dev libboost-date-time-dev libboost-filesystem-dev libboost-locale-dev libboost-program-options-dev libboost-regex-dev libboost-serialization-dev libboost-system-dev libboost-thread-dev python3 ccache doxygen graphviz +sudo apt update && sudo apt install build-essential cmake pkg-config libssl-dev libzmq3-dev libunbound-dev libsodium-dev libunwind8-dev liblzma-dev libreadline6-dev libexpat1-dev libpgm-dev qttools5-dev-tools libhidapi-dev libusb-1.0-0-dev libprotobuf-dev protobuf-compiler libudev-dev libboost-chrono-dev libboost-date-time-dev libboost-filesystem-dev libboost-locale-dev libboost-program-options-dev libboost-regex-dev libboost-serialization-dev libboost-system-dev libboost-thread-dev python3 ccache doxygen graphviz ``` Install all dependencies at once on Arch: ``` -sudo pacman -Syu --needed base-devel cmake boost openssl zeromq libpgm unbound libsodium libunwind xz readline ldns expat gtest python3 ccache doxygen graphviz qt5-tools hidapi libusb protobuf systemd +sudo pacman -Syu --needed base-devel cmake boost openssl zeromq libpgm unbound libsodium libunwind xz readline expat gtest python3 ccache doxygen graphviz qt5-tools hidapi libusb protobuf systemd ``` Install all dependencies at once on Fedora: ``` -sudo dnf install gcc gcc-c++ cmake pkgconf boost-devel openssl-devel zeromq-devel openpgm-devel unbound-devel libsodium-devel libunwind-devel xz-devel readline-devel ldns-devel expat-devel gtest-devel ccache doxygen graphviz qt5-linguist hidapi-devel libusbx-devel protobuf-devel protobuf-compiler systemd-devel +sudo dnf install gcc gcc-c++ cmake pkgconf boost-devel openssl-devel zeromq-devel openpgm-devel unbound-devel libsodium-devel libunwind-devel xz-devel readline-devel expat-devel gtest-devel ccache doxygen graphviz qt5-linguist hidapi-devel libusbx-devel protobuf-devel protobuf-compiler systemd-devel ``` Install all dependencies at once on openSUSE: ``` -sudo zypper ref && sudo zypper in cppzmq-devel ldns-devel libboost_chrono-devel libboost_date_time-devel libboost_filesystem-devel libboost_locale-devel libboost_program_options-devel libboost_regex-devel libboost_serialization-devel libboost_system-devel libboost_thread-devel libexpat-devel libminiupnpc-devel libsodium-devel libunwind-devel unbound-devel cmake doxygen ccache fdupes gcc-c++ libevent-devel libopenssl-devel pkgconf-pkg-config readline-devel xz-devel libqt5-qttools-devel patterns-devel-C-C++-devel_C_C++ +sudo zypper ref && sudo zypper in cppzmq-devel libboost_chrono-devel libboost_date_time-devel libboost_filesystem-devel libboost_locale-devel libboost_program_options-devel libboost_regex-devel libboost_serialization-devel libboost_system-devel libboost_thread-devel libexpat-devel libminiupnpc-devel libsodium-devel libunwind-devel unbound-devel cmake doxygen ccache fdupes gcc-c++ libevent-devel libopenssl-devel pkgconf-pkg-config readline-devel xz-devel libqt5-qttools-devel patterns-devel-C-C++-devel_C_C++ ``` Install all dependencies at once on macOS with the provided Brewfile: @@ -264,7 +267,7 @@ invokes cmake commands as needed. ```bash cd monero - git checkout release-v0.17 + git checkout release-v0.18 make ``` @@ -343,7 +346,7 @@ Tested on a Raspberry Pi Zero with a clean install of minimal Raspbian Stretch ( ```bash git clone https://github.com/monero-project/monero.git cd monero - git checkout v0.17.3.2 + git checkout v0.18.1.2 ``` * Build: @@ -462,10 +465,10 @@ application. cd monero ``` -* If you would like a specific [version/tag](https://github.com/monero-project/monero/tags), do a git checkout for that version. eg. 'v0.17.3.2'. If you don't care about the version and just want binaries from master, skip this step: +* If you would like a specific [version/tag](https://github.com/monero-project/monero/tags), do a git checkout for that version. eg. 'v0.18.1.2'. If you don't care about the version and just want binaries from master, skip this step: ```bash - git checkout v0.17.3.2 + git checkout v0.18.1.2 ``` * If you are on a 64-bit system, run: @@ -505,12 +508,12 @@ Monero is also available as a port or package as `monero-cli`. ### On OpenBSD: -You will need to add a few packages to your system. `pkg_add cmake gmake zeromq libiconv boost`. +You will need to add a few packages to your system. `pkg_add cmake gmake zeromq libiconv boost libunbound`. The `doxygen` and `graphviz` packages are optional and require the xbase set. -Running the test suite also requires `py-requests` package. +Running the test suite also requires `py3-requests` package. -Build monero: `env DEVELOPER_LOCAL_TOOLS=1 BOOST_ROOT=/usr/local gmake release-static` +Build monero: `gmake` Note: you may encounter the following error when compiling the latest version of Monero as a normal user: @@ -561,6 +564,8 @@ You can also cross-compile static binaries on Linux for Windows and macOS with t * ```make depends target=x86_64-linux-gnu``` for 64-bit linux binaries. * ```make depends target=x86_64-w64-mingw32``` for 64-bit windows binaries. * Requires: `python3 g++-mingw-w64-x86-64 wine1.6 bc` + * You also need to run: +```update-alternatives --set x86_64-w64-mingw32-g++ x86_64-w64-mingw32-g++-posix && update-alternatives --set x86_64-w64-mingw32-gcc x86_64-w64-mingw32-gcc-posix``` * ```make depends target=x86_64-apple-darwin11``` for macOS binaries. * Requires: `cmake imagemagick libcap-dev librsvg2-bin libz-dev libbz2-dev libtiff-tools python-dev` * ```make depends target=i686-linux-gnu``` for 32-bit linux binaries. @@ -579,12 +584,16 @@ You can also cross-compile static binaries on Linux for Windows and macOS with t * ```make depends target=aarch64-linux-android``` for 64bit android binaries -The required packages are the names for each toolchain on apt. Depending on your distro, they may have different names. +The required packages are the names for each toolchain on apt. Depending on your distro, they may have different names. The `depends` system has been tested on Ubuntu 18.04 and 20.04. Using `depends` might also be easier to compile Monero on Windows than using MSYS. Activate Windows Subsystem for Linux (WSL) with a distro (for example Ubuntu), install the apt build-essentials and follow the `depends` steps as depicted above. The produced binaries still link libc dynamically. If the binary is compiled on a current distribution, it might not run on an older distribution with an older installation of libc. Passing `-DBACKCOMPAT=ON` to cmake will make sure that the binary will run on systems having at least libc version 2.17. +### Gitian builds + +See [contrib/gitian/README.md](contrib/gitian/README.md). + ## Installing Monero from a package **DISCLAIMER: These packages are not part of this repository or maintained by this project's contributors, and as such, do not go through the same review process to ensure their trustworthiness and security.** diff --git a/cmake/32-bit-toolchain.cmake b/cmake/32-bit-toolchain.cmake index eb28953a437..0ae8394ab1a 100644 --- a/cmake/32-bit-toolchain.cmake +++ b/cmake/32-bit-toolchain.cmake @@ -1,4 +1,4 @@ -# Copyright (c) 2014-2022, The Monero Project +# Copyright (c) 2014-2023, The Monero Project # # All rights reserved. # diff --git a/cmake/64-bit-toolchain.cmake b/cmake/64-bit-toolchain.cmake index 2ec22b8e54c..1c1a95efd93 100644 --- a/cmake/64-bit-toolchain.cmake +++ b/cmake/64-bit-toolchain.cmake @@ -1,4 +1,4 @@ -# Copyright (c) 2014-2022, The Monero Project +# Copyright (c) 2014-2023, The Monero Project # # All rights reserved. # diff --git a/cmake/FindCcache.cmake b/cmake/FindCcache.cmake index d3f5f829bf6..c2cfe97897a 100644 --- a/cmake/FindCcache.cmake +++ b/cmake/FindCcache.cmake @@ -1,4 +1,4 @@ -# Copyright (c) 2014-2022, The Monero Project +# Copyright (c) 2014-2023, The Monero Project # # All rights reserved. # diff --git a/cmake/FindLibUSB.cmake b/cmake/FindLibUSB.cmake index 7f8a11460aa..f780628f80e 100644 --- a/cmake/FindLibUSB.cmake +++ b/cmake/FindLibUSB.cmake @@ -113,7 +113,7 @@ if ( LibUSB_FOUND ) if (APPLE OR LibUSB_VERSION_1.0.16 OR STATIC) if (APPLE) if(DEPENDS) - list(APPEND TEST_COMPILE_EXTRA_LIBRARIES "-framework Foundation -framework IOKit") + list(APPEND TEST_COMPILE_EXTRA_LIBRARIES "-framework Foundation -framework IOKit -framework Security") else() find_library(COREFOUNDATION CoreFoundation) find_library(IOKIT IOKit) diff --git a/cmake/FindUnbound.cmake b/cmake/FindUnbound.cmake index 611b047abc4..e6bfc8e2248 100644 --- a/cmake/FindUnbound.cmake +++ b/cmake/FindUnbound.cmake @@ -1,4 +1,4 @@ -# Copyright (c) 2014-2022, The Monero Project +# Copyright (c) 2014-2023, The Monero Project # All rights reserved. # # Redistribution and use in source and binary forms, with or without modification, are diff --git a/cmake/GitVersion.cmake b/cmake/GitVersion.cmake index 80f6b506d74..0dd7318816f 100644 --- a/cmake/GitVersion.cmake +++ b/cmake/GitVersion.cmake @@ -1,4 +1,4 @@ -# Copyright (c) 2014-2022, The Monero Project +# Copyright (c) 2014-2023, The Monero Project # # All rights reserved. # diff --git a/cmake/SetClangTidy.cmake b/cmake/SetClangTidy.cmake index 88dfe4be209..7288590bf3a 100644 --- a/cmake/SetClangTidy.cmake +++ b/cmake/SetClangTidy.cmake @@ -1,4 +1,4 @@ -# Copyright (c) 2014-2022, The Monero Project +# Copyright (c) 2014-2023, The Monero Project # # All rights reserved. # diff --git a/cmake/Version.cmake b/cmake/Version.cmake index e9ed9fba159..cfb0e32a7f5 100644 --- a/cmake/Version.cmake +++ b/cmake/Version.cmake @@ -1,4 +1,4 @@ -# Copyright (c) 2014-2022, The Monero Project +# Copyright (c) 2014-2023, The Monero Project # # All rights reserved. # diff --git a/cmake/test-libusb-version.c b/cmake/test-libusb-version.c index 9f22416b81a..f1e5d8ea8ca 100644 --- a/cmake/test-libusb-version.c +++ b/cmake/test-libusb-version.c @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2022, The Monero Project +// Copyright (c) 2014-2023, The Monero Project // // All rights reserved. // diff --git a/cmake/test-protobuf.cpp b/cmake/test-protobuf.cpp index f1665969e68..2b658a6058d 100644 --- a/cmake/test-protobuf.cpp +++ b/cmake/test-protobuf.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2022, The Monero Project +// Copyright (c) 2014-2023, The Monero Project // // All rights reserved. // diff --git a/cmake/test-static-assert.c b/cmake/test-static-assert.c index 1697bfeb97a..6d05d69ff1f 100644 --- a/cmake/test-static-assert.c +++ b/cmake/test-static-assert.c @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2022, The Monero Project +// Copyright (c) 2014-2023, The Monero Project // // All rights reserved. // diff --git a/cmake/test-static-assert.cpp b/cmake/test-static-assert.cpp index 1697bfeb97a..6d05d69ff1f 100644 --- a/cmake/test-static-assert.cpp +++ b/cmake/test-static-assert.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2022, The Monero Project +// Copyright (c) 2014-2023, The Monero Project // // All rights reserved. // diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index 10296514c90..631d899782e 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -1,4 +1,4 @@ -# Copyright (c) 2014-2022, The Monero Project +# Copyright (c) 2014-2023, The Monero Project # # All rights reserved. # diff --git a/contrib/brew/Brewfile b/contrib/brew/Brewfile index 36b073b2587..c74e7b2a255 100644 --- a/contrib/brew/Brewfile +++ b/contrib/brew/Brewfile @@ -25,7 +25,6 @@ brew "unbound" brew "libsodium" brew "miniupnpc" brew "readline" -brew "ldns" brew "expat" brew "ccache" brew "doxygen" diff --git a/contrib/depends/Makefile b/contrib/depends/Makefile index 3df1d677eef..4b60ee054cd 100644 --- a/contrib/depends/Makefile +++ b/contrib/depends/Makefile @@ -110,8 +110,7 @@ $(host_arch)_$(host_os)_id_string+=$(shell $(host_CXX) --version 2>/dev/null) $(host_arch)_$(host_os)_id_string+=$(shell $(host_RANLIB) --version 2>/dev/null) $(host_arch)_$(host_os)_id_string+=$(shell $(host_STRIP) --version 2>/dev/null) -qt_packages_$(NO_QT) = $(qt_packages) -packages += $($(host_arch)_$(host_os)_packages) $($(host_os)_packages) $(qt_packages_) +packages += $($(host_arch)_$(host_os)_packages) $($(host_os)_packages) native_packages += $($(host_arch)_$(host_os)_native_packages) $($(host_os)_native_packages) all_packages = $(packages) $(native_packages) @@ -122,17 +121,7 @@ $(host_arch)_$(host_os)_native_toolchain?=$($(host_os)_native_toolchain) include funcs.mk -CONF_PKGS := cmake-conf mxe-conf - -build-only-$(1)_$(3): CMAKE_RUNRESULT_FILE = $(PREFIX)/share/cmake/modules/TryRunResults.cmake -build-only-$(1)_$(3): CMAKE_TOOLCHAIN_FILE = $(PREFIX)/$(3)/share/cmake/mxe-conf.cmake -build-only-$(1)_$(3): CMAKE_TOOLCHAIN_DIR = $(PREFIX)/$(3)/share/cmake/mxe-conf.d -build-only-$(1)_$(3): CMAKE_STATIC_BOOL = $(if $(findstring shared,$(3)),OFF,ON) -build-only-$(1)_$(3): CMAKE_SHARED_BOOL = $(if $(findstring shared,$(3)),ON,OFF) - - toolchain_path=$($($(host_arch)_$(host_os)_native_toolchain)_prefixbin) -final_build_id_long+=$(shell $(build_SHA256SUM) config.site.in) final_build_id_long+=$(shell $(build_SHA256SUM) toolchain.cmake.in) final_build_id+=$(shell echo -n "$(final_build_id_long)" | $(build_SHA256SUM) | cut -c-$(HASH_LENGTH)) $(host_prefix)/.stamp_$(final_build_id): $(native_packages) $(packages) @@ -143,26 +132,6 @@ $(host_prefix)/.stamp_$(final_build_id): $(native_packages) $(packages) $(AT)cd $(@D); $(foreach package,$^, tar xf $($(package)_cached); ) $(AT)touch $@ -$(host_prefix)/share/config.site : config.site.in $(host_prefix)/.stamp_$(final_build_id) - $(AT)@mkdir -p $(@D) - $(AT)sed -e 's|@HOST@|$(host)|' \ - -e 's|@CC@|$(toolchain_path)$(host_CC)|' \ - -e 's|@CXX@|$(toolchain_path)$(host_CXX)|' \ - -e 's|@AR@|$(toolchain_path)$(host_AR)|' \ - -e 's|@RANLIB@|$(toolchain_path)$(host_RANLIB)|' \ - -e 's|@NM@|$(toolchain_path)$(host_NM)|' \ - -e 's|@STRIP@|$(toolchain_path)$(host_STRIP)|' \ - -e 's|@build_os@|$(build_os)|' \ - -e 's|@host_os@|$(host_os)|' \ - -e 's|@CFLAGS@|$(strip $(host_CFLAGS) $(host_$(release_type)_CFLAGS))|' \ - -e 's|@CXXFLAGS@|$(strip $(host_CXXFLAGS) $(host_$(release_type)_CXXFLAGS))|' \ - -e 's|@CPPFLAGS@|$(strip $(host_CPPFLAGS) $(host_$(release_type)_CPPFLAGS))|' \ - -e 's|@LDFLAGS@|$(strip $(host_LDFLAGS) $(host_$(release_type)_LDFLAGS))|' \ - -e 's|@allow_host_packages@|$(ALLOW_HOST_PACKAGES)|' \ - -e 's|@debug@|$(DEBUG)|' \ - $< > $@ - $(AT)touch $@ - $(host_prefix)/share/toolchain.cmake : toolchain.cmake.in $(host_prefix)/.stamp_$(final_build_id) $(AT)@mkdir -p $(@D) $(AT)sed -e 's|@HOST@|$(host)|' \ @@ -207,12 +176,10 @@ check-packages: check-sources: @$(foreach package,$(all_packages),$(call check_or_remove_sources,$(package));) -$(host_prefix)/share/config.site: check-packages $(host_prefix)/share/toolchain.cmake: check-packages check-packages: check-sources -install: check-packages $(host_prefix)/share/config.site install: check-packages $(host_prefix)/share/toolchain.cmake download-one: check-sources $(all_sources) diff --git a/contrib/depends/README.md b/contrib/depends/README.md index 1aa5b276fc6..2f1d8424a00 100644 --- a/contrib/depends/README.md +++ b/contrib/depends/README.md @@ -1,6 +1,6 @@ ### Usage -To build dependencies for the current arch+OS: +To build dependencies for the current arch+OS, from this working directory: ```bash make @@ -20,10 +20,12 @@ make HOST=x86_64-w64-mingw32 -j4 A toolchain will be generated that's suitable for plugging into Monero's cmake. In the above example, a dir named x86_64-w64-mingw32 will be -created. To use it for Monero: +created. To use it for Monero, from the top of the Monero source tree: ```bash -cmake -DCMAKE_TOOLCHAIN=`pwd`/contrib/depends/x86_64-w64-mingw32 +mkdir build +cd build +cmake -DCMAKE_TOOLCHAIN_FILE=$PWD/../contrib/depends/x86_64-w64-mingw32/share/toolchain.cmake .. ``` Common `host-platform-triplets` for cross compilation are: diff --git a/contrib/depends/config.site.in b/contrib/depends/config.site.in deleted file mode 100644 index dd91bcb2aa4..00000000000 --- a/contrib/depends/config.site.in +++ /dev/null @@ -1,81 +0,0 @@ -depends_prefix="`dirname ${ac_site_file}`/.." - -cross_compiling=maybe -host_alias=@HOST@ -ac_tool_prefix=${host_alias}- - -if test -z $with_boost; then - with_boost=$depends_prefix -fi -if test -z $with_qt_plugindir; then - with_qt_plugindir=$depends_prefix/plugins -fi -if test -z $with_qt_translationdir; then - with_qt_translationdir=$depends_prefix/translations -fi - -if test x@host_os@ = xdarwin; then - BREW=no - PORT=no -fi - -if test x@host_os@ = xmingw32; then - if test -z $with_qt_incdir; then - with_qt_incdir=$depends_prefix/include - fi - if test -z $with_qt_libdir; then - with_qt_libdir=$depends_prefix/lib - fi -fi - -PATH=$depends_prefix/native/bin:$PATH -PKG_CONFIG="`which pkg-config` --static" - -# These two need to remain exported because pkg-config does not see them -# otherwise. That means they must be unexported at the end of configure.ac to -# avoid ruining the cache. Sigh. -export PKG_CONFIG_PATH=$depends_prefix/share/pkgconfig:$depends_prefix/lib/pkgconfig -if test -z "@allow_host_packages@"; then - export PKGCONFIG_LIBDIR= -fi - -CPPFLAGS="-I$depends_prefix/include/ $CPPFLAGS" -LDFLAGS="-L$depends_prefix/lib $LDFLAGS" - -CC="@CC@" -CXX="@CXX@" -OBJC="${CC}" -CCACHE=$depends_prefix/native/bin/ccache -PYTHONPATH=$depends_prefix/native/lib/python/dist-packages:$PYTHONPATH - -if test -n "@AR@"; then - AR=@AR@ - ac_cv_path_ac_pt_AR=${AR} -fi - -if test -n "@RANLIB@"; then - RANLIB=@RANLIB@ - ac_cv_path_ac_pt_RANLIB=${RANLIB} -fi - -if test -n "@NM@"; then - NM=@NM@ - ac_cv_path_ac_pt_NM=${NM} -fi - -if test -n "@debug@"; then - enable_reduce_exports=no -fi - -if test -n "@CFLAGS@"; then - CFLAGS="@CFLAGS@ $CFLAGS" -fi -if test -n "@CXXFLAGS@"; then - CXXFLAGS="@CXXFLAGS@ $CXXFLAGS" -fi -if test -n "@CPPFLAGS@"; then - CPPFLAGS="@CPPFLAGS@ $CPPFLAGS" -fi -if test -n "@LDFLAGS@"; then - LDFLAGS="@LDFLAGS@ $LDFLAGS" -fi diff --git a/contrib/depends/funcs.mk b/contrib/depends/funcs.mk index 80412599085..66555aad7b0 100644 --- a/contrib/depends/funcs.mk +++ b/contrib/depends/funcs.mk @@ -143,8 +143,11 @@ $(1)_config_env+=PKG_CONFIG_PATH=$($($(1)_type)_prefix)/share/pkgconfig $(1)_config_env+=PATH="$(build_prefix)/bin:$(PATH)" $(1)_build_env+=PATH="$(build_prefix)/bin:$(PATH)" $(1)_stage_env+=PATH="$(build_prefix)/bin:$(PATH)" -$(1)_autoconf=./configure --host=$($($(1)_type)_host) --disable-dependency-tracking --prefix=$($($(1)_type)_prefix) $$($(1)_config_opts) CC="$$($(1)_cc)" CXX="$$($(1)_cxx)" +$(1)_autoconf=./configure --host=$($($(1)_type)_host) --prefix=$($($(1)_type)_prefix) $$($(1)_config_opts) CC="$$($(1)_cc)" CXX="$$($(1)_cxx)" +ifneq ($(1),libusb) +$(1)_autoconf += --disable-dependency-tracking +endif ifneq ($($(1)_nm),) $(1)_autoconf += NM="$$($(1)_nm)" endif diff --git a/contrib/depends/packages/graphviz.mk b/contrib/depends/packages/graphviz.mk deleted file mode 100644 index 1c4bc1b7160..00000000000 --- a/contrib/depends/packages/graphviz.mk +++ /dev/null @@ -1,30 +0,0 @@ -package=graphviz -$(package)_version=2.40.1 -$(package)_download_path=www.graphviz.org/pub/graphviz/stable/SOURCES/ -$(package)_file_name=$(package)-$($(package)_version).tar.gz -$(package)_sha256_hash=ca5218fade0204d59947126c38439f432853543b0818d9d728c589dfe7f3a421 - -define $(package)_preprocess_cmds - ./autogen.sh -endef - -define $(package)_set_vars - $(package)_config_opts=--disable-shared --enable-multibye --without-purify --without-curses - $(package)_config_opts_release=--disable-debug-mode - $(package)_config_opts_linux=--with-pic -endef - -define $(package)_config_cmds - $($(package)_autoconf) -endef - -define $(package)_build_cmds - $(MAKE) -endef - -define $(package)_stage_cmds - $(MAKE) DESTDIR=$($(package)_staging_dir) install -endef - -define $(package)_postprocess_cmds -endef diff --git a/contrib/depends/packages/hidapi.mk b/contrib/depends/packages/hidapi.mk index 97e204f17e9..64935a36a74 100644 --- a/contrib/depends/packages/hidapi.mk +++ b/contrib/depends/packages/hidapi.mk @@ -1,8 +1,8 @@ package=hidapi -$(package)_version=0.11.0 -$(package)_download_path=https://github.com/libusb/hidapi/archive +$(package)_version=0.13.1 +$(package)_download_path=https://github.com/libusb/hidapi/archive/refs/tags $(package)_file_name=$(package)-$($(package)_version).tar.gz -$(package)_sha256_hash=391d8e52f2d6a5cf76e2b0c079cfefe25497ba1d4659131297081fc0cd744632 +$(package)_sha256_hash=476a2c9a4dc7d1fc97dd223b84338dbea3809a84caea2dcd887d9778725490e3 $(package)_linux_dependencies=libusb eudev $(package)_patches=missing_win_include.patch diff --git a/contrib/depends/packages/ldns.mk b/contrib/depends/packages/ldns.mk deleted file mode 100644 index 90c63e82152..00000000000 --- a/contrib/depends/packages/ldns.mk +++ /dev/null @@ -1,34 +0,0 @@ -package=ldns -$(package)_version=1.7.1 -$(package)_download_path=https://www.nlnetlabs.nl/downloads/$(package)/ -$(package)_file_name=$(package)-$($(package)_version).tar.gz -$(package)_sha256_hash=8ac84c16bdca60e710eea75782356f3ac3b55680d40e1530d7cea474ac208229 -$(package)_dependencies=openssl - -define $(package)_set_vars - $(package)_config_opts=--disable-shared --enable-static --with-drill - $(package)_config_opts+=--with-ssl=$(host_prefix) - $(package)_config_opts_release=--disable-debug-mode - $(package)_config_opts_linux=--with-pic -endef - -define $(package)_preprocess_cmds - cp -f $(BASEDIR)/config.guess $(BASEDIR)/config.sub . -endef - -define $(package)_config_cmds - $($(package)_autoconf) -endef - -define $(package)_build_cmds - $(MAKE) -endef - -define $(package)_stage_cmds - $(MAKE) DESTDIR=$($(package)_staging_dir) install-h install-lib -endef - -define $(package)_postprocess_cmds - rm lib/*.la -endef - diff --git a/contrib/depends/packages/libICE.mk b/contrib/depends/packages/libICE.mk deleted file mode 100644 index a897d9aed77..00000000000 --- a/contrib/depends/packages/libICE.mk +++ /dev/null @@ -1,23 +0,0 @@ -package=libICE -$(package)_version=1.0.9 -$(package)_download_path=https://xorg.freedesktop.org/releases/individual/lib/ -$(package)_file_name=$(package)-$($(package)_version).tar.bz2 -$(package)_sha256_hash=8f7032f2c1c64352b5423f6b48a8ebdc339cc63064af34d66a6c9aa79759e202 -$(package)_dependencies=xtrans xproto - -define $(package)_set_vars - $(package)_config_opts=--disable-static --disable-docs --disable-specs --without-xsltproc - $(package)_config_opts_linux=--with-pic -endef - -define $(package)_config_cmds - $($(package)_autoconf) -endef - -define $(package)_build_cmds - $(MAKE) -endef - -define $(package)_stage_cmds - $(MAKE) DESTDIR=$($(package)_staging_dir) install -endef diff --git a/contrib/depends/packages/libSM.mk b/contrib/depends/packages/libSM.mk deleted file mode 100644 index 83fcd4cdb56..00000000000 --- a/contrib/depends/packages/libSM.mk +++ /dev/null @@ -1,23 +0,0 @@ -package=libSM -$(package)_version=1.2.2 -$(package)_download_path=https://xorg.freedesktop.org/releases/individual/lib/ -$(package)_file_name=$(package)-$($(package)_version).tar.bz2 -$(package)_sha256_hash=0baca8c9f5d934450a70896c4ad38d06475521255ca63b717a6510fdb6e287bd -$(package)_dependencies=xtrans xproto libICE - -define $(package)_set_vars - $(package)_config_opts=--without-libuuid --without-xsltproc --disable-docs --disable-static - $(package)_config_opts_linux=--with-pic -endef - -define $(package)_config_cmds - $($(package)_autoconf) -endef - -define $(package)_build_cmds - $(MAKE) -endef - -define $(package)_stage_cmds - $(MAKE) DESTDIR=$($(package)_staging_dir) install -endef diff --git a/contrib/depends/packages/libusb.mk b/contrib/depends/packages/libusb.mk index 348c410a7c9..c1d9fe6a9c2 100644 --- a/contrib/depends/packages/libusb.mk +++ b/contrib/depends/packages/libusb.mk @@ -1,8 +1,8 @@ package=libusb -$(package)_version=1.0.22 -$(package)_download_path=https://sourceforge.net/projects/libusb/files/libusb-1.0/libusb-$($(package)_version)/ +$(package)_version=1.0.26 +$(package)_download_path=https://github.com/libusb/libusb/releases/download/v$($(package)_version) $(package)_file_name=$(package)-$($(package)_version).tar.bz2 -$(package)_sha256_hash=75aeb9d59a4fdb800d329a545c2e6799f732362193b465ea198f2aa275518157 +$(package)_sha256_hash=12ce7a61fc9854d1d2a1ffe095f7b5fac19ddba095c259e6067a46500381b5a5 define $(package)_preprocess_cmds autoreconf -i diff --git a/contrib/depends/packages/native_biplist.mk b/contrib/depends/packages/native_biplist.mk deleted file mode 100644 index 3c6e8900f66..00000000000 --- a/contrib/depends/packages/native_biplist.mk +++ /dev/null @@ -1,20 +0,0 @@ -package=native_biplist -$(package)_version=0.9 -$(package)_download_path=https://pypi.python.org/packages/source/b/biplist -$(package)_file_name=biplist-$($(package)_version).tar.gz -$(package)_sha256_hash=b57cadfd26e4754efdf89e9e37de87885f9b5c847b2615688ca04adfaf6ca604 -$(package)_install_libdir=$(build_prefix)/lib/python/dist-packages -$(package)_patches=sorted_list.patch - -define $(package)_preprocess_cmds - patch -p1 < $($(package)_patch_dir)/sorted_list.patch -endef - -define $(package)_build_cmds - python setup.py build -endef - -define $(package)_stage_cmds - mkdir -p $($(package)_install_libdir) && \ - python setup.py install --root=$($(package)_staging_dir) --prefix=$(build_prefix) --install-lib=$($(package)_install_libdir) -endef diff --git a/contrib/depends/packages/native_cdrkit.mk b/contrib/depends/packages/native_cdrkit.mk deleted file mode 100644 index 8243458ec85..00000000000 --- a/contrib/depends/packages/native_cdrkit.mk +++ /dev/null @@ -1,26 +0,0 @@ -package=native_cdrkit -$(package)_version=1.1.11 -$(package)_download_path=https://distro.ibiblio.org/fatdog/source/600/c -$(package)_file_name=cdrkit-$($(package)_version).tar.bz2 -$(package)_sha256_hash=b50d64c214a65b1a79afe3a964c691931a4233e2ba605d793eb85d0ac3652564 -$(package)_patches=cdrkit-deterministic.patch - -define $(package)_preprocess_cmds - patch -p1 < $($(package)_patch_dir)/cdrkit-deterministic.patch -endef - -define $(package)_config_cmds - cmake -DCMAKE_INSTALL_PREFIX=$(build_prefix) -endef - -define $(package)_build_cmds - $(MAKE) genisoimage -endef - -define $(package)_stage_cmds - $(MAKE) DESTDIR=$($(package)_staging_dir) -C genisoimage install -endef - -define $(package)_postprocess_cmds - rm bin/isovfy bin/isoinfo bin/isodump bin/isodebug bin/devdump -endef diff --git a/contrib/depends/packages/native_cmake-unused.mk b/contrib/depends/packages/native_cmake-unused.mk deleted file mode 100644 index c9ab7571141..00000000000 --- a/contrib/depends/packages/native_cmake-unused.mk +++ /dev/null @@ -1,23 +0,0 @@ -package=native_cmake -$(package)_version=3.14.0 -$(package)_version_dot=v3.14 -$(package)_download_path=https://cmake.org/files/$($(package)_version_dot)/ -$(package)_file_name=cmake-$($(package)_version).tar.gz -$(package)_sha256_hash=aa76ba67b3c2af1946701f847073f4652af5cbd9f141f221c97af99127e75502 - -define $(package)_set_vars -$(package)_config_opts= -endef - -define $(package)_config_cmds - ./bootstrap &&\ - ./configure $($(package)_config_opts) -endef - -define $(package)_build_cmd - $(MAKE) -endef - -define $(package)_stage_cmds - $(MAKE) DESTDIR=$($(package)_staging_dir) install -endef diff --git a/contrib/depends/packages/native_ds_store.mk b/contrib/depends/packages/native_ds_store.mk deleted file mode 100644 index f0c6176592c..00000000000 --- a/contrib/depends/packages/native_ds_store.mk +++ /dev/null @@ -1,17 +0,0 @@ -package=native_ds_store -$(package)_version=1.1.0 -$(package)_download_path=https://github.com/al45tair/ds_store/archive/ -$(package)_download_file=v$($(package)_version).tar.gz -$(package)_file_name=$(package)-$($(package)_version).tar.gz -$(package)_sha256_hash=a9f4c0755c6be7224ff7029e188dd262e830bb81e801424841db9eb0780ec8ed -$(package)_install_libdir=$(build_prefix)/lib/python/dist-packages -$(package)_dependencies=native_biplist - -define $(package)_build_cmds - python setup.py build -endef - -define $(package)_stage_cmds - mkdir -p $($(package)_install_libdir) && \ - python setup.py install --root=$($(package)_staging_dir) --prefix=$(build_prefix) --install-lib=$($(package)_install_libdir) -endef diff --git a/contrib/depends/packages/native_libdmg-hfsplus.mk b/contrib/depends/packages/native_libdmg-hfsplus.mk deleted file mode 100644 index a4ffb6046cc..00000000000 --- a/contrib/depends/packages/native_libdmg-hfsplus.mk +++ /dev/null @@ -1,22 +0,0 @@ -package=native_libdmg-hfsplus -$(package)_version=0.1 -$(package)_download_path=https://github.com/theuni/libdmg-hfsplus/archive -$(package)_file_name=libdmg-hfsplus-v$($(package)_version).tar.gz -$(package)_sha256_hash=6569a02eb31c2827080d7d59001869ea14484c281efab0ae7f2b86af5c3120b3 -$(package)_build_subdir=build - -define $(package)_preprocess_cmds - mkdir build -endef - -define $(package)_config_cmds - cmake -DCMAKE_INSTALL_PREFIX:PATH=$(build_prefix)/bin .. -endef - -define $(package)_build_cmds - $(MAKE) -C dmg -endef - -define $(package)_stage_cmds - $(MAKE) DESTDIR=$($(package)_staging_dir) -C dmg install -endef diff --git a/contrib/depends/packages/native_mac_alias.mk b/contrib/depends/packages/native_mac_alias.mk deleted file mode 100644 index 48bd90fb694..00000000000 --- a/contrib/depends/packages/native_mac_alias.mk +++ /dev/null @@ -1,21 +0,0 @@ -package=native_mac_alias -$(package)_version=1.1.0 -$(package)_download_path=https://github.com/al45tair/mac_alias/archive/ -$(package)_download_file=v$($(package)_version).tar.gz -$(package)_file_name=$(package)-$($(package)_version).tar.gz -$(package)_sha256_hash=b10cb44ecb64fc25283fae7a9cf365d2829377d84e37b9c21100aca8757509be -$(package)_install_libdir=$(build_prefix)/lib/python/dist-packages -$(package)_patches=python3.patch - -define $(package)_preprocess_cmds - patch -p1 < $($(package)_patch_dir)/python3.patch -endef - -define $(package)_build_cmds - python setup.py build -endef - -define $(package)_stage_cmds - mkdir -p $($(package)_install_libdir) && \ - python setup.py install --root=$($(package)_staging_dir) --prefix=$(build_prefix) --install-lib=$($(package)_install_libdir) -endef diff --git a/contrib/depends/packages/openssl.mk b/contrib/depends/packages/openssl.mk index 0d8478d4930..990b8509391 100644 --- a/contrib/depends/packages/openssl.mk +++ b/contrib/depends/packages/openssl.mk @@ -1,9 +1,8 @@ package=openssl -$(package)_version=1.1.1l +$(package)_version=1.1.1t $(package)_download_path=https://www.openssl.org/source $(package)_file_name=$(package)-$($(package)_version).tar.gz -$(package)_sha256_hash=0b7a3e5e59c34827fe0c3a74b7ec8baef302b98fa80088d7f9153aa16fa76bd1 -$(package)_patches=fix_darwin.patch +$(package)_sha256_hash=8dee9b24bdb1dcbf0c3d1e9b02fb8f6bf22165e807f45adeb7c9677536859d3b define $(package)_set_vars $(package)_config_env=AR="$($(package)_ar)" ARFLAGS=$($(package)_arflags) RANLIB="$($(package)_ranlib)" CC="$($(package)_cc)" @@ -50,8 +49,7 @@ $(package)_config_opts_x86_64_freebsd=BSD-x86_64 endef define $(package)_preprocess_cmds - sed -i.old 's|"engines", "apps", "test", "util", "tools", "fuzz"|"engines", "tools"|' Configure && \ - patch -p1 < $($(package)_patch_dir)/fix_darwin.patch + sed -i.old 's|"engines", "apps", "test", "util", "tools", "fuzz"|"engines", "tools"|' Configure endef define $(package)_config_cmds diff --git a/contrib/depends/packages/packages.mk b/contrib/depends/packages/packages.mk index 11e2cb7be41..d2d1eca856d 100644 --- a/contrib/depends/packages/packages.mk +++ b/contrib/depends/packages/packages.mk @@ -1,4 +1,4 @@ -packages:=boost openssl zeromq libiconv expat ldns unbound +packages:=boost openssl zeromq libiconv expat unbound # ccache is useless in gitian builds ifneq ($(GITIAN),1) @@ -20,7 +20,6 @@ freebsd_packages = ncurses readline sodium linux_packages = eudev ncurses readline sodium $(hardware_packages) linux_native_packages = $(hardware_native_packages) -qt_packages = qt ifeq ($(build_tests),ON) packages += gtest diff --git a/contrib/depends/packages/qt.mk b/contrib/depends/packages/qt.mk deleted file mode 100644 index 76c50f3fdd9..00000000000 --- a/contrib/depends/packages/qt.mk +++ /dev/null @@ -1,175 +0,0 @@ -PACKAGE=qt -$(package)_version=5.15.1 -$(package)_download_path=https://download.qt.io/official_releases/qt/5.15/$($(package)_version)/submodules -$(package)_suffix=everywhere-src-$($(package)_version).tar.xz -$(package)_file_name=qtbase-$($(package)_suffix) -$(package)_sha256_hash=33960404d579675b7210de103ed06a72613bfc4305443e278e2d32a3eb1f3d8c -$(package)_build_subdir=qtbase -$(package)_qt_libs=corelib -$(package)_patches=fix_qt_pkgconfig.patch fix_no_printer.patch fix_rcc_determinism.patch no-xlib.patch - -$(package)_qttranslations_file_name=qttranslations-$($(package)_suffix) -$(package)_qttranslations_sha256_hash=46e0c0e3a511fbcc803a4146204062e47f6ed43b34d98a3c27372a03b8746bd8 - -$(package)_qttools_file_name=qttools-$($(package)_suffix) -$(package)_qttools_sha256_hash=c98ee5f0f980bf68cbf0c94d62434816a92441733de50bd9adbe9b9055f03498 - -$(package)_extra_sources = $($(package)_qttranslations_file_name) -$(package)_extra_sources += $($(package)_qttools_file_name) - -define $(package)_set_vars -$(package)_config_opts_release = -release -$(package)_config_opts_debug = -debug -$(package)_config_opts += -bindir $(build_prefix)/bin -$(package)_config_opts += -c++std c++11 -$(package)_config_opts += -confirm-license -$(package)_config_opts += -dbus-runtime -$(package)_config_opts += -hostprefix $(build_prefix) -$(package)_config_opts += -no-compile-examples -$(package)_config_opts += -no-cups -$(package)_config_opts += -no-egl -$(package)_config_opts += -no-eglfs -$(package)_config_opts += -no-evdev -$(package)_config_opts += -no-gui -$(package)_config_opts += -no-freetype -$(package)_config_opts += -no-gif -$(package)_config_opts += -no-glib -$(package)_config_opts += -no-icu -$(package)_config_opts += -no-ico -$(package)_config_opts += -no-iconv -$(package)_config_opts += -no-kms -$(package)_config_opts += -no-linuxfb -$(package)_config_opts += -no-libjpeg -$(package)_config_opts += -no-libudev -$(package)_config_opts += -no-mtdev -$(package)_config_opts += -no-openvg -$(package)_config_opts += -no-reduce-relocations -$(package)_config_opts += -no-sql-db2 -$(package)_config_opts += -no-sql-ibase -$(package)_config_opts += -no-sql-oci -$(package)_config_opts += -no-sql-tds -$(package)_config_opts += -no-sql-mysql -$(package)_config_opts += -no-sql-odbc -$(package)_config_opts += -no-sql-psql -$(package)_config_opts += -no-sql-sqlite -$(package)_config_opts += -no-sql-sqlite2 -$(package)_config_opts += -no-use-gold-linker -$(package)_config_opts += -nomake examples -$(package)_config_opts += -nomake tests -$(package)_config_opts += -opensource -$(package)_config_opts += -no-openssl -$(package)_config_opts += -optimized-qmake -$(package)_config_opts += -pch -$(package)_config_opts += -pkg-config -$(package)_config_opts += -prefix $(host_prefix) -$(package)_config_opts += -no-libpng -$(package)_config_opts += -qt-pcre -$(package)_config_opts += -qt-harfbuzz -$(package)_config_opts += -no-zlib -$(package)_config_opts += -static -$(package)_config_opts += -silent -$(package)_config_opts += -v -$(package)_config_opts += -no-feature-bearermanagement -$(package)_config_opts += -no-feature-colordialog -$(package)_config_opts += -no-feature-dial -$(package)_config_opts += -no-feature-filesystemwatcher -$(package)_config_opts += -no-feature-fontcombobox -$(package)_config_opts += -no-feature-ftp -$(package)_config_opts += -no-feature-image_heuristic_mask -$(package)_config_opts += -no-feature-keysequenceedit -$(package)_config_opts += -no-feature-lcdnumber -$(package)_config_opts += -no-feature-pdf -$(package)_config_opts += -no-feature-printdialog -$(package)_config_opts += -no-feature-printer -$(package)_config_opts += -no-feature-printpreviewdialog -$(package)_config_opts += -no-feature-printpreviewwidget -$(package)_config_opts += -no-feature-sessionmanager -$(package)_config_opts += -no-feature-sql -$(package)_config_opts += -no-feature-statemachine -$(package)_config_opts += -no-feature-syntaxhighlighter -$(package)_config_opts += -no-feature-textbrowser -$(package)_config_opts += -no-feature-textodfwriter -$(package)_config_opts += -no-feature-topleveldomain -$(package)_config_opts += -no-feature-udpsocket -$(package)_config_opts += -no-feature-undocommand -$(package)_config_opts += -no-feature-undogroup -$(package)_config_opts += -no-feature-undostack -$(package)_config_opts += -no-feature-undoview -$(package)_config_opts += -no-feature-vnc -$(package)_config_opts += -no-feature-wizard -$(package)_config_opts_linux = -no-fontconfig -$(package)_config_opts_linux += -no-opengl -$(package)_config_opts_linux += -no-xcb -$(package)_config_opts_linux += -no-feature-xlib -endef - -define $(package)_fetch_cmds -$(call fetch_file,$(package),$($(package)_download_path),$($(package)_download_file),$($(package)_file_name),$($(package)_sha256_hash)) && \ -$(call fetch_file,$(package),$($(package)_download_path),$($(package)_qttranslations_file_name),$($(package)_qttranslations_file_name),$($(package)_qttranslations_sha256_hash)) && \ -$(call fetch_file,$(package),$($(package)_download_path),$($(package)_qttools_file_name),$($(package)_qttools_file_name),$($(package)_qttools_sha256_hash)) -endef - -define $(package)_extract_cmds - mkdir -p $($(package)_extract_dir) && \ - echo "$($(package)_sha256_hash) $($(package)_source)" > $($(package)_extract_dir)/.$($(package)_file_name).hash && \ - echo "$($(package)_qttranslations_sha256_hash) $($(package)_source_dir)/$($(package)_qttranslations_file_name)" >> $($(package)_extract_dir)/.$($(package)_file_name).hash && \ - echo "$($(package)_qttools_sha256_hash) $($(package)_source_dir)/$($(package)_qttools_file_name)" >> $($(package)_extract_dir)/.$($(package)_file_name).hash && \ - $(build_SHA256SUM) -c $($(package)_extract_dir)/.$($(package)_file_name).hash && \ - mkdir qtbase && \ - tar --strip-components=1 -xf $($(package)_source) -C qtbase && \ - mkdir qttranslations && \ - tar --strip-components=1 -xf $($(package)_source_dir)/$($(package)_qttranslations_file_name) -C qttranslations && \ - mkdir qttools && \ - tar --strip-components=1 -xf $($(package)_source_dir)/$($(package)_qttools_file_name) -C qttools -endef - - -define $(package)_preprocess_cmds - sed -i.old "s|FT_Get_Font_Format|FT_Get_X11_Font_Format|" qtbase/src/platformsupport/fontdatabases/freetype/qfontengine_ft.cpp && \ - sed -i.old "s|updateqm.commands = \$$$$\$$$$LRELEASE|updateqm.commands = $($(package)_extract_dir)/qttools/bin/lrelease|" qttranslations/translations/translations.pro && \ - sed -i.old "/updateqm.depends =/d" qttranslations/translations/translations.pro && \ - sed -i.old "s/src_plugins.depends = src_sql src_network/src_plugins.depends = src_network/" qtbase/src/src.pro && \ - cp -r qtbase/mkspecs/linux-arm-gnueabi-g++ qtbase/mkspecs/bitcoin-linux-g++ && \ - sed -i.old "s/arm-linux-gnueabi-/$(host)-/g" qtbase/mkspecs/bitcoin-linux-g++/qmake.conf && \ - patch -p1 -i $($(package)_patch_dir)/fix_qt_pkgconfig.patch && \ - patch -p1 -i $($(package)_patch_dir)/fix_no_printer.patch && \ - echo "!host_build: QMAKE_CFLAGS += $($(package)_cflags) $($(package)_cppflags)" >> qtbase/mkspecs/common/gcc-base.conf && \ - echo "!host_build: QMAKE_CXXFLAGS += $($(package)_cxxflags) $($(package)_cppflags)" >> qtbase/mkspecs/common/gcc-base.conf && \ - echo "!host_build: QMAKE_LFLAGS += $($(package)_ldflags)" >> qtbase/mkspecs/common/gcc-base.conf && \ - patch -p1 -i $($(package)_patch_dir)/no-xlib.patch && \ - echo "QMAKE_LINK_OBJECT_MAX = 10" >> qtbase/mkspecs/win32-g++/qmake.conf && \ - echo "QMAKE_LINK_OBJECT_SCRIPT = object_script" >> qtbase/mkspecs/win32-g++/qmake.conf && \ - sed -i.old "s|QMAKE_CFLAGS += |!host_build: QMAKE_CFLAGS = $($(package)_cflags) $($(package)_cppflags) |" qtbase/mkspecs/win32-g++/qmake.conf && \ - sed -i.old "s|QMAKE_CXXFLAGS += |!host_build: QMAKE_CXXFLAGS = $($(package)_cxxflags) $($(package)_cppflags) |" qtbase/mkspecs/win32-g++/qmake.conf && \ - sed -i.old "0,/^QMAKE_LFLAGS_/s|^QMAKE_LFLAGS_|!host_build: QMAKE_LFLAGS = $($(package)_ldflags)\n&|" qtbase/mkspecs/win32-g++/qmake.conf && \ - sed -i.old "s/LIBRARY_PATH/(CROSS_)?\0/g" qtbase/mkspecs/features/toolchain.prf -endef - -define $(package)_config_cmds - export PKG_CONFIG_SYSROOT_DIR=/ && \ - export PKG_CONFIG_LIBDIR=$(host_prefix)/lib/pkgconfig && \ - export PKG_CONFIG_PATH=$(host_prefix)/share/pkgconfig && \ - ./configure $($(package)_config_opts) && \ - echo "CONFIG += force_bootstrap" >> mkspecs/qconfig.pri && \ - $(MAKE) sub-src-clean && \ - cd ../qttranslations && ../qtbase/bin/qmake qttranslations.pro -o Makefile && \ - cd translations && ../../qtbase/bin/qmake translations.pro -o Makefile && cd ../.. &&\ - cd qttools/src/linguist/lrelease/ && ../../../../qtbase/bin/qmake lrelease.pro -o Makefile -endef - -define $(package)_build_cmds - $(MAKE) -C src $(addprefix sub-,$($(package)_qt_libs)) && \ - $(MAKE) -C ../qttools/src/linguist/lrelease && \ - $(MAKE) -C ../qttranslations -endef - -define $(package)_stage_cmds - $(MAKE) -C src INSTALL_ROOT=$($(package)_staging_dir) $(addsuffix -install_subtargets,$(addprefix sub-,$($(package)_qt_libs))) && cd .. &&\ - $(MAKE) -C qttools/src/linguist/lrelease INSTALL_ROOT=$($(package)_staging_dir) install_target && \ - $(MAKE) -C qttranslations INSTALL_ROOT=$($(package)_staging_dir) install_subtargets -endef - -define $(package)_postprocess_cmds - rm -rf native/mkspecs/ native/lib/ lib/cmake/ && \ - rm -f lib/lib*.la lib/*.prl plugins/*/*.prl -endef diff --git a/contrib/depends/packages/unbound.mk b/contrib/depends/packages/unbound.mk index a85c47e4e28..421c51f7f59 100644 --- a/contrib/depends/packages/unbound.mk +++ b/contrib/depends/packages/unbound.mk @@ -3,7 +3,9 @@ $(package)_version=1.15.0 $(package)_download_path=https://www.nlnetlabs.nl/downloads/$(package)/ $(package)_file_name=$(package)-$($(package)_version).tar.gz $(package)_sha256_hash=a480dc6c8937447b98d161fe911ffc76cfaffa2da18788781314e81339f1126f -$(package)_dependencies=openssl expat ldns +$(package)_dependencies=openssl expat +$(package)_patches=disable-glibc-reallocarray.patch + define $(package)_set_vars $(package)_config_opts=--disable-shared --enable-static --without-pyunbound --prefix=$(host_prefix) --with-libexpat=$(host_prefix) --with-ssl=$(host_prefix) --with-libevent=no --without-pythonmodule --disable-flto --with-pthreads --with-libunbound-only @@ -12,8 +14,13 @@ define $(package)_set_vars $(package)_build_opts_mingw32=LDFLAGS="$($(package)_ldflags) -lpthread" endef +define $(package)_preprocess_cmds + patch -p1 < $($(package)_patch_dir)/disable-glibc-reallocarray.patch &&\ + autoconf +endef + define $(package)_config_cmds - $($(package)_autoconf) + $($(package)_autoconf) ac_cv_func_getentropy=no endef define $(package)_build_cmds diff --git a/contrib/depends/packages/xproto.mk b/contrib/depends/packages/xproto.mk deleted file mode 100644 index 52fe253c77c..00000000000 --- a/contrib/depends/packages/xproto.mk +++ /dev/null @@ -1,21 +0,0 @@ -package=xproto -$(package)_version=7.0.26 -$(package)_download_path=https://xorg.freedesktop.org/releases/individual/proto -$(package)_file_name=$(package)-$($(package)_version).tar.bz2 -$(package)_sha256_hash=636162c1759805a5a0114a369dffdeccb8af8c859ef6e1445f26a4e6e046514f - -define $(package)_set_vars -$(package)_config_opts=--disable-shared -endef - -define $(package)_config_cmds - $($(package)_autoconf) -endef - -define $(package)_build_cmds - $(MAKE) -endef - -define $(package)_stage_cmds - $(MAKE) DESTDIR=$($(package)_staging_dir) install -endef diff --git a/contrib/depends/patches/cmake/cmake-1-fixes.patch b/contrib/depends/patches/cmake/cmake-1-fixes.patch deleted file mode 100644 index 062c067676d..00000000000 --- a/contrib/depends/patches/cmake/cmake-1-fixes.patch +++ /dev/null @@ -1,67 +0,0 @@ -This file is part of MXE. See LICENSE.md for licensing information. - -Contains ad hoc patches for cross building. - -From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 -From: Tony Theodore -Date: Fri, 12 Aug 2016 02:01:20 +1000 -Subject: [PATCH 1/3] fix windres invocation options - -windres doesn't recognise various gcc flags like -mms-bitfields, --fopenmp, -mthreads etc. (basically not `-D` or `-I`) - -diff --git a/Modules/Platform/Windows-windres.cmake b/Modules/Platform/Windows-windres.cmake -index 1111111..2222222 100644 ---- a/Modules/Platform/Windows-windres.cmake -+++ b/Modules/Platform/Windows-windres.cmake -@@ -1 +1 @@ --set(CMAKE_RC_COMPILE_OBJECT " -O coff ") -+set(CMAKE_RC_COMPILE_OBJECT " -O coff ") - -From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 -From: Tony Theodore -Date: Tue, 25 Jul 2017 20:34:56 +1000 -Subject: [PATCH 2/3] add option to disable -isystem - -taken from (not accepted): -https://gitlab.kitware.com/cmake/cmake/merge_requests/895 - -see also: -https://gitlab.kitware.com/cmake/cmake/issues/16291 -https://gitlab.kitware.com/cmake/cmake/issues/16919 - -diff --git a/Modules/Compiler/GNU.cmake b/Modules/Compiler/GNU.cmake -index 1111111..2222222 100644 ---- a/Modules/Compiler/GNU.cmake -+++ b/Modules/Compiler/GNU.cmake -@@ -42,7 +42,7 @@ macro(__compiler_gnu lang) - string(APPEND CMAKE_${lang}_FLAGS_RELWITHDEBINFO_INIT " -O2 -g -DNDEBUG") - set(CMAKE_${lang}_CREATE_PREPROCESSED_SOURCE " -E > ") - set(CMAKE_${lang}_CREATE_ASSEMBLY_SOURCE " -S -o ") -- if(NOT APPLE OR NOT CMAKE_${lang}_COMPILER_VERSION VERSION_LESS 4) # work around #4462 -+ if(NOT APPLE OR NOT CMAKE_${lang}_COMPILER_VERSION VERSION_LESS 4 AND (NOT MXE_DISABLE_INCLUDE_SYSTEM_FLAG)) # work around #4462 - set(CMAKE_INCLUDE_SYSTEM_FLAG_${lang} "-isystem ") - endif() - endmacro() - -From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 -From: Tony Theodore -Date: Tue, 15 Aug 2017 15:25:06 +1000 -Subject: [PATCH 3/3] add CPACK_NSIS_EXECUTABLE variable - - -diff --git a/Source/CPack/cmCPackNSISGenerator.cxx b/Source/CPack/cmCPackNSISGenerator.cxx -index 1111111..2222222 100644 ---- a/Source/CPack/cmCPackNSISGenerator.cxx -+++ b/Source/CPack/cmCPackNSISGenerator.cxx -@@ -384,7 +384,9 @@ int cmCPackNSISGenerator::InitializeInternal() - } - #endif - -- nsisPath = cmSystemTools::FindProgram("makensis", path, false); -+ this->SetOptionIfNotSet("CPACK_NSIS_EXECUTABLE", "makensis"); -+ nsisPath = cmSystemTools::FindProgram( -+ this->GetOption("CPACK_NSIS_EXECUTABLE"), path, false); - - if (nsisPath.empty()) { - cmCPackLogger( diff --git a/contrib/depends/patches/native_biplist/sorted_list.patch b/contrib/depends/patches/native_biplist/sorted_list.patch deleted file mode 100644 index 89abdb1b71f..00000000000 --- a/contrib/depends/patches/native_biplist/sorted_list.patch +++ /dev/null @@ -1,29 +0,0 @@ ---- a/biplist/__init__.py 2014-10-26 19:03:11.000000000 +0000 -+++ b/biplist/__init__.py 2016-07-19 19:30:17.663521999 +0000 -@@ -541,7 +541,7 @@ - return HashableWrapper(n) - elif isinstance(root, dict): - n = {} -- for key, value in iteritems(root): -+ for key, value in sorted(iteritems(root)): - n[self.wrapRoot(key)] = self.wrapRoot(value) - return HashableWrapper(n) - elif isinstance(root, list): -@@ -616,7 +616,7 @@ - elif isinstance(obj, dict): - size = proc_size(len(obj)) - self.incrementByteCount('dictBytes', incr=1+size) -- for key, value in iteritems(obj): -+ for key, value in sorted(iteritems(obj)): - check_key(key) - self.computeOffsets(key, asReference=True) - self.computeOffsets(value, asReference=True) -@@ -714,7 +714,7 @@ - keys = [] - values = [] - objectsToWrite = [] -- for key, value in iteritems(obj): -+ for key, value in sorted(iteritems(obj)): - keys.append(key) - values.append(value) - for key in keys: diff --git a/contrib/depends/patches/native_cdrkit/cdrkit-deterministic.patch b/contrib/depends/patches/native_cdrkit/cdrkit-deterministic.patch deleted file mode 100644 index 8ab0993dc4d..00000000000 --- a/contrib/depends/patches/native_cdrkit/cdrkit-deterministic.patch +++ /dev/null @@ -1,86 +0,0 @@ ---- cdrkit-1.1.11.old/genisoimage/tree.c 2008-10-21 19:57:47.000000000 -0400 -+++ cdrkit-1.1.11/genisoimage/tree.c 2013-12-06 00:23:18.489622668 -0500 -@@ -1139,8 +1139,9 @@ - scan_directory_tree(struct directory *this_dir, char *path, - struct directory_entry *de) - { -- DIR *current_dir; -+ int current_file; - char whole_path[PATH_MAX]; -+ struct dirent **d_list; - struct dirent *d_entry; - struct directory *parent; - int dflag; -@@ -1164,7 +1165,8 @@ - this_dir->dir_flags |= DIR_WAS_SCANNED; - - errno = 0; /* Paranoia */ -- current_dir = opendir(path); -+ //current_dir = opendir(path); -+ current_file = scandir(path, &d_list, NULL, alphasort); - d_entry = NULL; - - /* -@@ -1173,12 +1175,12 @@ - */ - old_path = path; - -- if (current_dir) { -+ if (current_file >= 0) { - errno = 0; -- d_entry = readdir(current_dir); -+ d_entry = d_list[0]; - } - -- if (!current_dir || !d_entry) { -+ if (current_file < 0 || !d_entry) { - int ret = 1; - - #ifdef USE_LIBSCHILY -@@ -1191,8 +1193,8 @@ - de->isorec.flags[0] &= ~ISO_DIRECTORY; - ret = 0; - } -- if (current_dir) -- closedir(current_dir); -+ if(d_list) -+ free(d_list); - return (ret); - } - #ifdef ABORT_DEEP_ISO_ONLY -@@ -1208,7 +1210,7 @@ - errmsgno(EX_BAD, "use Rock Ridge extensions via -R or -r,\n"); - errmsgno(EX_BAD, "or allow deep ISO9660 directory nesting via -D.\n"); - } -- closedir(current_dir); -+ free(d_list); - return (1); - } - #endif -@@ -1250,13 +1252,13 @@ - * The first time through, skip this, since we already asked - * for the first entry when we opened the directory. - */ -- if (dflag) -- d_entry = readdir(current_dir); -+ if (dflag && current_file >= 0) -+ d_entry = d_list[current_file]; - dflag++; - -- if (!d_entry) -+ if (current_file < 0) - break; -- -+ current_file--; - /* OK, got a valid entry */ - - /* If we do not want all files, then pitch the backups. */ -@@ -1348,7 +1350,7 @@ - insert_file_entry(this_dir, whole_path, d_entry->d_name); - #endif /* APPLE_HYB */ - } -- closedir(current_dir); -+ free(d_list); - - #ifdef APPLE_HYB - /* diff --git a/contrib/depends/patches/native_mac_alias/python3.patch b/contrib/depends/patches/native_mac_alias/python3.patch deleted file mode 100644 index 1a32340be5b..00000000000 --- a/contrib/depends/patches/native_mac_alias/python3.patch +++ /dev/null @@ -1,72 +0,0 @@ -diff -dur a/mac_alias/alias.py b/mac_alias/alias.py ---- a/mac_alias/alias.py 2015-10-19 12:12:48.000000000 +0200 -+++ b/mac_alias/alias.py 2016-04-03 12:13:12.037159417 +0200 -@@ -243,10 +243,10 @@ - alias = Alias() - alias.appinfo = appinfo - -- alias.volume = VolumeInfo (volname.replace('/',':'), -+ alias.volume = VolumeInfo (volname.decode().replace('/',':'), - voldate, fstype, disktype, - volattrs, volfsid) -- alias.target = TargetInfo (kind, filename.replace('/',':'), -+ alias.target = TargetInfo (kind, filename.decode().replace('/',':'), - folder_cnid, cnid, - crdate, creator_code, type_code) - alias.target.levels_from = levels_from -@@ -261,9 +261,9 @@ - b.read(1) - - if tag == TAG_CARBON_FOLDER_NAME: -- alias.target.folder_name = value.replace('/',':') -+ alias.target.folder_name = value.decode().replace('/',':') - elif tag == TAG_CNID_PATH: -- alias.target.cnid_path = struct.unpack(b'>%uI' % (length // 4), -+ alias.target.cnid_path = struct.unpack('>%uI' % (length // 4), - value) - elif tag == TAG_CARBON_PATH: - alias.target.carbon_path = value -@@ -298,9 +298,9 @@ - alias.target.creation_date \ - = mac_epoch + datetime.timedelta(seconds=seconds) - elif tag == TAG_POSIX_PATH: -- alias.target.posix_path = value -+ alias.target.posix_path = value.decode() - elif tag == TAG_POSIX_PATH_TO_MOUNTPOINT: -- alias.volume.posix_path = value -+ alias.volume.posix_path = value.decode() - elif tag == TAG_RECURSIVE_ALIAS_OF_DISK_IMAGE: - alias.volume.disk_image_alias = Alias.from_bytes(value) - elif tag == TAG_USER_HOME_LENGTH_PREFIX: -@@ -422,13 +422,13 @@ - # (so doing so is ridiculous, and nothing could rely on it). - b.write(struct.pack(b'>h28pI2shI64pII4s4shhI2s10s', - self.target.kind, -- carbon_volname, voldate, -+ carbon_volname, int(voldate), - self.volume.fs_type, - self.volume.disk_type, - self.target.folder_cnid, - carbon_filename, - self.target.cnid, -- crdate, -+ int(crdate), - self.target.creator_code, - self.target.type_code, - self.target.levels_from, -@@ -449,12 +449,12 @@ - - b.write(struct.pack(b'>hhQhhQ', - TAG_HIGH_RES_VOLUME_CREATION_DATE, -- 8, long(voldate * 65536), -+ 8, int(voldate * 65536), - TAG_HIGH_RES_CREATION_DATE, -- 8, long(crdate * 65536))) -+ 8, int(crdate * 65536))) - - if self.target.cnid_path: -- cnid_path = struct.pack(b'>%uI' % len(self.target.cnid_path), -+ cnid_path = struct.pack('>%uI' % len(self.target.cnid_path), - *self.target.cnid_path) - b.write(struct.pack(b'>hh', TAG_CNID_PATH, - len(cnid_path))) diff --git a/contrib/depends/patches/openssl/fix_darwin.patch b/contrib/depends/patches/openssl/fix_darwin.patch deleted file mode 100644 index a917daa1294..00000000000 --- a/contrib/depends/patches/openssl/fix_darwin.patch +++ /dev/null @@ -1,60 +0,0 @@ -From 96ac8f13f4d0ee96baf5724d9f96c44c34b8606c Mon Sep 17 00:00:00 2001 -From: David Carlier -Date: Tue, 24 Aug 2021 22:40:14 +0100 -Subject: [PATCH] Darwin platform allows to build on releases before - Yosemite/ios 8. - -issue #16407 #16408 - -Reviewed-by: Paul Dale -Reviewed-by: Tomas Mraz -(Merged from https://github.com/openssl/openssl/pull/16409) ---- - crypto/rand/rand_unix.c | 5 +---- - include/crypto/rand.h | 10 ++++++++++ - 2 files changed, 11 insertions(+), 4 deletions(-) - -diff --git a/crypto/rand/rand_unix.c b/crypto/rand/rand_unix.c -index 43f1069d151d..0f4525106af7 100644 ---- a/crypto/rand/rand_unix.c -+++ b/crypto/rand/rand_unix.c -@@ -34,9 +34,6 @@ - #if defined(__OpenBSD__) - # include - #endif --#if defined(__APPLE__) --# include --#endif - - #if defined(OPENSSL_SYS_UNIX) || defined(__DJGPP__) - # include -@@ -381,7 +378,7 @@ static ssize_t syscall_random(void *buf, size_t buflen) - if (errno != ENOSYS) - return -1; - } --# elif defined(__APPLE__) -+# elif defined(OPENSSL_APPLE_CRYPTO_RANDOM) - if (CCRandomGenerateBytes(buf, buflen) == kCCSuccess) - return (ssize_t)buflen; - -diff --git a/include/crypto/rand.h b/include/crypto/rand.h -index 5350d3a93119..674f840fd13c 100644 ---- a/include/crypto/rand.h -+++ b/include/crypto/rand.h -@@ -20,6 +20,16 @@ - - # include - -+# if defined(__APPLE__) && !defined(OPENSSL_NO_APPLE_CRYPTO_RANDOM) -+# include -+# if (defined(__MAC_OS_X_VERSION_MIN_REQUIRED) && __MAC_OS_X_VERSION_MIN_REQUIRED >= 101000) || \ -+ (defined(__IPHONE_OS_VERSION_MIN_REQUIRED) && __IPHONE_OS_VERSION_MIN_REQUIRED >= 80000) -+# define OPENSSL_APPLE_CRYPTO_RANDOM 1 -+# include -+# include -+# endif -+# endif -+ - /* forward declaration */ - typedef struct rand_pool_st RAND_POOL; - diff --git a/contrib/depends/patches/qt/fix_no_printer.patch b/contrib/depends/patches/qt/fix_no_printer.patch deleted file mode 100644 index 13723561384..00000000000 --- a/contrib/depends/patches/qt/fix_no_printer.patch +++ /dev/null @@ -1,19 +0,0 @@ ---- x/qtbase/src/plugins/platforms/cocoa/qprintengine_mac_p.h -+++ y/qtbase/src/plugins/platforms/cocoa/qprintengine_mac_p.h -@@ -52,6 +52,7 @@ - // - - #include -+#include - - #ifndef QT_NO_PRINTER - ---- x/qtbase/src/plugins/plugins.pro -+++ y/qtbase/src/plugins/plugins.pro -@@ -9,6 +9,3 @@ qtHaveModule(gui) { - !android:qtConfig(library): SUBDIRS *= generic - } - qtHaveModule(widgets): SUBDIRS += styles -- --!winrt:qtHaveModule(printsupport): \ -- SUBDIRS += printsupport diff --git a/contrib/depends/patches/qt/fix_qt_pkgconfig.patch b/contrib/depends/patches/qt/fix_qt_pkgconfig.patch deleted file mode 100644 index 73f4d89f735..00000000000 --- a/contrib/depends/patches/qt/fix_qt_pkgconfig.patch +++ /dev/null @@ -1,11 +0,0 @@ ---- old/qtbase/mkspecs/features/qt_module.prf -+++ new/qtbase/mkspecs/features/qt_module.prf -@@ -269,7 +269,7 @@ load(qt_installs) - load(qt_targets) - - # this builds on top of qt_common --!internal_module:if(unix|mingw):!if(darwin:debug_and_release:CONFIG(debug, debug|release)) { -+if(unix|mingw):!if(darwin:debug_and_release:CONFIG(debug, debug|release)) { - CONFIG += create_pc - QMAKE_PKGCONFIG_DESTDIR = pkgconfig - host_build: \ diff --git a/contrib/depends/patches/qt/fix_rcc_determinism.patch b/contrib/depends/patches/qt/fix_rcc_determinism.patch deleted file mode 100644 index c1b07fe23af..00000000000 --- a/contrib/depends/patches/qt/fix_rcc_determinism.patch +++ /dev/null @@ -1,15 +0,0 @@ ---- old/qtbase/src/tools/rcc/rcc.cpp -+++ new/qtbase/src/tools/rcc/rcc.cpp -@@ -207,7 +207,11 @@ void RCCFileInfo::writeDataInfo(RCCResourceLibrary &lib) - if (lib.formatVersion() >= 2) { - // last modified time stamp - const QDateTime lastModified = m_fileInfo.lastModified(); -- lib.writeNumber8(quint64(lastModified.isValid() ? lastModified.toMSecsSinceEpoch() : 0)); -+ quint64 lastmod = quint64(lastModified.isValid() ? lastModified.toMSecsSinceEpoch() : 0); -+ static const quint64 sourceDate = 1000 * qgetenv("QT_RCC_SOURCE_DATE_OVERRIDE").toULongLong(); -+ if (sourceDate != 0) -+ lastmod = sourceDate; -+ lib.writeNumber8(lastmod); - if (text || pass1) - lib.writeChar('\n'); - } diff --git a/contrib/depends/patches/qt/no-xlib.patch b/contrib/depends/patches/qt/no-xlib.patch deleted file mode 100644 index 6800d398c5c..00000000000 --- a/contrib/depends/patches/qt/no-xlib.patch +++ /dev/null @@ -1,69 +0,0 @@ -From 9563cef873ae82e06f60708d706d054717e801ce Mon Sep 17 00:00:00 2001 -From: Carl Dong -Date: Thu, 18 Jul 2019 17:22:05 -0400 -Subject: [PATCH] Wrap xlib related code blocks in #if's - -They are not necessary to compile QT. ---- - qtbase/src/plugins/platforms/xcb/qxcbcursor.cpp | 8 ++++++++ - 1 file changed, 8 insertions(+) - -diff --git a/qtbase/src/plugins/platforms/xcb/qxcbcursor.cpp b/qtbase/src/plugins/platforms/xcb/qxcbcursor.cpp -index 7c62c2e2b3..c05c6c0a07 100644 ---- a/qtbase/src/plugins/platforms/xcb/qxcbcursor.cpp -+++ b/qtbase/src/plugins/platforms/xcb/qxcbcursor.cpp -@@ -49,7 +49,9 @@ - #include - #include - #include -+#if QT_CONFIG(xcb_xlib) && QT_CONFIG(library) - #include -+#endif - #include - #include - -@@ -391,6 +393,7 @@ void QXcbCursor::changeCursor(QCursor *cursor, QWindow *window) - xcb_flush(xcb_connection()); - } - -+#if QT_CONFIG(xcb_xlib) && QT_CONFIG(library) - static int cursorIdForShape(int cshape) - { - int cursorId = 0; -@@ -444,6 +447,7 @@ static int cursorIdForShape(int cshape) - } - return cursorId; - } -+#endif - - xcb_cursor_t QXcbCursor::createNonStandardCursor(int cshape) - { -@@ -556,7 +560,9 @@ static xcb_cursor_t loadCursor(void *dpy, int cshape) - xcb_cursor_t QXcbCursor::createFontCursor(int cshape) - { - xcb_connection_t *conn = xcb_connection(); -+#if QT_CONFIG(xcb_xlib) && QT_CONFIG(library) - int cursorId = cursorIdForShape(cshape); -+#endif - xcb_cursor_t cursor = XCB_NONE; - - // Try Xcursor first -@@ -586,6 +592,7 @@ xcb_cursor_t QXcbCursor::createFontCursor(int cshape) - // Non-standard X11 cursors are created from bitmaps - cursor = createNonStandardCursor(cshape); - -+#if QT_CONFIG(xcb_xlib) && QT_CONFIG(library) - // Create a glpyh cursor if everything else failed - if (!cursor && cursorId) { - cursor = xcb_generate_id(conn); -@@ -593,6 +600,7 @@ xcb_cursor_t QXcbCursor::createFontCursor(int cshape) - cursorId, cursorId + 1, - 0xFFFF, 0xFFFF, 0xFFFF, 0, 0, 0); - } -+#endif - - if (cursor && cshape >= 0 && cshape < Qt::LastCursor && connection()->hasXFixes()) { - const char *name = cursorNames[cshape].front(); ---- -2.22.0 - diff --git a/contrib/depends/patches/unbound/disable-glibc-reallocarray.patch b/contrib/depends/patches/unbound/disable-glibc-reallocarray.patch new file mode 100644 index 00000000000..d66a821ad59 --- /dev/null +++ b/contrib/depends/patches/unbound/disable-glibc-reallocarray.patch @@ -0,0 +1,14 @@ +diff --git a/configure.ac b/configure.ac +index 5c7da197..e2b25288 100644 +--- a/configure.ac ++++ b/configure.ac +@@ -1702,6 +1702,9 @@ AC_LINK_IFELSE([AC_LANG_SOURCE(AC_INCLUDES_DEFAULT + #ifndef _OPENBSD_SOURCE + #define _OPENBSD_SOURCE 1 + #endif ++#ifdef __linux__ ++# error reallocarray() is currently disabled on Linux to support glibc < 2.26 ++#endif + #include + int main(void) { + void* p = reallocarray(NULL, 10, 100); diff --git a/contrib/depends/protobuf.mk b/contrib/depends/protobuf.mk deleted file mode 100644 index 54d3fd92451..00000000000 --- a/contrib/depends/protobuf.mk +++ /dev/null @@ -1,29 +0,0 @@ -package=protobuf -$(package)_version=$(native_$(package)_version) -$(package)_download_path=$(native_$(package)_download_path) -$(package)_file_name=$(native_$(package)_file_name) -$(package)_sha256_hash=$(native_$(package)_sha256_hash) -$(package)_dependencies=native_$(package) -$(package)_cxxflags=-std=c++11 - -define $(package)_set_vars - $(package)_config_opts=--disable-shared --with-protoc=$(build_prefix)/bin/protoc - $(package)_config_opts_linux=--with-pic -endef - -define $(package)_config_cmds - $($(package)_autoconf) -endef - -define $(package)_build_cmds - $(MAKE) -C src libprotobuf.la -endef - -define $(package)_stage_cmds - $(MAKE) DESTDIR=$($(package)_staging_dir) -C src install-libLTLIBRARIES install-nobase_includeHEADERS &&\ - $(MAKE) DESTDIR=$($(package)_staging_dir) install-pkgconfigDATA -endef - -define $(package)_postprocess_cmds - rm lib/libprotoc.a -endef diff --git a/contrib/depends/toolchain.cmake.in b/contrib/depends/toolchain.cmake.in index 5f950132966..570065560e5 100644 --- a/contrib/depends/toolchain.cmake.in +++ b/contrib/depends/toolchain.cmake.in @@ -27,8 +27,6 @@ SET(Terminfo_LIBRARY @prefix@/lib/libtinfo.a) SET(UNBOUND_INCLUDE_DIR @prefix@/include) SET(UNBOUND_LIBRARIES @prefix@/lib/libunbound.a) -SET(LRELEASE_PATH @prefix@/native/bin CACHE FILEPATH "path to lrelease" FORCE) - if(NOT CMAKE_SYSTEM_NAME STREQUAL "Android") SET(LIBUNWIND_INCLUDE_DIR @prefix@/include) SET(LIBUNWIND_LIBRARIES @prefix@/lib/libunwind.a) diff --git a/contrib/epee/CMakeLists.txt b/contrib/epee/CMakeLists.txt index a61e5044500..9cfd09b5b28 100644 --- a/contrib/epee/CMakeLists.txt +++ b/contrib/epee/CMakeLists.txt @@ -1,4 +1,4 @@ -# Copyright (c) 2014-2022, The Monero Project +# Copyright (c) 2014-2023, The Monero Project # # All rights reserved. # diff --git a/contrib/epee/include/byte_slice.h b/contrib/epee/include/byte_slice.h index 57f6c410a45..fa45613e477 100644 --- a/contrib/epee/include/byte_slice.h +++ b/contrib/epee/include/byte_slice.h @@ -1,4 +1,4 @@ -// Copyright (c) 2019-2022, The Monero Project +// Copyright (c) 2019-2023, The Monero Project // // All rights reserved. // diff --git a/contrib/epee/include/byte_stream.h b/contrib/epee/include/byte_stream.h index e7993133a3f..4f565326a79 100644 --- a/contrib/epee/include/byte_stream.h +++ b/contrib/epee/include/byte_stream.h @@ -1,4 +1,4 @@ -// Copyright (c) 2020-2022, The Monero Project +// Copyright (c) 2020-2023, The Monero Project // // All rights reserved. diff --git a/contrib/epee/include/file_io_utils.h b/contrib/epee/include/file_io_utils.h index da05520c1b9..de95e58c3ea 100644 --- a/contrib/epee/include/file_io_utils.h +++ b/contrib/epee/include/file_io_utils.h @@ -29,6 +29,7 @@ #include #include +#include namespace epee { diff --git a/contrib/epee/include/fnv1.h b/contrib/epee/include/fnv1.h index 6a699f4d87d..8c11cd18764 100644 --- a/contrib/epee/include/fnv1.h +++ b/contrib/epee/include/fnv1.h @@ -1,4 +1,4 @@ -// Copyright (c) 2018-2022, The Monero Project +// Copyright (c) 2018-2023, The Monero Project // // All rights reserved. diff --git a/contrib/epee/include/hex.h b/contrib/epee/include/hex.h index af0a581a9fe..d9eb93d51d1 100644 --- a/contrib/epee/include/hex.h +++ b/contrib/epee/include/hex.h @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2022, The Monero Project +// Copyright (c) 2017-2023, The Monero Project // // All rights reserved. // diff --git a/contrib/epee/include/int-util.h b/contrib/epee/include/int-util.h index 3789076e8ec..d3fbd84ca98 100644 --- a/contrib/epee/include/int-util.h +++ b/contrib/epee/include/int-util.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2022, The Monero Project +// Copyright (c) 2014-2023, The Monero Project // // All rights reserved. // diff --git a/contrib/epee/include/memwipe.h b/contrib/epee/include/memwipe.h index ebb9373c912..a27e0bb5ec2 100644 --- a/contrib/epee/include/memwipe.h +++ b/contrib/epee/include/memwipe.h @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2022, The Monero Project +// Copyright (c) 2017-2023, The Monero Project // // All rights reserved. // diff --git a/contrib/epee/include/mlocker.h b/contrib/epee/include/mlocker.h index 47ca0032ac8..7d8e2c7a5d7 100644 --- a/contrib/epee/include/mlocker.h +++ b/contrib/epee/include/mlocker.h @@ -1,4 +1,4 @@ -// Copyright (c) 2018-2022, The Monero Project +// Copyright (c) 2018-2023, The Monero Project // // All rights reserved. diff --git a/contrib/epee/include/net/abstract_http_client.h b/contrib/epee/include/net/abstract_http_client.h index 5270824e15f..46b3747cd7e 100644 --- a/contrib/epee/include/net/abstract_http_client.h +++ b/contrib/epee/include/net/abstract_http_client.h @@ -72,7 +72,6 @@ namespace http virtual bool is_connected(bool *ssl = NULL) = 0; virtual bool invoke(const boost::string_ref uri, const boost::string_ref method, const boost::string_ref body, std::chrono::milliseconds timeout, const http_response_info** ppresponse_info = NULL, const fields_list& additional_params = fields_list()) = 0; virtual bool invoke_get(const boost::string_ref uri, std::chrono::milliseconds timeout, const std::string& body = std::string(), const http_response_info** ppresponse_info = NULL, const fields_list& additional_params = fields_list()) = 0; - virtual bool invoke_post(const boost::string_ref uri, const std::string& body, std::chrono::milliseconds timeout, const http_response_info** ppresponse_info = NULL, const fields_list& additional_params = fields_list()) = 0; virtual uint64_t get_bytes_sent() const = 0; virtual uint64_t get_bytes_received() const = 0; }; diff --git a/contrib/epee/include/net/abstract_tcp_server2.h b/contrib/epee/include/net/abstract_tcp_server2.h index 51aa9f275f3..bc0da66e299 100644 --- a/contrib/epee/include/net/abstract_tcp_server2.h +++ b/contrib/epee/include/net/abstract_tcp_server2.h @@ -44,12 +44,16 @@ #include #include #include +#include #include #include +#include +#include #include #include #include +#include #include "byte_slice.h" #include "net_utils_base.h" #include "syncobj.h" @@ -87,7 +91,172 @@ namespace net_utils { public: typedef typename t_protocol_handler::connection_context t_connection_context; + private: + using connection_t = connection; + using connection_ptr = boost::shared_ptr; + using ssl_support_t = epee::net_utils::ssl_support_t; + using timer_t = boost::asio::steady_timer; + using duration_t = timer_t::duration; + using ec_t = boost::system::error_code; + using handshake_t = boost::asio::ssl::stream_base::handshake_type; + + using io_context_t = boost::asio::io_service; + using strand_t = boost::asio::io_service::strand; + using socket_t = boost::asio::ip::tcp::socket; + + using network_throttle_t = epee::net_utils::network_throttle; + using network_throttle_manager_t = epee::net_utils::network_throttle_manager; + + unsigned int host_count(int delta = 0); + duration_t get_default_timeout(); + duration_t get_timeout_from_bytes_read(size_t bytes) const; + + void state_status_check(); + + void start_timer(duration_t duration, bool add = {}); + void async_wait_timer(); + void cancel_timer(); + + void start_handshake(); + void start_read(); + void start_write(); + void start_shutdown(); + void cancel_socket(); + + void cancel_handler(); + + void interrupt(); + void on_interrupted(); + + void terminate(); + void on_terminating(); + + bool send(epee::byte_slice message); + bool start_internal( + bool is_income, + bool is_multithreaded, + boost::optional real_remote + ); + + enum status_t { + TERMINATED, + RUNNING, + INTERRUPTED, + TERMINATING, + WASTED, + }; + + struct state_t { + struct stat_t { + struct { + network_throttle_t throttle{"speed_in", "throttle_speed_in"}; + } in; + struct { + network_throttle_t throttle{"speed_out", "throttle_speed_out"}; + } out; + }; + + struct data_t { + struct { + std::array buffer; + } read; + struct { + std::deque queue; + bool wait_consume; + } write; + }; + + struct ssl_t { + bool enabled; + bool forced; + bool detected; + bool handshaked; + }; + + struct socket_status_t { + bool connected; + + bool wait_handshake; + bool cancel_handshake; + + bool wait_read; + bool handle_read; + bool cancel_read; + + bool wait_write; + bool handle_write; + bool cancel_write; + + bool wait_shutdown; + bool cancel_shutdown; + }; + + struct timer_status_t { + bool wait_expire; + bool cancel_expire; + bool reset_expire; + }; + + struct timers_status_t { + struct throttle_t { + timer_status_t in; + timer_status_t out; + }; + + timer_status_t general; + throttle_t throttle; + }; + + struct protocol_t { + size_t reference_counter; + bool released; + bool initialized; + + bool wait_release; + bool wait_init; + size_t wait_callback; + }; + + std::mutex lock; + std::condition_variable_any condition; + status_t status; + socket_status_t socket; + ssl_t ssl; + timers_status_t timers; + protocol_t protocol; + stat_t stat; + data_t data; + }; + + struct timers_t { + timers_t(io_context_t &io_context): + general(io_context), + throttle(io_context) + {} + struct throttle_t { + throttle_t(io_context_t &io_context): + in(io_context), + out(io_context) + {} + timer_t in; + timer_t out; + }; + + timer_t general; + throttle_t throttle; + }; + io_context_t &m_io_context; + t_connection_type m_connection_type; + t_connection_context m_conn_context{}; + strand_t m_strand; + timers_t m_timers; + connection_ptr self{}; + bool m_local{}; + std::string m_host{}; + state_t m_state{}; + t_protocol_handler m_handler; + public: struct shared_state : connection_basic_shared_state, t_protocol_handler::config_type { shared_state() @@ -119,7 +288,7 @@ namespace net_utils // `real_remote` is the actual endpoint (if connection is to proxy, etc.) bool start(bool is_income, bool is_multithreaded, network_address real_remote); - void get_context(t_connection_context& context_){context_ = context;} + void get_context(t_connection_context& context_){context_ = m_conn_context;} void call_back_starter(); @@ -141,58 +310,6 @@ namespace net_utils virtual bool add_ref(); virtual bool release(); //------------------------------------------------------ - bool do_send_chunk(byte_slice chunk); ///< will send (or queue) a part of data. internal use only - - boost::shared_ptr > safe_shared_from_this(); - bool shutdown(); - /// Handle completion of a receive operation. - void handle_receive(const boost::system::error_code& e, - std::size_t bytes_transferred); - - /// Handle completion of a read operation. - void handle_read(const boost::system::error_code& e, - std::size_t bytes_transferred); - - /// Handle completion of a write operation. - void handle_write(const boost::system::error_code& e, size_t cb); - - /// reset connection timeout timer and callback - void reset_timer(boost::posix_time::milliseconds ms, bool add); - boost::posix_time::milliseconds get_default_timeout(); - boost::posix_time::milliseconds get_timeout_from_bytes_read(size_t bytes); - - /// host connection count tracking - unsigned int host_count(const std::string &host, int delta = 0); - - /// Buffer for incoming data. - boost::array buffer_; - size_t buffer_ssl_init_fill; - - t_connection_context context; - - // TODO what do they mean about wait on destructor?? --rfree : - //this should be the last one, because it could be wait on destructor, while other activities possible on other threads - t_protocol_handler m_protocol_handler; - //typename t_protocol_handler::config_type m_dummy_config; - size_t m_reference_count = 0; // reference count managed through add_ref/release support - boost::shared_ptr > m_self_ref; // the reference to hold - critical_section m_self_refs_lock; - critical_section m_chunking_lock; // held while we add small chunks of the big do_send() to small do_send_chunk() - critical_section m_shutdown_lock; // held while shutting down - - t_connection_type m_connection_type; - - // for calculate speed (last 60 sec) - network_throttle m_throttle_speed_in; - network_throttle m_throttle_speed_out; - boost::mutex m_throttle_speed_in_mutex; - boost::mutex m_throttle_speed_out_mutex; - - boost::asio::deadline_timer m_timer; - bool m_local; - bool m_ready_to_close; - std::string m_host; - public: void setRpcStation(); }; diff --git a/contrib/epee/include/net/abstract_tcp_server2.inl b/contrib/epee/include/net/abstract_tcp_server2.inl index 0c3b457bcde..81aa725d1e2 100644 --- a/contrib/epee/include/net/abstract_tcp_server2.inl +++ b/contrib/epee/include/net/abstract_tcp_server2.inl @@ -76,859 +76,1057 @@ namespace net_utils /************************************************************************/ /* */ /************************************************************************/ - template - connection::connection( boost::asio::io_service& io_service, - std::shared_ptr state, - t_connection_type connection_type, - ssl_support_t ssl_support - ) - : connection(boost::asio::ip::tcp::socket{io_service}, std::move(state), connection_type, ssl_support) + template + unsigned int connection::host_count(int delta) { + static std::mutex hosts_mutex; + std::lock_guard guard(hosts_mutex); + static std::map hosts; + unsigned int &val = hosts[m_host]; + if (delta > 0) + MTRACE("New connection from host " << m_host << ": " << val); + else if (delta < 0) + MTRACE("Closed connection from host " << m_host << ": " << val); + CHECK_AND_ASSERT_THROW_MES(delta >= 0 || val >= (unsigned)-delta, "Count would go negative"); + CHECK_AND_ASSERT_THROW_MES(delta <= 0 || val <= std::numeric_limits::max() - (unsigned)delta, "Count would wrap"); + val += delta; + return val; } - template - connection::connection( boost::asio::ip::tcp::socket&& sock, - std::shared_ptr state, - t_connection_type connection_type, - ssl_support_t ssl_support - ) - : - connection_basic(std::move(sock), state, ssl_support), - m_protocol_handler(this, check_and_get(state), context), - buffer_ssl_init_fill(0), - m_connection_type( connection_type ), - m_throttle_speed_in("speed_in", "throttle_speed_in"), - m_throttle_speed_out("speed_out", "throttle_speed_out"), - m_timer(GET_IO_SERVICE(socket_)), - m_local(false), - m_ready_to_close(false) + template + typename connection::duration_t connection::get_default_timeout() { - MDEBUG("test, connection constructor set m_connection_type="< AGGRESSIVE_TIMEOUT_THRESHOLD ? + std::min(std::max(count, 1u) - 1, 8u) : + 0 + ); + return ( + m_local ? + std::chrono::milliseconds(DEFAULT_TIMEOUT_MS_LOCAL >> shift) : + std::chrono::milliseconds(DEFAULT_TIMEOUT_MS_REMOTE >> shift) + ); } - //--------------------------------------------------------------------------------- - template - connection::~connection() noexcept(false) + template + typename connection::duration_t connection::get_timeout_from_bytes_read(size_t bytes) const { - if(!m_was_shutdown) - { - _dbg3("[sock " << socket().native_handle() << "] Socket destroyed without shutdown."); - shutdown(); - } - - _dbg3("[sock " << socket().native_handle() << "] Socket destroyed"); + return std::chrono::duration_cast::duration_t>( + std::chrono::duration( + bytes * TIMEOUT_EXTRA_MS_PER_BYTE + ) + ); } - //--------------------------------------------------------------------------------- - template - boost::shared_ptr > connection::safe_shared_from_this() + + template + void connection::state_status_check() { - try - { - return connection::shared_from_this(); - } - catch (const boost::bad_weak_ptr&) - { - // It happens when the connection is being deleted - return boost::shared_ptr >(); + switch (m_state.status) + { + case status_t::RUNNING: + interrupt(); + break; + case status_t::INTERRUPTED: + on_interrupted(); + break; + case status_t::TERMINATING: + on_terminating(); + break; + default: + break; } } - //--------------------------------------------------------------------------------- - template - bool connection::start(bool is_income, bool is_multithreaded) - { - TRY_ENTRY(); - boost::system::error_code ec; - auto remote_ep = socket().remote_endpoint(ec); - CHECK_AND_NO_ASSERT_MES(!ec, false, "Failed to get remote endpoint: " << ec.message() << ':' << ec.value()); - CHECK_AND_NO_ASSERT_MES(remote_ep.address().is_v4() || remote_ep.address().is_v6(), false, "only IPv4 and IPv6 supported here"); - - if (remote_ep.address().is_v4()) - { - const unsigned long ip_ = boost::asio::detail::socket_ops::host_to_network_long(remote_ep.address().to_v4().to_ulong()); - return start(is_income, is_multithreaded, ipv4_network_address{uint32_t(ip_), remote_ep.port()}); - } - else - { - const auto ip_ = remote_ep.address().to_v6(); - return start(is_income, is_multithreaded, ipv6_network_address{ip_, remote_ep.port()}); - } - CATCH_ENTRY_L0("connection::start()", false); - } - //--------------------------------------------------------------------------------- - template - bool connection::start(bool is_income, bool is_multithreaded, network_address real_remote) + template + void connection::start_timer(duration_t duration, bool add) { - TRY_ENTRY(); - - // Use safe_shared_from_this, because of this is public method and it can be called on the object being deleted - auto self = safe_shared_from_this(); - if(!self) - return false; - - m_is_multithreaded = is_multithreaded; - m_local = real_remote.is_loopback() || real_remote.is_local(); - - // create a random uuid, we don't need crypto strength here - const boost::uuids::uuid random_uuid = boost::uuids::random_generator()(); - - context = t_connection_context{}; - bool ssl = m_ssl_support == epee::net_utils::ssl_support_t::e_ssl_support_enabled; - context.set_details(random_uuid, std::move(real_remote), is_income, ssl); - - boost::system::error_code ec; - auto local_ep = socket().local_endpoint(ec); - CHECK_AND_NO_ASSERT_MES(!ec, false, "Failed to get local endpoint: " << ec.message() << ':' << ec.value()); - - _dbg3("[sock " << socket_.native_handle() << "] new connection from " << print_connection_context_short(context) << - " to " << local_ep.address().to_string() << ':' << local_ep.port() << - ", total sockets objects " << get_state().sock_count); - - if(static_cast(get_state()).pfilter && !static_cast(get_state()).pfilter->is_remote_host_allowed(context.m_remote_address)) - { - _dbg2("[sock " << socket().native_handle() << "] host denied " << context.m_remote_address.host_str() << ", shutdowning connection"); - close(); - return false; + if (m_state.timers.general.wait_expire) { + m_state.timers.general.cancel_expire = true; + m_state.timers.general.reset_expire = true; + ec_t ec; + m_timers.general.expires_from_now( + std::min( + duration + (add ? m_timers.general.expires_from_now() : duration_t{}), + get_default_timeout() + ), + ec + ); + } + else { + ec_t ec; + m_timers.general.expires_from_now( + std::min( + duration + (add ? m_timers.general.expires_from_now() : duration_t{}), + get_default_timeout() + ), + ec + ); + async_wait_timer(); } - - m_host = context.m_remote_address.host_str(); - try { host_count(m_host, 1); } catch(...) { /* ignore */ } - - m_protocol_handler.after_init_connection(); - - reset_timer(boost::posix_time::milliseconds(m_local ? NEW_CONNECTION_TIMEOUT_LOCAL : NEW_CONNECTION_TIMEOUT_REMOTE), false); - - // first read on the raw socket to detect SSL for the server - buffer_ssl_init_fill = 0; - if (is_income && m_ssl_support != epee::net_utils::ssl_support_t::e_ssl_support_disabled) - socket().async_receive(boost::asio::buffer(buffer_), - strand_.wrap( - std::bind(&connection::handle_receive, self, - std::placeholders::_1, - std::placeholders::_2))); - else - async_read_some(boost::asio::buffer(buffer_), - strand_.wrap( - std::bind(&connection::handle_read, self, - std::placeholders::_1, - std::placeholders::_2))); -#if !defined(_WIN32) || !defined(__i686) - // not supported before Windows7, too lazy for runtime check - // Just exclude for 32bit windows builds - //set ToS flag - int tos = get_tos_flag(); - boost::asio::detail::socket_option::integer< IPPROTO_IP, IP_TOS > - optionTos( tos ); - socket().set_option( optionTos ); - //_dbg1("Set ToS flag to " << tos); -#endif - - boost::asio::ip::tcp::no_delay noDelayOption(false); - socket().set_option(noDelayOption); - - return true; - - CATCH_ENTRY_L0("connection::start()", false); } - //--------------------------------------------------------------------------------- - template - bool connection::request_callback() - { - TRY_ENTRY(); - _dbg2("[" << print_connection_context_short(context) << "] request_callback"); - // Use safe_shared_from_this, because of this is public method and it can be called on the object being deleted - auto self = safe_shared_from_this(); - if(!self) - return false; - strand_.post(boost::bind(&connection::call_back_starter, self)); - CATCH_ENTRY_L0("connection::request_callback()", false); - return true; - } - //--------------------------------------------------------------------------------- - template - boost::asio::io_service& connection::get_io_service() + template + void connection::async_wait_timer() { - return GET_IO_SERVICE(socket()); + if (m_state.timers.general.wait_expire) + return; + m_state.timers.general.wait_expire = true; + auto self = connection::shared_from_this(); + m_timers.general.async_wait([this, self](const ec_t & ec){ + std::lock_guard guard(m_state.lock); + m_state.timers.general.wait_expire = false; + if (m_state.timers.general.cancel_expire) { + m_state.timers.general.cancel_expire = false; + if (m_state.timers.general.reset_expire) { + m_state.timers.general.reset_expire = false; + async_wait_timer(); + } + else if (m_state.status == status_t::INTERRUPTED) + on_interrupted(); + else if (m_state.status == status_t::TERMINATING) + on_terminating(); + } + else if (m_state.status == status_t::RUNNING) + interrupt(); + else if (m_state.status == status_t::INTERRUPTED) + terminate(); + }); } - //--------------------------------------------------------------------------------- - template - bool connection::add_ref() - { - TRY_ENTRY(); - // Use safe_shared_from_this, because of this is public method and it can be called on the object being deleted - auto self = safe_shared_from_this(); - if(!self) - return false; - //_dbg3("[sock " << socket().native_handle() << "] add_ref, m_peer_number=" << mI->m_peer_number); - CRITICAL_REGION_LOCAL(self->m_self_refs_lock); - //_dbg3("[sock " << socket().native_handle() << "] add_ref 2, m_peer_number=" << mI->m_peer_number); - ++m_reference_count; - m_self_ref = std::move(self); - return true; - CATCH_ENTRY_L0("connection::add_ref()", false); - } - //--------------------------------------------------------------------------------- - template - bool connection::release() - { - TRY_ENTRY(); - boost::shared_ptr > back_connection_copy; - LOG_TRACE_CC(context, "[sock " << socket().native_handle() << "] release"); - CRITICAL_REGION_BEGIN(m_self_refs_lock); - CHECK_AND_ASSERT_MES(m_reference_count, false, "[sock " << socket().native_handle() << "] m_reference_count already at 0 at connection::release() call"); - // is this the last reference? - if (--m_reference_count == 0) { - // move the held reference to a local variable, keeping the object alive until the function terminates - std::swap(back_connection_copy, m_self_ref); - } - CRITICAL_REGION_END(); - return true; - CATCH_ENTRY_L0("connection::release()", false); - } - //--------------------------------------------------------------------------------- - template - void connection::call_back_starter() + template + void connection::cancel_timer() { - TRY_ENTRY(); - _dbg2("[" << print_connection_context_short(context) << "] fired_callback"); - m_protocol_handler.handle_qued_callback(); - CATCH_ENTRY_L0("connection::call_back_starter()", void()); + if (!m_state.timers.general.wait_expire) + return; + m_state.timers.general.cancel_expire = true; + m_state.timers.general.reset_expire = false; + ec_t ec; + m_timers.general.cancel(ec); } - //--------------------------------------------------------------------------------- - template - void connection::save_dbg_log() - { - std::string address, port; - boost::system::error_code e; - boost::asio::ip::tcp::endpoint endpoint = socket().remote_endpoint(e); - if (e) - { - address = ""; - port = ""; - } - else - { - address = endpoint.address().to_string(); - port = boost::lexical_cast(endpoint.port()); - } - MDEBUG(" connection type " << to_string( m_connection_type ) << " " - << socket().local_endpoint().address().to_string() << ":" << socket().local_endpoint().port() - << " <--> " << context.m_remote_address.str() << " (via " << address << ":" << port << ")"); - } - //--------------------------------------------------------------------------------- - template - void connection::handle_read(const boost::system::error_code& e, - std::size_t bytes_transferred) + template + void connection::start_handshake() { - TRY_ENTRY(); - //_info("[sock " << socket().native_handle() << "] Async read calledback."); - - if (m_was_shutdown) - return; + if (m_state.socket.wait_handshake) + return; + static_assert( + epee::net_utils::get_ssl_magic_size() <= sizeof(m_state.data.read.buffer), + "" + ); + auto self = connection::shared_from_this(); + if (!m_state.ssl.forced && !m_state.ssl.detected) { + m_state.socket.wait_read = true; + boost::asio::async_read( + connection_basic::socket_.next_layer(), + boost::asio::buffer( + m_state.data.read.buffer.data(), + m_state.data.read.buffer.size() + ), + boost::asio::transfer_exactly(epee::net_utils::get_ssl_magic_size()), + m_strand.wrap( + [this, self](const ec_t &ec, size_t bytes_transferred){ + std::lock_guard guard(m_state.lock); + m_state.socket.wait_read = false; + if (m_state.socket.cancel_read) { + m_state.socket.cancel_read = false; + state_status_check(); + } + else if (ec.value()) { + terminate(); + } + else if ( + !epee::net_utils::is_ssl( + static_cast( + m_state.data.read.buffer.data() + ), + bytes_transferred + ) + ) { + m_state.ssl.enabled = false; + m_state.socket.handle_read = true; + connection_basic::strand_.post( + [this, self, bytes_transferred]{ + bool success = m_handler.handle_recv( + reinterpret_cast(m_state.data.read.buffer.data()), + bytes_transferred + ); + std::lock_guard guard(m_state.lock); + m_state.socket.handle_read = false; + if (m_state.status == status_t::INTERRUPTED) + on_interrupted(); + else if (m_state.status == status_t::TERMINATING) + on_terminating(); + else if (!success) + interrupt(); + else { + start_read(); + } + } + ); + } + else { + m_state.ssl.detected = true; + start_handshake(); + } + } + ) + ); + return; + } - if (!e) - { - double current_speed_down; - { - CRITICAL_REGION_LOCAL(m_throttle_speed_in_mutex); - m_throttle_speed_in.handle_trafic_exact(bytes_transferred); - current_speed_down = m_throttle_speed_in.get_current_speed(); - } - context.m_current_speed_down = current_speed_down; - context.m_max_speed_down = std::max(context.m_max_speed_down, current_speed_down); - - { - CRITICAL_REGION_LOCAL( epee::net_utils::network_throttle_manager::network_throttle_manager::m_lock_get_global_throttle_in ); - epee::net_utils::network_throttle_manager::network_throttle_manager::get_global_throttle_in().handle_trafic_exact(bytes_transferred); - } - - double delay=0; // will be calculated - how much we should sleep to obey speed limit etc - - - if (speed_limit_is_enabled()) { - do // keep sleeping if we should sleep - { - { //_scope_dbg1("CRITICAL_REGION_LOCAL"); - CRITICAL_REGION_LOCAL( epee::net_utils::network_throttle_manager::m_lock_get_global_throttle_in ); - delay = epee::net_utils::network_throttle_manager::get_global_throttle_in().get_sleep_time_after_tick( bytes_transferred ); - } - - if (m_was_shutdown) - return; - - delay *= 0.5; - long int ms = (long int)(delay * 100); - if (ms > 0) { - reset_timer(boost::posix_time::milliseconds(ms + 1), true); - boost::this_thread::sleep_for(boost::chrono::milliseconds(ms)); - } - } while(delay > 0); - } // any form of sleeping - - //_info("[sock " << socket().native_handle() << "] RECV " << bytes_transferred); - logger_handle_net_read(bytes_transferred); - context.m_last_recv = time(NULL); - context.m_recv_cnt += bytes_transferred; - m_ready_to_close = false; - bool recv_res = m_protocol_handler.handle_recv(buffer_.data(), bytes_transferred); - if(!recv_res) - { - //_info("[sock " << socket().native_handle() << "] protocol_want_close"); - //some error in protocol, protocol handler ask to close connection - m_want_close_connection = true; - bool do_shutdown = false; - CRITICAL_REGION_BEGIN(m_send_que_lock); - if(!m_send_que.size()) - do_shutdown = true; - CRITICAL_REGION_END(); - if(do_shutdown) - shutdown(); - }else - { - reset_timer(get_timeout_from_bytes_read(bytes_transferred), false); - async_read_some(boost::asio::buffer(buffer_), - strand_.wrap( - boost::bind(&connection::handle_read, connection::shared_from_this(), - boost::asio::placeholders::error, - boost::asio::placeholders::bytes_transferred))); - //_info("[sock " << socket().native_handle() << "]Async read requested."); + m_state.socket.wait_handshake = true; + auto on_handshake = [this, self](const ec_t &ec, size_t bytes_transferred){ + std::lock_guard guard(m_state.lock); + m_state.socket.wait_handshake = false; + if (m_state.socket.cancel_handshake) { + m_state.socket.cancel_handshake = false; + state_status_check(); } - }else - { - _dbg3("[sock " << socket().native_handle() << "] Some not success at read: " << e.message() << ':' << e.value()); - if(e.value() != 2) - { - _dbg3("[sock " << socket().native_handle() << "] Some problems at read: " << e.message() << ':' << e.value()); - shutdown(); + else if (ec.value()) { + ec_t ec; + connection_basic::socket_.next_layer().shutdown( + socket_t::shutdown_both, + ec + ); + connection_basic::socket_.next_layer().close(ec); + m_state.socket.connected = false; + interrupt(); } - else - { - _dbg3("[sock " << socket().native_handle() << "] peer closed connection"); - bool do_shutdown = false; - CRITICAL_REGION_BEGIN(m_send_que_lock); - if(!m_send_que.size()) - do_shutdown = true; - CRITICAL_REGION_END(); - if (m_ready_to_close || do_shutdown) - shutdown(); + else { + m_state.ssl.handshaked = true; + start_write(); + start_read(); } - m_ready_to_close = true; - } - // If an error occurs then no new asynchronous operations are started. This - // means that all shared_ptr references to the connection object will - // disappear and the object will be destroyed automatically after this - // handler returns. The connection class's destructor closes the socket. - CATCH_ENTRY_L0("connection::handle_read", void()); + }; + const auto handshake = handshake_t::server; + static_cast( + connection_basic::get_state() + ).ssl_options().configure(connection_basic::socket_, handshake); + m_strand.post( + [this, self, on_handshake]{ + connection_basic::socket_.async_handshake( + handshake, + boost::asio::buffer( + m_state.data.read.buffer.data(), + m_state.ssl.forced ? 0 : + epee::net_utils::get_ssl_magic_size() + ), + m_strand.wrap(on_handshake) + ); + } + ); } - //--------------------------------------------------------------------------------- - template - void connection::handle_receive(const boost::system::error_code& e, - std::size_t bytes_transferred) - { - TRY_ENTRY(); - if (m_was_shutdown) return; - - if (e) - { - // offload the error case - handle_read(e, bytes_transferred); + template + void connection::start_read() + { + if (m_state.timers.throttle.in.wait_expire || m_state.socket.wait_read || + m_state.socket.handle_read + ) { return; } + auto self = connection::shared_from_this(); + if (m_connection_type != e_connection_type_RPC) { + auto calc_duration = []{ + CRITICAL_REGION_LOCAL( + network_throttle_manager_t::m_lock_get_global_throttle_in + ); + return std::chrono::duration_cast::duration_t>( + std::chrono::duration( + std::min( + network_throttle_manager_t::get_global_throttle_in( + ).get_sleep_time_after_tick(1), + 1.0 + ) + ) + ); + }; + const auto duration = calc_duration(); + if (duration > duration_t{}) { + ec_t ec; + m_timers.throttle.in.expires_from_now(duration, ec); + m_state.timers.throttle.in.wait_expire = true; + m_timers.throttle.in.async_wait([this, self](const ec_t &ec){ + std::lock_guard guard(m_state.lock); + m_state.timers.throttle.in.wait_expire = false; + if (m_state.timers.throttle.in.cancel_expire) { + m_state.timers.throttle.in.cancel_expire = false; + state_status_check(); + } + else if (ec.value()) + interrupt(); + else + start_read(); + }); + return; + } + } + m_state.socket.wait_read = true; + auto on_read = [this, self](const ec_t &ec, size_t bytes_transferred){ + std::lock_guard guard(m_state.lock); + m_state.socket.wait_read = false; + if (m_state.socket.cancel_read) { + m_state.socket.cancel_read = false; + state_status_check(); + } + else if (ec.value()) + terminate(); + else { + { + m_state.stat.in.throttle.handle_trafic_exact(bytes_transferred); + const auto speed = m_state.stat.in.throttle.get_current_speed(); + m_conn_context.m_current_speed_down = speed; + m_conn_context.m_max_speed_down = std::max( + m_conn_context.m_max_speed_down, + speed + ); + { + CRITICAL_REGION_LOCAL( + network_throttle_manager_t::m_lock_get_global_throttle_in + ); + network_throttle_manager_t::get_global_throttle_in( + ).handle_trafic_exact(bytes_transferred); + } + connection_basic::logger_handle_net_read(bytes_transferred); + m_conn_context.m_last_recv = time(NULL); + m_conn_context.m_recv_cnt += bytes_transferred; + start_timer(get_timeout_from_bytes_read(bytes_transferred), true); + } - buffer_ssl_init_fill += bytes_transferred; - MTRACE("we now have " << buffer_ssl_init_fill << "/" << get_ssl_magic_size() << " bytes needed to detect SSL"); - if (buffer_ssl_init_fill < get_ssl_magic_size()) - { - socket().async_receive(boost::asio::buffer(buffer_.data() + buffer_ssl_init_fill, buffer_.size() - buffer_ssl_init_fill), - strand_.wrap( - boost::bind(&connection::handle_receive, connection::shared_from_this(), - boost::asio::placeholders::error, - boost::asio::placeholders::bytes_transferred))); + // Post handle_recv to a separate `strand_`, distinct from `m_strand` + // which is listening for reads/writes. This avoids a circular dep. + // handle_recv can queue many writes, and `m_strand` will process those + // writes until the connection terminates without deadlocking waiting + // for handle_recv. + m_state.socket.handle_read = true; + connection_basic::strand_.post( + [this, self, bytes_transferred]{ + bool success = m_handler.handle_recv( + reinterpret_cast(m_state.data.read.buffer.data()), + bytes_transferred + ); + std::lock_guard guard(m_state.lock); + m_state.socket.handle_read = false; + if (m_state.status == status_t::INTERRUPTED) + on_interrupted(); + else if (m_state.status == status_t::TERMINATING) + on_terminating(); + else if (!success) + interrupt(); + else { + start_read(); + } + } + ); + } + }; + if (!m_state.ssl.enabled) + connection_basic::socket_.next_layer().async_read_some( + boost::asio::buffer( + m_state.data.read.buffer.data(), + m_state.data.read.buffer.size() + ), + m_strand.wrap(on_read) + ); + else + m_strand.post( + [this, self, on_read]{ + connection_basic::socket_.async_read_some( + boost::asio::buffer( + m_state.data.read.buffer.data(), + m_state.data.read.buffer.size() + ), + m_strand.wrap(on_read) + ); + } + ); + } + + template + void connection::start_write() + { + if (m_state.timers.throttle.out.wait_expire || m_state.socket.wait_write || + m_state.data.write.queue.empty() || + (m_state.ssl.enabled && !m_state.ssl.handshaked) + ) { return; } + auto self = connection::shared_from_this(); + if (m_connection_type != e_connection_type_RPC) { + auto calc_duration = [this]{ + CRITICAL_REGION_LOCAL( + network_throttle_manager_t::m_lock_get_global_throttle_out + ); + return std::chrono::duration_cast::duration_t>( + std::chrono::duration( + std::min( + network_throttle_manager_t::get_global_throttle_out( + ).get_sleep_time_after_tick( + m_state.data.write.queue.back().size() + ), + 1.0 + ) + ) + ); + }; + const auto duration = calc_duration(); + if (duration > duration_t{}) { + ec_t ec; + m_timers.throttle.out.expires_from_now(duration, ec); + m_state.timers.throttle.out.wait_expire = true; + m_timers.throttle.out.async_wait([this, self](const ec_t &ec){ + std::lock_guard guard(m_state.lock); + m_state.timers.throttle.out.wait_expire = false; + if (m_state.timers.throttle.out.cancel_expire) { + m_state.timers.throttle.out.cancel_expire = false; + state_status_check(); + } + else if (ec.value()) + interrupt(); + else + start_write(); + }); + } + } - // detect SSL - if (m_ssl_support == epee::net_utils::ssl_support_t::e_ssl_support_autodetect) - { - if (is_ssl((const unsigned char*)buffer_.data(), buffer_ssl_init_fill)) - { - MDEBUG("That looks like SSL"); - m_ssl_support = epee::net_utils::ssl_support_t::e_ssl_support_enabled; // read/write to the SSL socket + m_state.socket.wait_write = true; + auto on_write = [this, self](const ec_t &ec, size_t bytes_transferred){ + std::lock_guard guard(m_state.lock); + m_state.socket.wait_write = false; + if (m_state.socket.cancel_write) { + m_state.socket.cancel_write = false; + m_state.data.write.queue.clear(); + state_status_check(); } - else - { - MDEBUG("That does not look like SSL"); - m_ssl_support = epee::net_utils::ssl_support_t::e_ssl_support_disabled; // read/write to the raw socket + else if (ec.value()) { + m_state.data.write.queue.clear(); + interrupt(); } - } + else { + { + m_state.stat.out.throttle.handle_trafic_exact(bytes_transferred); + const auto speed = m_state.stat.out.throttle.get_current_speed(); + m_conn_context.m_current_speed_up = speed; + m_conn_context.m_max_speed_down = std::max( + m_conn_context.m_max_speed_down, + speed + ); + { + CRITICAL_REGION_LOCAL( + network_throttle_manager_t::m_lock_get_global_throttle_out + ); + network_throttle_manager_t::get_global_throttle_out( + ).handle_trafic_exact(bytes_transferred); + } + connection_basic::logger_handle_net_write(bytes_transferred); + m_conn_context.m_last_send = time(NULL); + m_conn_context.m_send_cnt += bytes_transferred; - if (m_ssl_support == epee::net_utils::ssl_support_t::e_ssl_support_enabled) - { - // Handshake - if (!handshake(boost::asio::ssl::stream_base::server, boost::asio::const_buffer(buffer_.data(), buffer_ssl_init_fill))) - { - MERROR("SSL handshake failed"); - m_want_close_connection = true; - m_ready_to_close = true; - bool do_shutdown = false; - CRITICAL_REGION_BEGIN(m_send_que_lock); - if(!m_send_que.size()) - do_shutdown = true; - CRITICAL_REGION_END(); - if(do_shutdown) - shutdown(); - return; + start_timer(get_default_timeout(), true); + } + assert(bytes_transferred == m_state.data.write.queue.back().size()); + m_state.data.write.queue.pop_back(); + m_state.condition.notify_all(); + start_write(); } - } + }; + if (!m_state.ssl.enabled) + boost::asio::async_write( + connection_basic::socket_.next_layer(), + boost::asio::buffer( + m_state.data.write.queue.back().data(), + m_state.data.write.queue.back().size() + ), + m_strand.wrap(on_write) + ); else - { - handle_read(e, buffer_ssl_init_fill); - return; - } + m_strand.post( + [this, self, on_write]{ + boost::asio::async_write( + connection_basic::socket_, + boost::asio::buffer( + m_state.data.write.queue.back().data(), + m_state.data.write.queue.back().size() + ), + m_strand.wrap(on_write) + ); + } + ); + } - async_read_some(boost::asio::buffer(buffer_), - strand_.wrap( - boost::bind(&connection::handle_read, connection::shared_from_this(), - boost::asio::placeholders::error, - boost::asio::placeholders::bytes_transferred))); - - // If an error occurs then no new asynchronous operations are started. This - // means that all shared_ptr references to the connection object will - // disappear and the object will be destroyed automatically after this - // handler returns. The connection class's destructor closes the socket. - CATCH_ENTRY_L0("connection::handle_receive", void()); + template + void connection::start_shutdown() + { + if (m_state.socket.wait_shutdown) + return; + auto self = connection::shared_from_this(); + m_state.socket.wait_shutdown = true; + auto on_shutdown = [this, self](const ec_t &ec){ + std::lock_guard guard(m_state.lock); + m_state.socket.wait_shutdown = false; + if (m_state.socket.cancel_shutdown) { + m_state.socket.cancel_shutdown = false; + switch (m_state.status) + { + case status_t::RUNNING: + interrupt(); + break; + case status_t::INTERRUPTED: + terminate(); + break; + case status_t::TERMINATING: + on_terminating(); + break; + default: + break; + } + } + else if (ec.value()) + terminate(); + else { + cancel_timer(); + on_interrupted(); + } + }; + m_strand.post( + [this, self, on_shutdown]{ + connection_basic::socket_.async_shutdown( + m_strand.wrap(on_shutdown) + ); + } + ); + start_timer(get_default_timeout()); } - //--------------------------------------------------------------------------------- - template - bool connection::call_run_once_service_io() + + template + void connection::cancel_socket() { - TRY_ENTRY(); - if(!m_is_multithreaded) - { - //single thread model, we can wait in blocked call - size_t cnt = GET_IO_SERVICE(socket()).run_one(); - if(!cnt)//service is going to quit - return false; - }else - { - //multi thread model, we can't(!) wait in blocked call - //so we make non blocking call and releasing CPU by calling sleep(0); - //if no handlers were called - //TODO: Maybe we need to have have critical section + event + callback to upper protocol to - //ask it inside(!) critical region if we still able to go in event wait... - size_t cnt = GET_IO_SERVICE(socket()).poll_one(); - if(!cnt) - misc_utils::sleep_no_w(1); + bool wait_socket = false; + if (m_state.socket.wait_handshake) + wait_socket = m_state.socket.cancel_handshake = true; + if (m_state.timers.throttle.in.wait_expire) { + m_state.timers.throttle.in.cancel_expire = true; + ec_t ec; + m_timers.throttle.in.cancel(ec); + } + if (m_state.socket.wait_read) + wait_socket = m_state.socket.cancel_read = true; + if (m_state.timers.throttle.out.wait_expire) { + m_state.timers.throttle.out.cancel_expire = true; + ec_t ec; + m_timers.throttle.out.cancel(ec); + } + if (m_state.socket.wait_write) + wait_socket = m_state.socket.cancel_write = true; + if (m_state.socket.wait_shutdown) + wait_socket = m_state.socket.cancel_shutdown = true; + if (wait_socket) { + ec_t ec; + connection_basic::socket_.next_layer().cancel(ec); } - - return true; - CATCH_ENTRY_L0("connection::call_run_once_service_io", false); } - //--------------------------------------------------------------------------------- - template - bool connection::do_send(byte_slice message) { - TRY_ENTRY(); - - // Use safe_shared_from_this, because of this is public method and it can be called on the object being deleted - auto self = safe_shared_from_this(); - if (!self) return false; - if (m_was_shutdown) return false; - // TODO avoid copy - - std::uint8_t const* const message_data = message.data(); - const std::size_t message_size = message.size(); - - const double factor = 32; // TODO config - typedef long long signed int t_safe; // my t_size to avoid any overunderflow in arithmetic - const t_safe chunksize_good = (t_safe)( 1024 * std::max(1.0,factor) ); - const t_safe chunksize_max = chunksize_good * 2 ; - const bool allow_split = (m_connection_type == e_connection_type_RPC) ? false : true; // do not split RPC data - - CHECK_AND_ASSERT_MES(! (chunksize_max<0), false, "Negative chunksize_max" ); // make sure it is unsigned before removin sign with cast: - long long unsigned int chunksize_max_unsigned = static_cast( chunksize_max ) ; - - if (allow_split && (message_size > chunksize_max_unsigned)) { - { // LOCK: chunking - epee::critical_region_t send_guard(m_chunking_lock); // *** critical *** - - MDEBUG("do_send() will SPLIT into small chunks, from packet="< + void connection::cancel_handler() + { + if (m_state.protocol.released || m_state.protocol.wait_release) + return; + m_state.protocol.wait_release = true; + m_state.lock.unlock(); + m_handler.release_protocol(); + m_state.lock.lock(); + m_state.protocol.wait_release = false; + m_state.protocol.released = true; + if (m_state.status == status_t::INTERRUPTED) + on_interrupted(); + else if (m_state.status == status_t::TERMINATING) + on_terminating(); + } - MDEBUG("do_send() DONE SPLIT from packet="<::do_send", false); - } // do_send() + template + void connection::on_terminating() + { + assert(m_state.status == status_t::TERMINATING); + if (m_state.timers.general.wait_expire) + return; + if (m_state.socket.wait_handshake) + return; + if (m_state.timers.throttle.in.wait_expire) + return; + if (m_state.socket.wait_read) + return; + if (m_state.socket.handle_read) + return; + if (m_state.timers.throttle.out.wait_expire) + return; + if (m_state.socket.wait_write) + return; + if (m_state.socket.wait_shutdown) + return; + if (m_state.protocol.wait_init) + return; + if (m_state.protocol.wait_callback) + return; + if (m_state.protocol.wait_release) + return; + if (m_state.socket.connected) { + ec_t ec; + connection_basic::socket_.next_layer().shutdown( + socket_t::shutdown_both, + ec + ); + connection_basic::socket_.next_layer().close(ec); + m_state.socket.connected = false; + } + m_state.status = status_t::WASTED; + } - //--------------------------------------------------------------------------------- - template - bool connection::do_send_chunk(byte_slice chunk) + template + bool connection::send(epee::byte_slice message) { - TRY_ENTRY(); - // Use safe_shared_from_this, because of this is public method and it can be called on the object being deleted - auto self = safe_shared_from_this(); - if(!self) - return false; - if(m_was_shutdown) + std::lock_guard guard(m_state.lock); + if (m_state.status != status_t::RUNNING || m_state.socket.wait_handshake) return false; - double current_speed_up; - { - CRITICAL_REGION_LOCAL(m_throttle_speed_out_mutex); - m_throttle_speed_out.handle_trafic_exact(chunk.size()); - current_speed_up = m_throttle_speed_out.get_current_speed(); - } - context.m_current_speed_up = current_speed_up; - context.m_max_speed_up = std::max(context.m_max_speed_up, current_speed_up); - - //_info("[sock " << socket().native_handle() << "] SEND " << cb); - context.m_last_send = time(NULL); - context.m_send_cnt += chunk.size(); - //some data should be wrote to stream - //request complete - - // No sleeping here; sleeping is done once and for all in "handle_write" - - m_send_que_lock.lock(); // *** critical *** - epee::misc_utils::auto_scope_leave_caller scope_exit_handler = epee::misc_utils::create_scope_leave_handler([&](){m_send_que_lock.unlock();}); - - long int retry=0; - const long int retry_limit = 5*4; - while (m_send_que.size() > ABSTRACT_SERVER_SEND_QUE_MAX_COUNT) - { - retry++; - - /* if ( ::cryptonote::core::get_is_stopping() ) { // TODO re-add fast stop - _fact("ABORT queue wait due to stopping"); - return false; // aborted - }*/ + // Wait for the write queue to fall below the max. If it doesn't after a + // randomized delay, drop the connection. + auto wait_consume = [this] { + auto random_delay = []{ using engine = std::mt19937; - - engine rng; std::random_device dev; - std::seed_seq::result_type rand[engine::state_size]{}; // Use complete bit space - + std::seed_seq::result_type rand[ + engine::state_size // Use complete bit space + ]{}; std::generate_n(rand, engine::state_size, std::ref(dev)); std::seed_seq seed(rand, rand + engine::state_size); - rng.seed(seed); - - long int ms = 250 + (rng() % 50); - MDEBUG("Sleeping because QUEUE is FULL, in " << __FUNCTION__ << " for " << ms << " ms before packet_size="< retry_limit) { - MWARNING("send que size is more than ABSTRACT_SERVER_SEND_QUE_MAX_COUNT(" << ABSTRACT_SERVER_SEND_QUE_MAX_COUNT << "), shutting down connection"); - shutdown(); - return false; + engine rng(seed); + return std::chrono::milliseconds( + std::uniform_int_distribution<>(5000, 6000)(rng) + ); + }; + if (m_state.data.write.queue.size() <= ABSTRACT_SERVER_SEND_QUE_MAX_COUNT) + return true; + m_state.data.write.wait_consume = true; + bool success = m_state.condition.wait_for( + m_state.lock, + random_delay(), + [this]{ + return ( + m_state.status != status_t::RUNNING || + m_state.data.write.queue.size() <= + ABSTRACT_SERVER_SEND_QUE_MAX_COUNT + ); + } + ); + m_state.data.write.wait_consume = false; + if (!success) { + terminate(); + return false; + } + else + return m_state.status == status_t::RUNNING; + }; + auto wait_sender = [this] { + m_state.condition.wait( + m_state.lock, + [this] { + return ( + m_state.status != status_t::RUNNING || + !m_state.data.write.wait_consume + ); } + ); + return m_state.status == status_t::RUNNING; + }; + if (!wait_sender()) + return false; + constexpr size_t CHUNK_SIZE = 32 * 1024; + if (m_connection_type == e_connection_type_RPC || + message.size() <= 2 * CHUNK_SIZE + ) { + if (!wait_consume()) + return false; + m_state.data.write.queue.emplace_front(std::move(message)); + start_write(); } - - m_send_que.push_back(std::move(chunk)); - - if(m_send_que.size() > 1) - { // active operation should be in progress, nothing to do, just wait last operation callback - auto size_now = m_send_que.back().size(); - MDEBUG("do_send_chunk() NOW just queues: packet="< + bool connection::start_internal( + bool is_income, + bool is_multithreaded, + boost::optional real_remote + ) + { + std::unique_lock guard(m_state.lock); + if (m_state.status != status_t::TERMINATED) + return false; + if (!real_remote) { + ec_t ec; + auto endpoint = connection_basic::socket_.next_layer().remote_endpoint( + ec + ); + if (ec.value()) + return false; + real_remote = ( + endpoint.address().is_v6() ? + network_address{ + ipv6_network_address{endpoint.address().to_v6(), endpoint.port()} + } : + network_address{ + ipv4_network_address{ + uint32_t{ + boost::asio::detail::socket_ops::host_to_network_long( + endpoint.address().to_v4().to_ulong() + ) + }, + endpoint.port() + } } - - auto size_now = m_send_que.front().size(); - MDEBUG("do_send_chunk() NOW SENSD: packet="<::handle_write, self, std::placeholders::_1, std::placeholders::_2) - ) - ); - //_dbg3("(chunk): " << size_now); - //logger_handle_net_write(size_now); - //_info("[sock " << socket().native_handle() << "] Async send requested " << m_send_que.front().size()); + ); } - - //do_send_handler_stop( ptr , cb ); // empty function - + auto *filter = static_cast( + connection_basic::get_state() + ).pfilter; + if (filter && !filter->is_remote_host_allowed(*real_remote)) + return false; + ec_t ec; + #if !defined(_WIN32) || !defined(__i686) + connection_basic::socket_.next_layer().set_option( + boost::asio::detail::socket_option::integer{ + connection_basic::get_tos_flag() + }, + ec + ); + if (ec.value()) + return false; + #endif + connection_basic::socket_.next_layer().set_option( + boost::asio::ip::tcp::no_delay{false}, + ec + ); + if (ec.value()) + return false; + connection_basic::m_is_multithreaded = is_multithreaded; + m_conn_context.set_details( + boost::uuids::random_generator()(), + *real_remote, + is_income, + connection_basic::m_ssl_support == ssl_support_t::e_ssl_support_enabled + ); + m_host = real_remote->host_str(); + try { host_count(1); } catch(...) { /* ignore */ } + m_local = real_remote->is_loopback() || real_remote->is_local(); + m_state.ssl.enabled = ( + connection_basic::m_ssl_support != ssl_support_t::e_ssl_support_disabled + ); + m_state.ssl.forced = ( + connection_basic::m_ssl_support == ssl_support_t::e_ssl_support_enabled + ); + m_state.socket.connected = true; + m_state.status = status_t::RUNNING; + start_timer( + std::chrono::milliseconds( + m_local ? NEW_CONNECTION_TIMEOUT_LOCAL : NEW_CONNECTION_TIMEOUT_REMOTE + ) + ); + m_state.protocol.wait_init = true; + guard.unlock(); + m_handler.after_init_connection(); + guard.lock(); + m_state.protocol.wait_init = false; + m_state.protocol.initialized = true; + if (m_state.status == status_t::INTERRUPTED) + on_interrupted(); + else if (m_state.status == status_t::TERMINATING) + on_terminating(); + else if (!is_income || !m_state.ssl.enabled) + start_read(); + else + start_handshake(); return true; + } - CATCH_ENTRY_L0("connection::do_send_chunk", false); - } // do_send_chunk - //--------------------------------------------------------------------------------- - template - boost::posix_time::milliseconds connection::get_default_timeout() + template + connection::connection( + io_context_t &io_context, + std::shared_ptr shared_state, + t_connection_type connection_type, + ssl_support_t ssl_support + ): + connection( + std::move(socket_t{io_context}), + std::move(shared_state), + connection_type, + ssl_support + ) { - unsigned count; - try { count = host_count(m_host); } catch (...) { count = 0; } - const unsigned shift = get_state().sock_count > AGGRESSIVE_TIMEOUT_THRESHOLD ? std::min(std::max(count, 1u) - 1, 8u) : 0; - boost::posix_time::milliseconds timeout(0); - if (m_local) - timeout = boost::posix_time::milliseconds(DEFAULT_TIMEOUT_MS_LOCAL >> shift); - else - timeout = boost::posix_time::milliseconds(DEFAULT_TIMEOUT_MS_REMOTE >> shift); - return timeout; } - //--------------------------------------------------------------------------------- - template - boost::posix_time::milliseconds connection::get_timeout_from_bytes_read(size_t bytes) + + template + connection::connection( + socket_t &&socket, + std::shared_ptr shared_state, + t_connection_type connection_type, + ssl_support_t ssl_support + ): + connection_basic(std::move(socket), shared_state, ssl_support), + m_handler(this, *shared_state, m_conn_context), + m_connection_type(connection_type), + m_io_context{GET_IO_SERVICE(connection_basic::socket_)}, + m_strand{m_io_context}, + m_timers{m_io_context} { - boost::posix_time::milliseconds ms = (boost::posix_time::milliseconds)(unsigned)(bytes * TIMEOUT_EXTRA_MS_PER_BYTE); - const auto cur = m_timer.expires_from_now().total_milliseconds(); - if (cur > 0) - ms += (boost::posix_time::milliseconds)cur; - if (ms > get_default_timeout()) - ms = get_default_timeout(); - return ms; } - //--------------------------------------------------------------------------------- - template - unsigned int connection::host_count(const std::string &host, int delta) + + template + connection::~connection() noexcept(false) { - static boost::mutex hosts_mutex; - CRITICAL_REGION_LOCAL(hosts_mutex); - static std::map hosts; - unsigned int &val = hosts[host]; - if (delta > 0) - MTRACE("New connection from host " << host << ": " << val); - else if (delta < 0) - MTRACE("Closed connection from host " << host << ": " << val); - CHECK_AND_ASSERT_THROW_MES(delta >= 0 || val >= (unsigned)-delta, "Count would go negative"); - CHECK_AND_ASSERT_THROW_MES(delta <= 0 || val <= std::numeric_limits::max() - (unsigned)delta, "Count would wrap"); - val += delta; - return val; + std::lock_guard guard(m_state.lock); + assert(m_state.status == status_t::TERMINATED || + m_state.status == status_t::WASTED || + m_io_context.stopped() + ); + if (m_state.status != status_t::WASTED) + return; + try { host_count(-1); } catch (...) { /* ignore */ } } - //--------------------------------------------------------------------------------- - template - void connection::reset_timer(boost::posix_time::milliseconds ms, bool add) + + template + bool connection::start( + bool is_income, + bool is_multithreaded + ) { - const auto tms = ms.total_milliseconds(); - if (tms < 0 || (add && tms == 0)) - { - MWARNING("Ignoring negative timeout " << ms); - return; - } - MTRACE((add ? "Adding" : "Setting") << " " << ms << " expiry"); - auto self = safe_shared_from_this(); - if(!self) - { - MERROR("Resetting timer on a dead object"); - return; - } - if (m_was_shutdown) - { - MERROR("Setting timer on a shut down object"); - return; - } - if (add) - { - const auto cur = m_timer.expires_from_now().total_milliseconds(); - if (cur > 0) - ms += (boost::posix_time::milliseconds)cur; - } - m_timer.expires_from_now(ms); - m_timer.async_wait([=](const boost::system::error_code& ec) - { - if(ec == boost::asio::error::operation_aborted) - return; - MDEBUG(context << "connection timeout, closing"); - self->close(); - }); + return start_internal(is_income, is_multithreaded, {}); } - //--------------------------------------------------------------------------------- - template - bool connection::shutdown() + + template + bool connection::start( + bool is_income, + bool is_multithreaded, + network_address real_remote + ) { - CRITICAL_REGION_BEGIN(m_shutdown_lock); - if (m_was_shutdown) - return true; - m_was_shutdown = true; - // Initiate graceful connection closure. - m_timer.cancel(); - boost::system::error_code ignored_ec; - if (m_ssl_support == epee::net_utils::ssl_support_t::e_ssl_support_enabled) - { - const shared_state &state = static_cast(get_state()); - if (!state.stop_signal_sent) - socket_.shutdown(ignored_ec); - } - socket().shutdown(boost::asio::ip::tcp::socket::shutdown_both, ignored_ec); - if (!m_host.empty()) - { - try { host_count(m_host, -1); } catch (...) { /* ignore */ } - m_host = ""; - } - CRITICAL_REGION_END(); - m_protocol_handler.release_protocol(); - return true; + return start_internal(is_income, is_multithreaded, real_remote); } - //--------------------------------------------------------------------------------- - template - bool connection::close() + + template + void connection::save_dbg_log() { - TRY_ENTRY(); - auto self = safe_shared_from_this(); - if(!self) - return false; - //_info("[sock " << socket().native_handle() << "] Que Shutdown called."); - m_timer.cancel(); - size_t send_que_size = 0; - CRITICAL_REGION_BEGIN(m_send_que_lock); - send_que_size = m_send_que.size(); - CRITICAL_REGION_END(); - m_want_close_connection = true; - if(!send_que_size) - { - shutdown(); + std::lock_guard guard(m_state.lock); + std::string address; + std::string port; + ec_t ec; + auto endpoint = connection_basic::socket().remote_endpoint(ec); + if (ec.value()) { + address = ""; + port = ""; } - - return true; - CATCH_ENTRY_L0("connection::close", false); + else { + address = endpoint.address().to_string(); + port = std::to_string(endpoint.port()); + } + MDEBUG( + " connection type " << std::to_string(m_connection_type) << + " " << connection_basic::socket().local_endpoint().address().to_string() << + ":" << connection_basic::socket().local_endpoint().port() << + " <--> " << m_conn_context.m_remote_address.str() << + " (via " << address << ":" << port << ")" + ); } - //--------------------------------------------------------------------------------- - template - bool connection::send_done() + + template + bool connection::speed_limit_is_enabled() const { - if (m_ready_to_close) - return close(); - m_ready_to_close = true; - return true; + return m_connection_type != e_connection_type_RPC; } - //--------------------------------------------------------------------------------- - template - bool connection::cancel() + + template + bool connection::cancel() { return close(); } - //--------------------------------------------------------------------------------- - template - void connection::handle_write(const boost::system::error_code& e, size_t cb) + + template + bool connection::do_send(byte_slice message) { - TRY_ENTRY(); - LOG_TRACE_CC(context, "[sock " << socket().native_handle() << "] Async send calledback " << cb); + return send(std::move(message)); + } - if (e) - { - _dbg1("[sock " << socket().native_handle() << "] Some problems at write: " << e.message() << ':' << e.value()); - shutdown(); - return; - } - logger_handle_net_write(cb); + template + bool connection::send_done() + { + return true; + } - // The single sleeping that is needed for correctly handling "out" speed throttling - if (speed_limit_is_enabled()) { - sleep_before_packet(cb, 1, 1); - } + template + bool connection::close() + { + std::lock_guard guard(m_state.lock); + if (m_state.status != status_t::RUNNING) + return false; + terminate(); + return true; + } - bool do_shutdown = false; - CRITICAL_REGION_BEGIN(m_send_que_lock); - if(m_send_que.empty()) - { - _erro("[sock " << socket().native_handle() << "] m_send_que.size() == 0 at handle_write!"); - return; + template + bool connection::call_run_once_service_io() + { + if(connection_basic::m_is_multithreaded) { + if (!m_io_context.poll_one()) + misc_utils::sleep_no_w(1); } - - m_send_que.pop_front(); - if(m_send_que.empty()) - { - if(m_want_close_connection) - { - do_shutdown = true; - } - }else - { - //have more data to send - reset_timer(get_default_timeout(), false); - auto size_now = m_send_que.front().size(); - MDEBUG("handle_write() NOW SENDS: packet="<::handle_write, connection::shared_from_this(), std::placeholders::_1, std::placeholders::_2) - ) - ); - //_dbg3("(normal)" << size_now); + else { + if (!m_io_context.run_one()) + return false; } - CRITICAL_REGION_END(); + return true; + } - if(do_shutdown) - { - shutdown(); - } - CATCH_ENTRY_L0("connection::handle_write", void()); + template + bool connection::request_callback() + { + std::lock_guard guard(m_state.lock); + if (m_state.status != status_t::RUNNING) + return false; + auto self = connection::shared_from_this(); + ++m_state.protocol.wait_callback; + connection_basic::strand_.post([this, self]{ + m_handler.handle_qued_callback(); + std::lock_guard guard(m_state.lock); + --m_state.protocol.wait_callback; + if (m_state.status == status_t::INTERRUPTED) + on_interrupted(); + else if (m_state.status == status_t::TERMINATING) + on_terminating(); + }); + return true; } - //--------------------------------------------------------------------------------- - template - void connection::setRpcStation() + template + typename connection::io_context_t &connection::get_io_service() { - m_connection_type = e_connection_type_RPC; - MDEBUG("set m_connection_type = RPC "); + return m_io_context; } + template + bool connection::add_ref() + { + try { + auto self = connection::shared_from_this(); + std::lock_guard guard(m_state.lock); + this->self = std::move(self); + ++m_state.protocol.reference_counter; + return true; + } + catch (boost::bad_weak_ptr &exception) { + return false; + } + } - template - bool connection::speed_limit_is_enabled() const { - return m_connection_type != e_connection_type_RPC ; - } + template + bool connection::release() + { + connection_ptr self; + std::lock_guard guard(m_state.lock); + if (!(--m_state.protocol.reference_counter)) + self = std::move(this->self); + return true; + } - /************************************************************************/ - /* */ - /************************************************************************/ + template + void connection::setRpcStation() + { + std::lock_guard guard(m_state.lock); + m_connection_type = e_connection_type_RPC; + } template boosted_tcp_server::boosted_tcp_server( t_connection_type connection_type ) : diff --git a/contrib/epee/include/net/buffer.h b/contrib/epee/include/net/buffer.h index c5305a8b717..8e53512ac5b 100644 --- a/contrib/epee/include/net/buffer.h +++ b/contrib/epee/include/net/buffer.h @@ -1,4 +1,4 @@ -// Copyright (c) 2018-2022, The Monero Project +// Copyright (c) 2018-2023, The Monero Project // // All rights reserved. diff --git a/contrib/epee/include/net/connection_basic.hpp b/contrib/epee/include/net/connection_basic.hpp index a26d538cd1f..e3093de51ad 100644 --- a/contrib/epee/include/net/connection_basic.hpp +++ b/contrib/epee/include/net/connection_basic.hpp @@ -8,7 +8,7 @@ // ! (how ever if in some wonderful juristdictions that is not the case, then why not make another sub-class withat that members and licence it as epee part) // ! Working on above premise, IF this is valid in your juristdictions, then consider this code as released as: -// Copyright (c) 2014-2022, The Monero Project +// Copyright (c) 2014-2023, The Monero Project // // All rights reserved. // diff --git a/contrib/epee/include/net/enums.h b/contrib/epee/include/net/enums.h index 8c402da2021..125ef0db0af 100644 --- a/contrib/epee/include/net/enums.h +++ b/contrib/epee/include/net/enums.h @@ -1,4 +1,4 @@ -// Copyright (c) 2018-2022, The Monero Project +// Copyright (c) 2018-2023, The Monero Project // // All rights reserved. @@ -64,3 +64,13 @@ namespace net_utils } // net_utils } // epee +namespace std +{ + template<> struct hash + { + std::size_t operator()(const epee::net_utils::zone _z) const + { + return static_cast(_z); + } + }; +} // std diff --git a/contrib/epee/include/net/http_auth.h b/contrib/epee/include/net/http_auth.h index 97407ae2cf9..ee5a79a7d67 100644 --- a/contrib/epee/include/net/http_auth.h +++ b/contrib/epee/include/net/http_auth.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2022, The Monero Project +// Copyright (c) 2014-2023, The Monero Project // // All rights reserved. // diff --git a/contrib/epee/include/net/http_base.h b/contrib/epee/include/net/http_base.h index 4af4da7908a..15fd30bf32b 100644 --- a/contrib/epee/include/net/http_base.h +++ b/contrib/epee/include/net/http_base.h @@ -55,20 +55,8 @@ namespace net_utils http_method_unknown }; - enum http_content_type - { - http_content_type_text_html, - http_content_type_image_gif, - http_content_type_other, - http_content_type_not_set - }; - typedef std::list > fields_list; - std::string get_value_from_fields_list(const std::string& param_name, const net_utils::http::fields_list& fields); - - std::string get_value_from_uri_line(const std::string& param_name, const std::string& uri); - static inline void add_field(std::string& out, const boost::string_ref name, const boost::string_ref value) { out.append(name.data(), name.size()).append(": "); diff --git a/contrib/epee/include/net/http_client.h b/contrib/epee/include/net/http_client.h index 9ce30b62014..8cee399cb5e 100644 --- a/contrib/epee/include/net/http_client.h +++ b/contrib/epee/include/net/http_client.h @@ -32,7 +32,6 @@ #include #include #include -//#include #include #include #include @@ -48,57 +47,13 @@ #include "net_parse_helpers.h" #include "syncobj.h" -//#include "shlwapi.h" - -//#pragma comment(lib, "shlwapi.lib") - #undef MONERO_DEFAULT_LOG_CATEGORY #define MONERO_DEFAULT_LOG_CATEGORY "net.http" -extern epee::critical_section gregexp_lock; - - namespace epee { namespace net_utils { - - /*struct url - { - public: - void parse(const std::string& url_s) - { - const string prot_end("://"); - string::const_iterator prot_i = search(url_s.begin(), url_s.end(), - prot_end.begin(), prot_end.end()); - protocol_.reserve(distance(url_s.begin(), prot_i)); - transform(url_s.begin(), prot_i, - back_inserter(protocol_), - ptr_fun(tolower)); // protocol is icase - if( prot_i == url_s.end() ) - return; - advance(prot_i, prot_end.length()); - string::const_iterator path_i = find(prot_i, url_s.end(), '/'); - host_.reserve(distance(prot_i, path_i)); - transform(prot_i, path_i, - back_inserter(host_), - ptr_fun(tolower)); // host is icase - string::const_iterator query_i = find(path_i, url_s.end(), '?'); - path_.assign(path_i, query_i); - if( query_i != url_s.end() ) - ++query_i; - query_.assign(query_i, url_s.end()); - } - - std::string protocol_; - std::string host_; - std::string path_; - std::string query_; - };*/ - - - - //--------------------------------------------------------------------------- namespace http { @@ -135,7 +90,6 @@ namespace net_utils http_response_info m_response_info; size_t m_len_in_summary; size_t m_len_in_remain; - //std::string* m_ptarget_buffer; boost::shared_ptr m_pcontent_encoding_handler; reciev_machine_state m_state; chunked_state m_chunked_state; @@ -300,12 +254,6 @@ namespace net_utils return false; } //--------------------------------------------------------------------------- - inline bool invoke_post(const boost::string_ref uri, const std::string& body, std::chrono::milliseconds timeout, const http_response_info** ppresponse_info = NULL, const fields_list& additional_params = fields_list()) override - { - CRITICAL_REGION_LOCAL(m_lock); - return invoke(uri, "POST", body, timeout, ppresponse_info, additional_params); - } - //--------------------------------------------------------------------------- bool test(const std::string &s, std::chrono::milliseconds timeout) // TEST FUNC ONLY { CRITICAL_REGION_LOCAL(m_lock); diff --git a/contrib/epee/include/net/http_server_handlers_map2.h b/contrib/epee/include/net/http_server_handlers_map2.h index ffb3f3b7ea5..848b8ffc4ad 100644 --- a/contrib/epee/include/net/http_server_handlers_map2.h +++ b/contrib/epee/include/net/http_server_handlers_map2.h @@ -63,15 +63,11 @@ bool handled = false; \ if(false) return true; //just a stub to have "else if" -#define MAP_URI2(pattern, callback) else if(std::string::npos != query_info.m_URI.find(pattern)) return callback(query_info, response_info, &m_conn_context); - -#define MAP_URI_AUTO_XML2(s_pattern, callback_f, command_type) //TODO: don't think i ever again will use xml - ambiguous and "overtagged" format - #define MAP_URI_AUTO_JON2_IF(s_pattern, callback_f, command_type, cond) \ else if((query_info.m_URI == s_pattern) && (cond)) \ { \ handled = true; \ - uint64_t ticks = misc_utils::get_tick_count(); \ + uint64_t ticks = epee::misc_utils::get_tick_count(); \ boost::value_initialized req; \ bool parse_res = epee::serialization::load_t_from_json(static_cast(req), query_info.m_body); \ if (!parse_res) \ @@ -107,7 +103,7 @@ else if(query_info.m_URI == s_pattern) \ { \ handled = true; \ - uint64_t ticks = misc_utils::get_tick_count(); \ + uint64_t ticks = epee::misc_utils::get_tick_count(); \ boost::value_initialized req; \ bool parse_res = epee::serialization::load_t_from_binary(static_cast(req), epee::strspan(query_info.m_body)); \ if (!parse_res) \ @@ -117,7 +113,7 @@ response_info.m_response_comment = "Bad request"; \ return true; \ } \ - uint64_t ticks1 = misc_utils::get_tick_count(); \ + uint64_t ticks1 = epee::misc_utils::get_tick_count(); \ boost::value_initialized resp;\ MINFO(m_conn_context << "calling " << s_pattern); \ bool res = false; \ @@ -129,7 +125,7 @@ response_info.m_response_comment = "Internal Server Error"; \ return true; \ } \ - uint64_t ticks2 = misc_utils::get_tick_count(); \ + uint64_t ticks2 = epee::misc_utils::get_tick_count(); \ epee::byte_slice buffer; \ epee::serialization::store_t_to_binary(static_cast(resp), buffer, 64 * 1024); \ uint64_t ticks3 = epee::misc_utils::get_tick_count(); \ @@ -139,8 +135,6 @@ MDEBUG( s_pattern << "() processed with " << ticks1-ticks << "/"<< ticks2-ticks1 << "/" << ticks3-ticks2 << "ms"); \ } -#define CHAIN_URI_MAP2(callback) else {callback(query_info, response_info, m_conn_context);handled = true;} - #define END_URI_MAP2() return handled;} @@ -225,26 +219,6 @@ #define MAP_JON_RPC_WE(method_name, callback_f, command_type) MAP_JON_RPC_WE_IF(method_name, callback_f, command_type, true) -#define MAP_JON_RPC_WERI(method_name, callback_f, command_type) \ - else if(callback_name == method_name) \ -{ \ - PREPARE_OBJECTS_FROM_JSON(command_type) \ - epee::json_rpc::error_response fail_resp = AUTO_VAL_INIT(fail_resp); \ - fail_resp.jsonrpc = "2.0"; \ - fail_resp.id = req.id; \ - MINFO(m_conn_context << "calling RPC method " << method_name); \ - bool res = false; \ - try { res = callback_f(req.params, resp.result, fail_resp.error, response_info, &m_conn_context); } \ - catch (const std::exception &e) { MERROR(m_conn_context << "Failed to " << #callback_f << "(): " << e.what()); } \ - if (!res) \ - { \ - epee::serialization::store_t_to_json(static_cast(fail_resp), response_info.m_body); \ - return true; \ - } \ - FINALIZE_OBJECTS_TO_JSON(method_name) \ - return true;\ -} - #define MAP_JON_RPC(method_name, callback_f, command_type) \ else if(callback_name == method_name) \ { \ diff --git a/contrib/epee/include/net/net_helper.h b/contrib/epee/include/net/net_helper.h index 0a35797fddf..3cb3d8c669b 100644 --- a/contrib/epee/include/net/net_helper.h +++ b/contrib/epee/include/net/net_helper.h @@ -29,8 +29,6 @@ #pragma once -//#include -//#include #include #include #include @@ -52,11 +50,6 @@ #undef MONERO_DEFAULT_LOG_CATEGORY #define MONERO_DEFAULT_LOG_CATEGORY "net" -#ifndef MAKE_IP -#define MAKE_IP( a1, a2, a3, a4 ) (a1|(a2<<8)|(a3<<16)|(a4<<24)) -#endif - - namespace epee { namespace net_utils @@ -106,7 +99,6 @@ namespace net_utils m_ssl_socket(new boost::asio::ssl::stream(m_io_service, m_ctx)), m_connector(direct_connect{}), m_ssl_options(epee::net_utils::ssl_support_t::e_ssl_support_autodetect), - m_initialized(true), m_connected(false), m_deadline(m_io_service, std::chrono::steady_clock::time_point::max()), m_shutdowned(false), @@ -149,12 +141,6 @@ namespace net_utils m_ssl_options = std::move(ssl_options); } - inline - bool connect(const std::string& addr, int port, std::chrono::milliseconds timeout) - { - return connect(addr, std::to_string(port), timeout); - } - inline try_connect_result_t try_connect(const std::string& addr, const std::string& port, std::chrono::milliseconds timeout) { @@ -333,64 +319,6 @@ namespace net_utils return true; } - inline - bool send(const void* data, size_t sz) - { - try - { - /* - m_deadline.expires_from_now(boost::posix_time::milliseconds(m_reciev_timeout)); - - // Set up the variable that receives the result of the asynchronous - // operation. The error code is set to would_block to signal that the - // operation is incomplete. Asio guarantees that its asynchronous - // operations will never fail with would_block, so any other value in - // ec indicates completion. - boost::system::error_code ec = boost::asio::error::would_block; - - // Start the asynchronous operation itself. The boost::lambda function - // object is used as a callback and will update the ec variable when the - // operation completes. The blocking_udp_client.cpp example shows how you - // can use boost::bind rather than boost::lambda. - boost::asio::async_write(m_socket, boost::asio::buffer(data, sz), boost::lambda::var(ec) = boost::lambda::_1); - - // Block until the asynchronous operation has completed. - while (ec == boost::asio::error::would_block) - { - m_io_service.run_one(); - } - */ - boost::system::error_code ec; - - size_t writen = write(data, sz, ec); - - if (!writen || ec) - { - LOG_PRINT_L3("Problems at write: " << ec.message()); - m_connected = false; - return false; - }else - { - m_deadline.expires_at(std::chrono::steady_clock::time_point::max()); - m_bytes_sent += sz; - } - } - - catch(const boost::system::system_error& er) - { - LOG_ERROR("Some problems at send, message: " << er.what()); - m_connected = false; - return false; - } - catch(...) - { - LOG_ERROR("Some fatal problems."); - return false; - } - - return true; - } - bool is_connected(bool *ssl = NULL) { if (!m_connected || !m_ssl_socket->next_layer().is_open()) @@ -487,79 +415,6 @@ namespace net_utils } - inline bool recv_n(std::string& buff, int64_t sz, std::chrono::milliseconds timeout) - { - - try - { - // Set a deadline for the asynchronous operation. Since this function uses - // a composed operation (async_read_until), the deadline applies to the - // entire operation, rather than individual reads from the socket. - m_deadline.expires_from_now(timeout); - - // Set up the variable that receives the result of the asynchronous - // operation. The error code is set to would_block to signal that the - // operation is incomplete. Asio guarantees that its asynchronous - // operations will never fail with would_block, so any other value in - // ec indicates completion. - //boost::system::error_code ec = boost::asio::error::would_block; - - // Start the asynchronous operation itself. The boost::lambda function - // object is used as a callback and will update the ec variable when the - // operation completes. The blocking_udp_client.cpp example shows how you - // can use boost::bind rather than boost::lambda. - - buff.resize(static_cast(sz)); - boost::system::error_code ec = boost::asio::error::would_block; - size_t bytes_transfered = 0; - - - handler_obj hndlr(ec, bytes_transfered); - async_read((char*)buff.data(), buff.size(), boost::asio::transfer_at_least(buff.size()), hndlr); - - // Block until the asynchronous operation has completed. - while (ec == boost::asio::error::would_block && !m_shutdowned) - { - m_io_service.run_one(); - } - - if (ec) - { - LOG_PRINT_L3("Problems at read: " << ec.message()); - m_connected = false; - return false; - }else - { - m_deadline.expires_at(std::chrono::steady_clock::time_point::max()); - } - - m_bytes_received += bytes_transfered; - if(bytes_transfered != buff.size()) - { - LOG_ERROR("Transferred mismatch with transfer_at_least value: m_bytes_transferred=" << bytes_transfered << " at_least value=" << buff.size()); - return false; - } - - return true; - } - - catch(const boost::system::system_error& er) - { - LOG_ERROR("Some problems at read, message: " << er.what()); - m_connected = false; - return false; - } - catch(...) - { - LOG_ERROR("Some fatal problems at read."); - return false; - } - - - - return false; - } - bool shutdown() { m_deadline.cancel(); @@ -579,16 +434,6 @@ namespace net_utils m_connected = false; return true; } - - boost::asio::io_service& get_io_service() - { - return m_io_service; - } - - boost::asio::ip::tcp::socket& get_socket() - { - return m_ssl_socket->next_layer(); - } uint64_t get_bytes_sent() const { @@ -648,16 +493,6 @@ namespace net_utils } protected: - bool write(const void* data, size_t sz, boost::system::error_code& ec) - { - bool success; - if(m_ssl_options.support != ssl_support_t::e_ssl_support_disabled) - success = boost::asio::write(*m_ssl_socket, boost::asio::buffer(data, sz), ec); - else - success = boost::asio::write(m_ssl_socket->next_layer(), boost::asio::buffer(data, sz), ec); - return success; - } - void async_write(const void* data, size_t sz, boost::system::error_code& ec) { if(m_ssl_options.support != ssl_support_t::e_ssl_support_disabled) @@ -681,126 +516,12 @@ namespace net_utils std::shared_ptr> m_ssl_socket; std::function m_connector; ssl_options_t m_ssl_options; - bool m_initialized; bool m_connected; boost::asio::steady_timer m_deadline; std::atomic m_shutdowned; std::atomic m_bytes_sent; std::atomic m_bytes_received; }; - - - /************************************************************************/ - /* */ - /************************************************************************/ - class async_blocked_mode_client: public blocked_mode_client - { - public: - async_blocked_mode_client():m_send_deadline(blocked_mode_client::m_io_service) - { - - // No deadline is required until the first socket operation is started. We - // set the deadline to positive infinity so that the actor takes no action - // until a specific deadline is set. - m_send_deadline.expires_at(boost::posix_time::pos_infin); - - // Start the persistent actor that checks for deadline expiry. - check_send_deadline(); - } - ~async_blocked_mode_client() - { - m_send_deadline.cancel(); - } - - bool shutdown() - { - blocked_mode_client::shutdown(); - m_send_deadline.cancel(); - return true; - } - - inline - bool send(const void* data, size_t sz) - { - try - { - /* - m_send_deadline.expires_from_now(boost::posix_time::milliseconds(m_reciev_timeout)); - - // Set up the variable that receives the result of the asynchronous - // operation. The error code is set to would_block to signal that the - // operation is incomplete. Asio guarantees that its asynchronous - // operations will never fail with would_block, so any other value in - // ec indicates completion. - boost::system::error_code ec = boost::asio::error::would_block; - - // Start the asynchronous operation itself. The boost::lambda function - // object is used as a callback and will update the ec variable when the - // operation completes. The blocking_udp_client.cpp example shows how you - // can use boost::bind rather than boost::lambda. - boost::asio::async_write(m_socket, boost::asio::buffer(data, sz), boost::lambda::var(ec) = boost::lambda::_1); - - // Block until the asynchronous operation has completed. - while(ec == boost::asio::error::would_block) - { - m_io_service.run_one(); - }*/ - - boost::system::error_code ec; - - size_t writen = write(data, sz, ec); - - if (!writen || ec) - { - LOG_PRINT_L3("Problems at write: " << ec.message()); - return false; - }else - { - m_send_deadline.expires_at(boost::posix_time::pos_infin); - } - } - - catch(const boost::system::system_error& er) - { - LOG_ERROR("Some problems at connect, message: " << er.what()); - return false; - } - catch(...) - { - LOG_ERROR("Some fatal problems."); - return false; - } - - return true; - } - - - private: - - boost::asio::deadline_timer m_send_deadline; - - void check_send_deadline() - { - // Check whether the deadline has passed. We compare the deadline against - // the current time since a new asynchronous operation may have moved the - // deadline before this actor had a chance to run. - if (m_send_deadline.expires_at() <= boost::asio::deadline_timer::traits_type::now()) - { - // The deadline has passed. The socket is closed so that any outstanding - // asynchronous operations are cancelled. This allows the blocked - // connect(), read_line() or write_line() functions to return. - LOG_PRINT_L3("Timed out socket"); - m_ssl_socket->next_layer().close(); - - // There is no longer an active deadline. The expiry is set to positive - // infinity so that the actor takes no action until a new deadline is set. - m_send_deadline.expires_at(boost::posix_time::pos_infin); - } - - // Put the actor back to sleep. - m_send_deadline.async_wait(boost::bind(&async_blocked_mode_client::check_send_deadline, this)); - } - }; } } diff --git a/contrib/epee/include/net/net_ssl.h b/contrib/epee/include/net/net_ssl.h index 108e6771bd6..c6ef925baa7 100644 --- a/contrib/epee/include/net/net_ssl.h +++ b/contrib/epee/include/net/net_ssl.h @@ -110,6 +110,11 @@ namespace net_utils //! Search against internal fingerprints. Always false if `behavior() != user_certificate_check`. bool has_fingerprint(boost::asio::ssl::verify_context &ctx) const; + //! configure ssl_stream handshake verification + void configure( + boost::asio::ssl::stream &socket, + boost::asio::ssl::stream_base::handshake_type type, + const std::string& host = {}) const; boost::asio::ssl::context create_context() const; /*! \note If `this->support == autodetect && this->verification != none`, @@ -146,6 +151,33 @@ namespace net_utils bool create_ec_ssl_certificate(EVP_PKEY *&pkey, X509 *&cert); bool create_rsa_ssl_certificate(EVP_PKEY *&pkey, X509 *&cert); + /** + * @brief Create a human-readable X509 certificate fingerprint + * + * Example output: "12:A3:92:19:87:D2:A2:A5:77:94:82:29:B9:5A:91:01:AB:5F:75:16:9A:BA:CD:3D:D3:69:3D:6A:87:DC:E8:0E" + * + * @param[in] cert The certificate which will be used to create the fingerprint + * @param[in] fdig The digest algorithm to use, defaults to SHA-256 b/c that is what ssl_options_t uses + * @return The human-readable fingerprint string + * + * @throw boost::system_error if there is an OpenSSL error + */ + std::string get_hr_ssl_fingerprint(const X509 *cert, const EVP_MD *fdig = EVP_sha256()); + + /** + * @brief Create a human-readable fingerprint from the contents of an X509 certificate + * + * Should be equivalent to the command `openssl x509 -in -fingerprint -sha256 -noout` + * Example output: "12:A3:92:19:87:D2:A2:A5:77:94:82:29:B9:5A:91:01:AB:5F:75:16:9A:BA:CD:3D:D3:69:3D:6A:87:DC:E8:0E" + * + * @param[in] cert_path The path to an X509 certificate which will be used to create the fingerprint + * @param[in] fdig The digest algorithm to use, defaults to SHA-256 b/c that is what ssl_options_t uses + * @return The human-readable fingerprint string + * + * @throw boost::system_error if there is an OpenSSL error or file I/O error + */ + std::string get_hr_ssl_fingerprint_from_file(const std::string& cert_path, const EVP_MD *fdig = EVP_sha256()); + //! Store private key for `ssl` at `base + ".key"` unencrypted and certificate for `ssl` at `base + ".crt"`. boost::system::error_code store_ssl_keys(boost::asio::ssl::context& ssl, const boost::filesystem::path& base); } diff --git a/contrib/epee/include/net/network_throttle-detail.hpp b/contrib/epee/include/net/network_throttle-detail.hpp index 0a6dc4a207b..596ef29f558 100644 --- a/contrib/epee/include/net/network_throttle-detail.hpp +++ b/contrib/epee/include/net/network_throttle-detail.hpp @@ -2,7 +2,7 @@ /// @author rfree (current maintainer in monero.cc project) /// @brief implementaion for throttling of connection (count and rate-limit speed etc) -// Copyright (c) 2014-2022, The Monero Project +// Copyright (c) 2014-2023, The Monero Project // // All rights reserved. // diff --git a/contrib/epee/include/net/network_throttle.hpp b/contrib/epee/include/net/network_throttle.hpp index 75023161068..90e214aa7d4 100644 --- a/contrib/epee/include/net/network_throttle.hpp +++ b/contrib/epee/include/net/network_throttle.hpp @@ -2,7 +2,7 @@ /// @author rfree (current maintainer in monero.cc project) /// @brief interface for throttling of connection (count and rate-limit speed etc) -// Copyright (c) 2014-2022, The Monero Project +// Copyright (c) 2014-2023, The Monero Project // // All rights reserved. // diff --git a/contrib/epee/include/rolling_median.h b/contrib/epee/include/rolling_median.h index e230fd9740c..c9caddfec57 100644 --- a/contrib/epee/include/rolling_median.h +++ b/contrib/epee/include/rolling_median.h @@ -1,4 +1,4 @@ -// Copyright (c) 2019-2022, The Monero Project +// Copyright (c) 2019-2023, The Monero Project // // All rights reserved. // diff --git a/contrib/epee/include/span.h b/contrib/epee/include/span.h index 99c2ebb4f28..82936a77714 100644 --- a/contrib/epee/include/span.h +++ b/contrib/epee/include/span.h @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2022, The Monero Project +// Copyright (c) 2017-2023, The Monero Project // // All rights reserved. // diff --git a/contrib/epee/include/storages/levin_abstract_invoke2.h b/contrib/epee/include/storages/levin_abstract_invoke2.h index 7fd786a53fd..9d3dda0d654 100644 --- a/contrib/epee/include/storages/levin_abstract_invoke2.h +++ b/contrib/epee/include/storages/levin_abstract_invoke2.h @@ -56,33 +56,6 @@ namespace epee { namespace net_utils { - template - bool invoke_remote_command2(const epee::net_utils::connection_context_base context, int command, const t_arg& out_struct, t_result& result_struct, t_transport& transport) - { - const boost::uuids::uuid &conn_id = context.m_connection_id; - typename serialization::portable_storage stg; - out_struct.store(stg); - levin::message_writer to_send{16 * 1024}; - std::string buff_to_recv; - stg.store_to_binary(to_send.buffer); - - int res = transport.invoke(command, std::move(to_send), buff_to_recv, conn_id); - if( res <=0 ) - { - LOG_PRINT_L1("Failed to invoke command " << command << " return code " << res); - return false; - } - typename serialization::portable_storage stg_ret; - if(!stg_ret.load_from_binary(buff_to_recv, &default_levin_limits)) - { - on_levin_traffic(context, true, false, true, buff_to_recv.size(), command); - LOG_ERROR("Failed to load_from_binary on command " << command); - return false; - } - on_levin_traffic(context, true, false, false, buff_to_recv.size(), command); - return result_struct.load(stg_ret); - } - template bool async_invoke_remote_command2(const epee::net_utils::connection_context_base &context, int command, const t_arg& out_struct, t_transport& transport, const callback_t &cb, size_t inv_timeout = LEVIN_DEFAULT_TIMEOUT_PRECONFIGURED) { @@ -216,65 +189,20 @@ namespace epee return handle_invoke_map(true, command, in_buff, fake_str, context, handled); \ } - -#define CHAIN_LEVIN_INVOKE_MAP() \ - int invoke(int command, const epee::span in_buff, epee::byte_stream& buff_out, epee::net_utils::connection_context_base& context) \ - { \ - bool handled = false; \ - return handle_invoke_map(false, command, in_buff, buff_out, context, handled); \ - } - -#define CHAIN_LEVIN_NOTIFY_MAP() \ - int notify(int command, const epee::span in_buff, epee::net_utils::connection_context_base& context) \ - { \ - bool handled = false; std::string fake_str;\ - return handle_invoke_map(true, command, in_buff, fake_str, context, handled); \ - } - -#define CHAIN_LEVIN_NOTIFY_STUB() \ - int notify(int command, const epee::span in_buff, epee::net_utils::connection_context_base& context) \ - { \ - return -1; \ - } - #define BEGIN_INVOKE_MAP2(owner_type) \ template int handle_invoke_map(bool is_notify, int command, const epee::span in_buff, epee::byte_stream& buff_out, t_context& context, bool& handled) \ { \ try { \ typedef owner_type internal_owner_type_name; -#define HANDLE_INVOKE2(command_id, func, type_name_in, typename_out) \ - if(!is_notify && command_id == command) \ - {handled=true;return epee::net_utils::buff_to_t_adapter(this, command, in_buff, buff_out, std::bind(func, this, std::placeholders::_1, std::placeholders::_2, std::placeholders::_3, std::placeholders::_4), context);} - #define HANDLE_INVOKE_T2(COMMAND, func) \ if(!is_notify && COMMAND::ID == command) \ {handled=true;return epee::net_utils::buff_to_t_adapter(command, in_buff, buff_out, std::bind(func, this, std::placeholders::_1, std::placeholders::_2, std::placeholders::_3, std::placeholders::_4), context);} - -#define HANDLE_NOTIFY2(command_id, func, type_name_in) \ - if(is_notify && command_id == command) \ - {handled=true;return epee::net_utils::buff_to_t_adapter(this, command, in_buff, std::bind(func, this, std::placeholders::_1, std::placeholders::_2, std::placeholders::_3), context);} - #define HANDLE_NOTIFY_T2(NOTIFY, func) \ if(is_notify && NOTIFY::ID == command) \ {handled=true;return epee::net_utils::buff_to_t_adapter(this, command, in_buff, std::bind(func, this, std::placeholders::_1, std::placeholders::_2, std::placeholders::_3), context);} - -#define CHAIN_INVOKE_MAP2(func) \ - { \ - int res = func(is_notify, command, in_buff, buff_out, context, handled); \ - if(handled) \ - return res; \ - } - -#define CHAIN_INVOKE_MAP_TO_OBJ2(obj) \ - { \ - int res = obj.handle_invoke_map(is_notify, command, in_buff, buff_out, context, handled); \ - if(handled) \ - return res; \ - } - #define CHAIN_INVOKE_MAP_TO_OBJ_FORCE_CONTEXT(obj, context_type) \ { \ int res = obj.handle_invoke_map(is_notify, command, in_buff, buff_out, static_cast(context), handled); \ diff --git a/contrib/epee/include/storages/parserse_base_utils.h b/contrib/epee/include/storages/parserse_base_utils.h index 898813ff9fd..4658eba77ad 100644 --- a/contrib/epee/include/storages/parserse_base_utils.h +++ b/contrib/epee/include/storages/parserse_base_utils.h @@ -30,6 +30,7 @@ #include #include +#include namespace epee { diff --git a/contrib/epee/include/storages/portable_storage_bin_utils.h b/contrib/epee/include/storages/portable_storage_bin_utils.h index 476c81ea19f..d4309025100 100644 --- a/contrib/epee/include/storages/portable_storage_bin_utils.h +++ b/contrib/epee/include/storages/portable_storage_bin_utils.h @@ -1,4 +1,4 @@ -// Copyright (c) 2019-2022, The Monero Project +// Copyright (c) 2019-2023, The Monero Project // // All rights reserved. // diff --git a/contrib/epee/include/storages/portable_storage_from_json.h b/contrib/epee/include/storages/portable_storage_from_json.h index 3021598f5f9..69192ca6bd7 100644 --- a/contrib/epee/include/storages/portable_storage_from_json.h +++ b/contrib/epee/include/storages/portable_storage_from_json.h @@ -51,7 +51,6 @@ namespace epee { CHECK_AND_ASSERT_THROW_MES(recursion < EPEE_JSON_RECURSION_LIMIT_INTERNAL, "Wrong JSON data: recursion limitation (" << EPEE_JSON_RECURSION_LIMIT_INTERNAL << ") exceeded"); - std::string::const_iterator sub_element_start; std::string name; typename t_storage::harray h_array = nullptr; enum match_state diff --git a/contrib/epee/include/string_coding.h b/contrib/epee/include/string_coding.h index 82050ef96bd..0d9c6c244ee 100644 --- a/contrib/epee/include/string_coding.h +++ b/contrib/epee/include/string_coding.h @@ -34,100 +34,6 @@ namespace epee { namespace string_encoding { - inline std::string convert_to_ansii(const std::wstring& str_from) - { - - std::string res(str_from.begin(), str_from.end()); - return res; - /* - std::string result; - std::locale loc; - for(unsigned int i= 0; i < str_from.size(); ++i) - { - result += std::use_facet >(loc).narrow(str_from[i]); - } - return result; - */ - - //return boost::lexical_cast(str_from); - /* - std::string str_trgt; - if(!str_from.size()) - return str_trgt; - int cb = ::WideCharToMultiByte( code_page, 0, str_from.data(), (__int32)str_from.size(), 0, 0, 0, 0 ); - if(!cb) - return str_trgt; - str_trgt.resize(cb); - ::WideCharToMultiByte( code_page, 0, str_from.data(), (int)str_from.size(), - (char*)str_trgt.data(), (int)str_trgt.size(), 0, 0); - return str_trgt;*/ - } - - inline std::string convert_to_ansii(const std::string& str_from) - { - return str_from; - } - - inline std::wstring convert_to_unicode(const std::string& str_from) - { - std::wstring result; - std::locale loc; - for(unsigned int i= 0; i < str_from.size(); ++i) - { - result += std::use_facet >(loc).widen(str_from[i]); - } - return result; - - //return boost::lexical_cast(str_from); - /* - std::wstring str_trgt; - if(!str_from.size()) - return str_trgt; - - int cb = ::MultiByteToWideChar( code_page, 0, str_from.data(), (int)str_from.size(), 0, 0 ); - if(!cb) - return str_trgt; - - str_trgt.resize(cb); - ::MultiByteToWideChar( code_page, 0, str_from.data(),(int)str_from.size(), - (wchar_t*)str_trgt.data(),(int)str_trgt.size()); - return str_trgt;*/ - } - inline std::wstring convert_to_unicode(const std::wstring& str_from) - { - return str_from; - } - - template - inline target_string convert_to_t(const std::wstring& str_from); - - template<> - inline std::string convert_to_t(const std::wstring& str_from) - { - return convert_to_ansii(str_from); - } - - template<> - inline std::wstring convert_to_t(const std::wstring& str_from) - { - return str_from; - } - - template - inline target_string convert_to_t(const std::string& str_from); - - template<> - inline std::string convert_to_t(const std::string& str_from) - { - return str_from; - } - - template<> - inline std::wstring convert_to_t(const std::string& str_from) - { - return convert_to_unicode(str_from); - } - inline std::string& base64_chars() { diff --git a/contrib/epee/include/wipeable_string.h b/contrib/epee/include/wipeable_string.h index 65977cd97b6..bd1e9f12b61 100644 --- a/contrib/epee/include/wipeable_string.h +++ b/contrib/epee/include/wipeable_string.h @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2022, The Monero Project +// Copyright (c) 2017-2023, The Monero Project // // All rights reserved. // diff --git a/contrib/epee/src/CMakeLists.txt b/contrib/epee/src/CMakeLists.txt index 808b9f09e21..71ff94f0028 100644 --- a/contrib/epee/src/CMakeLists.txt +++ b/contrib/epee/src/CMakeLists.txt @@ -1,4 +1,4 @@ -# Copyright (c) 2014-2022, The Monero Project +# Copyright (c) 2014-2023, The Monero Project # # All rights reserved. # @@ -37,7 +37,6 @@ monero_add_library(epee byte_slice.cpp byte_stream.cpp hex.cpp abstract_http_cli misc_language.cpp file_io_utils.cpp net_parse_helpers.cpp - http_base.cpp ${EPEE_HEADERS_PUBLIC} ) diff --git a/contrib/epee/src/buffer.cpp b/contrib/epee/src/buffer.cpp index dd66190746a..7d054c7c282 100644 --- a/contrib/epee/src/buffer.cpp +++ b/contrib/epee/src/buffer.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2018-2022, The Monero Project +// Copyright (c) 2018-2023, The Monero Project // // All rights reserved. diff --git a/contrib/epee/src/byte_slice.cpp b/contrib/epee/src/byte_slice.cpp index 72aa3976844..82ca12f547e 100644 --- a/contrib/epee/src/byte_slice.cpp +++ b/contrib/epee/src/byte_slice.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2019-2022, The Monero Project +// Copyright (c) 2019-2023, The Monero Project // // All rights reserved. // diff --git a/contrib/epee/src/byte_stream.cpp b/contrib/epee/src/byte_stream.cpp index b674b5fe0c1..39166615cb6 100644 --- a/contrib/epee/src/byte_stream.cpp +++ b/contrib/epee/src/byte_stream.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2020-2022, The Monero Project +// Copyright (c) 2020-2023, The Monero Project // // All rights reserved. diff --git a/contrib/epee/src/connection_basic.cpp b/contrib/epee/src/connection_basic.cpp index b0a30f47f9a..a42295d88c3 100644 --- a/contrib/epee/src/connection_basic.cpp +++ b/contrib/epee/src/connection_basic.cpp @@ -2,7 +2,7 @@ /// @author rfree (current maintainer in monero.cc project) /// @brief base for connection, contains e.g. the ratelimit hooks -// Copyright (c) 2014-2022, The Monero Project +// Copyright (c) 2014-2023, The Monero Project // // All rights reserved. // diff --git a/contrib/epee/src/file_io_utils.cpp b/contrib/epee/src/file_io_utils.cpp index a8348431c7d..c0798a5100f 100644 --- a/contrib/epee/src/file_io_utils.cpp +++ b/contrib/epee/src/file_io_utils.cpp @@ -29,7 +29,7 @@ #include #include #include -#ifdef WIN32 +#ifdef _WIN32 #include #include "string_tools.h" #endif @@ -70,7 +70,7 @@ namespace file_io_utils bool save_string_to_file(const std::string& path_to_file, const std::string& str) { -#ifdef WIN32 +#ifdef _WIN32 std::wstring wide_path; try { wide_path = string_tools::utf8_to_utf16(path_to_file); } catch (...) { return false; } HANDLE file_handle = CreateFileW(wide_path.c_str(), GENERIC_WRITE, 0, NULL, CREATE_ALWAYS, FILE_ATTRIBUTE_NORMAL, NULL); @@ -104,7 +104,7 @@ namespace file_io_utils bool load_file_to_string(const std::string& path_to_file, std::string& target_str, size_t max_size) { -#ifdef WIN32 +#ifdef _WIN32 std::wstring wide_path; try { wide_path = string_tools::utf8_to_utf16(path_to_file); } catch (...) { return false; } HANDLE file_handle = CreateFileW(wide_path.c_str(), GENERIC_READ, 0, NULL, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, NULL); @@ -153,7 +153,7 @@ namespace file_io_utils bool get_file_size(const std::string& path_to_file, uint64_t &size) { -#ifdef WIN32 +#ifdef _WIN32 std::wstring wide_path; try { wide_path = string_tools::utf8_to_utf16(path_to_file); } catch (...) { return false; } HANDLE file_handle = CreateFileW(wide_path.c_str(), GENERIC_READ, 0, NULL, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, NULL); diff --git a/contrib/epee/src/hex.cpp b/contrib/epee/src/hex.cpp index c62406c3984..edc7d46f400 100644 --- a/contrib/epee/src/hex.cpp +++ b/contrib/epee/src/hex.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2022, The Monero Project +// Copyright (c) 2017-2023, The Monero Project // // All rights reserved. // diff --git a/contrib/epee/src/http_auth.cpp b/contrib/epee/src/http_auth.cpp index 98278cdfbe3..5071d26854c 100644 --- a/contrib/epee/src/http_auth.cpp +++ b/contrib/epee/src/http_auth.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2022, The Monero Project +// Copyright (c) 2014-2023, The Monero Project // // All rights reserved. // diff --git a/contrib/epee/src/http_base.cpp b/contrib/epee/src/http_base.cpp deleted file mode 100644 index f6d7568c524..00000000000 --- a/contrib/epee/src/http_base.cpp +++ /dev/null @@ -1,71 +0,0 @@ -// Copyright (c) 2006-2013, Andrey N. Sabelnikov, www.sabelnikov.net -// All rights reserved. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are met: -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// * Neither the name of the Andrey N. Sabelnikov nor the -// names of its contributors may be used to endorse or promote products -// derived from this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER BE LIABLE FOR ANY -// DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES -// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND -// ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// - -#include "net/http_base.h" -#include "memwipe.h" -#include "string_tools.h" - -#include -#include -#include - -#undef MONERO_DEFAULT_LOG_CATEGORY -#define MONERO_DEFAULT_LOG_CATEGORY "net.http" - -namespace epee -{ -namespace net_utils -{ -namespace http -{ - std::string get_value_from_fields_list(const std::string& param_name, const net_utils::http::fields_list& fields) - { - fields_list::const_iterator it = fields.begin(); - for(; it != fields.end(); ++it) - if(!string_tools::compare_no_case(param_name, it->first)) - break; - - if(it==fields.end()) - return std::string(); - - return it->second; - } - - std::string get_value_from_uri_line(const std::string& param_name, const std::string& uri) - { - std::string buff = "([\\?|&])"; - buff += param_name + "=([^&]*)"; - boost::regex match_param(buff.c_str(), boost::regex::icase | boost::regex::normal); - boost::smatch result; - if(boost::regex_search(uri, result, match_param, boost::match_default) && result[0].matched) - { - return result[2]; - } - return std::string(); - } -} -} -} diff --git a/contrib/epee/src/int-util.cpp b/contrib/epee/src/int-util.cpp index 3a2924c5a84..a330ff9ac7f 100644 --- a/contrib/epee/src/int-util.cpp +++ b/contrib/epee/src/int-util.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2019-2022, The Monero Project +// Copyright (c) 2019-2023, The Monero Project // // All rights reserved. // diff --git a/contrib/epee/src/levin_base.cpp b/contrib/epee/src/levin_base.cpp index a64f2aa172a..111fe6651ee 100644 --- a/contrib/epee/src/levin_base.cpp +++ b/contrib/epee/src/levin_base.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2019-2022, The Monero Project +// Copyright (c) 2019-2023, The Monero Project // // All rights reserved. // diff --git a/contrib/epee/src/memwipe.c b/contrib/epee/src/memwipe.c index e96b9459839..27dfb28a99a 100644 --- a/contrib/epee/src/memwipe.c +++ b/contrib/epee/src/memwipe.c @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2022, The Monero Project +// Copyright (c) 2017-2023, The Monero Project // // All rights reserved. // diff --git a/contrib/epee/src/mlocker.cpp b/contrib/epee/src/mlocker.cpp index ed71e386c12..1bf764f1879 100644 --- a/contrib/epee/src/mlocker.cpp +++ b/contrib/epee/src/mlocker.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2018-2022, The Monero Project +// Copyright (c) 2018-2023, The Monero Project // // All rights reserved. diff --git a/contrib/epee/src/mlog.cpp b/contrib/epee/src/mlog.cpp index 092d4177761..4ca1a3632a8 100644 --- a/contrib/epee/src/mlog.cpp +++ b/contrib/epee/src/mlog.cpp @@ -338,11 +338,21 @@ bool is_stdout_a_tty() return is_a_tty.load(std::memory_order_relaxed); } +static bool is_nocolor() +{ + static const char *no_color_var = getenv("NO_COLOR"); + static const bool no_color = no_color_var && *no_color_var; // apparently, NO_COLOR=0 means no color too (as per no-color.org) + return no_color; +} + void set_console_color(int color, bool bright) { if (!is_stdout_a_tty()) return; + if (is_nocolor()) + return; + switch(color) { case console_color_default: @@ -461,6 +471,9 @@ void reset_console_color() { if (!is_stdout_a_tty()) return; + if (is_nocolor()) + return; + #ifdef WIN32 HANDLE h_stdout = GetStdHandle(STD_OUTPUT_HANDLE); SetConsoleTextAttribute(h_stdout, FOREGROUND_RED | FOREGROUND_GREEN | FOREGROUND_BLUE); diff --git a/contrib/epee/src/net_ssl.cpp b/contrib/epee/src/net_ssl.cpp index 7dfb56068a3..3822eb16d98 100644 --- a/contrib/epee/src/net_ssl.cpp +++ b/contrib/epee/src/net_ssl.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2018-2022, The Monero Project +// Copyright (c) 2018-2023, The Monero Project // // All rights reserved. @@ -30,14 +30,17 @@ #include #include #include +#include #include #include +#include +#include #include #include #include #include "misc_log_ex.h" -#include "net/net_helper.h" #include "net/net_ssl.h" +#include "net/net_utils_base.h" #include "file_io_utils.h" // to validate .crt and .key paths #undef MONERO_DEFAULT_LOG_CATEGORY @@ -488,12 +491,10 @@ bool ssl_options_t::has_fingerprint(boost::asio::ssl::verify_context &ctx) const return false; } -bool ssl_options_t::handshake( +void ssl_options_t::configure( boost::asio::ssl::stream &socket, boost::asio::ssl::stream_base::handshake_type type, - boost::asio::const_buffer buffer, - const std::string& host, - std::chrono::milliseconds timeout) const + const std::string& host) const { socket.next_layer().set_option(boost::asio::ip::tcp::no_delay(true)); @@ -538,30 +539,98 @@ bool ssl_options_t::handshake( return true; }); } +} + +bool ssl_options_t::handshake( + boost::asio::ssl::stream &socket, + boost::asio::ssl::stream_base::handshake_type type, + boost::asio::const_buffer buffer, + const std::string& host, + std::chrono::milliseconds timeout) const +{ + configure(socket, type, host); + + auto start_handshake = [&]{ + using ec_t = boost::system::error_code; + using timer_t = boost::asio::steady_timer; + using strand_t = boost::asio::io_service::strand; + using socket_t = boost::asio::ip::tcp::socket; + + auto &io_context = GET_IO_SERVICE(socket); + if (io_context.stopped()) + io_context.reset(); + strand_t strand(io_context); + timer_t deadline(io_context, timeout); + + struct state_t { + std::mutex lock; + std::condition_variable_any condition; + ec_t result; + bool wait_timer; + bool wait_handshake; + bool cancel_timer; + bool cancel_handshake; + }; + state_t state{}; + + state.wait_timer = true; + auto on_timer = [&](const ec_t &ec){ + std::lock_guard guard(state.lock); + state.wait_timer = false; + state.condition.notify_all(); + if (!state.cancel_timer) { + state.cancel_handshake = true; + ec_t ec; + socket.next_layer().cancel(ec); + } + }; + + state.wait_handshake = true; + auto on_handshake = [&](const ec_t &ec, size_t bytes_transferred){ + std::lock_guard guard(state.lock); + state.wait_handshake = false; + state.condition.notify_all(); + state.result = ec; + if (!state.cancel_handshake) { + state.cancel_timer = true; + ec_t ec; + deadline.cancel(ec); + } + }; + + deadline.async_wait(on_timer); + strand.post( + [&]{ + socket.async_handshake( + type, + boost::asio::buffer(buffer), + strand.wrap(on_handshake) + ); + } + ); - auto& io_service = GET_IO_SERVICE(socket); - boost::asio::steady_timer deadline(io_service, timeout); - deadline.async_wait([&socket](const boost::system::error_code& error) { - if (error != boost::asio::error::operation_aborted) + while (!io_context.stopped()) { - socket.next_layer().close(); + io_context.poll_one(); + std::lock_guard guard(state.lock); + state.condition.wait_for( + state.lock, + std::chrono::milliseconds(30), + [&]{ + return !state.wait_timer && !state.wait_handshake; + } + ); + if (!state.wait_timer && !state.wait_handshake) + break; } - }); - - boost::system::error_code ec = boost::asio::error::would_block; - socket.async_handshake(type, boost::asio::buffer(buffer), boost::lambda::var(ec) = boost::lambda::_1); - if (io_service.stopped()) - { - io_service.reset(); - } - while (ec == boost::asio::error::would_block && !io_service.stopped()) - { - // should poll_one(), can't run_one() because it can block if there is - // another worker thread executing io_service's tasks - // TODO: once we get Boost 1.66+, replace with run_one_for/run_until - std::this_thread::sleep_for(std::chrono::milliseconds(30)); - io_service.poll_one(); - } + if (state.result.value()) { + ec_t ec; + socket.next_layer().shutdown(socket_t::shutdown_both, ec); + socket.next_layer().close(ec); + } + return state.result; + }; + const auto ec = start_handshake(); if (ec) { @@ -572,6 +641,56 @@ bool ssl_options_t::handshake( return true; } +std::string get_hr_ssl_fingerprint(const X509 *cert, const EVP_MD *fdig) +{ + unsigned int j; + unsigned int n; + unsigned char md[EVP_MAX_MD_SIZE]; + std::string fingerprint; + + CHECK_AND_ASSERT_THROW_MES(cert && fdig, "Pointer args to get_hr_ssl_fingerprint cannot be null"); + + if (!X509_digest(cert, fdig, md, &n)) + { + const unsigned long ssl_err_val = static_cast(ERR_get_error()); + const boost::system::error_code ssl_err_code = boost::asio::error::ssl_errors(static_cast(ssl_err_val)); + MERROR("Failed to create SSL fingerprint: " << ERR_reason_error_string(ssl_err_val)); + throw boost::system::system_error(ssl_err_code, ERR_reason_error_string(ssl_err_val)); + } + fingerprint.resize(n * 3 - 1); + char *out = &fingerprint[0]; + for (j = 0; j < n; ++j) + { + snprintf(out, 3 + (j + 1 < n), "%02X%s", md[j], (j + 1 == n) ? "" : ":"); + out += 3; + } + return fingerprint; +} + +std::string get_hr_ssl_fingerprint_from_file(const std::string& cert_path, const EVP_MD *fdig) { + // Open file for reading + FILE* fp = fopen(cert_path.c_str(), "r"); + if (!fp) + { + const boost::system::error_code err_code(errno, boost::system::system_category()); + throw boost::system::system_error(err_code, "Failed to open certificate file '" + cert_path + "'"); + } + std::unique_ptr file(fp, &fclose); + + // Extract certificate structure from file + X509* ssl_cert_handle = PEM_read_X509(file.get(), NULL, NULL, NULL); + if (!ssl_cert_handle) { + const unsigned long ssl_err_val = static_cast(ERR_get_error()); + const boost::system::error_code ssl_err_code = boost::asio::error::ssl_errors(static_cast(ssl_err_val)); + MERROR("OpenSSL error occurred while loading certificate at '" + cert_path + "'"); + throw boost::system::system_error(ssl_err_code, ERR_reason_error_string(ssl_err_val)); + } + std::unique_ptr ssl_cert(ssl_cert_handle, &X509_free); + + // Get the fingerprint from X509 structure + return get_hr_ssl_fingerprint(ssl_cert.get(), fdig); +} + bool ssl_support_from_string(ssl_support_t &ssl, boost::string_ref s) { if (s == "enabled") @@ -636,6 +755,29 @@ boost::system::error_code store_ssl_keys(boost::asio::ssl::context& ssl, const b return boost::asio::error::ssl_errors(ERR_get_error()); if (std::fclose(file.release()) != 0) return {errno, boost::system::system_category()}; + + // write SHA-256 fingerprint file + const boost::filesystem::path fp_file{base.string() + ".fingerprint"}; + file.reset(std::fopen(fp_file.string().c_str(), "w")); + if (!file) + return {errno, boost::system::system_category()}; + const auto fp_perms = (boost::filesystem::owner_read | boost::filesystem::group_read | boost::filesystem::others_read); + boost::filesystem::permissions(fp_file, fp_perms, error); + if (error) + return error; + try + { + const std::string fingerprint = get_hr_ssl_fingerprint(ssl_cert); + if (fingerprint.length() != fwrite(fingerprint.c_str(), sizeof(char), fingerprint.length(), file.get())) + return {errno, boost::system::system_category()}; + } + catch (const boost::system::system_error& fperr) + { + return fperr.code(); + } + if (std::fclose(file.release()) != 0) + return {errno, boost::system::system_category()}; + return error; } diff --git a/contrib/epee/src/network_throttle-detail.cpp b/contrib/epee/src/network_throttle-detail.cpp index 9785721204f..56c869d8e6e 100644 --- a/contrib/epee/src/network_throttle-detail.cpp +++ b/contrib/epee/src/network_throttle-detail.cpp @@ -2,7 +2,7 @@ /// @author rfree (current maintainer in monero.cc project) /// @brief implementaion for throttling of connection (count and rate-limit speed etc) -// Copyright (c) 2014-2022, The Monero Project +// Copyright (c) 2014-2023, The Monero Project // // All rights reserved. // diff --git a/contrib/epee/src/network_throttle.cpp b/contrib/epee/src/network_throttle.cpp index 3f250a54216..bbff08fe8b1 100644 --- a/contrib/epee/src/network_throttle.cpp +++ b/contrib/epee/src/network_throttle.cpp @@ -26,7 +26,7 @@ Throttling work by: */ -// Copyright (c) 2014-2022, The Monero Project +// Copyright (c) 2014-2023, The Monero Project // // All rights reserved. // diff --git a/contrib/epee/src/portable_storage.cpp b/contrib/epee/src/portable_storage.cpp index b922cc9e383..cedc58e46aa 100644 --- a/contrib/epee/src/portable_storage.cpp +++ b/contrib/epee/src/portable_storage.cpp @@ -49,7 +49,7 @@ namespace serialization byte_stream ss; ss.reserve(initial_buffer_size); store_to_binary(ss); - target = epee::byte_slice{std::move(ss)}; + target = epee::byte_slice{std::move(ss), false}; return true; CATCH_ENTRY("portable_storage::store_to_binary", false); } diff --git a/contrib/epee/src/readline_buffer.cpp b/contrib/epee/src/readline_buffer.cpp index 1047d16968f..ac68d1fdbf4 100644 --- a/contrib/epee/src/readline_buffer.cpp +++ b/contrib/epee/src/readline_buffer.cpp @@ -238,6 +238,10 @@ static char** attempted_completion(const char* text, int start, int end) static void install_line_handler() { +#if RL_READLINE_VERSION >= 0x0801 + rl_variable_bind("enable-bracketed-paste", "off"); +#endif + rl_attempted_completion_function = attempted_completion; rl_callback_handler_install("", handle_line); stifle_history(500); diff --git a/contrib/epee/src/wipeable_string.cpp b/contrib/epee/src/wipeable_string.cpp index b016f2f480f..aea1cc97007 100644 --- a/contrib/epee/src/wipeable_string.cpp +++ b/contrib/epee/src/wipeable_string.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2022, The Monero Project +// Copyright (c) 2017-2023, The Monero Project // // All rights reserved. // diff --git a/contrib/gitian/DOCKRUN.md b/contrib/gitian/DOCKRUN.md index 96998b1fe3f..06e4894a243 100644 --- a/contrib/gitian/DOCKRUN.md +++ b/contrib/gitian/DOCKRUN.md @@ -57,7 +57,8 @@ The dockrun.sh script will do everything to build the binaries. Just specify the version to build as its only argument, e.g. ```bash -./dockrun.sh v0.17.3.0 +VERSION=v0.18.1.0 +./dockrun.sh $VERSION ``` The build should run to completion with no errors, and will display the SHA256 checksums @@ -78,7 +79,7 @@ e.g. ```bash # Run build processes with 8 threads -OPT="-j 8" ./dockrun.sh v0.17.3.0 +OPT="-j 8" ./dockrun.sh $VERSION ``` Post-build @@ -98,16 +99,16 @@ more builder/var/install-linux.log more builder/var/build-linux.log ``` -You can find the compiled archives inside of the container at the following directory (be sure to replace `v0.17.3.0` with the version being built): +You can find the compiled archives inside of the container at the following directory: ```bash docker exec -it gitrun /bin/bash -ls -la out/v0.17.3.0/ +ls -la out/$VERSION/ ``` -To copy the compiled archives to the local host out of the Docker container, you can run the following (be sure to replace `v0.17.3.0` with the version being built): +To copy the compiled archives to the local host out of the Docker container, you can run the following: ```bash mkdir out -docker cp gitrun:/home/ubuntu/out/v0.17.3.0 out +docker cp gitrun:/home/ubuntu/out/$VERSION out ``` diff --git a/contrib/gitian/README.md b/contrib/gitian/README.md index 9852b07ba35..5211b84094e 100644 --- a/contrib/gitian/README.md +++ b/contrib/gitian/README.md @@ -133,10 +133,11 @@ Common setup part: su - gitianuser GH_USER=YOUR_GITHUB_USER_NAME -VERSION=v0.17.2.0 +VERSION=v0.18.0.0 ``` Where `GH_USER` is your GitHub user name and `VERSION` is the version tag you want to build. +The `gitian-build.py`'s `--setup` switch will also refresh the environment of any stale files and submodules. Setup for LXC: diff --git a/contrib/gitian/gitian-android.yml b/contrib/gitian/gitian-android.yml index 23cb7d0e8a4..7e9ca817828 100644 --- a/contrib/gitian/gitian-android.yml +++ b/contrib/gitian/gitian-android.yml @@ -1,5 +1,5 @@ --- -name: "monero-android-0.17" +name: "monero-android-0.18" enable_cache: true suites: - "bionic" diff --git a/contrib/gitian/gitian-build.py b/contrib/gitian/gitian-build.py index 859c2c645a0..859f460a78b 100755 --- a/contrib/gitian/gitian-build.py +++ b/contrib/gitian/gitian-build.py @@ -31,8 +31,10 @@ def setup(): subprocess.check_call(['git', 'checkout', 'c0f77ca018cb5332bfd595e0aff0468f77542c23']) os.makedirs('inputs', exist_ok=True) os.chdir('inputs') - if not os.path.isdir('monero'): - subprocess.check_call(['git', 'clone', args.url, 'monero']) + if os.path.isdir('monero'): + # Remove the potentially stale monero dir. Otherwise you might face submodule mismatches. + subprocess.check_call(['rm', 'monero', '-fR']) + subprocess.check_call(['git', 'clone', args.url, 'monero']) os.chdir('..') make_image_prog = ['bin/make-base-vm', '--suite', 'bionic', '--arch', 'amd64'] if args.docker: diff --git a/contrib/gitian/gitian-freebsd.yml b/contrib/gitian/gitian-freebsd.yml index 134823b9508..7a17f0750c6 100644 --- a/contrib/gitian/gitian-freebsd.yml +++ b/contrib/gitian/gitian-freebsd.yml @@ -1,5 +1,5 @@ --- -name: "monero-freebsd-0.17" +name: "monero-freebsd-0.18" enable_cache: true suites: - "bionic" diff --git a/contrib/gitian/gitian-linux.yml b/contrib/gitian/gitian-linux.yml index 7ab628fbc59..63d2bc5d2bf 100644 --- a/contrib/gitian/gitian-linux.yml +++ b/contrib/gitian/gitian-linux.yml @@ -1,5 +1,5 @@ --- -name: "monero-linux-0.17" +name: "monero-linux-0.18" enable_cache: true suites: - "bionic" diff --git a/contrib/gitian/gitian-osx.yml b/contrib/gitian/gitian-osx.yml index b4929e8220c..648688bcd71 100644 --- a/contrib/gitian/gitian-osx.yml +++ b/contrib/gitian/gitian-osx.yml @@ -1,5 +1,5 @@ --- -name: "monero-osx-0.17" +name: "monero-osx-0.18" enable_cache: true suites: - "bionic" diff --git a/contrib/gitian/gitian-win.yml b/contrib/gitian/gitian-win.yml index 7d5a249c8f3..4c607898eb5 100644 --- a/contrib/gitian/gitian-win.yml +++ b/contrib/gitian/gitian-win.yml @@ -1,5 +1,5 @@ --- -name: "monero-win-0.17" +name: "monero-win-0.18" enable_cache: true suites: - "bionic" diff --git a/docs/CONTRIBUTING.md b/docs/CONTRIBUTING.md index a50114d1a20..4fb92af473a 100644 --- a/docs/CONTRIBUTING.md +++ b/docs/CONTRIBUTING.md @@ -56,7 +56,7 @@ the previous paragraph is here. ## License Copyright (c) 2009-2015 Pieter Hintjens. -Copyright (c) 2017-2022, The Monero Project +Copyright (c) 2017-2023, The Monero Project This Specification is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation; either version 3 of the License, or (at your option) any later version. diff --git a/docs/PORTABLE_STORAGE.md b/docs/PORTABLE_STORAGE.md index 675ca818c6b..70e7ff954d8 100644 --- a/docs/PORTABLE_STORAGE.md +++ b/docs/PORTABLE_STORAGE.md @@ -158,7 +158,7 @@ that most will be familiar with): ```json { - "short_quote": "Give me liberty or give me death!", + "short_quote": "Give me liberty or give me death", "long_quote": "Monero is more than just a technology. It's also what the technology stands for.", "signed_32bit_int": 20140418, "array_of_bools": [true, false, true, true], @@ -169,9 +169,51 @@ that most will be familiar with): } ``` -This would translate to: +This object would translate into the following bytes when serialized into epee portable storage format. The bytes are represented in hex, with comments and whitespace added for readability. -![Epee binary storage format example](/docs/images/storage_binary_example.png) +``` +01 11 01 01 01 01 02 01 // Signature +01 // Version +14 // Varint number of section entries (5) +0b // Length of next section key (11) +73 68 6f 72 74 5f 71 75 6f 74 65 // Section key ("short_quote") +0a // Type code (STRING) +80 // Varint length of string (32) +47 69 76 65 20 6d 65 20 6c 69 62 65 72 74 79 20 // STRING value ("Give me liberty ") +6f 72 20 67 69 76 65 20 6d 65 20 64 65 61 74 68 // STRING value cont. ("or give me death") +0a // Length of next section key (10) +6c 6f 6e 67 5f 71 75 6f 74 65 // Section key ("long_quote") +0a // Type code (STRING) +41 01 // Varint length of string (80). Note it's 2 bytes +4d 6f 6e 65 72 6f 20 69 73 20 6d 6f 72 65 20 74 // STRING value ("Monero is more t") +68 61 6e 20 6a 75 73 74 20 61 20 74 65 63 68 6e // STRING value cont. ("han just a techn") +6f 6c 6f 67 79 2e 20 49 74 27 73 20 61 6c 73 6f // STRING value cont. ("ology. It's also") +20 77 68 61 74 20 74 68 65 20 74 65 63 68 6e 6f // STRING value cont. (" what the techno") +6c 6f 67 79 20 73 74 61 6e 64 73 20 66 6f 72 2e // STRING value cont. ("logy stands for.") +10 // Length of next section key (16) +73 69 67 6e 65 64 5f 33 32 62 69 74 5f 69 6e 74 // Section key ("signed_32bit_int") +02 // type code (INT32) +82 51 33 01 // INT32 value (20140418) +0e // Length of next section key (14) +61 72 72 61 79 5f 6f 66 5f 62 6f 6f 6c 73 // Section key ("array_of_bools") +8b // Type code (BOOL | FLAG_ARRAY) +10 // Varint number of array elements (4) +01 00 01 01 // Array BOOL values [true, false, true, true] +0e // Length of next section key (14) +6e 65 73 74 65 64 5f 73 65 63 74 69 6f 6e // Section key ("nested_section") +0c // Type code (OBJECT) +08 // Varint number of inner section entries (2) +06 // Length of first inner section key (6) +64 6f 75 62 6c 65 // Section key ("double") +09 // Type code (DOUBLE) +9a 99 99 99 99 99 1b c0 // DOUBLE value (-6.9) +12 // Length of second inner section key (18) +75 6e 73 69 67 6e 65 64 5f 36 34 62 69 74 5f 69 // Section key ("unsigned_64bit_i") +6e 74 // Section key cont ("nt") +05 // Type code (UINT64) +c7 71 ac b5 af 98 32 9a // UINT64 value (11111111111111111111) + +``` ## Monero specifics diff --git a/docs/RELEASE_CHECKLIST.md b/docs/RELEASE_CHECKLIST.md index 6323669850d..cfd04f98d02 100644 --- a/docs/RELEASE_CHECKLIST.md +++ b/docs/RELEASE_CHECKLIST.md @@ -1,15 +1,20 @@ +# Monero hard-fork release check-list + - [ ] Security audit - [ ] Code audit - [ ] Ledger integration - - [ ] Implemented in Monero codebase (if needed) - - [ ] Ledger app integration coded by Ledger + - [ ] Ledger notified + - [ ] Pull request made against Monero codebase (if needed) + - [ ] Pull request merged into Monero codebase (if needed) + - [ ] Ledger app integration coded - [ ] Ledger Monero app update available - [ ] Trezor integration - - [ ] Implemented in Monero codebase (if needed) - - [ ] Trezor app integration coded by Trezor - - [ ] Trezor firmware update available (if needed) + - [ ] Trezor notified + - [ ] Pull request made against Monero codebase (if needed) + - [ ] Pull request merged into Monero codebase (if needed) + - [ ] Trezor firmware update coded + - [ ] Trezor firmware update available - [ ] Fork height set - - [ ] Monero-announce mailer notice - [ ] Twitter announcement - [ ] Reddit announcement - [ ] Getmonero.org announcement @@ -26,13 +31,15 @@ - [ ] Edge Wallet - [ ] Exodus - [ ] XMRWallet + - [ ] Feather Wallet - [ ] Notify exchanges - - [ ] https://web.getmonero.org/community/merchants/#exchanges + - [ ] https://www.getmonero.org/community/merchants/#exchanges - [ ] Notify 3rd party payment processors - - [ ] https://web.getmonero.org/community/merchants/#payment-gateways + - [ ] https://www.getmonero.org/community/merchants/#payment-gateways + - [ ] BTCPayServer - [ ] Notify mining pools - [ ] https://miningpoolstats.stream/monero -- [ ] Release tagged +- [ ] Release branch created - [ ] Update src/version.cpp.in with new version AND new name (if necessary) - [ ] Update Gitian YML files in contrib/gitian/ to the new version number - [ ] Update README.md with new fork table entry (or at least update the Recommended Monero version) @@ -46,16 +53,21 @@ - [ ] Trezor - [ ] Release-specific testing - [ ] RPC testing/update RPC documentation +- [ ] Stagenet forked +- [ ] Stagenet testing/verification + - [ ] Ledger + - [ ] Trezor + - [ ] Release-specific testing - [ ] CLI reproducible builds validated - [ ] CLI released - - [ ] https://web.getmonero.org/downloads/ updated + - [ ] https://www.getmonero.org/downloads/ updated - [ ] Update hashes.txt on website - [ ] Update downloads.yml on website - [ ] Update auto-update DNS records - [ ] Update redirects on downloads box - [ ] Update seed nodes - [ ] GUI released - - [ ] https://web.getmonero.org/downloads/ updated + - [ ] https://www.getmonero.org/downloads/ updated - [ ] Update hashes.txt on website - [ ] Update hashes.txt.sig on website - [ ] Update downloads.yml on website diff --git a/docs/images/storage_binary_example.png b/docs/images/storage_binary_example.png deleted file mode 100644 index 1baa20cc110..00000000000 Binary files a/docs/images/storage_binary_example.png and /dev/null differ diff --git a/external/CMakeLists.txt b/external/CMakeLists.txt index 5b7f69a5684..8deadc7ba63 100644 --- a/external/CMakeLists.txt +++ b/external/CMakeLists.txt @@ -1,4 +1,4 @@ -# Copyright (c) 2014-2019, The Monero Project +# Copyright (c) 2014-2023, The Monero Project # # All rights reserved. # diff --git a/external/boost/archive/portable_binary_archive.hpp b/external/boost/archive/portable_binary_archive.hpp index 7ae01a225fa..b1d6ae73e6d 100644 --- a/external/boost/archive/portable_binary_archive.hpp +++ b/external/boost/archive/portable_binary_archive.hpp @@ -44,9 +44,16 @@ reverse_bytes(signed char size, char *address){ char * first = address; char * last = first + size - 1; for(;first < last;++first, --last){ +#if defined(__GNUC__) && !defined(__clang__) +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wstringop-overflow=" +#endif char x = *last; *last = *first; *first = x; +#if defined(__GNUC__) && !defined(__clang__) +#pragma GCC diagnostic pop +#endif } } diff --git a/external/db_drivers/CMakeLists.txt b/external/db_drivers/CMakeLists.txt index a9b29e0d763..d4ce7aa26df 100644 --- a/external/db_drivers/CMakeLists.txt +++ b/external/db_drivers/CMakeLists.txt @@ -1,4 +1,4 @@ -# Copyright (c) 2014-2019, The Monero Project +# Copyright (c) 2014-2023, The Monero Project # # All rights reserved. # diff --git a/external/db_drivers/liblmdb/CMakeLists.txt b/external/db_drivers/liblmdb/CMakeLists.txt index 562ebb1eb14..323437def5f 100644 --- a/external/db_drivers/liblmdb/CMakeLists.txt +++ b/external/db_drivers/liblmdb/CMakeLists.txt @@ -1,4 +1,4 @@ -# Copyright (c) 2014-2019, The Monero Project +# Copyright (c) 2014-2023, The Monero Project # # All rights reserved. # diff --git a/external/db_drivers/liblmdb/mdb.c b/external/db_drivers/liblmdb/mdb.c index 6314d577546..bf60c70133c 100644 --- a/external/db_drivers/liblmdb/mdb.c +++ b/external/db_drivers/liblmdb/mdb.c @@ -1467,6 +1467,8 @@ struct MDB_env { #endif /** Failed to update the meta page. Probably an I/O error. */ #define MDB_FATAL_ERROR 0x80000000U + /** using a raw block device */ +#define MDB_RAWPART 0x40000000U /** Some fields are initialized. */ #define MDB_ENV_ACTIVE 0x20000000U /** me_txkey is set */ @@ -4038,6 +4040,8 @@ mdb_txn_commit(MDB_txn *txn) return rc; } +static int ESECT mdb_env_map(MDB_env *env, void *addr); + /** Read the environment parameters of a DB environment before * mapping it into memory. * @param[in] env the environment handle @@ -4054,6 +4058,31 @@ mdb_env_read_header(MDB_env *env, int prev, MDB_meta *meta) int i, rc, off; enum { Size = sizeof(pbuf) }; + if (env->me_flags & MDB_RAWPART) { +#define VM_ALIGN 0x200000 + env->me_mapsize += VM_ALIGN-1; + env->me_mapsize &= ~(VM_ALIGN-1); + env->me_psize = env->me_os_psize; + rc = mdb_env_map(env, NULL); + if (rc) { + DPRINTF(("mdb_env_map: %s", mdb_strerror(rc))); + return rc; + } + p = (MDB_page *)env->me_map; + for (i=0; imp_flags, P_META)) + return ENOENT; + if (env->me_metas[i]->mm_magic != MDB_MAGIC) + return MDB_INVALID; + if (env->me_metas[i]->mm_version != MDB_DATA_VERSION) + return MDB_VERSION_MISMATCH; + if (i == 0 || env->me_metas[i]->mm_txnid > meta->mm_txnid) + *meta = *env->me_metas[i]; + p = (MDB_page *)((char *)p + env->me_psize); + } + return 0; + } + /* We don't know the page size yet, so use a minimum value. * Read both meta pages so we can use the latest one. */ @@ -4081,6 +4110,8 @@ mdb_env_read_header(MDB_env *env, int prev, MDB_meta *meta) p = (MDB_page *)&pbuf; if (!F_ISSET(p->mp_flags, P_META)) { + if (env->me_flags & MDB_RAWPART) + return ENOENT; DPRINTF(("page %"Yu" not a meta page", p->mp_pgno)); return MDB_INVALID; } @@ -4148,6 +4179,18 @@ mdb_env_init_meta(MDB_env *env, MDB_meta *meta) psize = env->me_psize; + if ((env->me_flags & (MDB_RAWPART|MDB_WRITEMAP)) == (MDB_RAWPART|MDB_WRITEMAP)) { + p = (MDB_page *)env->me_map; + p->mp_pgno = 0; + p->mp_flags = P_META; + *(MDB_meta *)METADATA(p) = *meta; + q = (MDB_page *)((char *)p + psize); + q->mp_pgno = 1; + q->mp_flags = P_META; + *(MDB_meta *)METADATA(q) = *meta; + return 0; + } + p = calloc(NUM_METAS, psize); if (!p) return ENOMEM; @@ -4410,7 +4453,7 @@ mdb_env_map(MDB_env *env, void *addr) int prot = PROT_READ; if (flags & MDB_WRITEMAP) { prot |= PROT_WRITE; - if (ftruncate(env->me_fd, env->me_mapsize) < 0) + if (!(flags & MDB_RAWPART) && ftruncate(env->me_fd, env->me_mapsize) < 0) return ErrCode(); } env->me_map = mmap(addr, env->me_mapsize, prot, MAP_SHARED, @@ -5448,6 +5491,17 @@ mdb_env_open(MDB_env *env, const char *path, unsigned int flags, mdb_mode_t mode if (rc) goto leave; #endif +#endif +#ifndef _WIN32 + { + struct stat st; + flags &= ~MDB_RAWPART; + if (!stat(path, &st) && (S_ISBLK(st.st_mode) || S_ISCHR(st.st_mode))) { + flags |= MDB_RAWPART | MDB_NOSUBDIR; + if (!env->me_mapsize) + env->me_mapsize = DEFAULT_MAPSIZE; + } + } #endif flags |= MDB_ENV_ACTIVE; /* tell mdb_env_close0() to clean up */ @@ -7668,7 +7722,7 @@ mdb_cursor_put(MDB_cursor *mc, MDB_val *key, MDB_val *data, offset *= 4; /* space for 4 more */ break; } - /* FALLTHRU: Big enough MDB_DUPFIXED sub-page */ + /* FALLTHRU *//* Big enough MDB_DUPFIXED sub-page */ case MDB_CURRENT: fp->mp_flags |= P_DIRTY; COPY_PGNO(fp->mp_pgno, mp->mp_pgno); diff --git a/external/easylogging++/CMakeLists.txt b/external/easylogging++/CMakeLists.txt index 462f774bf46..f3352d8371e 100644 --- a/external/easylogging++/CMakeLists.txt +++ b/external/easylogging++/CMakeLists.txt @@ -1,4 +1,4 @@ -# Copyright (c) 2014-2019, The Monero Project +# Copyright (c) 2014-2023, The Monero Project # # All rights reserved. # diff --git a/external/easylogging++/easylogging++.cc b/external/easylogging++/easylogging++.cc index a765ee8cc9f..891936b6bbb 100644 --- a/external/easylogging++/easylogging++.cc +++ b/external/easylogging++/easylogging++.cc @@ -149,6 +149,11 @@ static el::Color colorFromLevel(el::Level level) static void setConsoleColor(el::Color color, bool bright) { + static const char *no_color_var = getenv("NO_COLOR"); + static const bool no_color = no_color_var && *no_color_var; // apparently, NO_COLOR=0 means no color too (as per no-color.org) + if (no_color) + return; + #if ELPP_OS_WINDOWS HANDLE h_stdout = GetStdHandle(STD_OUTPUT_HANDLE); switch (color) diff --git a/external/randomx b/external/randomx index 85c527a6230..261d58c77fc 160000 --- a/external/randomx +++ b/external/randomx @@ -1 +1 @@ -Subproject commit 85c527a62301b7b8be89d941020308b1cb92b75c +Subproject commit 261d58c77fc5547c0aa7fdfeb58421ba7e0e6e1c diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 3335d3c2110..fed042ce299 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -1,4 +1,4 @@ -# Copyright (c) 2014-2022, The Monero Project +# Copyright (c) 2014-2023, The Monero Project # # All rights reserved. # diff --git a/src/blockchain_db/CMakeLists.txt b/src/blockchain_db/CMakeLists.txt index 14ed7629513..b1525d6d72a 100644 --- a/src/blockchain_db/CMakeLists.txt +++ b/src/blockchain_db/CMakeLists.txt @@ -1,4 +1,4 @@ -# Copyright (c) 2014-2022, The Monero Project +# Copyright (c) 2014-2023, The Monero Project # # All rights reserved. # diff --git a/src/blockchain_db/blockchain_db.cpp b/src/blockchain_db/blockchain_db.cpp index ab73e255c90..3d52d5fb921 100644 --- a/src/blockchain_db/blockchain_db.cpp +++ b/src/blockchain_db/blockchain_db.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2022, The Monero Project +// Copyright (c) 2014-2023, The Monero Project // // All rights reserved. // diff --git a/src/blockchain_db/blockchain_db.h b/src/blockchain_db/blockchain_db.h index 263948fa2a9..835d6dcad63 100644 --- a/src/blockchain_db/blockchain_db.h +++ b/src/blockchain_db/blockchain_db.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2022, The Monero Project +// Copyright (c) 2014-2023, The Monero Project // // All rights reserved. // @@ -1883,16 +1883,18 @@ class db_txn_guard } virtual ~db_txn_guard() { - if (active) - stop(); + stop(); } void stop() { - if (readonly) - db->block_rtxn_stop(); - else - db->block_wtxn_stop(); - active = false; + if (active) + { + if (readonly) + db->block_rtxn_stop(); + else + db->block_wtxn_stop(); + active = false; + } } void abort() { diff --git a/src/blockchain_db/lmdb/db_lmdb.cpp b/src/blockchain_db/lmdb/db_lmdb.cpp index e2ac9df0b48..4178c862bb6 100644 --- a/src/blockchain_db/lmdb/db_lmdb.cpp +++ b/src/blockchain_db/lmdb/db_lmdb.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2022, The Monero Project +// Copyright (c) 2014-2023, The Monero Project // All rights reserved. // // Redistribution and use in source and binary forms, with or without modification, are @@ -25,13 +25,6 @@ // STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF // THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -#ifndef _WIN32 -#include -#include -#include -#include -#endif - #include "db_lmdb.h" #include @@ -472,6 +465,32 @@ void mdb_txn_safe::increment_txns(int i) num_active_txns += i; } +#define TXN_PREFIX(flags); \ + mdb_txn_safe auto_txn; \ + mdb_txn_safe* txn_ptr = &auto_txn; \ + if (m_batch_active) \ + txn_ptr = m_write_txn; \ + else \ + { \ + if (auto mdb_res = lmdb_txn_begin(m_env, NULL, flags, auto_txn)) \ + throw0(DB_ERROR(lmdb_error(std::string("Failed to create a transaction for the db in ")+__FUNCTION__+": ", mdb_res).c_str())); \ + } \ + +#define TXN_PREFIX_RDONLY() \ + MDB_txn *m_txn; \ + mdb_txn_cursors *m_cursors; \ + mdb_txn_safe auto_txn; \ + bool my_rtxn = block_rtxn_start(&m_txn, &m_cursors); \ + if (my_rtxn) auto_txn.m_tinfo = m_tinfo.get(); \ + else auto_txn.uncheck() +#define TXN_POSTFIX_RDONLY() + +#define TXN_POSTFIX_SUCCESS() \ + do { \ + if (! m_batch_active) \ + auto_txn.commit(); \ + } while(0) + void lmdb_resized(MDB_env *env, int isactive) { mdb_txn_safe::prevent_new_txns(); @@ -720,21 +739,20 @@ uint64_t BlockchainLMDB::get_estimated_batch_size(uint64_t batch_num_blocks, uin } else { - MDB_txn *rtxn; - mdb_txn_cursors *rcurs; - bool my_rtxn = block_rtxn_start(&rtxn, &rcurs); - for (uint64_t block_num = block_start; block_num <= block_stop; ++block_num) { - // we have access to block weight, which will be greater or equal to block size, - // so use this as a proxy. If it's too much off, we might have to check actual size, - // which involves reading more data, so is not really wanted - size_t block_weight = get_block_weight(block_num); - total_block_size += block_weight; - // Track number of blocks being totalled here instead of assuming, in case - // some blocks were to be skipped for being outliers. - ++num_blocks_used; + TXN_PREFIX_RDONLY(); + for (uint64_t block_num = block_start; block_num <= block_stop; ++block_num) + { + // we have access to block weight, which will be greater or equal to block size, + // so use this as a proxy. If it's too much off, we might have to check actual size, + // which involves reading more data, so is not really wanted + size_t block_weight = get_block_weight(block_num); + total_block_size += block_weight; + // Track number of blocks being totalled here instead of assuming, in case + // some blocks were to be skipped for being outliers. + ++num_blocks_used; + } } - if (my_rtxn) block_rtxn_stop(); avg_block_size = total_block_size / (num_blocks_used ? num_blocks_used : 1); MDEBUG("average block size across recent " << num_blocks_used << " blocks: " << avg_block_size); } @@ -1303,26 +1321,6 @@ BlockchainLMDB::BlockchainLMDB(bool batch_transactions): BlockchainDB() m_hardfork = nullptr; } -void BlockchainLMDB::check_mmap_support() -{ -#ifndef _WIN32 - const boost::filesystem::path mmap_test_file = m_folder / boost::filesystem::unique_path(); - int mmap_test_fd = ::open(mmap_test_file.string().c_str(), O_RDWR | O_CREAT, 0600); - if (mmap_test_fd < 0) - throw0(DB_ERROR((std::string("Failed to check for mmap support: open failed: ") + strerror(errno)).c_str())); - epee::misc_utils::auto_scope_leave_caller scope_exit_handler = epee::misc_utils::create_scope_leave_handler([mmap_test_fd, &mmap_test_file]() { - ::close(mmap_test_fd); - boost::filesystem::remove(mmap_test_file.string()); - }); - if (write(mmap_test_fd, "mmaptest", 8) != 8) - throw0(DB_ERROR((std::string("Failed to check for mmap support: write failed: ") + strerror(errno)).c_str())); - void *mmap_res = mmap(NULL, 8, PROT_READ, MAP_SHARED, mmap_test_fd, 0); - if (mmap_res == MAP_FAILED) - throw0(DB_ERROR("This filesystem does not support mmap: use --data-dir to place the blockchain on a filesystem which does")); - munmap(mmap_res, 8); -#endif -} - void BlockchainLMDB::open(const std::string& filename, const int db_flags) { int result; @@ -1334,14 +1332,8 @@ void BlockchainLMDB::open(const std::string& filename, const int db_flags) throw0(DB_OPEN_FAILURE("Attempted to open db, but it's already open")); boost::filesystem::path direc(filename); - if (boost::filesystem::exists(direc)) - { - if (!boost::filesystem::is_directory(direc)) - throw0(DB_OPEN_FAILURE("LMDB needs a directory path, but a file was passed")); - } - else - { - if (!boost::filesystem::create_directories(direc)) + if (!boost::filesystem::exists(direc) && + !boost::filesystem::create_directories(direc)) { throw0(DB_OPEN_FAILURE(std::string("Failed to create directory ").append(filename).c_str())); } @@ -1364,9 +1356,6 @@ void BlockchainLMDB::open(const std::string& filename, const int db_flags) m_folder = filename; - try { check_mmap_support(); } - catch(...) { MERROR("Failed to check for mmap support, proceeding"); } - #ifdef __OpenBSD__ if ((mdb_flags & MDB_WRITEMAP) == 0) { MCLOG_RED(el::Level::Info, "global", "Running on OpenBSD: forcing WRITEMAP"); @@ -1714,32 +1703,6 @@ void BlockchainLMDB::unlock() check_open(); } -#define TXN_PREFIX(flags); \ - mdb_txn_safe auto_txn; \ - mdb_txn_safe* txn_ptr = &auto_txn; \ - if (m_batch_active) \ - txn_ptr = m_write_txn; \ - else \ - { \ - if (auto mdb_res = lmdb_txn_begin(m_env, NULL, flags, auto_txn)) \ - throw0(DB_ERROR(lmdb_error(std::string("Failed to create a transaction for the db in ")+__FUNCTION__+": ", mdb_res).c_str())); \ - } \ - -#define TXN_PREFIX_RDONLY() \ - MDB_txn *m_txn; \ - mdb_txn_cursors *m_cursors; \ - mdb_txn_safe auto_txn; \ - bool my_rtxn = block_rtxn_start(&m_txn, &m_cursors); \ - if (my_rtxn) auto_txn.m_tinfo = m_tinfo.get(); \ - else auto_txn.uncheck() -#define TXN_POSTFIX_RDONLY() - -#define TXN_POSTFIX_SUCCESS() \ - do { \ - if (! m_batch_active) \ - auto_txn.commit(); \ - } while(0) - // The below two macros are for DB access within block add/remove, whether // regular batch txn is in use or not. m_write_txn is used as a batch txn, even @@ -3959,13 +3922,20 @@ void BlockchainLMDB::block_rtxn_stop() const LOG_PRINT_L3("BlockchainLMDB::" << __func__); mdb_txn_reset(m_tinfo->m_ti_rtxn); memset(&m_tinfo->m_ti_rflags, 0, sizeof(m_tinfo->m_ti_rflags)); + /* cancel out the increment from rtxn_start */ + mdb_txn_safe::increment_txns(-1); } bool BlockchainLMDB::block_rtxn_start() const { MDB_txn *mtxn; mdb_txn_cursors *mcur; - return block_rtxn_start(&mtxn, &mcur); + /* auto_txn is only used for the create gate */ + mdb_txn_safe auto_txn; + bool ret = block_rtxn_start(&mtxn, &mcur); + if (ret) + auto_txn.increment_txns(1); /* remember there is an active readtxn */ + return ret; } void BlockchainLMDB::block_wtxn_start() diff --git a/src/blockchain_db/lmdb/db_lmdb.h b/src/blockchain_db/lmdb/db_lmdb.h index 20edab2e9a0..c352458b46b 100644 --- a/src/blockchain_db/lmdb/db_lmdb.h +++ b/src/blockchain_db/lmdb/db_lmdb.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2022, The Monero Project +// Copyright (c) 2014-2023, The Monero Project // All rights reserved. // // Redistribution and use in source and binary forms, with or without modification, are @@ -359,7 +359,6 @@ class BlockchainLMDB : public BlockchainDB static int compare_string(const MDB_val *a, const MDB_val *b); private: - void check_mmap_support(); void do_resize(uint64_t size_increase=0); bool need_resize(uint64_t threshold_size=0) const; diff --git a/src/blockchain_db/locked_txn.h b/src/blockchain_db/locked_txn.h index beab8af2179..6170d7b5acd 100644 --- a/src/blockchain_db/locked_txn.h +++ b/src/blockchain_db/locked_txn.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2022, The Monero Project +// Copyright (c) 2014-2023, The Monero Project // // All rights reserved. // diff --git a/src/blockchain_db/testdb.h b/src/blockchain_db/testdb.h index fe8078d5e01..946f2627027 100644 --- a/src/blockchain_db/testdb.h +++ b/src/blockchain_db/testdb.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2022, The Monero Project +// Copyright (c) 2014-2023, The Monero Project // // All rights reserved. // diff --git a/src/blockchain_utilities/CMakeLists.txt b/src/blockchain_utilities/CMakeLists.txt index 8122d903492..224a9e63c8c 100644 --- a/src/blockchain_utilities/CMakeLists.txt +++ b/src/blockchain_utilities/CMakeLists.txt @@ -1,4 +1,4 @@ -# Copyright (c) 2014-2022, The Monero Project +# Copyright (c) 2014-2023, The Monero Project # # All rights reserved. # diff --git a/src/blockchain_utilities/README.md b/src/blockchain_utilities/README.md index a7d47e2337e..4fcd138c445 100644 --- a/src/blockchain_utilities/README.md +++ b/src/blockchain_utilities/README.md @@ -1,6 +1,6 @@ # Monero Blockchain Utilities -Copyright (c) 2014-2022, The Monero Project +Copyright (c) 2014-2023, The Monero Project ## Introduction diff --git a/src/blockchain_utilities/blockchain_ancestry.cpp b/src/blockchain_utilities/blockchain_ancestry.cpp index b0964e4a36e..66dd7813b72 100644 --- a/src/blockchain_utilities/blockchain_ancestry.cpp +++ b/src/blockchain_utilities/blockchain_ancestry.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2022, The Monero Project +// Copyright (c) 2014-2023, The Monero Project // // All rights reserved. // diff --git a/src/blockchain_utilities/blockchain_blackball.cpp b/src/blockchain_utilities/blockchain_blackball.cpp index dee0f7f2a33..83051e29022 100644 --- a/src/blockchain_utilities/blockchain_blackball.cpp +++ b/src/blockchain_utilities/blockchain_blackball.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2022, The Monero Project +// Copyright (c) 2014-2023, The Monero Project // // All rights reserved. // diff --git a/src/blockchain_utilities/blockchain_depth.cpp b/src/blockchain_utilities/blockchain_depth.cpp index b98a1f8e2b7..6a06e0a96b8 100644 --- a/src/blockchain_utilities/blockchain_depth.cpp +++ b/src/blockchain_utilities/blockchain_depth.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2022, The Monero Project +// Copyright (c) 2014-2023, The Monero Project // // All rights reserved. // diff --git a/src/blockchain_utilities/blockchain_export.cpp b/src/blockchain_utilities/blockchain_export.cpp index 82fe524de4a..3d7b3f61af5 100644 --- a/src/blockchain_utilities/blockchain_export.cpp +++ b/src/blockchain_utilities/blockchain_export.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2022, The Monero Project +// Copyright (c) 2014-2023, The Monero Project // // All rights reserved. // diff --git a/src/blockchain_utilities/blockchain_import.cpp b/src/blockchain_utilities/blockchain_import.cpp index f8cca638d4a..f75a1e827fc 100644 --- a/src/blockchain_utilities/blockchain_import.cpp +++ b/src/blockchain_utilities/blockchain_import.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2022, The Monero Project +// Copyright (c) 2014-2023, The Monero Project // // All rights reserved. // @@ -40,7 +40,6 @@ #include "blocks/blocks.h" #include "cryptonote_basic/cryptonote_format_utils.h" #include "serialization/binary_utils.h" // dump_binary(), parse_binary() -#include "serialization/json_utils.h" // dump_json() #include "include_base_utils.h" #include "cryptonote_core/cryptonote_core.h" diff --git a/src/blockchain_utilities/blockchain_prune.cpp b/src/blockchain_utilities/blockchain_prune.cpp index 4a91cf7ccc1..1e4b48b734c 100644 --- a/src/blockchain_utilities/blockchain_prune.cpp +++ b/src/blockchain_utilities/blockchain_prune.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2018-2022, The Monero Project +// Copyright (c) 2018-2023, The Monero Project // // All rights reserved. @@ -52,9 +52,11 @@ using namespace cryptonote; static std::string db_path; // default to fast:1 -static uint64_t records_per_sync = 128; +static uint64_t records_per_sync = 16 * 65536; static const size_t slack = 512 * 1024 * 1024; +static std::vector is_v1; + static std::error_code replace_file(const boost::filesystem::path& replacement_name, const boost::filesystem::path& replaced_name) { std::error_code ec = tools::replace_file(replacement_name.string(), replaced_name.string()); @@ -89,6 +91,14 @@ static void close(MDB_env *env) mdb_env_close(env); } +static void mark_v1_tx(const MDB_val &k, const MDB_val &v) +{ + const uint64_t tx_id = *(const uint64_t*)k.mv_data; + if (tx_id >= is_v1.size()) + is_v1.resize(tx_id + 1, false); + is_v1[tx_id] = cryptonote::is_v1_tx(cryptonote::blobdata_ref{(const char*)v.mv_data, v.mv_size}); +} + static void add_size(MDB_env *env, uint64_t bytes) { try @@ -136,7 +146,7 @@ static void check_resize(MDB_env *env, size_t bytes) add_size(env, size_used + bytes + 2 * slack - mei.me_mapsize); } -static bool resize_point(size_t nrecords, MDB_env *env, MDB_txn **txn, size_t &bytes) +static bool resize_point(size_t &nrecords, MDB_env *env, MDB_txn **txn, size_t &bytes) { if (nrecords % records_per_sync && bytes <= slack / 2) return false; @@ -146,10 +156,11 @@ static bool resize_point(size_t nrecords, MDB_env *env, MDB_txn **txn, size_t &b dbr = mdb_txn_begin(env, NULL, 0, txn); if (dbr) throw std::runtime_error("Failed to create LMDB transaction: " + std::string(mdb_strerror(dbr))); bytes = 0; + nrecords = 0; return true; } -static void copy_table(MDB_env *env0, MDB_env *env1, const char *table, unsigned int flags, unsigned int putflags, int (*cmp)(const MDB_val*, const MDB_val*)=0) +static void copy_table(MDB_env *env0, MDB_env *env1, const char *table, unsigned int flags, unsigned int putflags, int (*cmp)(const MDB_val*, const MDB_val*)=0, void (*f)(const MDB_val&, const MDB_val&) = 0) { MDB_dbi dbi0, dbi1; MDB_txn *txn0, *txn1; @@ -200,6 +211,11 @@ static void copy_table(MDB_env *env0, MDB_env *env1, const char *table, unsigned dbr = mdb_cursor_open(txn1, dbi1, &cur1); if (dbr) throw std::runtime_error("Failed to create LMDB cursor: " + std::string(mdb_strerror(dbr))); + if (flags & MDB_DUPSORT) + putflags |= MDB_APPENDDUP; + else + putflags |= MDB_APPEND; + MDB_val k; MDB_val v; MDB_cursor_op op = MDB_FIRST; @@ -214,7 +230,8 @@ static void copy_table(MDB_env *env0, MDB_env *env1, const char *table, unsigned throw std::runtime_error("Failed to enumerate " + std::string(table) + " records: " + std::string(mdb_strerror(ret))); bytes += k.mv_size + v.mv_size; - if (resize_point(++nrecords, env1, &txn1, bytes)) + ++nrecords; + if (resize_point(nrecords, env1, &txn1, bytes)) { dbr = mdb_cursor_open(txn1, dbi1, &cur1); if (dbr) throw std::runtime_error("Failed to create LMDB cursor: " + std::string(mdb_strerror(dbr))); @@ -223,6 +240,9 @@ static void copy_table(MDB_env *env0, MDB_env *env1, const char *table, unsigned ret = mdb_cursor_put(cur1, &k, &v, putflags); if (ret) throw std::runtime_error("Failed to write " + std::string(table) + " record: " + std::string(mdb_strerror(ret))); + + if (f) + (*f)(k, v); } mdb_cursor_close(cur1); @@ -235,17 +255,6 @@ static void copy_table(MDB_env *env0, MDB_env *env1, const char *table, unsigned mdb_dbi_close(env0, dbi0); } -static bool is_v1_tx(MDB_cursor *c_txs_pruned, MDB_val *tx_id) -{ - MDB_val v; - int ret = mdb_cursor_get(c_txs_pruned, tx_id, &v, MDB_SET); - if (ret) - throw std::runtime_error("Failed to find transaction pruned data: " + std::string(mdb_strerror(ret))); - if (v.mv_size == 0) - throw std::runtime_error("Invalid transaction pruned data"); - return cryptonote::is_v1_tx(cryptonote::blobdata_ref{(const char*)v.mv_data, v.mv_size}); -} - static void prune(MDB_env *env0, MDB_env *env1) { MDB_dbi dbi0_blocks, dbi0_txs_pruned, dbi0_txs_prunable, dbi0_tx_indices, dbi1_txs_prunable, dbi1_txs_prunable_tip, dbi1_properties; @@ -324,7 +333,10 @@ static void prune(MDB_env *env0, MDB_env *env1) mdb_dbi_close(env0, dbi0_blocks); const uint64_t blockchain_height = stats.ms_entries; size_t nrecords = 0, bytes = 0; + std::vector prunable_needed; + // go through all txes tx indices, recording which ones should have their prunable part retained + MINFO("Marking prunable txes"); MDB_cursor_op op = MDB_FIRST; while (1) { @@ -336,7 +348,8 @@ static void prune(MDB_env *env0, MDB_env *env1) const txindex *ti = (const txindex*)v.mv_data; const uint64_t block_height = ti->data.block_id; - MDB_val_set(kk, ti->data.tx_id); + const uint64_t tx_id = ti->data.tx_id; + MDB_val_set(kk, tx_id); if (block_height + CRYPTONOTE_PRUNING_TIP_BLOCKS >= blockchain_height) { MDEBUG(block_height << "/" << blockchain_height << " is in tip"); @@ -344,22 +357,23 @@ static void prune(MDB_env *env0, MDB_env *env1) dbr = mdb_cursor_put(cur1_txs_prunable_tip, &kk, &vv, 0); if (dbr) throw std::runtime_error("Failed to write prunable tx tip data: " + std::string(mdb_strerror(dbr))); bytes += kk.mv_size + vv.mv_size; - } - if (tools::has_unpruned_block(block_height, blockchain_height, pruning_seed) || is_v1_tx(cur0_txs_pruned, &kk)) - { - MDB_val vv; - dbr = mdb_cursor_get(cur0_txs_prunable, &kk, &vv, MDB_SET); - if (dbr) throw std::runtime_error("Failed to read prunable tx data: " + std::string(mdb_strerror(dbr))); - bytes += kk.mv_size + vv.mv_size; - if (resize_point(++nrecords, env1, &txn1, bytes)) + + ++nrecords; + if (resize_point(nrecords, env1, &txn1, bytes)) { dbr = mdb_cursor_open(txn1, dbi1_txs_prunable, &cur1_txs_prunable); if (dbr) throw std::runtime_error("Failed to create LMDB cursor: " + std::string(mdb_strerror(dbr))); dbr = mdb_cursor_open(txn1, dbi1_txs_prunable_tip, &cur1_txs_prunable_tip); if (dbr) throw std::runtime_error("Failed to create LMDB cursor: " + std::string(mdb_strerror(dbr))); } - dbr = mdb_cursor_put(cur1_txs_prunable, &kk, &vv, 0); - if (dbr) throw std::runtime_error("Failed to write prunable tx data: " + std::string(mdb_strerror(dbr))); + } + if (tx_id >= is_v1.size()) + throw std::runtime_error("tx_id out of range of is_v1 vector"); + if (tools::has_unpruned_block(block_height, blockchain_height, pruning_seed) || is_v1[tx_id]) + { + if (tx_id >= prunable_needed.size()) + prunable_needed.resize(tx_id + 1, false); + prunable_needed[tx_id] = true; } else { @@ -367,6 +381,37 @@ static void prune(MDB_env *env0, MDB_env *env1) } } + // go through prunable parts, carrying over those we need + MINFO("Copying retained prunable data"); + op = MDB_FIRST; + while (1) + { + int ret = mdb_cursor_get(cur0_txs_prunable, &k, &v, op); + op = MDB_NEXT; + if (ret == MDB_NOTFOUND) + break; + if (ret) throw std::runtime_error("Failed to enumerate records: " + std::string(mdb_strerror(ret))); + + const uint64_t tx_id = *(const uint64_t*)k.mv_data; + if (tx_id >= prunable_needed.size()) + throw std::runtime_error("tx_id out of range of prunable_needed vector"); + if (prunable_needed[tx_id]) + { + dbr = mdb_cursor_put(cur1_txs_prunable, &k, &v, MDB_APPEND); + if (dbr) throw std::runtime_error("Failed to write prunable tx data: " + std::string(mdb_strerror(dbr))); + + bytes += k.mv_size + v.mv_size; + ++nrecords; + if (resize_point(nrecords, env1, &txn1, bytes)) + { + dbr = mdb_cursor_open(txn1, dbi1_txs_prunable, &cur1_txs_prunable); + if (dbr) throw std::runtime_error("Failed to create LMDB cursor: " + std::string(mdb_strerror(dbr))); + dbr = mdb_cursor_open(txn1, dbi1_txs_prunable_tip, &cur1_txs_prunable_tip); + if (dbr) throw std::runtime_error("Failed to create LMDB cursor: " + std::string(mdb_strerror(dbr))); + } + } + } + mdb_cursor_close(cur1_txs_prunable_tip); mdb_cursor_close(cur1_txs_prunable); mdb_cursor_close(cur0_txs_prunable); @@ -419,7 +464,7 @@ static bool parse_db_sync_mode(std::string db_sync_mode, uint64_t &db_flags) else if(options[0] == "fastest") { db_flags = DBF_FASTEST; - records_per_sync = 1000; // default to fastest:async:1000 + // default to fastest:async:N } else return false; @@ -455,7 +500,7 @@ int main(int argc, char* argv[]) const command_line::arg_descriptor arg_db_sync_mode = { "db-sync-mode" , "Specify sync option, using format [safe|fast|fastest]:[nrecords_per_sync]." - , "fast:1000" + , "fast:" + std::to_string(records_per_sync) }; const command_line::arg_descriptor arg_copy_pruned_database = {"copy-pruned-database", "Copy database anyway if already pruned"}; @@ -601,26 +646,27 @@ int main(int argc, char* argv[]) MDB_env *env0 = NULL, *env1 = NULL; open(env0, paths[0], db_flags, true); open(env1, paths[1], db_flags, false); - copy_table(env0, env1, "blocks", MDB_INTEGERKEY, MDB_APPEND); - copy_table(env0, env1, "block_info", MDB_INTEGERKEY | MDB_DUPSORT| MDB_DUPFIXED, MDB_APPENDDUP, BlockchainLMDB::compare_uint64); + copy_table(env0, env1, "blocks", MDB_INTEGERKEY, 0); + copy_table(env0, env1, "block_info", MDB_INTEGERKEY | MDB_DUPSORT| MDB_DUPFIXED, 0, BlockchainLMDB::compare_uint64); copy_table(env0, env1, "block_heights", MDB_INTEGERKEY | MDB_DUPSORT| MDB_DUPFIXED, 0, BlockchainLMDB::compare_hash32); //copy_table(env0, env1, "txs", MDB_INTEGERKEY); - copy_table(env0, env1, "txs_pruned", MDB_INTEGERKEY, MDB_APPEND); - copy_table(env0, env1, "txs_prunable_hash", MDB_INTEGERKEY | MDB_DUPSORT | MDB_DUPFIXED, MDB_APPEND); + copy_table(env0, env1, "txs_pruned", MDB_INTEGERKEY, 0, NULL, &mark_v1_tx); + copy_table(env0, env1, "txs_prunable_hash", MDB_INTEGERKEY | MDB_DUPSORT | MDB_DUPFIXED, 0); // not copied: prunable, prunable_tip copy_table(env0, env1, "tx_indices", MDB_INTEGERKEY | MDB_DUPSORT | MDB_DUPFIXED, 0, BlockchainLMDB::compare_hash32); - copy_table(env0, env1, "tx_outputs", MDB_INTEGERKEY, MDB_APPEND); - copy_table(env0, env1, "output_txs", MDB_INTEGERKEY | MDB_DUPSORT | MDB_DUPFIXED, MDB_APPENDDUP, BlockchainLMDB::compare_uint64); - copy_table(env0, env1, "output_amounts", MDB_INTEGERKEY | MDB_DUPSORT | MDB_DUPFIXED, MDB_APPENDDUP, BlockchainLMDB::compare_uint64); - copy_table(env0, env1, "spent_keys", MDB_INTEGERKEY | MDB_DUPSORT | MDB_DUPFIXED, MDB_NODUPDATA, BlockchainLMDB::compare_hash32); - copy_table(env0, env1, "txpool_meta", 0, MDB_NODUPDATA, BlockchainLMDB::compare_hash32); - copy_table(env0, env1, "txpool_blob", 0, MDB_NODUPDATA, BlockchainLMDB::compare_hash32); - copy_table(env0, env1, "hf_versions", MDB_INTEGERKEY, MDB_APPEND); + copy_table(env0, env1, "tx_outputs", MDB_INTEGERKEY, 0); + copy_table(env0, env1, "output_txs", MDB_INTEGERKEY | MDB_DUPSORT | MDB_DUPFIXED, 0, BlockchainLMDB::compare_uint64); + copy_table(env0, env1, "output_amounts", MDB_INTEGERKEY | MDB_DUPSORT | MDB_DUPFIXED, 0, BlockchainLMDB::compare_uint64); + copy_table(env0, env1, "spent_keys", MDB_INTEGERKEY | MDB_DUPSORT | MDB_DUPFIXED, 0, BlockchainLMDB::compare_hash32); + copy_table(env0, env1, "txpool_meta", 0, 0, BlockchainLMDB::compare_hash32); + copy_table(env0, env1, "txpool_blob", 0, 0, BlockchainLMDB::compare_hash32); + copy_table(env0, env1, "alt_blocks", 0, 0, BlockchainLMDB::compare_hash32); + copy_table(env0, env1, "hf_versions", MDB_INTEGERKEY, 0); copy_table(env0, env1, "properties", 0, 0, BlockchainLMDB::compare_string); if (already_pruned) { - copy_table(env0, env1, "txs_prunable", MDB_INTEGERKEY, MDB_APPEND, BlockchainLMDB::compare_uint64); - copy_table(env0, env1, "txs_prunable_tip", MDB_INTEGERKEY | MDB_DUPSORT | MDB_DUPFIXED, MDB_NODUPDATA, BlockchainLMDB::compare_uint64); + copy_table(env0, env1, "txs_prunable", MDB_INTEGERKEY, 0, BlockchainLMDB::compare_uint64); + copy_table(env0, env1, "txs_prunable_tip", MDB_INTEGERKEY | MDB_DUPSORT | MDB_DUPFIXED, 0, BlockchainLMDB::compare_uint64); } else { diff --git a/src/blockchain_utilities/blockchain_prune_known_spent_data.cpp b/src/blockchain_utilities/blockchain_prune_known_spent_data.cpp index 05aaf42ee9a..4da9c15c181 100644 --- a/src/blockchain_utilities/blockchain_prune_known_spent_data.cpp +++ b/src/blockchain_utilities/blockchain_prune_known_spent_data.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2022, The Monero Project +// Copyright (c) 2014-2023, The Monero Project // // All rights reserved. // diff --git a/src/blockchain_utilities/blockchain_stats.cpp b/src/blockchain_utilities/blockchain_stats.cpp index 3009b50247f..5e4245ebd56 100644 --- a/src/blockchain_utilities/blockchain_stats.cpp +++ b/src/blockchain_utilities/blockchain_stats.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2022, The Monero Project +// Copyright (c) 2014-2023, The Monero Project // // All rights reserved. // @@ -46,6 +46,77 @@ using namespace cryptonote; static bool stop_requested = false; +static bool do_inputs, do_outputs, do_ringsize, do_hours, do_emission, do_fees, do_diff; + +static struct tm prevtm, currtm; +static uint64_t prevsz, currsz; +static uint64_t prevtxs, currtxs; +static uint64_t currblks; +static uint64_t h; +static uint64_t totins, totouts, totrings; +static boost::multiprecision::uint128_t prevemission, prevfees; +static boost::multiprecision::uint128_t emission, fees; +static boost::multiprecision::uint128_t totdiff, mindiff, maxdiff; + +#define MAX_INOUT 0xffffffff +#define MAX_RINGS 0xffffffff + +static uint32_t minins = MAX_INOUT, maxins; +static uint32_t minouts = MAX_INOUT, maxouts; +static uint32_t minrings = MAX_RINGS, maxrings; +static uint32_t io, tottxs; +static uint32_t txhr[24]; + +static void doprint() +{ + char timebuf[64]; + + strftime(timebuf, sizeof(timebuf), "%Y-%m-%d", &prevtm); + prevtm = currtm; + std::cout << timebuf << "\t" << currblks << "\t" << h << "\t" << currtxs << "\t" << prevtxs + currtxs << "\t" << currsz << "\t" << prevsz + currsz; + prevsz += currsz; + currsz = 0; + prevtxs += currtxs; + currtxs = 0; + if (!tottxs) + tottxs = 1; + if (do_emission) { + std::cout << "\t" << print_money(emission) << "\t" << print_money(prevemission + emission); + prevemission += emission; + emission = 0; + } + if (do_fees) { + std::cout << "\t" << print_money(fees) << "\t" << print_money(prevfees + fees); + prevfees += fees; + fees = 0; + } + if (do_diff) { + std::cout << "\t" << (maxdiff ? mindiff : 0) << "\t" << maxdiff << "\t" << totdiff / currblks; + mindiff = 0; maxdiff = 0; totdiff = 0; + } + if (do_inputs) { + std::cout << "\t" << (maxins ? minins : 0) << "\t" << maxins << "\t" << totins * 1.0 / tottxs; + minins = MAX_INOUT; maxins = 0; totins = 0; + } + if (do_outputs) { + std::cout << "\t" << (maxouts ? minouts : 0) << "\t" << maxouts << "\t" << totouts * 1.0 / tottxs; + minouts = MAX_INOUT; maxouts = 0; totouts = 0; + } + if (do_ringsize) { + std::cout << "\t" << (maxrings ? minrings : 0) << "\t" << maxrings << "\t" << totrings * 1.0 / tottxs; + minrings = MAX_RINGS; maxrings = 0; totrings = 0; + } + if (do_hours) { + for (int i=0; i<24; i++) { + std::cout << "\t" << txhr[i]; + txhr[i] = 0; + } + } + currblks = 0; + tottxs = 0; + std::cout << ENDL; +} + int main(int argc, char* argv[]) { TRY_ENTRY(); @@ -123,13 +194,13 @@ int main(int argc, char* argv[]) network_type net_type = opt_testnet ? TESTNET : opt_stagenet ? STAGENET : MAINNET; block_start = command_line::get_arg(vm, arg_block_start); block_stop = command_line::get_arg(vm, arg_block_stop); - bool do_inputs = command_line::get_arg(vm, arg_inputs); - bool do_outputs = command_line::get_arg(vm, arg_outputs); - bool do_ringsize = command_line::get_arg(vm, arg_ringsize); - bool do_hours = command_line::get_arg(vm, arg_hours); - bool do_emission = command_line::get_arg(vm, arg_emission); - bool do_fees = command_line::get_arg(vm, arg_fees); - bool do_diff = command_line::get_arg(vm, arg_diff); + do_inputs = command_line::get_arg(vm, arg_inputs); + do_outputs = command_line::get_arg(vm, arg_outputs); + do_ringsize = command_line::get_arg(vm, arg_ringsize); + do_hours = command_line::get_arg(vm, arg_hours); + do_emission = command_line::get_arg(vm, arg_emission); + do_fees = command_line::get_arg(vm, arg_fees); + do_diff = command_line::get_arg(vm, arg_diff); LOG_PRINT_L0("Initializing source blockchain (BlockchainDB)"); std::unique_ptr core_storage; @@ -211,25 +282,7 @@ plot 'stats.csv' index "DATA" using (timecolumn(1,"%Y-%m-%d")):4 with lines, '' } std::cout << ENDL; -#define MAX_INOUT 0xffffffff -#define MAX_RINGS 0xffffffff - - struct tm prevtm = {0}, currtm; - uint64_t prevsz = 0, currsz = 0; - uint64_t prevtxs = 0, currtxs = 0; - uint64_t currblks = 0; - uint64_t totins = 0, totouts = 0, totrings = 0; - boost::multiprecision::uint128_t prevemission = 0, prevfees = 0; - boost::multiprecision::uint128_t emission = 0, fees = 0; - boost::multiprecision::uint128_t totdiff = 0, mindiff = 0, maxdiff = 0; - uint32_t minins = MAX_INOUT, maxins = 0; - uint32_t minouts = MAX_INOUT, maxouts = 0; - uint32_t minrings = MAX_RINGS, maxrings = 0; - uint32_t io, tottxs = 0; - uint32_t txhr[24] = {0}; - unsigned int i; - - for (uint64_t h = block_start; h < block_stop; ++h) + for (h = block_start; h < block_stop; ++h) { cryptonote::blobdata bd = db->get_block_blob_from_height(h); cryptonote::block blk; @@ -239,7 +292,6 @@ plot 'stats.csv' index "DATA" using (timecolumn(1,"%Y-%m-%d")):4 with lines, '' return 1; } time_t tt = blk.timestamp; - char timebuf[64]; epee::misc_utils::get_gmt_time(tt, currtm); if (!prevtm.tm_year) prevtm = currtm; @@ -247,54 +299,9 @@ plot 'stats.csv' index "DATA" using (timecolumn(1,"%Y-%m-%d")):4 with lines, '' if (currtm.tm_mday > prevtm.tm_mday || (currtm.tm_mday == 1 && prevtm.tm_mday > 27)) { // check for timestamp fudging around month ends - if (prevtm.tm_mday == 1 && currtm.tm_mday > 27) - goto skip; - strftime(timebuf, sizeof(timebuf), "%Y-%m-%d", &prevtm); - prevtm = currtm; - std::cout << timebuf << "\t" << currblks << "\t" << h << "\t" << currtxs << "\t" << prevtxs + currtxs << "\t" << currsz << "\t" << prevsz + currsz; - prevsz += currsz; - currsz = 0; - prevtxs += currtxs; - currtxs = 0; - if (!tottxs) - tottxs = 1; - if (do_emission) { - std::cout << "\t" << print_money(emission) << "\t" << print_money(prevemission + emission); - prevemission += emission; - emission = 0; - } - if (do_fees) { - std::cout << "\t" << print_money(fees) << "\t" << print_money(prevfees + fees); - prevfees += fees; - fees = 0; - } - if (do_diff) { - std::cout << "\t" << (maxdiff ? mindiff : 0) << "\t" << maxdiff << "\t" << totdiff / currblks; - mindiff = 0; maxdiff = 0; totdiff = 0; - } - if (do_inputs) { - std::cout << "\t" << (maxins ? minins : 0) << "\t" << maxins << "\t" << totins * 1.0 / tottxs; - minins = MAX_INOUT; maxins = 0; totins = 0; - } - if (do_outputs) { - std::cout << "\t" << (maxouts ? minouts : 0) << "\t" << maxouts << "\t" << totouts * 1.0 / tottxs; - minouts = MAX_INOUT; maxouts = 0; totouts = 0; - } - if (do_ringsize) { - std::cout << "\t" << (maxrings ? minrings : 0) << "\t" << maxrings << "\t" << totrings * 1.0 / tottxs; - minrings = MAX_RINGS; maxrings = 0; totrings = 0; - } - if (do_hours) { - for (i=0; i<24; i++) { - std::cout << "\t" << txhr[i]; - txhr[i] = 0; - } - } - currblks = 0; - tottxs = 0; - std::cout << ENDL; + if (!(prevtm.tm_mday == 1 && currtm.tm_mday > 27)) + doprint(); } -skip: currsz += bd.size(); uint64_t coinbase_amount; uint64_t tx_fee_amount = 0; @@ -371,6 +378,8 @@ plot 'stats.csv' index "DATA" using (timecolumn(1,"%Y-%m-%d")):4 with lines, '' if (stop_requested) break; } + if (currblks) + doprint(); core_storage->deinit(); return 0; diff --git a/src/blockchain_utilities/blockchain_usage.cpp b/src/blockchain_utilities/blockchain_usage.cpp index 129a9be21ef..a5228eb925f 100644 --- a/src/blockchain_utilities/blockchain_usage.cpp +++ b/src/blockchain_utilities/blockchain_usage.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2022, The Monero Project +// Copyright (c) 2014-2023, The Monero Project // // All rights reserved. // diff --git a/src/blockchain_utilities/blockchain_utilities.h b/src/blockchain_utilities/blockchain_utilities.h index 47bbb0fafd7..13e2ef42894 100644 --- a/src/blockchain_utilities/blockchain_utilities.h +++ b/src/blockchain_utilities/blockchain_utilities.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2022, The Monero Project +// Copyright (c) 2014-2023, The Monero Project // // All rights reserved. // diff --git a/src/blockchain_utilities/blocksdat_file.cpp b/src/blockchain_utilities/blocksdat_file.cpp index 606805a0899..6a469289db9 100644 --- a/src/blockchain_utilities/blocksdat_file.cpp +++ b/src/blockchain_utilities/blocksdat_file.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2022, The Monero Project +// Copyright (c) 2014-2023, The Monero Project // // All rights reserved. // diff --git a/src/blockchain_utilities/blocksdat_file.h b/src/blockchain_utilities/blocksdat_file.h index ce224baa6be..557b395a4a3 100644 --- a/src/blockchain_utilities/blocksdat_file.h +++ b/src/blockchain_utilities/blocksdat_file.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2022, The Monero Project +// Copyright (c) 2014-2023, The Monero Project // // All rights reserved. // diff --git a/src/blockchain_utilities/bootstrap_file.cpp b/src/blockchain_utilities/bootstrap_file.cpp index 71477912a9d..9375b63b022 100644 --- a/src/blockchain_utilities/bootstrap_file.cpp +++ b/src/blockchain_utilities/bootstrap_file.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2022, The Monero Project +// Copyright (c) 2014-2023, The Monero Project // // All rights reserved. // @@ -28,7 +28,6 @@ #include "bootstrap_serialization.h" #include "serialization/binary_utils.h" // dump_binary(), parse_binary() -#include "serialization/json_utils.h" // dump_json() #include "bootstrap_file.h" diff --git a/src/blockchain_utilities/bootstrap_file.h b/src/blockchain_utilities/bootstrap_file.h index 0f27761727f..7a1085a56fd 100644 --- a/src/blockchain_utilities/bootstrap_file.h +++ b/src/blockchain_utilities/bootstrap_file.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2022, The Monero Project +// Copyright (c) 2014-2023, The Monero Project // // All rights reserved. // diff --git a/src/blockchain_utilities/bootstrap_serialization.h b/src/blockchain_utilities/bootstrap_serialization.h index 261810a6bb5..d6ab90a99f0 100644 --- a/src/blockchain_utilities/bootstrap_serialization.h +++ b/src/blockchain_utilities/bootstrap_serialization.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2022, The Monero Project +// Copyright (c) 2014-2023, The Monero Project // // All rights reserved. // diff --git a/src/blocks/CMakeLists.txt b/src/blocks/CMakeLists.txt index db8fe5f9428..60b27baf3a8 100644 --- a/src/blocks/CMakeLists.txt +++ b/src/blocks/CMakeLists.txt @@ -1,4 +1,4 @@ -# Copyright (c) 2014-2022, The Monero Project +# Copyright (c) 2014-2023, The Monero Project # # All rights reserved. # diff --git a/src/blocks/checkpoints.dat b/src/blocks/checkpoints.dat index e75e379f2db..2ed1d630fc7 100644 Binary files a/src/blocks/checkpoints.dat and b/src/blocks/checkpoints.dat differ diff --git a/src/checkpoints/CMakeLists.txt b/src/checkpoints/CMakeLists.txt index 665441f6231..6800857573f 100644 --- a/src/checkpoints/CMakeLists.txt +++ b/src/checkpoints/CMakeLists.txt @@ -1,4 +1,4 @@ -# Copyright (c) 2014-2022, The Monero Project +# Copyright (c) 2014-2023, The Monero Project # # All rights reserved. # diff --git a/src/checkpoints/checkpoints.cpp b/src/checkpoints/checkpoints.cpp index 27e77cae8f3..70ae58cf0a9 100644 --- a/src/checkpoints/checkpoints.cpp +++ b/src/checkpoints/checkpoints.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2022, The Monero Project +// Copyright (c) 2014-2023, The Monero Project // // All rights reserved. // @@ -239,6 +239,7 @@ namespace cryptonote ADD_CHECKPOINT2(2046000, "5e867f0b8baefed9244a681df97fc885d8ab36c3dfcd24c7a3abf3b8ac8b8314", "0x9cb8b6ff2978c6"); ADD_CHECKPOINT2(2092500, "c4e00820c9c7989b49153d5e90ae095a18a11d990e82fcc3be54e6ed785472b5", "0xb4e585a31369cb"); ADD_CHECKPOINT2(2182500, "0d22b5f81982eff21d094af9e821dc2007e6342069e3b1a37b15d97646353124", "0xead4a874083492"); + ADD_CHECKPOINT2(2661600, "41c9060e8426012238e8a26da26fcb90797436896cc70886a894c2c560bcccf2", "0x2e0d87526ff161f"); return true; } diff --git a/src/checkpoints/checkpoints.h b/src/checkpoints/checkpoints.h index 07daeb4c0b8..7a93213c930 100644 --- a/src/checkpoints/checkpoints.h +++ b/src/checkpoints/checkpoints.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2022, The Monero Project +// Copyright (c) 2014-2023, The Monero Project // // All rights reserved. // diff --git a/src/common/CMakeLists.txt b/src/common/CMakeLists.txt index b712ee6b1ec..7f0b0c18f8d 100644 --- a/src/common/CMakeLists.txt +++ b/src/common/CMakeLists.txt @@ -1,4 +1,4 @@ -# Copyright (c) 2014-2022, The Monero Project +# Copyright (c) 2014-2023, The Monero Project # # All rights reserved. # diff --git a/src/common/aligned.c b/src/common/aligned.c index 3e33bfa8070..9ffa01f92be 100644 --- a/src/common/aligned.c +++ b/src/common/aligned.c @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2022, The Monero Project +// Copyright (c) 2017-2023, The Monero Project // // All rights reserved. // diff --git a/src/common/aligned.h b/src/common/aligned.h index 33242a15142..7c201e000b0 100644 --- a/src/common/aligned.h +++ b/src/common/aligned.h @@ -1,4 +1,4 @@ -// Copyright (c) 2018-2022, The Monero Project +// Copyright (c) 2018-2023, The Monero Project // // All rights reserved. diff --git a/src/common/apply_permutation.h b/src/common/apply_permutation.h index ceccdfe96f1..1f8fce66f98 100644 --- a/src/common/apply_permutation.h +++ b/src/common/apply_permutation.h @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2022, The Monero Project +// Copyright (c) 2017-2023, The Monero Project // // All rights reserved. // diff --git a/src/common/base58.cpp b/src/common/base58.cpp index f50bd10fb3d..efa1d3fc8e8 100644 --- a/src/common/base58.cpp +++ b/src/common/base58.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2022, The Monero Project +// Copyright (c) 2014-2023, The Monero Project // // All rights reserved. // diff --git a/src/common/base58.h b/src/common/base58.h index fa97ab98ca7..7e37a57eaf9 100644 --- a/src/common/base58.h +++ b/src/common/base58.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2022, The Monero Project +// Copyright (c) 2014-2023, The Monero Project // // All rights reserved. // diff --git a/src/common/combinator.cpp b/src/common/combinator.cpp index 72b139737d7..ea68f34145a 100644 --- a/src/common/combinator.cpp +++ b/src/common/combinator.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2018-2022, The Monero Project +// Copyright (c) 2018-2023, The Monero Project // // All rights reserved. diff --git a/src/common/combinator.h b/src/common/combinator.h index 0d35e478602..6ef244a1ed1 100644 --- a/src/common/combinator.h +++ b/src/common/combinator.h @@ -1,4 +1,4 @@ -// Copyright (c) 2018-2022, The Monero Project +// Copyright (c) 2018-2023, The Monero Project // // All rights reserved. @@ -34,6 +34,7 @@ #include #include #include +#include namespace tools { diff --git a/src/common/command_line.cpp b/src/common/command_line.cpp index 30ded6f3324..1380622b210 100644 --- a/src/common/command_line.cpp +++ b/src/common/command_line.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2022, The Monero Project +// Copyright (c) 2014-2023, The Monero Project // // All rights reserved. // diff --git a/src/common/command_line.h b/src/common/command_line.h index d80a1b0dfc6..bd5a32edb05 100644 --- a/src/common/command_line.h +++ b/src/common/command_line.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2022, The Monero Project +// Copyright (c) 2014-2023, The Monero Project // // All rights reserved. // diff --git a/src/common/common_fwd.h b/src/common/common_fwd.h index 4853c23c95c..04aa3ce0a65 100644 --- a/src/common/common_fwd.h +++ b/src/common/common_fwd.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2022, The Monero Project +// Copyright (c) 2014-2023, The Monero Project // // All rights reserved. // diff --git a/src/serialization/json_utils.h b/src/common/data_cache.h similarity index 72% rename from src/serialization/json_utils.h rename to src/common/data_cache.h index 63f4bc04314..5e70da11587 100644 --- a/src/serialization/json_utils.h +++ b/src/common/data_cache.h @@ -28,20 +28,38 @@ // // Parts of this file are originally copyright (c) 2012-2013 The Cryptonote developers -#pragma once +#pragma once -#include -#include "json_archive.h" +#include +#include -namespace serialization { - -template -std::string dump_json(T &v) +namespace tools { - std::stringstream ostr; - json_archive oar(ostr); - assert(serialization::serialize(oar, v)); - return ostr.str(); -}; + template + class data_cache + { + public: + void add(const T& value) + { + std::lock_guard lock(m); + if (data.insert(value).second) + { + T& old_value = buf[counter++ % MAX_SIZE]; + data.erase(old_value); + old_value = value; + } + } + + bool has(const T& value) const + { + std::lock_guard lock(m); + return (data.find(value) != data.end()); + } -} // namespace serialization + private: + mutable std::mutex m; + std::unordered_set data; + T buf[MAX_SIZE] = {}; + size_t counter = 0; + }; +} diff --git a/src/common/dns_utils.cpp b/src/common/dns_utils.cpp index 6ab6ff4fe5d..324566afd0b 100644 --- a/src/common/dns_utils.cpp +++ b/src/common/dns_utils.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2022, The Monero Project +// Copyright (c) 2014-2023, The Monero Project // // All rights reserved. // @@ -30,6 +30,8 @@ // check local first (in the event of static or in-source compilation of libunbound) #include "unbound.h" +#include +#include #include #include "include_base_utils.h" #include "common/threadpool.h" @@ -102,7 +104,6 @@ get_builtin_ds(void) { static const char * const ds[] = { - ". IN DS 19036 8 2 49AAC11D7B6F6446702E54A1607371607A1A41855200FD2CE1CDDE32F24E8FB5\n", ". IN DS 20326 8 2 E06D44B80B8F1D39A95C0B0D7C65D08458E880409BBC683457104237C7F8EC8D\n", NULL }; @@ -327,11 +328,6 @@ std::vector DNSResolver::get_record(const std::string& url, int rec dnssec_available = false; dnssec_valid = false; - if (!check_address_syntax(url.c_str())) - { - return addresses; - } - // destructor takes care of cleanup ub_result_ptr result; @@ -414,16 +410,6 @@ DNSResolver DNSResolver::create() return DNSResolver(); } -bool DNSResolver::check_address_syntax(const char *addr) const -{ - // if string doesn't contain a dot, we won't consider it a url for now. - if (strchr(addr,'.') == NULL) - { - return false; - } - return true; -} - namespace dns_utils { @@ -521,7 +507,7 @@ bool load_txt_records_from_dns(std::vector &good_records, const std // send all requests in parallel std::deque avail(dns_urls.size(), false), valid(dns_urls.size(), false); - tools::threadpool& tpool = tools::threadpool::getInstance(); + tools::threadpool& tpool = tools::threadpool::getInstanceForIO(); tools::threadpool::waiter waiter(tpool); for (size_t n = 0; n < dns_urls.size(); ++n) { diff --git a/src/common/dns_utils.h b/src/common/dns_utils.h index f9507b42aa7..87c6ec14095 100644 --- a/src/common/dns_utils.h +++ b/src/common/dns_utils.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2022, The Monero Project +// Copyright (c) 2014-2023, The Monero Project // // All rights reserved. // @@ -159,15 +159,6 @@ class DNSResolver // TODO: modify this to accommodate DNSSEC std::vector get_record(const std::string& url, int record_type, boost::optional (*reader)(const char *,size_t), bool& dnssec_available, bool& dnssec_valid); - /** - * @brief Checks a string to see if it looks like a URL - * - * @param addr the string to be checked - * - * @return true if it looks enough like a URL, false if not - */ - bool check_address_syntax(const char *addr) const; - DNSResolverData *m_data; }; // class DNSResolver diff --git a/src/common/download.cpp b/src/common/download.cpp index 01d4a9aaba4..7d23568e2bd 100644 --- a/src/common/download.cpp +++ b/src/common/download.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2022, The Monero Project +// Copyright (c) 2017-2023, The Monero Project // // All rights reserved. // diff --git a/src/common/download.h b/src/common/download.h index 52af4dc6ae8..ee73530893b 100644 --- a/src/common/download.h +++ b/src/common/download.h @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2022, The Monero Project +// Copyright (c) 2017-2023, The Monero Project // // All rights reserved. // diff --git a/src/common/error.cpp b/src/common/error.cpp index f5e402ef405..0be5586ed26 100644 --- a/src/common/error.cpp +++ b/src/common/error.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2018-2022, The Monero Project +// Copyright (c) 2018-2023, The Monero Project // All rights reserved. // diff --git a/src/common/error.h b/src/common/error.h index d271517f431..b79731162f9 100644 --- a/src/common/error.h +++ b/src/common/error.h @@ -1,4 +1,4 @@ -// Copyright (c) 2018-2022, The Monero Project +// Copyright (c) 2018-2023, The Monero Project // All rights reserved. // diff --git a/src/common/expect.h b/src/common/expect.h index 6f2d8e2914b..64a753f457a 100644 --- a/src/common/expect.h +++ b/src/common/expect.h @@ -1,4 +1,4 @@ -// Copyright (c) 2018-2022, The Monero Project +// Copyright (c) 2018-2023, The Monero Project // // All rights reserved. diff --git a/src/common/http_connection.h b/src/common/http_connection.h index f8281fe1eff..cf9a68d8d8d 100644 --- a/src/common/http_connection.h +++ b/src/common/http_connection.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2022, The Monero Project +// Copyright (c) 2014-2023, The Monero Project // // All rights reserved. // diff --git a/src/common/i18n.cpp b/src/common/i18n.cpp index 6d904a2e380..352c7879fee 100644 --- a/src/common/i18n.cpp +++ b/src/common/i18n.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2022, The Monero Project +// Copyright (c) 2014-2023, The Monero Project // // All rights reserved. // diff --git a/src/common/i18n.h b/src/common/i18n.h index 1ed78bf8fd0..55506ce02b3 100644 --- a/src/common/i18n.h +++ b/src/common/i18n.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2022, The Monero Project +// Copyright (c) 2014-2023, The Monero Project // // All rights reserved. // diff --git a/src/common/json_util.h b/src/common/json_util.h index 6d2cfedb435..e512f4df8cf 100644 --- a/src/common/json_util.h +++ b/src/common/json_util.h @@ -1,4 +1,4 @@ -// Copyright (c) 2016-2022, The Monero Project +// Copyright (c) 2016-2023, The Monero Project // // All rights reserved. // diff --git a/src/common/notify.cpp b/src/common/notify.cpp index 28f38d8c39e..e1d6ee7f5f6 100644 --- a/src/common/notify.cpp +++ b/src/common/notify.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2018-2022, The Monero Project +// Copyright (c) 2018-2023, The Monero Project // // All rights reserved. diff --git a/src/common/notify.h b/src/common/notify.h index 7ff721c8aa0..fa743858590 100644 --- a/src/common/notify.h +++ b/src/common/notify.h @@ -1,4 +1,4 @@ -// Copyright (c) 2018-2022, The Monero Project +// Copyright (c) 2018-2023, The Monero Project // // All rights reserved. diff --git a/src/common/password.cpp b/src/common/password.cpp index e6dff95ea44..7ba4546d3cc 100644 --- a/src/common/password.cpp +++ b/src/common/password.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2022, The Monero Project +// Copyright (c) 2014-2023, The Monero Project // // All rights reserved. // diff --git a/src/common/password.h b/src/common/password.h index 26b6616abea..2d889f4e656 100644 --- a/src/common/password.h +++ b/src/common/password.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2022, The Monero Project +// Copyright (c) 2014-2023, The Monero Project // // All rights reserved. // diff --git a/src/common/perf_timer.cpp b/src/common/perf_timer.cpp index 30164a557e7..2b67154e5b9 100644 --- a/src/common/perf_timer.cpp +++ b/src/common/perf_timer.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2016-2022, The Monero Project +// Copyright (c) 2016-2023, The Monero Project // // All rights reserved. // diff --git a/src/common/perf_timer.h b/src/common/perf_timer.h index c6120b06d29..31df89fee13 100644 --- a/src/common/perf_timer.h +++ b/src/common/perf_timer.h @@ -1,4 +1,4 @@ -// Copyright (c) 2016-2022, The Monero Project +// Copyright (c) 2016-2023, The Monero Project // // All rights reserved. // diff --git a/src/common/pod-class.h b/src/common/pod-class.h index c6fce75e696..3cdde72a73b 100644 --- a/src/common/pod-class.h +++ b/src/common/pod-class.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2022, The Monero Project +// Copyright (c) 2014-2023, The Monero Project // // All rights reserved. // diff --git a/src/common/pruning.cpp b/src/common/pruning.cpp index 5cae238ae67..48c36542549 100644 --- a/src/common/pruning.cpp +++ b/src/common/pruning.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2018-2022, The Monero Project +// Copyright (c) 2018-2023, The Monero Project // // All rights reserved. diff --git a/src/common/pruning.h b/src/common/pruning.h index 6530dcf2e69..a0d33bf6674 100644 --- a/src/common/pruning.h +++ b/src/common/pruning.h @@ -1,4 +1,4 @@ -// Copyright (c) 2018-2022, The Monero Project +// Copyright (c) 2018-2023, The Monero Project // // All rights reserved. diff --git a/src/common/rpc_client.h b/src/common/rpc_client.h index a603a7baf3e..f6c1ea10020 100644 --- a/src/common/rpc_client.h +++ b/src/common/rpc_client.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2022, The Monero Project +// Copyright (c) 2014-2023, The Monero Project // // All rights reserved. // diff --git a/src/common/scoped_message_writer.h b/src/common/scoped_message_writer.h index cf4e6855f00..5814d7ff5ee 100644 --- a/src/common/scoped_message_writer.h +++ b/src/common/scoped_message_writer.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2022, The Monero Project +// Copyright (c) 2014-2023, The Monero Project // // All rights reserved. // diff --git a/src/common/sfinae_helpers.h b/src/common/sfinae_helpers.h index c5974fe8257..fe9163c0860 100644 --- a/src/common/sfinae_helpers.h +++ b/src/common/sfinae_helpers.h @@ -1,4 +1,4 @@ -// Copyright (c) 2016-2022, The Monero Project +// Copyright (c) 2016-2023, The Monero Project // // All rights reserved. // diff --git a/src/common/spawn.cpp b/src/common/spawn.cpp index 22e84040739..1bdf0ddda97 100644 --- a/src/common/spawn.cpp +++ b/src/common/spawn.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2018-2022, The Monero Project +// Copyright (c) 2018-2023, The Monero Project // // All rights reserved. diff --git a/src/common/spawn.h b/src/common/spawn.h index f3c8dbb6e9e..55b2333b16d 100644 --- a/src/common/spawn.h +++ b/src/common/spawn.h @@ -1,4 +1,4 @@ -// Copyright (c) 2018-2022, The Monero Project +// Copyright (c) 2018-2023, The Monero Project // // All rights reserved. diff --git a/src/common/stack_trace.cpp b/src/common/stack_trace.cpp index 130ba4d81ce..bef14b2442f 100644 --- a/src/common/stack_trace.cpp +++ b/src/common/stack_trace.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2016-2022, The Monero Project +// Copyright (c) 2016-2023, The Monero Project // // All rights reserved. // diff --git a/src/common/stack_trace.h b/src/common/stack_trace.h index dc818215477..77af0c58f2a 100644 --- a/src/common/stack_trace.h +++ b/src/common/stack_trace.h @@ -1,4 +1,4 @@ -// Copyright (c) 2016-2022, The Monero Project +// Copyright (c) 2016-2023, The Monero Project // // All rights reserved. // diff --git a/src/common/threadpool.cpp b/src/common/threadpool.cpp index 5d2acc0a68d..ff6277af539 100644 --- a/src/common/threadpool.cpp +++ b/src/common/threadpool.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2022, The Monero Project +// Copyright (c) 2017-2023, The Monero Project // // All rights reserved. // diff --git a/src/common/threadpool.h b/src/common/threadpool.h index ce1bc579993..321f6682a60 100644 --- a/src/common/threadpool.h +++ b/src/common/threadpool.h @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2022, The Monero Project +// Copyright (c) 2017-2023, The Monero Project // // All rights reserved. // @@ -31,6 +31,7 @@ #include #include #include +#include #include #include #include @@ -42,10 +43,14 @@ namespace tools class threadpool { public: - static threadpool& getInstance() { + static threadpool& getInstanceForCompute() { static threadpool instance; return instance; } + static threadpool& getInstanceForIO() { + static threadpool instance(8); + return instance; + } static threadpool *getNewForUnitTests(unsigned max_threads = 0) { return new threadpool(max_threads); } diff --git a/src/common/unordered_containers_boost_serialization.h b/src/common/unordered_containers_boost_serialization.h index bc6dd8e7afc..49bbe1956a5 100644 --- a/src/common/unordered_containers_boost_serialization.h +++ b/src/common/unordered_containers_boost_serialization.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2022, The Monero Project +// Copyright (c) 2014-2023, The Monero Project // // All rights reserved. // diff --git a/src/common/updates.cpp b/src/common/updates.cpp index 654ad068cf4..15efb94c1b7 100644 --- a/src/common/updates.cpp +++ b/src/common/updates.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2022, The Monero Project +// Copyright (c) 2017-2023, The Monero Project // // All rights reserved. // diff --git a/src/common/updates.h b/src/common/updates.h index 64e4eea50a6..da1c8353b71 100644 --- a/src/common/updates.h +++ b/src/common/updates.h @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2022, The Monero Project +// Copyright (c) 2017-2023, The Monero Project // // All rights reserved. // diff --git a/src/common/utf8.h b/src/common/utf8.h index 434de15e7bd..d21509c9c7a 100644 --- a/src/common/utf8.h +++ b/src/common/utf8.h @@ -1,4 +1,4 @@ -// Copyright (c) 2019-2022, The Monero Project +// Copyright (c) 2019-2023, The Monero Project // // All rights reserved. diff --git a/src/common/util.cpp b/src/common/util.cpp index 89dcf4feffa..a0074f44cab 100644 --- a/src/common/util.cpp +++ b/src/common/util.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2022, The Monero Project +// Copyright (c) 2014-2023, The Monero Project // // All rights reserved. // @@ -85,7 +85,7 @@ using namespace epee; #include #include #include -#include +#include #undef MONERO_DEFAULT_LOG_CATEGORY #define MONERO_DEFAULT_LOG_CATEGORY "util" @@ -941,14 +941,7 @@ std::string get_nix_version_display_string() bool sha256sum(const uint8_t *data, size_t len, crypto::hash &hash) { - SHA256_CTX ctx; - if (!SHA256_Init(&ctx)) - return false; - if (!SHA256_Update(&ctx, data, len)) - return false; - if (!SHA256_Final((unsigned char*)hash.data, &ctx)) - return false; - return true; + return EVP_Digest(data, len, (unsigned char*) hash.data, NULL, EVP_sha256(), NULL) != 0; } bool sha256sum(const std::string &filename, crypto::hash &hash) @@ -961,8 +954,8 @@ std::string get_nix_version_display_string() if (!f) return false; std::ifstream::pos_type file_size = f.tellg(); - SHA256_CTX ctx; - if (!SHA256_Init(&ctx)) + std::unique_ptr ctx(EVP_MD_CTX_new(), &EVP_MD_CTX_free); + if (!EVP_DigestInit_ex(ctx.get(), EVP_sha256(), nullptr)) return false; size_t size_left = file_size; f.seekg(0, std::ios::beg); @@ -973,12 +966,12 @@ std::string get_nix_version_display_string() f.read(buf, read_size); if (!f || !f.good()) return false; - if (!SHA256_Update(&ctx, buf, read_size)) + if (!EVP_DigestUpdate(ctx.get(), buf, read_size)) return false; size_left -= read_size; } f.close(); - if (!SHA256_Final((unsigned char*)hash.data, &ctx)) + if (!EVP_DigestFinal_ex(ctx.get(), (unsigned char*)hash.data, nullptr)) return false; return true; } @@ -1074,7 +1067,7 @@ std::string get_nix_version_display_string() time_t tt = ts; struct tm tm; misc_utils::get_gmt_time(tt, tm); - strftime(buffer, sizeof(buffer), "%Y-%m-%d %H:%M:%S", &tm); + strftime(buffer, sizeof(buffer), "%Y-%m-%d %H:%M:%SZ", &tm); return std::string(buffer); } diff --git a/src/common/util.h b/src/common/util.h index 25f5ceb479a..05748e0206a 100644 --- a/src/common/util.h +++ b/src/common/util.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2022, The Monero Project +// Copyright (c) 2014-2023, The Monero Project // // All rights reserved. // @@ -231,7 +231,27 @@ namespace tools bool is_privacy_preserving_network(const std::string &address); int vercmp(const char *v0, const char *v1); // returns < 0, 0, > 0, similar to strcmp, but more human friendly than lexical - does not attempt to validate + /** + * \brief Creates a SHA-256 digest of a data buffer + * + * \param[in] data pointer to the buffer + * \param[in] len size of the buffer in bytes + * \param[out] hash where message digest will be written to + * + * \returns true if successful, false otherwise + */ bool sha256sum(const uint8_t *data, size_t len, crypto::hash &hash); + + /** + * \brief Creates a SHA-256 digest of a file's contents, equivalent to the sha256sum command in Linux + * + * \param[in] filename path to target file + * \param[out] hash where message digest will be written to + * + * \returns true if successful, false if the file can not be opened or there is an OpenSSL failure + * + * \throws ios_base::failure if after the file is successfully opened, an error occurs during reading + */ bool sha256sum(const std::string &filename, crypto::hash &hash); boost::optional is_hdd(const char *path); diff --git a/src/common/varint.h b/src/common/varint.h index 9f8b9a4abec..58d3bbb8472 100644 --- a/src/common/varint.h +++ b/src/common/varint.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2022, The Monero Project +// Copyright (c) 2014-2023, The Monero Project // // All rights reserved. // diff --git a/src/crypto/CMakeLists.txt b/src/crypto/CMakeLists.txt index 595c7f96682..e17584c9065 100644 --- a/src/crypto/CMakeLists.txt +++ b/src/crypto/CMakeLists.txt @@ -1,4 +1,4 @@ -# Copyright (c) 2014-2022, The Monero Project +# Copyright (c) 2014-2023, The Monero Project # # All rights reserved. # diff --git a/src/crypto/blake256.c b/src/crypto/blake256.c index 7e302bcad24..83161fbb55a 100644 --- a/src/crypto/blake256.c +++ b/src/crypto/blake256.c @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2022, The Monero Project +// Copyright (c) 2014-2023, The Monero Project // // All rights reserved. // diff --git a/src/crypto/blake256.h b/src/crypto/blake256.h index f727bddeef8..5bde2cb2f0c 100644 --- a/src/crypto/blake256.h +++ b/src/crypto/blake256.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2022, The Monero Project +// Copyright (c) 2014-2023, The Monero Project // // All rights reserved. // diff --git a/src/crypto/c_threads.h b/src/crypto/c_threads.h index c5431cb8dcd..e6c846db610 100644 --- a/src/crypto/c_threads.h +++ b/src/crypto/c_threads.h @@ -1,4 +1,4 @@ -// Copyright (c) 2019-2022, The Monero Project +// Copyright (c) 2019-2023, The Monero Project // // All rights reserved. // @@ -30,29 +30,41 @@ #pragma once #ifdef _WIN32 + #include -#define CTHR_MUTEX_TYPE HANDLE -#define CTHR_MUTEX_INIT NULL -#define CTHR_MUTEX_LOCK(x) do { if (x == NULL) { \ - HANDLE p = CreateMutex(NULL, FALSE, NULL); \ - if (InterlockedCompareExchangePointer((PVOID*)&x, (PVOID)p, NULL) != NULL) \ - CloseHandle(p); \ - } WaitForSingleObject(x, INFINITE); } while(0) -#define CTHR_MUTEX_UNLOCK(x) ReleaseMutex(x) + +#define CTHR_RWLOCK_TYPE SRWLOCK +#define CTHR_RWLOCK_INIT SRWLOCK_INIT +#define CTHR_RWLOCK_LOCK_WRITE(x) AcquireSRWLockExclusive(&x) +#define CTHR_RWLOCK_UNLOCK_WRITE(x) ReleaseSRWLockExclusive(&x) +#define CTHR_RWLOCK_LOCK_READ(x) AcquireSRWLockShared(&x) +#define CTHR_RWLOCK_UNLOCK_READ(x) ReleaseSRWLockShared(&x) +#define CTHR_RWLOCK_TRYLOCK_READ(x) TryAcquireSRWLockShared(&x) + #define CTHR_THREAD_TYPE HANDLE -#define CTHR_THREAD_RTYPE void -#define CTHR_THREAD_RETURN return -#define CTHR_THREAD_CREATE(thr, func, arg) thr = (HANDLE)_beginthread(func, 0, arg) -#define CTHR_THREAD_JOIN(thr) WaitForSingleObject(thr, INFINITE) +#define CTHR_THREAD_RTYPE unsigned __stdcall +#define CTHR_THREAD_RETURN _endthreadex(0); return 0; +#define CTHR_THREAD_CREATE(thr, func, arg) ((thr = (HANDLE)_beginthreadex(0, 0, func, arg, 0, 0)) != 0L) +#define CTHR_THREAD_JOIN(thr) do { WaitForSingleObject(thr, INFINITE); CloseHandle(thr); } while(0) +#define CTHR_THREAD_CLOSE(thr) CloseHandle((HANDLE)thr); + #else + #include -#define CTHR_MUTEX_TYPE pthread_mutex_t -#define CTHR_MUTEX_INIT PTHREAD_MUTEX_INITIALIZER -#define CTHR_MUTEX_LOCK(x) pthread_mutex_lock(&x) -#define CTHR_MUTEX_UNLOCK(x) pthread_mutex_unlock(&x) + +#define CTHR_RWLOCK_TYPE pthread_rwlock_t +#define CTHR_RWLOCK_INIT PTHREAD_RWLOCK_INITIALIZER +#define CTHR_RWLOCK_LOCK_WRITE(x) pthread_rwlock_wrlock(&x) +#define CTHR_RWLOCK_UNLOCK_WRITE(x) pthread_rwlock_unlock(&x) +#define CTHR_RWLOCK_LOCK_READ(x) pthread_rwlock_rdlock(&x) +#define CTHR_RWLOCK_UNLOCK_READ(x) pthread_rwlock_unlock(&x) +#define CTHR_RWLOCK_TRYLOCK_READ(x) (pthread_rwlock_tryrdlock(&x) == 0) + #define CTHR_THREAD_TYPE pthread_t #define CTHR_THREAD_RTYPE void * #define CTHR_THREAD_RETURN return NULL -#define CTHR_THREAD_CREATE(thr, func, arg) pthread_create(&thr, NULL, func, arg) +#define CTHR_THREAD_CREATE(thr, func, arg) (pthread_create(&thr, NULL, func, arg) == 0) #define CTHR_THREAD_JOIN(thr) pthread_join(thr, NULL) +#define CTHR_THREAD_CLOSE(thr) + #endif diff --git a/src/crypto/chacha.h b/src/crypto/chacha.h index 74f05cbe848..80170dea643 100644 --- a/src/crypto/chacha.h +++ b/src/crypto/chacha.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2022, The Monero Project +// Copyright (c) 2014-2023, The Monero Project // // All rights reserved. // diff --git a/src/crypto/crypto-ops-data.c b/src/crypto/crypto-ops-data.c index 1a85de60d5c..e31cd6b9cdd 100644 --- a/src/crypto/crypto-ops-data.c +++ b/src/crypto/crypto-ops-data.c @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2022, The Monero Project +// Copyright (c) 2014-2023, The Monero Project // // All rights reserved. // diff --git a/src/crypto/crypto-ops.c b/src/crypto/crypto-ops.c index 4b392d4727b..e9a5cc6f67d 100644 --- a/src/crypto/crypto-ops.c +++ b/src/crypto/crypto-ops.c @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2022, The Monero Project +// Copyright (c) 2014-2023, The Monero Project // // All rights reserved. // @@ -38,7 +38,6 @@ DISABLE_VS_WARNINGS(4146 4244) /* Predeclarations */ -static void fe_mul(fe, const fe, const fe); static void fe_sq(fe, const fe); static void ge_madd(ge_p1p1 *, const ge_p3 *, const ge_precomp *); static void ge_msub(ge_p1p1 *, const ge_p3 *, const ge_precomp *); @@ -72,7 +71,7 @@ uint64_t load_4(const unsigned char *in) h = 0 */ -static void fe_0(fe h) { +void fe_0(fe h) { h[0] = 0; h[1] = 0; h[2] = 0; @@ -375,7 +374,7 @@ Can get away with 11 carries, but then data flow is much deeper. With tighter constraints on inputs can squeeze carries into int32. */ -static void fe_mul(fe h, const fe f, const fe g) { +void fe_mul(fe h, const fe f, const fe g) { int32_t f0 = f[0]; int32_t f1 = f[1]; int32_t f2 = f[2]; diff --git a/src/crypto/crypto-ops.h b/src/crypto/crypto-ops.h index e4901e08020..49bc8e081ea 100644 --- a/src/crypto/crypto-ops.h +++ b/src/crypto/crypto-ops.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2022, The Monero Project +// Copyright (c) 2014-2023, The Monero Project // // All rights reserved. // @@ -30,6 +30,8 @@ #pragma once +#include + /* From fe.h */ typedef int32_t fe[10]; @@ -161,5 +163,7 @@ void ge_sub(ge_p1p1 *r, const ge_p3 *p, const ge_cached *q); void fe_add(fe h, const fe f, const fe g); void fe_tobytes(unsigned char *, const fe); void fe_invert(fe out, const fe z); +void fe_mul(fe out, const fe, const fe); +void fe_0(fe h); int ge_p3_is_point_at_infinity_vartime(const ge_p3 *p); diff --git a/src/crypto/crypto.cpp b/src/crypto/crypto.cpp index 77a36069a5e..0ab4cad368e 100644 --- a/src/crypto/crypto.cpp +++ b/src/crypto/crypto.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2022, The Monero Project +// Copyright (c) 2014-2023, The Monero Project // // All rights reserved. // diff --git a/src/crypto/crypto.h b/src/crypto/crypto.h index d8cd6c6a01f..81d6e1803b1 100644 --- a/src/crypto/crypto.h +++ b/src/crypto/crypto.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2022, The Monero Project +// Copyright (c) 2014-2023, The Monero Project // // All rights reserved. // @@ -335,8 +335,16 @@ namespace crypto { inline bool operator<(const public_key &p1, const public_key &p2) { return memcmp(&p1, &p2, sizeof(public_key)) < 0; } inline bool operator>(const public_key &p1, const public_key &p2) { return p2 < p1; } + inline bool operator<(const key_image &p1, const key_image &p2) { return memcmp(&p1, &p2, sizeof(key_image)) < 0; } + inline bool operator>(const key_image &p1, const key_image &p2) { return p2 < p1; } } +// type conversions for easier calls to sc_add(), sc_sub(), hash functions +inline unsigned char* to_bytes(crypto::ec_scalar &scalar) { return &reinterpret_cast(scalar); } +inline const unsigned char* to_bytes(const crypto::ec_scalar &scalar) { return &reinterpret_cast(scalar); } +inline unsigned char* to_bytes(crypto::ec_point &point) { return &reinterpret_cast(point); } +inline const unsigned char* to_bytes(const crypto::ec_point &point) { return &reinterpret_cast(point); } + CRYPTO_MAKE_HASHABLE(public_key) CRYPTO_MAKE_HASHABLE_CONSTANT_TIME(secret_key) CRYPTO_MAKE_HASHABLE_CONSTANT_TIME(public_key_memsafe) diff --git a/src/crypto/crypto_ops_builder/README.md b/src/crypto/crypto_ops_builder/README.md index 831c6a63c37..aae9e88cdc3 100644 --- a/src/crypto/crypto_ops_builder/README.md +++ b/src/crypto/crypto_ops_builder/README.md @@ -1,6 +1,6 @@ # Monero -Copyright (c) 2014-2022, The Monero Project +Copyright (c) 2014-2023, The Monero Project ## Crypto Ops Builder diff --git a/src/crypto/crypto_ops_builder/crypto-ops-data.c b/src/crypto/crypto_ops_builder/crypto-ops-data.c index 4785f975f9f..ac1cf82e568 100644 --- a/src/crypto/crypto_ops_builder/crypto-ops-data.c +++ b/src/crypto/crypto_ops_builder/crypto-ops-data.c @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2022, The Monero Project +// Copyright (c) 2014-2023, The Monero Project // // All rights reserved. // diff --git a/src/crypto/crypto_ops_builder/crypto-ops-old.c b/src/crypto/crypto_ops_builder/crypto-ops-old.c index 5d632809ea8..10c25bd1eb3 100644 --- a/src/crypto/crypto_ops_builder/crypto-ops-old.c +++ b/src/crypto/crypto_ops_builder/crypto-ops-old.c @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2022, The Monero Project +// Copyright (c) 2014-2023, The Monero Project // // All rights reserved. // diff --git a/src/crypto/crypto_ops_builder/crypto-ops.h b/src/crypto/crypto_ops_builder/crypto-ops.h index 568bf2a37c6..171343676cc 100644 --- a/src/crypto/crypto_ops_builder/crypto-ops.h +++ b/src/crypto/crypto_ops_builder/crypto-ops.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2022, The Monero Project +// Copyright (c) 2014-2023, The Monero Project // // All rights reserved. // diff --git a/src/crypto/crypto_ops_builder/ref10CommentedCombined/MakeCryptoOps.py b/src/crypto/crypto_ops_builder/ref10CommentedCombined/MakeCryptoOps.py index 16b6c0ba910..5bb88f0a98b 100644 --- a/src/crypto/crypto_ops_builder/ref10CommentedCombined/MakeCryptoOps.py +++ b/src/crypto/crypto_ops_builder/ref10CommentedCombined/MakeCryptoOps.py @@ -15,7 +15,7 @@ a = "" license = textwrap.dedent("""\ - // Copyright (c) 2014-2022, The Monero Project + // Copyright (c) 2014-2023, The Monero Project // // All rights reserved. // diff --git a/src/crypto/crypto_ops_builder/ref10CommentedCombined/crypto-ops.h b/src/crypto/crypto_ops_builder/ref10CommentedCombined/crypto-ops.h index 8c0cbcda1d9..4badb59d6b1 100644 --- a/src/crypto/crypto_ops_builder/ref10CommentedCombined/crypto-ops.h +++ b/src/crypto/crypto_ops_builder/ref10CommentedCombined/crypto-ops.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2022, The Monero Project +// Copyright (c) 2014-2023, The Monero Project // // All rights reserved. // diff --git a/src/crypto/duration.h b/src/crypto/duration.h index 25d1c0b8cab..ad9f0a8c4a3 100644 --- a/src/crypto/duration.h +++ b/src/crypto/duration.h @@ -1,4 +1,4 @@ -// Copyright (c) 2020-2022, The Monero Project +// Copyright (c) 2020-2023, The Monero Project // // All rights reserved. diff --git a/src/crypto/generic-ops.h b/src/crypto/generic-ops.h index 5a5e09f9b62..da19157f6a6 100644 --- a/src/crypto/generic-ops.h +++ b/src/crypto/generic-ops.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2022, The Monero Project +// Copyright (c) 2014-2023, The Monero Project // // All rights reserved. // diff --git a/src/crypto/groestl.h b/src/crypto/groestl.h index 899660cb119..6fb0a26f46b 100644 --- a/src/crypto/groestl.h +++ b/src/crypto/groestl.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2022, The Monero Project +// Copyright (c) 2014-2023, The Monero Project // // All rights reserved. // diff --git a/src/crypto/groestl_tables.h b/src/crypto/groestl_tables.h index 556354c4760..c553b98f5dd 100644 --- a/src/crypto/groestl_tables.h +++ b/src/crypto/groestl_tables.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2022, The Monero Project +// Copyright (c) 2014-2023, The Monero Project // // All rights reserved. // diff --git a/src/crypto/hash-extra-blake.c b/src/crypto/hash-extra-blake.c index 1557269e699..f3e91a0a8ae 100644 --- a/src/crypto/hash-extra-blake.c +++ b/src/crypto/hash-extra-blake.c @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2022, The Monero Project +// Copyright (c) 2014-2023, The Monero Project // // All rights reserved. // diff --git a/src/crypto/hash-extra-groestl.c b/src/crypto/hash-extra-groestl.c index 96230aed74c..7d8e84eaa65 100644 --- a/src/crypto/hash-extra-groestl.c +++ b/src/crypto/hash-extra-groestl.c @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2022, The Monero Project +// Copyright (c) 2014-2023, The Monero Project // // All rights reserved. // diff --git a/src/crypto/hash-extra-jh.c b/src/crypto/hash-extra-jh.c index 4d7481c07a3..f2180a5dcac 100644 --- a/src/crypto/hash-extra-jh.c +++ b/src/crypto/hash-extra-jh.c @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2022, The Monero Project +// Copyright (c) 2014-2023, The Monero Project // // All rights reserved. // @@ -36,7 +36,9 @@ #include "jh.h" #include "hash-ops.h" +#define JH_HASH_BITLEN HASH_SIZE * 8 + void hash_extra_jh(const void *data, size_t length, char *hash) { - int r = jh_hash(HASH_SIZE * 8, data, 8 * length, (uint8_t*)hash); - assert(SUCCESS == r); + // No need to check for failure b/c jh_hash only fails for invalid hash size + jh_hash(JH_HASH_BITLEN, data, 8 * length, (uint8_t*)hash); } diff --git a/src/crypto/hash-extra-skein.c b/src/crypto/hash-extra-skein.c index 9ea9c4faad0..07bfdccd0f4 100644 --- a/src/crypto/hash-extra-skein.c +++ b/src/crypto/hash-extra-skein.c @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2022, The Monero Project +// Copyright (c) 2014-2023, The Monero Project // // All rights reserved. // @@ -34,7 +34,9 @@ #include "hash-ops.h" #include "skein.h" +#define SKEIN_HASH_BITLEN HASH_SIZE * 8 + void hash_extra_skein(const void *data, size_t length, char *hash) { - int r = skein_hash(8 * HASH_SIZE, data, 8 * length, (uint8_t*)hash); - assert(SKEIN_SUCCESS == r); + // No need to check for failure b/c skein_hash only fails for invalid hash size + skein_hash(SKEIN_HASH_BITLEN, data, 8 * length, (uint8_t*)hash); } diff --git a/src/crypto/hash-ops.h b/src/crypto/hash-ops.h index b7ec80d7cb7..83cc78ce27a 100644 --- a/src/crypto/hash-ops.h +++ b/src/crypto/hash-ops.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2022, The Monero Project +// Copyright (c) 2014-2023, The Monero Project // // All rights reserved. // @@ -97,5 +97,9 @@ void rx_slow_hash_allocate_state(void); void rx_slow_hash_free_state(void); uint64_t rx_seedheight(const uint64_t height); void rx_seedheights(const uint64_t height, uint64_t *seed_height, uint64_t *next_height); -void rx_slow_hash(const uint64_t mainheight, const uint64_t seedheight, const char *seedhash, const void *data, size_t length, char *hash, int miners, int is_alt); -void rx_reorg(const uint64_t split_height); + +void rx_set_main_seedhash(const char *seedhash, size_t max_dataset_init_threads); +void rx_slow_hash(const char *seedhash, const void *data, size_t length, char *result_hash); + +void rx_set_miner_thread(uint32_t value, size_t max_dataset_init_threads); +uint32_t rx_get_miner_thread(void); diff --git a/src/crypto/hash.c b/src/crypto/hash.c index 7c761a1b923..b73a8587a68 100644 --- a/src/crypto/hash.c +++ b/src/crypto/hash.c @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2022, The Monero Project +// Copyright (c) 2014-2023, The Monero Project // // All rights reserved. // diff --git a/src/crypto/hash.h b/src/crypto/hash.h index 2812422e0b4..c1fccc50ce2 100644 --- a/src/crypto/hash.h +++ b/src/crypto/hash.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2022, The Monero Project +// Copyright (c) 2014-2023, The Monero Project // // All rights reserved. // diff --git a/src/crypto/hmac-keccak.c b/src/crypto/hmac-keccak.c index 771fcc27ecc..0bdbf6d0f0c 100644 --- a/src/crypto/hmac-keccak.c +++ b/src/crypto/hmac-keccak.c @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2022, The Monero Project +// Copyright (c) 2014-2023, The Monero Project // // All rights reserved. // diff --git a/src/crypto/hmac-keccak.h b/src/crypto/hmac-keccak.h index 6b3633617a5..9d92a693ac3 100644 --- a/src/crypto/hmac-keccak.h +++ b/src/crypto/hmac-keccak.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2022, The Monero Project +// Copyright (c) 2014-2023, The Monero Project // // All rights reserved. // diff --git a/src/crypto/initializer.h b/src/crypto/initializer.h index 90c09a087b8..57c15d3e763 100644 --- a/src/crypto/initializer.h +++ b/src/crypto/initializer.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2022, The Monero Project +// Copyright (c) 2014-2023, The Monero Project // // All rights reserved. // diff --git a/src/crypto/keccak.c b/src/crypto/keccak.c index f098cbdf039..6616d353088 100644 --- a/src/crypto/keccak.c +++ b/src/crypto/keccak.c @@ -123,7 +123,7 @@ void keccak(const uint8_t *in, size_t inlen, uint8_t *md, int mdlen) size_t i, rsiz, rsizw; static_assert(HASH_DATA_AREA <= sizeof(temp), "Bad keccak preconditions"); - if (mdlen <= 0 || (mdlen > 100 && sizeof(st) != (size_t)mdlen)) + if (mdlen <= 0 || (mdlen >= 100 && sizeof(st) != (size_t)mdlen)) { local_abort("Bad keccak use"); } diff --git a/src/crypto/random.c b/src/crypto/random.c index cfb637fb42a..854b2954738 100644 --- a/src/crypto/random.c +++ b/src/crypto/random.c @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2022, The Monero Project +// Copyright (c) 2014-2023, The Monero Project // // All rights reserved. // diff --git a/src/crypto/random.h b/src/crypto/random.h index d50f2943078..5bb80728af8 100644 --- a/src/crypto/random.h +++ b/src/crypto/random.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2022, The Monero Project +// Copyright (c) 2014-2023, The Monero Project // // All rights reserved. // diff --git a/src/crypto/rx-slow-hash.c b/src/crypto/rx-slow-hash.c index 40ef96ac930..4ba8f056125 100644 --- a/src/crypto/rx-slow-hash.c +++ b/src/crypto/rx-slow-hash.c @@ -1,4 +1,4 @@ -// Copyright (c) 2019-2022, The Monero Project +// Copyright (c) 2019-2023, The Monero Project // // All rights reserved. // @@ -43,32 +43,41 @@ #define RX_LOGCAT "randomx" +// Report large page allocation failures as debug messages +#define alloc_err_msg(x) mdebug(RX_LOGCAT, x); + +static CTHR_RWLOCK_TYPE main_dataset_lock = CTHR_RWLOCK_INIT; +static CTHR_RWLOCK_TYPE main_cache_lock = CTHR_RWLOCK_INIT; + +static randomx_dataset *main_dataset = NULL; +static randomx_cache *main_cache = NULL; +static char main_seedhash[HASH_SIZE]; +static int main_seedhash_set = 0; + +static CTHR_RWLOCK_TYPE secondary_cache_lock = CTHR_RWLOCK_INIT; + +static randomx_cache *secondary_cache = NULL; +static char secondary_seedhash[HASH_SIZE]; +static int secondary_seedhash_set = 0; + #if defined(_MSC_VER) #define THREADV __declspec(thread) #else #define THREADV __thread #endif -typedef struct rx_state { - CTHR_MUTEX_TYPE rs_mutex; - char rs_hash[HASH_SIZE]; - uint64_t rs_height; - randomx_cache *rs_cache; -} rx_state; - -static CTHR_MUTEX_TYPE rx_mutex = CTHR_MUTEX_INIT; -static CTHR_MUTEX_TYPE rx_dataset_mutex = CTHR_MUTEX_INIT; +static THREADV randomx_vm *main_vm_full = NULL; +static THREADV randomx_vm *main_vm_light = NULL; +static THREADV randomx_vm *secondary_vm_light = NULL; -static rx_state rx_s[2] = {{CTHR_MUTEX_INIT,{0},0,0},{CTHR_MUTEX_INIT,{0},0,0}}; +static THREADV uint32_t miner_thread = 0; -static randomx_dataset *rx_dataset; -static int rx_dataset_nomem; -static int rx_dataset_nolp; -static uint64_t rx_dataset_height; -static THREADV randomx_vm *rx_vm = NULL; +static bool is_main(const char* seedhash) { return main_seedhash_set && (memcmp(seedhash, main_seedhash, HASH_SIZE) == 0); } +static bool is_secondary(const char* seedhash) { return secondary_seedhash_set && (memcmp(seedhash, secondary_seedhash, HASH_SIZE) == 0); } static void local_abort(const char *msg) { + merror(RX_LOGCAT, "%s", msg); fprintf(stderr, "%s\n", msg); #ifdef NDEBUG _exit(1); @@ -77,6 +86,16 @@ static void local_abort(const char *msg) #endif } +static void hash2hex(const char* hash, char* hex) { + const char* d = "0123456789abcdef"; + for (int i = 0; i < HASH_SIZE; ++i) { + const uint8_t b = hash[i]; + hex[i * 2 + 0] = d[b >> 4]; + hex[i * 2 + 1] = d[b & 15]; + } + hex[HASH_SIZE * 2] = '\0'; +} + static inline int disabled_flags(void) { static int flags = -1; @@ -157,19 +176,6 @@ static unsigned int get_seedhash_epoch_blocks(void) return blocks; } -void rx_reorg(const uint64_t split_height) { - int i; - CTHR_MUTEX_LOCK(rx_mutex); - for (i=0; i<2; i++) { - if (split_height <= rx_s[i].rs_height) { - if (rx_s[i].rs_height == rx_dataset_height) - rx_dataset_height = 1; - rx_s[i].rs_height = 1; /* set to an invalid seed height */ - } - } - CTHR_MUTEX_UNLOCK(rx_mutex); -} - uint64_t rx_seedheight(const uint64_t height) { const uint64_t seedhash_epoch_lag = get_seedhash_epoch_lag(); const uint64_t seedhash_epoch_blocks = get_seedhash_epoch_blocks(); @@ -183,6 +189,103 @@ void rx_seedheights(const uint64_t height, uint64_t *seedheight, uint64_t *nexth *nextheight = rx_seedheight(height + get_seedhash_epoch_lag()); } +static void rx_alloc_dataset(randomx_flags flags, randomx_dataset** dataset, int ignore_env) +{ + if (*dataset) { + return; + } + + if (disabled_flags() & RANDOMX_FLAG_FULL_MEM) { + static int shown = 0; + if (!shown) { + shown = 1; + minfo(RX_LOGCAT, "RandomX dataset is disabled by MONERO_RANDOMX_UMASK environment variable."); + } + return; + } + + if (!ignore_env && !getenv("MONERO_RANDOMX_FULL_MEM")) { + static int shown = 0; + if (!shown) { + shown = 1; + minfo(RX_LOGCAT, "RandomX dataset is not enabled by default. Use MONERO_RANDOMX_FULL_MEM environment variable to enable it."); + } + return; + } + + *dataset = randomx_alloc_dataset((flags | RANDOMX_FLAG_LARGE_PAGES) & ~disabled_flags()); + if (!*dataset) { + alloc_err_msg("Couldn't allocate RandomX dataset using large pages"); + *dataset = randomx_alloc_dataset(flags & ~disabled_flags()); + if (!*dataset) { + merror(RX_LOGCAT, "Couldn't allocate RandomX dataset"); + } + } +} + +static void rx_alloc_cache(randomx_flags flags, randomx_cache** cache) +{ + if (*cache) { + return; + } + + *cache = randomx_alloc_cache((flags | RANDOMX_FLAG_LARGE_PAGES) & ~disabled_flags()); + if (!*cache) { + alloc_err_msg("Couldn't allocate RandomX cache using large pages"); + *cache = randomx_alloc_cache(flags & ~disabled_flags()); + if (!*cache) local_abort("Couldn't allocate RandomX cache"); + } +} + +static void rx_init_full_vm(randomx_flags flags, randomx_vm** vm) +{ + if (*vm || !main_dataset || (disabled_flags() & RANDOMX_FLAG_FULL_MEM)) { + return; + } + + if ((flags & RANDOMX_FLAG_JIT) && !miner_thread) { + flags |= RANDOMX_FLAG_SECURE; + } + + *vm = randomx_create_vm((flags | RANDOMX_FLAG_LARGE_PAGES | RANDOMX_FLAG_FULL_MEM) & ~disabled_flags(), NULL, main_dataset); + if (!*vm) { + static int shown = 0; + if (!shown) { + shown = 1; + alloc_err_msg("Couldn't allocate RandomX full VM using large pages (will print only once)"); + } + *vm = randomx_create_vm((flags | RANDOMX_FLAG_FULL_MEM) & ~disabled_flags(), NULL, main_dataset); + if (!*vm) { + merror(RX_LOGCAT, "Couldn't allocate RandomX full VM"); + } + } +} + +static void rx_init_light_vm(randomx_flags flags, randomx_vm** vm, randomx_cache* cache) +{ + if (*vm) { + randomx_vm_set_cache(*vm, cache); + return; + } + + if ((flags & RANDOMX_FLAG_JIT) && !miner_thread) { + flags |= RANDOMX_FLAG_SECURE; + } + + flags &= ~RANDOMX_FLAG_FULL_MEM; + + *vm = randomx_create_vm((flags | RANDOMX_FLAG_LARGE_PAGES) & ~disabled_flags(), cache, NULL); + if (!*vm) { + static int shown = 0; + if (!shown) { + shown = 1; + alloc_err_msg("Couldn't allocate RandomX light VM using large pages (will print only once)"); + } + *vm = randomx_create_vm(flags & ~disabled_flags(), cache, NULL); + if (!*vm) local_abort("Couldn't allocate RandomX light VM"); + } +} + typedef struct seedinfo { randomx_cache *si_cache; unsigned long si_start; @@ -191,187 +294,231 @@ typedef struct seedinfo { static CTHR_THREAD_RTYPE rx_seedthread(void *arg) { seedinfo *si = arg; - randomx_init_dataset(rx_dataset, si->si_cache, si->si_start, si->si_count); + randomx_init_dataset(main_dataset, si->si_cache, si->si_start, si->si_count); CTHR_THREAD_RETURN; } -static void rx_initdata(randomx_cache *rs_cache, const int miners, const uint64_t seedheight) { - if (miners > 1) { - unsigned long delta = randomx_dataset_item_count() / miners; - unsigned long start = 0; - int i; - seedinfo *si; - CTHR_THREAD_TYPE *st; - si = malloc(miners * sizeof(seedinfo)); - if (si == NULL) - local_abort("Couldn't allocate RandomX mining threadinfo"); - st = malloc(miners * sizeof(CTHR_THREAD_TYPE)); - if (st == NULL) { - free(si); - local_abort("Couldn't allocate RandomX mining threadlist"); - } - for (i=0; i seedheight) - is_alt = 1; - /* miner can be ahead of mainchain */ - else if (s_height < seedheight) - toggle ^= 1; - } - - toggle ^= (is_alt != 0); - - rx_sp = &rx_s[toggle]; - CTHR_MUTEX_LOCK(rx_sp->rs_mutex); - CTHR_MUTEX_UNLOCK(rx_mutex); - - cache = rx_sp->rs_cache; - if (cache == NULL) { - if (!(disabled_flags() & RANDOMX_FLAG_LARGE_PAGES)) { - cache = randomx_alloc_cache(flags | RANDOMX_FLAG_LARGE_PAGES); - if (cache == NULL) { - mdebug(RX_LOGCAT, "Couldn't use largePages for RandomX cache"); - } - } - if (cache == NULL) { - cache = randomx_alloc_cache(flags); - if (cache == NULL) - local_abort("Couldn't allocate RandomX cache"); - } +typedef struct thread_info { + char seedhash[HASH_SIZE]; + size_t max_threads; +} thread_info; + +static CTHR_THREAD_RTYPE rx_set_main_seedhash_thread(void *arg) { + thread_info* info = arg; + + CTHR_RWLOCK_LOCK_WRITE(main_dataset_lock); + CTHR_RWLOCK_LOCK_WRITE(main_cache_lock); + + // Double check that seedhash wasn't already updated + if (is_main(info->seedhash)) { + CTHR_RWLOCK_UNLOCK_WRITE(main_cache_lock); + CTHR_RWLOCK_UNLOCK_WRITE(main_dataset_lock); + free(info); + CTHR_THREAD_RETURN; } - if (rx_sp->rs_height != seedheight || rx_sp->rs_cache == NULL || memcmp(seedhash, rx_sp->rs_hash, HASH_SIZE)) { - randomx_init_cache(cache, seedhash, HASH_SIZE); - rx_sp->rs_cache = cache; - rx_sp->rs_height = seedheight; - memcpy(rx_sp->rs_hash, seedhash, HASH_SIZE); + memcpy(main_seedhash, info->seedhash, HASH_SIZE); + main_seedhash_set = 1; + + char buf[HASH_SIZE * 2 + 1]; + hash2hex(main_seedhash, buf); + minfo(RX_LOGCAT, "RandomX new main seed hash is %s", buf); + + const randomx_flags flags = enabled_flags() & ~disabled_flags(); + rx_alloc_dataset(flags, &main_dataset, 0); + rx_alloc_cache(flags, &main_cache); + + randomx_init_cache(main_cache, info->seedhash, HASH_SIZE); + minfo(RX_LOGCAT, "RandomX main cache initialized"); + + CTHR_RWLOCK_UNLOCK_WRITE(main_cache_lock); + + // From this point, rx_slow_hash can calculate hashes in light mode, but dataset is not initialized yet + rx_init_dataset(info->max_threads); + + CTHR_RWLOCK_UNLOCK_WRITE(main_dataset_lock); + + free(info); + CTHR_THREAD_RETURN; +} + +void rx_set_main_seedhash(const char *seedhash, size_t max_dataset_init_threads) { + // Early out if seedhash didn't change + if (is_main(seedhash)) { + return; } - if (rx_vm == NULL) { - if ((flags & RANDOMX_FLAG_JIT) && !miners) { - flags |= RANDOMX_FLAG_SECURE & ~disabled_flags(); - } - if (miners && (disabled_flags() & RANDOMX_FLAG_FULL_MEM)) { - miners = 0; - } - if (miners) { - CTHR_MUTEX_LOCK(rx_dataset_mutex); - if (!rx_dataset_nomem) { - if (rx_dataset == NULL) { - if (!(disabled_flags() & RANDOMX_FLAG_LARGE_PAGES)) { - rx_dataset = randomx_alloc_dataset(RANDOMX_FLAG_LARGE_PAGES); - if (rx_dataset == NULL) { - mdebug(RX_LOGCAT, "Couldn't use largePages for RandomX dataset"); - } - } - if (rx_dataset == NULL) - rx_dataset = randomx_alloc_dataset(RANDOMX_FLAG_DEFAULT); - if (rx_dataset != NULL) - rx_initdata(rx_sp->rs_cache, miners, seedheight); - } - } - if (rx_dataset != NULL) - flags |= RANDOMX_FLAG_FULL_MEM; - else { - miners = 0; - if (!rx_dataset_nomem) { - rx_dataset_nomem = 1; - mwarning(RX_LOGCAT, "Couldn't allocate RandomX dataset for miner"); + + // Update main cache and dataset in the background + thread_info* info = malloc(sizeof(thread_info)); + if (!info) local_abort("Couldn't allocate RandomX mining threadinfo"); + + memcpy(info->seedhash, seedhash, HASH_SIZE); + info->max_threads = max_dataset_init_threads; + + CTHR_THREAD_TYPE t; + if (!CTHR_THREAD_CREATE(t, rx_set_main_seedhash_thread, info)) { + local_abort("Couldn't start RandomX seed thread"); + } + CTHR_THREAD_CLOSE(t); +} + +void rx_slow_hash(const char *seedhash, const void *data, size_t length, char *result_hash) { + const randomx_flags flags = enabled_flags() & ~disabled_flags(); + int success = 0; + + // Fast path (seedhash == main_seedhash) + // Multiple threads can run in parallel in fast or light mode, 1-2 ms or 10-15 ms per hash per thread + if (is_main(seedhash)) { + // If CTHR_RWLOCK_TRYLOCK_READ fails it means dataset is being initialized now, so use the light mode + if (main_dataset && CTHR_RWLOCK_TRYLOCK_READ(main_dataset_lock)) { + // Double check that main_seedhash didn't change + if (is_main(seedhash)) { + rx_init_full_vm(flags, &main_vm_full); + if (main_vm_full) { + randomx_calculate_hash(main_vm_full, data, length, result_hash); + success = 1; } } - CTHR_MUTEX_UNLOCK(rx_dataset_mutex); - } - if (!(disabled_flags() & RANDOMX_FLAG_LARGE_PAGES) && !rx_dataset_nolp) { - rx_vm = randomx_create_vm(flags | RANDOMX_FLAG_LARGE_PAGES, rx_sp->rs_cache, rx_dataset); - if(rx_vm == NULL) { //large pages failed - mdebug(RX_LOGCAT, "Couldn't use largePages for RandomX VM"); - rx_dataset_nolp = 1; + CTHR_RWLOCK_UNLOCK_READ(main_dataset_lock); + } else { + CTHR_RWLOCK_LOCK_READ(main_cache_lock); + // Double check that main_seedhash didn't change + if (is_main(seedhash)) { + rx_init_light_vm(flags, &main_vm_light, main_cache); + randomx_calculate_hash(main_vm_light, data, length, result_hash); + success = 1; } + CTHR_RWLOCK_UNLOCK_READ(main_cache_lock); } - if (rx_vm == NULL) - rx_vm = randomx_create_vm(flags, rx_sp->rs_cache, rx_dataset); - if(rx_vm == NULL) {//fallback if everything fails - flags = RANDOMX_FLAG_DEFAULT | (miners ? RANDOMX_FLAG_FULL_MEM : 0); - rx_vm = randomx_create_vm(flags, rx_sp->rs_cache, rx_dataset); - } - if (rx_vm == NULL) - local_abort("Couldn't allocate RandomX VM"); - } else if (miners) { - CTHR_MUTEX_LOCK(rx_dataset_mutex); - if (rx_dataset != NULL && rx_dataset_height != seedheight) - rx_initdata(cache, miners, seedheight); - else if (rx_dataset == NULL) { - /* this is a no-op if the cache hasn't changed */ - randomx_vm_set_cache(rx_vm, rx_sp->rs_cache); + } + + if (success) { + return; + } + + char buf[HASH_SIZE * 2 + 1]; + + // Slow path (seedhash != main_seedhash, but seedhash == secondary_seedhash) + // Multiple threads can run in parallel in light mode, 10-15 ms per hash per thread + if (!secondary_cache) { + CTHR_RWLOCK_LOCK_WRITE(secondary_cache_lock); + if (!secondary_cache) { + hash2hex(seedhash, buf); + minfo(RX_LOGCAT, "RandomX new secondary seed hash is %s", buf); + + rx_alloc_cache(flags, &secondary_cache); + randomx_init_cache(secondary_cache, seedhash, HASH_SIZE); + minfo(RX_LOGCAT, "RandomX secondary cache updated"); + memcpy(secondary_seedhash, seedhash, HASH_SIZE); + secondary_seedhash_set = 1; } - CTHR_MUTEX_UNLOCK(rx_dataset_mutex); - } else { - /* this is a no-op if the cache hasn't changed */ - randomx_vm_set_cache(rx_vm, rx_sp->rs_cache); - } - /* mainchain users can run in parallel */ - if (!is_alt) - CTHR_MUTEX_UNLOCK(rx_sp->rs_mutex); - randomx_calculate_hash(rx_vm, data, length, hash); - /* altchain slot users always get fully serialized */ - if (is_alt) - CTHR_MUTEX_UNLOCK(rx_sp->rs_mutex); -} + CTHR_RWLOCK_UNLOCK_WRITE(secondary_cache_lock); + } + + CTHR_RWLOCK_LOCK_READ(secondary_cache_lock); + if (is_secondary(seedhash)) { + rx_init_light_vm(flags, &secondary_vm_light, secondary_cache); + randomx_calculate_hash(secondary_vm_light, data, length, result_hash); + success = 1; + } + CTHR_RWLOCK_UNLOCK_READ(secondary_cache_lock); -void rx_slow_hash_allocate_state(void) { + if (success) { + return; + } + + // Slowest path (seedhash != main_seedhash, seedhash != secondary_seedhash) + // Only one thread runs at a time and updates secondary_seedhash if needed, up to 200-500 ms per hash + CTHR_RWLOCK_LOCK_WRITE(secondary_cache_lock); + if (!is_secondary(seedhash)) { + hash2hex(seedhash, buf); + minfo(RX_LOGCAT, "RandomX new secondary seed hash is %s", buf); + + randomx_init_cache(secondary_cache, seedhash, HASH_SIZE); + minfo(RX_LOGCAT, "RandomX secondary cache updated"); + memcpy(secondary_seedhash, seedhash, HASH_SIZE); + secondary_seedhash_set = 1; + } + rx_init_light_vm(flags, &secondary_vm_light, secondary_cache); + randomx_calculate_hash(secondary_vm_light, data, length, result_hash); + CTHR_RWLOCK_UNLOCK_WRITE(secondary_cache_lock); } -void rx_slow_hash_free_state(void) { - if (rx_vm != NULL) { - randomx_destroy_vm(rx_vm); - rx_vm = NULL; +void rx_set_miner_thread(uint32_t value, size_t max_dataset_init_threads) { + miner_thread = value; + + // If dataset is not allocated yet, try to allocate and initialize it + CTHR_RWLOCK_LOCK_WRITE(main_dataset_lock); + if (main_dataset) { + CTHR_RWLOCK_UNLOCK_WRITE(main_dataset_lock); + return; } + + const randomx_flags flags = enabled_flags() & ~disabled_flags(); + rx_alloc_dataset(flags, &main_dataset, 1); + rx_init_dataset(max_dataset_init_threads); + + CTHR_RWLOCK_UNLOCK_WRITE(main_dataset_lock); +} + +uint32_t rx_get_miner_thread() { + return miner_thread; } -void rx_stop_mining(void) { - CTHR_MUTEX_LOCK(rx_dataset_mutex); - if (rx_dataset != NULL) { - randomx_dataset *rd = rx_dataset; - rx_dataset = NULL; - randomx_release_dataset(rd); +void rx_slow_hash_allocate_state() {} + +static void rx_destroy_vm(randomx_vm** vm) { + if (*vm) { + randomx_destroy_vm(*vm); + *vm = NULL; } - rx_dataset_nomem = 0; - rx_dataset_nolp = 0; - CTHR_MUTEX_UNLOCK(rx_dataset_mutex); +} + +void rx_slow_hash_free_state() { + rx_destroy_vm(&main_vm_full); + rx_destroy_vm(&main_vm_light); + rx_destroy_vm(&secondary_vm_light); } diff --git a/src/crypto/skein_port.h b/src/crypto/skein_port.h index 2b701e8cc0b..6ba83d3d766 100644 --- a/src/crypto/skein_port.h +++ b/src/crypto/skein_port.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2022, The Monero Project +// Copyright (c) 2014-2023, The Monero Project // // All rights reserved. // diff --git a/src/crypto/slow-hash.c b/src/crypto/slow-hash.c index 0de7db505de..6b975f627aa 100644 --- a/src/crypto/slow-hash.c +++ b/src/crypto/slow-hash.c @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2022, The Monero Project +// Copyright (c) 2014-2023, The Monero Project // // All rights reserved. // diff --git a/src/crypto/tree-hash.c b/src/crypto/tree-hash.c index 93a1bce4de2..0959b5050dd 100644 --- a/src/crypto/tree-hash.c +++ b/src/crypto/tree-hash.c @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2022, The Monero Project +// Copyright (c) 2014-2023, The Monero Project // // All rights reserved. // diff --git a/src/crypto/wallet/CMakeLists.txt b/src/crypto/wallet/CMakeLists.txt index ac1bdf7fdfb..e724b3e9652 100644 --- a/src/crypto/wallet/CMakeLists.txt +++ b/src/crypto/wallet/CMakeLists.txt @@ -1,4 +1,4 @@ -# Copyright (c) 2020-2022, The Monero Project +# Copyright (c) 2020-2023, The Monero Project # # All rights reserved. diff --git a/src/crypto/wallet/crypto.h b/src/crypto/wallet/crypto.h index cee0ca18eb5..27da956ec93 100644 --- a/src/crypto/wallet/crypto.h +++ b/src/crypto/wallet/crypto.h @@ -1,4 +1,4 @@ -// Copyright (c) 2020-2022, The Monero Project +// Copyright (c) 2020-2023, The Monero Project // // All rights reserved. diff --git a/src/crypto/wallet/empty.h.in b/src/crypto/wallet/empty.h.in index b884a57b536..bf54f54c584 100644 --- a/src/crypto/wallet/empty.h.in +++ b/src/crypto/wallet/empty.h.in @@ -1,4 +1,4 @@ -// Copyright (c) 2020-2022, The Monero Project +// Copyright (c) 2020-2023, The Monero Project // // All rights reserved. diff --git a/src/cryptonote_basic/CMakeLists.txt b/src/cryptonote_basic/CMakeLists.txt index 1414be1b262..864306cf7e0 100644 --- a/src/cryptonote_basic/CMakeLists.txt +++ b/src/cryptonote_basic/CMakeLists.txt @@ -1,4 +1,4 @@ -# Copyright (c) 2014-2022, The Monero Project +# Copyright (c) 2014-2023, The Monero Project # # All rights reserved. # diff --git a/src/cryptonote_basic/account.cpp b/src/cryptonote_basic/account.cpp index 2ac455fda25..9dc6e387d0d 100644 --- a/src/cryptonote_basic/account.cpp +++ b/src/cryptonote_basic/account.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2022, The Monero Project +// Copyright (c) 2014-2023, The Monero Project // // All rights reserved. // diff --git a/src/cryptonote_basic/account.h b/src/cryptonote_basic/account.h index 6e887db6d5e..7578aaf2ad7 100644 --- a/src/cryptonote_basic/account.h +++ b/src/cryptonote_basic/account.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2022, The Monero Project +// Copyright (c) 2014-2023, The Monero Project // // All rights reserved. // @@ -55,8 +55,6 @@ namespace cryptonote KV_SERIALIZE_VAL_POD_AS_BLOB_OPT(m_encryption_iv, default_iv) END_KV_SERIALIZE_MAP() - account_keys& operator=(account_keys const&) = default; - void encrypt(const crypto::chacha_key &key); void decrypt(const crypto::chacha_key &key); void encrypt_viewkey(const crypto::chacha_key &key); diff --git a/src/cryptonote_basic/account_boost_serialization.h b/src/cryptonote_basic/account_boost_serialization.h index ce0e4501aa1..ca1a1aa2971 100644 --- a/src/cryptonote_basic/account_boost_serialization.h +++ b/src/cryptonote_basic/account_boost_serialization.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2022, The Monero Project +// Copyright (c) 2014-2023, The Monero Project // // All rights reserved. // diff --git a/src/cryptonote_basic/blobdatatype.h b/src/cryptonote_basic/blobdatatype.h index 49735c034aa..98d05022c03 100644 --- a/src/cryptonote_basic/blobdatatype.h +++ b/src/cryptonote_basic/blobdatatype.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2022, The Monero Project +// Copyright (c) 2014-2023, The Monero Project // // All rights reserved. // diff --git a/src/cryptonote_basic/connection_context.cpp b/src/cryptonote_basic/connection_context.cpp index 4395bad9f9c..37cf31de4af 100644 --- a/src/cryptonote_basic/connection_context.cpp +++ b/src/cryptonote_basic/connection_context.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2020-2022, The Monero Project +// Copyright (c) 2020-2023, The Monero Project // // All rights reserved. diff --git a/src/cryptonote_basic/connection_context.h b/src/cryptonote_basic/connection_context.h index 818999a6085..e88c4ba7ea9 100644 --- a/src/cryptonote_basic/connection_context.h +++ b/src/cryptonote_basic/connection_context.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2022, The Monero Project +// Copyright (c) 2014-2023, The Monero Project // // All rights reserved. // diff --git a/src/cryptonote_basic/cryptonote_basic.h b/src/cryptonote_basic/cryptonote_basic.h index 12795879686..ae112c31f50 100644 --- a/src/cryptonote_basic/cryptonote_basic.h +++ b/src/cryptonote_basic/cryptonote_basic.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2022, The Monero Project +// Copyright (c) 2014-2023, The Monero Project // // All rights reserved. // @@ -182,7 +182,7 @@ namespace cryptonote BEGIN_SERIALIZE() VARINT_FIELD(version) - if(version == 0 || CURRENT_TRANSACTION_VERSION < version) return false; + if((version == 0 || CURRENT_TRANSACTION_VERSION < version)) return false; VARINT_FIELD(unlock_time) FIELD(vin) FIELD(vout) diff --git a/src/cryptonote_basic/cryptonote_basic_impl.cpp b/src/cryptonote_basic/cryptonote_basic_impl.cpp index c60a626899f..9bde20609aa 100644 --- a/src/cryptonote_basic/cryptonote_basic_impl.cpp +++ b/src/cryptonote_basic/cryptonote_basic_impl.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2022, The Monero Project +// Copyright (c) 2014-2023, The Monero Project // // All rights reserved. // diff --git a/src/cryptonote_basic/cryptonote_basic_impl.h b/src/cryptonote_basic/cryptonote_basic_impl.h index b423573c5a0..984bee19fab 100644 --- a/src/cryptonote_basic/cryptonote_basic_impl.h +++ b/src/cryptonote_basic/cryptonote_basic_impl.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2022, The Monero Project +// Copyright (c) 2014-2023, The Monero Project // // All rights reserved. // @@ -39,15 +39,6 @@ namespace cryptonote { /************************************************************************/ /* */ /************************************************************************/ - template - struct array_hasher: std::unary_function - { - std::size_t operator()(const t_array& val) const - { - return boost::hash_range(&val.data[0], &val.data[sizeof(val.data)]); - } - }; - #pragma pack(push, 1) struct public_address_outer_blob diff --git a/src/cryptonote_basic/cryptonote_boost_serialization.h b/src/cryptonote_basic/cryptonote_boost_serialization.h index 493c9d91d1c..954f429cb0c 100644 --- a/src/cryptonote_basic/cryptonote_boost_serialization.h +++ b/src/cryptonote_basic/cryptonote_boost_serialization.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2022, The Monero Project +// Copyright (c) 2014-2023, The Monero Project // // All rights reserved. // diff --git a/src/cryptonote_basic/cryptonote_format_utils.cpp b/src/cryptonote_basic/cryptonote_format_utils.cpp index f101f10c532..cdbcf48c2f3 100644 --- a/src/cryptonote_basic/cryptonote_format_utils.cpp +++ b/src/cryptonote_basic/cryptonote_format_utils.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2022, The Monero Project +// Copyright (c) 2014-2023, The Monero Project // // All rights reserved. // @@ -989,7 +989,7 @@ namespace cryptonote return true; } //--------------------------------------------------------------- - bool out_can_be_to_acc(const boost::optional& view_tag_opt, const crypto::key_derivation& derivation, const size_t output_index) + bool out_can_be_to_acc(const boost::optional& view_tag_opt, const crypto::key_derivation& derivation, const size_t output_index, hw::device* hwdev) { // If there is no view tag to check, the output can possibly belong to the account. // Will need to derive the output pub key to be certain whether or not the output belongs to the account. @@ -1002,7 +1002,15 @@ namespace cryptonote // Therefore can fail out early to avoid expensive crypto ops needlessly deriving output public key to // determine if output belongs to the account. crypto::view_tag derived_view_tag; - crypto::derive_view_tag(derivation, output_index, derived_view_tag); + if (hwdev != nullptr) + { + bool r = hwdev->derive_view_tag(derivation, output_index, derived_view_tag); + CHECK_AND_ASSERT_MES(r, false, "Failed to derive view tag"); + } + else + { + crypto::derive_view_tag(derivation, output_index, derived_view_tag); + } return view_tag == derived_view_tag; } //--------------------------------------------------------------- @@ -1012,7 +1020,7 @@ namespace cryptonote bool r = acc.get_device().generate_key_derivation(tx_pub_key, acc.m_view_secret_key, derivation); CHECK_AND_ASSERT_MES(r, false, "Failed to generate key derivation"); crypto::public_key pk; - if (out_can_be_to_acc(view_tag_opt, derivation, output_index)) + if (out_can_be_to_acc(view_tag_opt, derivation, output_index, &acc.get_device())) { r = acc.get_device().derive_public_key(derivation, output_index, acc.m_account_address.m_spend_public_key, pk); CHECK_AND_ASSERT_MES(r, false, "Failed to derive public key"); @@ -1026,7 +1034,7 @@ namespace cryptonote CHECK_AND_ASSERT_MES(output_index < additional_tx_pub_keys.size(), false, "wrong number of additional tx pubkeys"); r = acc.get_device().generate_key_derivation(additional_tx_pub_keys[output_index], acc.m_view_secret_key, derivation); CHECK_AND_ASSERT_MES(r, false, "Failed to generate key derivation"); - if (out_can_be_to_acc(view_tag_opt, derivation, output_index)) + if (out_can_be_to_acc(view_tag_opt, derivation, output_index, &acc.get_device())) { r = acc.get_device().derive_public_key(derivation, output_index, acc.m_account_address.m_spend_public_key, pk); CHECK_AND_ASSERT_MES(r, false, "Failed to derive public key"); @@ -1040,9 +1048,9 @@ namespace cryptonote { // try the shared tx pubkey crypto::public_key subaddress_spendkey; - if (out_can_be_to_acc(view_tag_opt, derivation, output_index)) + if (out_can_be_to_acc(view_tag_opt, derivation, output_index, &hwdev)) { - hwdev.derive_subaddress_public_key(out_key, derivation, output_index, subaddress_spendkey); + CHECK_AND_ASSERT_MES(hwdev.derive_subaddress_public_key(out_key, derivation, output_index, subaddress_spendkey), boost::none, "Failed to derive subaddress public key"); auto found = subaddresses.find(subaddress_spendkey); if (found != subaddresses.end()) return subaddress_receive_info{ found->second, derivation }; @@ -1052,9 +1060,9 @@ namespace cryptonote if (!additional_derivations.empty()) { CHECK_AND_ASSERT_MES(output_index < additional_derivations.size(), boost::none, "wrong number of additional derivations"); - if (out_can_be_to_acc(view_tag_opt, additional_derivations[output_index], output_index)) + if (out_can_be_to_acc(view_tag_opt, additional_derivations[output_index], output_index, &hwdev)) { - hwdev.derive_subaddress_public_key(out_key, additional_derivations[output_index], output_index, subaddress_spendkey); + CHECK_AND_ASSERT_MES(hwdev.derive_subaddress_public_key(out_key, additional_derivations[output_index], output_index, subaddress_spendkey), boost::none, "Failed to derive subaddress public key"); auto found = subaddresses.find(subaddress_spendkey); if (found != subaddresses.end()) return subaddress_receive_info{ found->second, additional_derivations[output_index] }; diff --git a/src/cryptonote_basic/cryptonote_format_utils.h b/src/cryptonote_basic/cryptonote_format_utils.h index 8f5459ca7b8..33fec238e91 100644 --- a/src/cryptonote_basic/cryptonote_format_utils.h +++ b/src/cryptonote_basic/cryptonote_format_utils.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2022, The Monero Project +// Copyright (c) 2014-2023, The Monero Project // // All rights reserved. // @@ -91,7 +91,7 @@ namespace cryptonote bool get_encrypted_payment_id_from_tx_extra_nonce(const blobdata& extra_nonce, crypto::hash8& payment_id); void set_tx_out(const uint64_t amount, const crypto::public_key& output_public_key, const bool use_view_tags, const crypto::view_tag& view_tag, tx_out& out); bool check_output_types(const transaction& tx, const uint8_t hf_version); - bool out_can_be_to_acc(const boost::optional& view_tag_opt, const crypto::key_derivation& derivation, const size_t output_index); + bool out_can_be_to_acc(const boost::optional& view_tag_opt, const crypto::key_derivation& derivation, const size_t output_index, hw::device *hwdev = nullptr); bool is_out_to_acc(const account_keys& acc, const crypto::public_key& output_public_key, const crypto::public_key& tx_pub_key, const std::vector& additional_tx_public_keys, size_t output_index, const boost::optional& view_tag_opt = boost::optional()); struct subaddress_receive_info { diff --git a/src/cryptonote_basic/cryptonote_format_utils_basic.cpp b/src/cryptonote_basic/cryptonote_format_utils_basic.cpp index 7431a1beb63..fccfeccba90 100644 --- a/src/cryptonote_basic/cryptonote_format_utils_basic.cpp +++ b/src/cryptonote_basic/cryptonote_format_utils_basic.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2022, The Monero Project +// Copyright (c) 2014-2023, The Monero Project // // All rights reserved. // diff --git a/src/cryptonote_basic/difficulty.cpp b/src/cryptonote_basic/difficulty.cpp index 165c1936ecb..a9777c5812c 100644 --- a/src/cryptonote_basic/difficulty.cpp +++ b/src/cryptonote_basic/difficulty.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2022, The Monero Project +// Copyright (c) 2014-2023, The Monero Project // // All rights reserved. // diff --git a/src/cryptonote_basic/difficulty.h b/src/cryptonote_basic/difficulty.h index ee9378eb911..93964646e99 100644 --- a/src/cryptonote_basic/difficulty.h +++ b/src/cryptonote_basic/difficulty.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2022, The Monero Project +// Copyright (c) 2014-2023, The Monero Project // // All rights reserved. // diff --git a/src/cryptonote_basic/events.h b/src/cryptonote_basic/events.h index 18ea9980075..476ff6a2266 100644 --- a/src/cryptonote_basic/events.h +++ b/src/cryptonote_basic/events.h @@ -1,4 +1,4 @@ -// Copyright (c) 2020-2022, The Monero Project +// Copyright (c) 2020-2023, The Monero Project // // All rights reserved. diff --git a/src/cryptonote_basic/fwd.h b/src/cryptonote_basic/fwd.h index 54f3c018498..dd0d3aec3d4 100644 --- a/src/cryptonote_basic/fwd.h +++ b/src/cryptonote_basic/fwd.h @@ -1,4 +1,4 @@ -// Copyright (c) 2020-2022, The Monero Project +// Copyright (c) 2020-2023, The Monero Project // // All rights reserved. diff --git a/src/cryptonote_basic/hardfork.cpp b/src/cryptonote_basic/hardfork.cpp index a0ea3633f4a..aa10e470928 100644 --- a/src/cryptonote_basic/hardfork.cpp +++ b/src/cryptonote_basic/hardfork.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2022, The Monero Project +// Copyright (c) 2014-2023, The Monero Project // // All rights reserved. // diff --git a/src/cryptonote_basic/hardfork.h b/src/cryptonote_basic/hardfork.h index 32f669c71ae..9b2fef0e16b 100644 --- a/src/cryptonote_basic/hardfork.h +++ b/src/cryptonote_basic/hardfork.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2022, The Monero Project +// Copyright (c) 2014-2023, The Monero Project // // All rights reserved. // @@ -231,6 +231,11 @@ namespace cryptonote */ uint64_t get_window_size() const { return window_size; } + /** + * @brief returns info for all known hard forks + */ + const std::vector& get_hardforks() const { return heights; } + private: uint8_t get_block_version(uint64_t height) const; diff --git a/src/cryptonote_basic/merge_mining.cpp b/src/cryptonote_basic/merge_mining.cpp index b8d16aade92..0e649e09505 100644 --- a/src/cryptonote_basic/merge_mining.cpp +++ b/src/cryptonote_basic/merge_mining.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2020-2022, The Monero Project +// Copyright (c) 2020-2023, The Monero Project // // All rights reserved. diff --git a/src/cryptonote_basic/merge_mining.h b/src/cryptonote_basic/merge_mining.h index 8fe282a7613..acb70ebf0d6 100644 --- a/src/cryptonote_basic/merge_mining.h +++ b/src/cryptonote_basic/merge_mining.h @@ -1,4 +1,4 @@ -// Copyright (c) 2020-2022, The Monero Project +// Copyright (c) 2020-2023, The Monero Project // // All rights reserved. diff --git a/src/cryptonote_basic/miner.cpp b/src/cryptonote_basic/miner.cpp index 5b0db9518dd..249896ee8c0 100644 --- a/src/cryptonote_basic/miner.cpp +++ b/src/cryptonote_basic/miner.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2022, The Monero Project +// Copyright (c) 2014-2023, The Monero Project // // All rights reserved. // @@ -82,6 +82,7 @@ using namespace epee; #include "miner.h" +#include "crypto/hash.h" extern "C" void slow_hash_allocate_state(); @@ -436,7 +437,6 @@ namespace cryptonote { m_stop = true; } - extern "C" void rx_stop_mining(void); //----------------------------------------------------------------------------------------------------- bool miner::stop() { @@ -469,7 +469,6 @@ namespace cryptonote MINFO("Mining has been stopped, " << m_threads.size() << " finished" ); m_threads.clear(); m_threads_autodetect.clear(); - rx_stop_mining(); return true; } //----------------------------------------------------------------------------------------------------- @@ -524,6 +523,8 @@ namespace cryptonote bool miner::worker_thread() { const uint32_t th_local_index = m_thread_index++; // atomically increment, getting value before increment + crypto::rx_set_miner_thread(th_local_index, tools::get_max_concurrency()); + MLOG_SET_THREAD_NAME(std::string("[miner ") + std::to_string(th_local_index) + "]"); MGINFO("Miner thread was started ["<< th_local_index << "]"); uint32_t nonce = m_starter_nonce + th_local_index; diff --git a/src/cryptonote_basic/miner.h b/src/cryptonote_basic/miner.h index 72dc12762ee..35b988bd775 100644 --- a/src/cryptonote_basic/miner.h +++ b/src/cryptonote_basic/miner.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2022, The Monero Project +// Copyright (c) 2014-2023, The Monero Project // // All rights reserved. // diff --git a/src/cryptonote_basic/subaddress_index.h b/src/cryptonote_basic/subaddress_index.h index 612d1e8a510..788040ae4fd 100644 --- a/src/cryptonote_basic/subaddress_index.h +++ b/src/cryptonote_basic/subaddress_index.h @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2022, The Monero Project +// Copyright (c) 2017-2023, The Monero Project // // All rights reserved. // diff --git a/src/cryptonote_basic/tx_extra.h b/src/cryptonote_basic/tx_extra.h index 141f72352fd..af6ee519770 100644 --- a/src/cryptonote_basic/tx_extra.h +++ b/src/cryptonote_basic/tx_extra.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2022, The Monero Project +// Copyright (c) 2014-2023, The Monero Project // // All rights reserved. // diff --git a/src/cryptonote_basic/verification_context.h b/src/cryptonote_basic/verification_context.h index 34157218fbc..11d54ae9f69 100644 --- a/src/cryptonote_basic/verification_context.h +++ b/src/cryptonote_basic/verification_context.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2022, The Monero Project +// Copyright (c) 2014-2023, The Monero Project // // All rights reserved. // @@ -42,7 +42,12 @@ namespace cryptonote static_assert(unsigned(relay_method::none) == 0, "default m_relay initialization is not to relay_method::none"); relay_method m_relay; // gives indication on how tx should be relayed (if at all) - bool m_verifivation_failed; //bad tx, should drop connection + bool m_verifivation_failed; //bad tx, tx should not enter mempool and connection should be dropped unless m_no_drop_offense + // Do not add to mempool, do not relay, but also do not punish the peer for sending or drop + // connections to them. Used for low fees, tx_extra too big, "relay-only rules". Not to be + // confused with breaking soft fork rules, because tx could be later added to the chain if mined + // because it does not violate consensus rules. + bool m_no_drop_offense; bool m_verifivation_impossible; //the transaction is related with an alternative blockchain bool m_added_to_pool; bool m_low_mixin; @@ -53,6 +58,7 @@ namespace cryptonote bool m_overspend; bool m_fee_too_low; bool m_too_few_outputs; + bool m_tx_extra_too_big; }; struct block_verification_context diff --git a/src/cryptonote_config.h b/src/cryptonote_config.h index f2a8e9b79d9..45da75b66a0 100644 --- a/src/cryptonote_config.h +++ b/src/cryptonote_config.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2022, The Monero Project +// Copyright (c) 2014-2023, The Monero Project // // All rights reserved. // @@ -206,6 +206,11 @@ #define DNS_BLOCKLIST_LIFETIME (86400 * 8) +//The limit is enough for the mandatory transaction content with 16 outputs (547 bytes), +//a custom tag (1 byte) and up to 32 bytes of custom data for each recipient. +// (1+32) + (1+1+16*32) + (1+16*32) = 1060 +#define MAX_TX_EXTRA_SIZE 1060 + // New constants are intended to go here namespace config { @@ -239,12 +244,16 @@ namespace config const unsigned char HASH_KEY_MEMORY = 'k'; const unsigned char HASH_KEY_MULTISIG[] = {'M', 'u', 'l', 't' , 'i', 's', 'i', 'g', 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }; const unsigned char HASH_KEY_MULTISIG_KEY_AGGREGATION[] = "Multisig_key_agg"; + const unsigned char HASH_KEY_CLSAG_ROUND_MULTISIG[] = "CLSAG_round_ms_merge_factor"; const unsigned char HASH_KEY_TXPROOF_V2[] = "TXPROOF_V2"; const unsigned char HASH_KEY_CLSAG_ROUND[] = "CLSAG_round"; const unsigned char HASH_KEY_CLSAG_AGG_0[] = "CLSAG_agg_0"; const unsigned char HASH_KEY_CLSAG_AGG_1[] = "CLSAG_agg_1"; const char HASH_KEY_MESSAGE_SIGNING[] = "MoneroMessageSignature"; const unsigned char HASH_KEY_MM_SLOT = 'm'; + const constexpr char HASH_KEY_MULTISIG_TX_PRIVKEYS_SEED[] = "multisig_tx_privkeys_seed"; + const constexpr char HASH_KEY_MULTISIG_TX_PRIVKEYS[] = "multisig_tx_privkeys"; + const constexpr char HASH_KEY_TXHASH_AND_MIXRING[] = "txhash_and_mixring"; // Multisig const uint32_t MULTISIG_MAX_SIGNERS{16}; diff --git a/src/cryptonote_core/CMakeLists.txt b/src/cryptonote_core/CMakeLists.txt index e272b94f060..373732b3bee 100644 --- a/src/cryptonote_core/CMakeLists.txt +++ b/src/cryptonote_core/CMakeLists.txt @@ -1,4 +1,4 @@ -# Copyright (c) 2014-2022, The Monero Project +# Copyright (c) 2014-2023, The Monero Project # # All rights reserved. # @@ -31,7 +31,9 @@ set(cryptonote_core_sources cryptonote_core.cpp tx_pool.cpp tx_sanity_check.cpp - cryptonote_tx_utils.cpp) + cryptonote_tx_utils.cpp + tx_verification_utils.cpp +) set(cryptonote_core_headers) @@ -49,7 +51,6 @@ target_link_libraries(cryptonote_core common cncrypto blockchain_db - multisig ringct device hardforks diff --git a/src/cryptonote_core/blockchain.cpp b/src/cryptonote_core/blockchain.cpp index 5b7b4353d0e..dc20bd1ff62 100644 --- a/src/cryptonote_core/blockchain.cpp +++ b/src/cryptonote_core/blockchain.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2022, The Monero Project +// Copyright (c) 2014-2023, The Monero Project // // All rights reserved. // @@ -48,7 +48,6 @@ #include "file_io_utils.h" #include "int-util.h" #include "common/threadpool.h" -#include "common/boost_serialization_helper.h" #include "warnings.h" #include "crypto/hash.h" #include "cryptonote_core.h" @@ -57,6 +56,7 @@ #include "common/notify.h" #include "common/varint.h" #include "common/pruning.h" +#include "common/data_cache.h" #include "time_helper.h" #undef MONERO_DEFAULT_LOG_CATEGORY @@ -98,7 +98,8 @@ Blockchain::Blockchain(tx_memory_pool& tx_pool) : m_difficulty_for_next_block(1), m_btc_valid(false), m_batch_success(true), - m_prepare_height(0) + m_prepare_height(0), + m_rct_ver_cache() { LOG_PRINT_L3("Blockchain::" << __func__); } @@ -456,6 +457,14 @@ bool Blockchain::init(BlockchainDB* db, const network_type nettype, bool offline if (!update_next_cumulative_weight_limit()) return false; } + + if (m_hardfork->get_current_version() >= RX_BLOCK_VERSION) + { + const crypto::hash seedhash = get_block_id_by_height(crypto::rx_seedheight(m_db->height())); + if (seedhash != crypto::null_hash) + rx_set_main_seedhash(seedhash.data, tools::get_max_concurrency()); + } + return true; } //------------------------------------------------------------------ @@ -570,6 +579,12 @@ void Blockchain::pop_blocks(uint64_t nblocks) if (stop_batch) m_db->batch_stop(); + + if (m_hardfork->get_current_version() >= RX_BLOCK_VERSION) + { + const crypto::hash seedhash = get_block_id_by_height(crypto::rx_seedheight(m_db->height())); + rx_set_main_seedhash(seedhash.data, tools::get_max_concurrency()); + } } //------------------------------------------------------------------ // This function tells BlockchainDB to remove the top block from the @@ -1239,18 +1254,20 @@ bool Blockchain::switch_to_alternative_blockchain(std::list } m_hardfork->reorganize_from_chain_height(split_height); - get_block_longhash_reorg(split_height); std::shared_ptr reorg_notify = m_reorg_notify; if (reorg_notify) reorg_notify->notify("%s", std::to_string(split_height).c_str(), "%h", std::to_string(m_db->height()).c_str(), "%n", std::to_string(m_db->height() - split_height).c_str(), "%d", std::to_string(discarded_blocks).c_str(), NULL); + const uint64_t new_height = m_db->height(); + const crypto::hash seedhash = get_block_id_by_height(crypto::rx_seedheight(new_height)); + crypto::hash prev_id; if (!get_block_hash(alt_chain.back().bl, prev_id)) MERROR("Failed to get block hash of an alternative chain's tip"); else - send_miner_notifications(prev_id, alt_chain.back().already_generated_coins); + send_miner_notifications(new_height, seedhash, prev_id, alt_chain.back().already_generated_coins); for (const auto& notifier : m_block_notifiers) { @@ -1262,6 +1279,9 @@ bool Blockchain::switch_to_alternative_blockchain(std::list } } + if (m_hardfork->get_current_version() >= RX_BLOCK_VERSION) + rx_set_main_seedhash(seedhash.data, tools::get_max_concurrency()); + MGINFO_GREEN("REORGANIZE SUCCESS! on height: " << split_height << ", new blockchain size: " << m_db->height()); return true; } @@ -2001,7 +2021,7 @@ bool Blockchain::handle_alternative_block(const block& b, const crypto::hash& id { seedhash = get_block_id_by_height(seedheight); } - get_altblock_longhash(bei.bl, proof_of_work, get_current_blockchain_height(), bei.height, seedheight, seedhash); + get_altblock_longhash(bei.bl, proof_of_work, seedhash); } else { get_block_longhash(this, bei.bl, proof_of_work, bei.height, 0); @@ -2044,7 +2064,7 @@ bool Blockchain::handle_alternative_block(const block& b, const crypto::hash& id cryptonote::blobdata blob; if (m_tx_pool.have_tx(txid, relay_category::legacy)) { - if (m_tx_pool.get_transaction_info(txid, td)) + if (m_tx_pool.get_transaction_info(txid, td, true/*include_sensitive_data*/)) { bei.block_cumulative_weight += td.weight; } @@ -3192,7 +3212,7 @@ bool Blockchain::have_tx_keyimges_as_spent(const transaction &tx) const } return false; } -bool Blockchain::expand_transaction_2(transaction &tx, const crypto::hash &tx_prefix_hash, const std::vector> &pubkeys) const +bool Blockchain::expand_transaction_2(transaction &tx, const crypto::hash &tx_prefix_hash, const std::vector> &pubkeys) { PERF_TIMER(expand_transaction_2); CHECK_AND_ASSERT_MES(tx.version == 2, false, "Transaction version is not 2"); @@ -3434,7 +3454,7 @@ bool Blockchain::check_tx_inputs(transaction& tx, tx_verification_context &tvc, std::vector < uint64_t > results; results.resize(tx.vin.size(), 0); - tools::threadpool& tpool = tools::threadpool::getInstance(); + tools::threadpool& tpool = tools::threadpool::getInstanceForCompute(); tools::threadpool::waiter waiter(tpool); int threads = tpool.get_max_concurrency(); @@ -3515,6 +3535,13 @@ bool Blockchain::check_tx_inputs(transaction& tx, tx_verification_context &tvc, false, "Transaction spends at least one output which is too young"); } + // Warn that new RCT types are present, and thus the cache is not being used effectively + static constexpr const std::uint8_t RCT_CACHE_TYPE = rct::RCTTypeBulletproofPlus; + if (tx.rct_signatures.type > RCT_CACHE_TYPE) + { + MWARNING("RCT cache is not caching new verification results. Please update RCT_CACHE_TYPE!"); + } + if (tx.version == 1) { if (threads > 1) @@ -3536,12 +3563,6 @@ bool Blockchain::check_tx_inputs(transaction& tx, tx_verification_context &tvc, } else { - if (!expand_transaction_2(tx, tx_prefix_hash, pubkeys)) - { - MERROR_VER("Failed to expand rct signatures!"); - return false; - } - // from version 2, check ringct signatures // obviously, the original and simple rct APIs use a mixRing that's indexes // in opposite orders, because it'd be too simple otherwise... @@ -3559,61 +3580,7 @@ bool Blockchain::check_tx_inputs(transaction& tx, tx_verification_context &tvc, case rct::RCTTypeCLSAG: case rct::RCTTypeBulletproofPlus: { - // check all this, either reconstructed (so should really pass), or not - { - if (pubkeys.size() != rv.mixRing.size()) - { - MERROR_VER("Failed to check ringct signatures: mismatched pubkeys/mixRing size"); - return false; - } - for (size_t i = 0; i < pubkeys.size(); ++i) - { - if (pubkeys[i].size() != rv.mixRing[i].size()) - { - MERROR_VER("Failed to check ringct signatures: mismatched pubkeys/mixRing size"); - return false; - } - } - - for (size_t n = 0; n < pubkeys.size(); ++n) - { - for (size_t m = 0; m < pubkeys[n].size(); ++m) - { - if (pubkeys[n][m].dest != rct::rct2pk(rv.mixRing[n][m].dest)) - { - MERROR_VER("Failed to check ringct signatures: mismatched pubkey at vin " << n << ", index " << m); - return false; - } - if (pubkeys[n][m].mask != rct::rct2pk(rv.mixRing[n][m].mask)) - { - MERROR_VER("Failed to check ringct signatures: mismatched commitment at vin " << n << ", index " << m); - return false; - } - } - } - } - - const size_t n_sigs = rct::is_rct_clsag(rv.type) ? rv.p.CLSAGs.size() : rv.p.MGs.size(); - if (n_sigs != tx.vin.size()) - { - MERROR_VER("Failed to check ringct signatures: mismatched MGs/vin sizes"); - return false; - } - for (size_t n = 0; n < tx.vin.size(); ++n) - { - bool error; - if (rct::is_rct_clsag(rv.type)) - error = memcmp(&boost::get(tx.vin[n]).k_image, &rv.p.CLSAGs[n].I, 32); - else - error = rv.p.MGs[n].II.empty() || memcmp(&boost::get(tx.vin[n]).k_image, &rv.p.MGs[n].II[0], 32); - if (error) - { - MERROR_VER("Failed to check ringct signatures: mismatched key image"); - return false; - } - } - - if (!rct::verRctNonSemanticsSimple(rv)) + if (!ver_rct_non_semantics_simple_cached(tx, pubkeys, m_rct_ver_cache, RCT_CACHE_TYPE)) { MERROR_VER("Failed to check ringct signatures!"); return false; @@ -3622,6 +3589,12 @@ bool Blockchain::check_tx_inputs(transaction& tx, tx_verification_context &tvc, } case rct::RCTTypeFull: { + if (!expand_transaction_2(tx, tx_prefix_hash, pubkeys)) + { + MERROR_VER("Failed to expand rct signatures!"); + return false; + } + // check all this, either reconstructed (so should really pass), or not { bool size_matches = true; @@ -3868,7 +3841,7 @@ void Blockchain::get_dynamic_base_fee_estimate_2021_scaling(uint64_t grace_block epee::misc_utils::rolling_median_t rm = m_long_term_block_weights_cache_rolling_median; for (size_t i = 0; i < grace_blocks; ++i) rm.insert(0); - const uint64_t Mlw_penalty_free_zone_for_wallet = std::max(rm.median(), CRYPTONOTE_BLOCK_GRANTED_FULL_REWARD_ZONE_V5); + const uint64_t Mlw_penalty_free_zone_for_wallet = std::max(rm.size() == 0 ? 0 : rm.median(), CRYPTONOTE_BLOCK_GRANTED_FULL_REWARD_ZONE_V5); // Msw: median over [100 - grace blocks] past + [grace blocks] future blocks CHECK_AND_ASSERT_THROW_MES(grace_blocks <= 100, "Grace blocks invalid In 2021 fee scaling estimate."); @@ -4552,11 +4525,15 @@ bool Blockchain::handle_block_to_main_chain(const block& bl, const crypto::hash& } } - send_miner_notifications(id, already_generated_coins); + const crypto::hash seedhash = get_block_id_by_height(crypto::rx_seedheight(new_height)); + send_miner_notifications(new_height, seedhash, id, already_generated_coins); for (const auto& notifier: m_block_notifiers) notifier(new_height - 1, {std::addressof(bl), 1}); + if (m_hardfork->get_current_version() >= RX_BLOCK_VERSION) + rx_set_main_seedhash(seedhash.data, tools::get_max_concurrency()); + return true; } //------------------------------------------------------------------ @@ -5137,7 +5114,7 @@ bool Blockchain::prepare_handle_incoming_blocks(const std::vectorheight(); const uint8_t major_version = m_hardfork->get_ideal_version(height); const difficulty_type diff = get_difficulty_for_next_block(); const uint64_t median_weight = m_current_block_cumul_weight_median; - crypto::hash seed_hash = crypto::null_hash; - if (m_hardfork->get_current_version() >= RX_BLOCK_VERSION) - { - uint64_t seed_height, next_height; - crypto::rx_seedheights(height, &seed_height, &next_height); - seed_hash = get_block_id_by_height(seed_height); - } - std::vector tx_backlog; m_tx_pool.get_block_template_backlog(tx_backlog); diff --git a/src/cryptonote_core/blockchain.h b/src/cryptonote_core/blockchain.h index 7a94f63589d..a45d3ec60aa 100644 --- a/src/cryptonote_core/blockchain.h +++ b/src/cryptonote_core/blockchain.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2022, The Monero Project +// Copyright (c) 2014-2023, The Monero Project // // All rights reserved. // @@ -57,6 +57,7 @@ #include "rpc/core_rpc_server_commands_defs.h" #include "cryptonote_basic/difficulty.h" #include "cryptonote_tx_utils.h" +#include "tx_verification_utils.h" #include "cryptonote_basic/verification_context.h" #include "crypto/hash.h" #include "checkpoints/checkpoints.h" @@ -158,6 +159,13 @@ namespace cryptonote */ bool deinit(); + /** + * @brief get a set of blockchain checkpoint hashes + * + * @return set of blockchain checkpoint hashes + */ + const checkpoints& get_checkpoints() const { return m_checkpoints; } + /** * @brief assign a set of blockchain checkpoint hashes * @@ -589,6 +597,15 @@ namespace cryptonote */ bool store_blockchain(); + /** + * @brief expands v2 transaction data from blockchain + * + * RingCT transactions do not transmit some of their data if it + * can be reconstituted by the receiver. This function expands + * that implicit data. + */ + static bool expand_transaction_2(transaction &tx, const crypto::hash &tx_prefix_hash, const std::vector> &pubkeys); + /** * @brief validates a transaction's inputs * @@ -892,6 +909,13 @@ namespace cryptonote */ uint64_t get_earliest_ideal_height_for_version(uint8_t version) const { return m_hardfork->get_earliest_ideal_height_for_version(version); } + /** + * @brief returns info for all known hard forks + * + * @return the hardforks + */ + const std::vector& get_hardforks() const { return m_hardfork->get_hardforks(); } + /** * @brief get information about hardfork voting for a version * @@ -1208,6 +1232,9 @@ namespace cryptonote uint64_t m_prepare_nblocks; std::vector *m_prepare_blocks; + // cache for verifying transaction RCT non semantics + mutable rct_ver_cache_t m_rct_ver_cache; + /** * @brief collects the keys for all outputs being "spent" as an input * @@ -1560,15 +1587,6 @@ namespace cryptonote */ void load_compiled_in_block_hashes(const GetCheckpointsCallback& get_checkpoints); - /** - * @brief expands v2 transaction data from blockchain - * - * RingCT transactions do not transmit some of their data if it - * can be reconstituted by the receiver. This function expands - * that implicit data. - */ - bool expand_transaction_2(transaction &tx, const crypto::hash &tx_prefix_hash, const std::vector> &pubkeys) const; - /** * @brief invalidates any cached block template */ @@ -1584,9 +1602,11 @@ namespace cryptonote /** * @brief sends new block notifications to ZMQ `miner_data` subscribers * + * @param height current blockchain height + * @param seed_hash seed hash to use for mining * @param prev_id hash of new blockchain tip * @param already_generated_coins total coins mined by the network so far */ - void send_miner_notifications(const crypto::hash &prev_id, uint64_t already_generated_coins); + void send_miner_notifications(uint64_t height, const crypto::hash &seed_hash, const crypto::hash &prev_id, uint64_t already_generated_coins); }; } // namespace cryptonote diff --git a/src/cryptonote_core/blockchain_storage_boost_serialization.h b/src/cryptonote_core/blockchain_storage_boost_serialization.h index d166caf764f..3c6f87886a9 100644 --- a/src/cryptonote_core/blockchain_storage_boost_serialization.h +++ b/src/cryptonote_core/blockchain_storage_boost_serialization.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2022, The Monero Project +// Copyright (c) 2014-2023, The Monero Project // // All rights reserved. // diff --git a/src/cryptonote_core/cryptonote_core.cpp b/src/cryptonote_core/cryptonote_core.cpp index a78f5d673f0..72ddd0dc9a2 100644 --- a/src/cryptonote_core/cryptonote_core.cpp +++ b/src/cryptonote_core/cryptonote_core.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2022, The Monero Project +// Copyright (c) 2014-2023, The Monero Project // // All rights reserved. // @@ -252,6 +252,10 @@ namespace cryptonote m_pprotocol = &m_protocol_stub; } //----------------------------------------------------------------------------------- + const checkpoints& core::get_checkpoints() const + { + return m_blockchain_storage.get_checkpoints(); + } void core::set_checkpoints(checkpoints&& chk_pts) { m_blockchain_storage.set_checkpoints(std::move(chk_pts)); @@ -1009,7 +1013,7 @@ namespace cryptonote CRITICAL_REGION_LOCAL(m_incoming_tx_lock); - tools::threadpool& tpool = tools::threadpool::getInstance(); + tools::threadpool& tpool = tools::threadpool::getInstanceForCompute(); tools::threadpool::waiter waiter(tpool); epee::span::const_iterator it = tx_blobs.begin(); for (size_t i = 0; i < tx_blobs.size(); i++, ++it) { @@ -1095,7 +1099,7 @@ namespace cryptonote else if(tvc[i].m_verifivation_impossible) {MERROR_VER("Transaction verification impossible: " << results[i].hash);} - if(tvc[i].m_added_to_pool) + if(tvc[i].m_added_to_pool && results[i].tx.extra.size() <= MAX_TX_EXTRA_SIZE) { MDEBUG("tx added: " << results[i].hash); valid_events = true; @@ -1406,21 +1410,66 @@ namespace cryptonote return true; } //----------------------------------------------------------------------------------------------- + bool core::notify_txpool_event(const epee::span tx_blobs, epee::span tx_hashes, epee::span txs, const std::vector &just_broadcasted) const + { + if (!m_zmq_pub) + return true; + + if (tx_blobs.size() != tx_hashes.size() || tx_blobs.size() != txs.size() || tx_blobs.size() != just_broadcasted.size()) + return false; + + /* Publish txs via ZMQ that are "just broadcasted" by the daemon. This is + done here in addition to `handle_incoming_txs` in order to guarantee txs + are pub'd via ZMQ when we know the daemon has/will broadcast to other + nodes & *after* the tx is visible in the pool. This should get called + when the user submits a tx to a daemon in the "fluff" epoch relaying txs + via a public network. */ + if (std::count(just_broadcasted.begin(), just_broadcasted.end(), true) == 0) + return true; + + std::vector results{}; + results.resize(tx_blobs.size()); + for (std::size_t i = 0; i < results.size(); ++i) + { + results[i].tx = std::move(txs[i]); + results[i].hash = std::move(tx_hashes[i]); + results[i].blob_size = tx_blobs[i].size(); + results[i].weight = results[i].tx.pruned ? get_pruned_transaction_weight(results[i].tx) : get_transaction_weight(results[i].tx, results[i].blob_size); + results[i].res = just_broadcasted[i]; + } + + m_zmq_pub(std::move(results)); + + return true; + } + //----------------------------------------------------------------------------------------------- void core::on_transactions_relayed(const epee::span tx_blobs, const relay_method tx_relay) { + // lock ensures duplicate txs aren't pub'd via zmq + CRITICAL_REGION_LOCAL(m_incoming_tx_lock); + std::vector tx_hashes{}; tx_hashes.resize(tx_blobs.size()); + std::vector txs{}; + txs.resize(tx_blobs.size()); + for (std::size_t i = 0; i < tx_blobs.size(); ++i) { - cryptonote::transaction tx{}; - if (!parse_and_validate_tx_from_blob(tx_blobs[i], tx, tx_hashes[i])) + if (!parse_and_validate_tx_from_blob(tx_blobs[i], txs[i], tx_hashes[i])) { LOG_ERROR("Failed to parse relayed transaction"); return; } } - m_mempool.set_relayed(epee::to_span(tx_hashes), tx_relay); + + std::vector just_broadcasted{}; + just_broadcasted.reserve(tx_hashes.size()); + + m_mempool.set_relayed(epee::to_span(tx_hashes), tx_relay, just_broadcasted); + + if (m_zmq_pub && matches_category(tx_relay, relay_category::legacy)) + notify_txpool_event(tx_blobs, epee::to_span(tx_hashes), epee::to_span(txs), just_broadcasted); } //----------------------------------------------------------------------------------------------- bool core::get_block_template(block& b, const account_public_address& adr, difficulty_type& diffic, uint64_t& height, uint64_t& expected_reward, const blobdata& ex_nonce, uint64_t &seed_height, crypto::hash &seed_hash) @@ -1607,10 +1656,6 @@ namespace cryptonote if (((size_t)-1) <= 0xffffffff && block_blob.size() >= 0x3fffffff) MWARNING("This block's size is " << block_blob.size() << ", closing on the 32 bit limit"); - // load json & DNS checkpoints every 10min/hour respectively, - // and verify them with respect to what blocks we already have - CHECK_AND_ASSERT_MES(update_checkpoints(), false, "One or more checkpoints loaded from json or dns conflicted with existing checkpoints."); - block lb; if (!b) { @@ -1682,6 +1727,11 @@ namespace cryptonote return true; } //----------------------------------------------------------------------------------------------- + bool core::get_pool_transactions_info(const std::vector& txids, std::vector>& txs, bool include_sensitive_txes) const + { + return m_mempool.get_transactions_info(txids, txs, include_sensitive_txes); + } + //----------------------------------------------------------------------------------------------- bool core::get_pool_transactions(std::vector& txs, bool include_sensitive_data) const { m_mempool.get_transactions(txs, include_sensitive_data); @@ -1694,6 +1744,11 @@ namespace cryptonote return true; } //----------------------------------------------------------------------------------------------- + bool core::get_pool_info(time_t start_time, bool include_sensitive_txes, size_t max_tx_count, std::vector>& added_txs, std::vector& remaining_added_txids, std::vector& removed_txs, bool& incremental) const + { + return m_mempool.get_pool_info(start_time, include_sensitive_txes, max_tx_count, added_txs, remaining_added_txids, removed_txs, incremental); + } + //----------------------------------------------------------------------------------------------- bool core::get_pool_transaction_stats(struct txpool_stats& stats, bool include_sensitive_data) const { m_mempool.get_transaction_stats(stats, include_sensitive_data); diff --git a/src/cryptonote_core/cryptonote_core.h b/src/cryptonote_core/cryptonote_core.h index 0b36730b6ef..e0655dfa2dc 100644 --- a/src/cryptonote_core/cryptonote_core.h +++ b/src/cryptonote_core/cryptonote_core.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2022, The Monero Project +// Copyright (c) 2014-2023, The Monero Project // // All rights reserved. // @@ -436,6 +436,13 @@ namespace cryptonote */ void set_cryptonote_protocol(i_cryptonote_protocol* pprotocol); + /** + * @copydoc Blockchain::get_checkpoints + * + * @note see Blockchain::get_checkpoints() + */ + const checkpoints& get_checkpoints() const; + /** * @copydoc Blockchain::set_checkpoints * @@ -503,6 +510,23 @@ namespace cryptonote bool get_pool_transaction_hashes(std::vector& txs, bool include_sensitive_txes = false) const; /** + * @copydoc tx_memory_pool::get_pool_transactions_info + * @param include_sensitive_txes include private transactions + * + * @note see tx_memory_pool::get_pool_transactions_info + */ + bool get_pool_transactions_info(const std::vector& txids, std::vector>& txs, bool include_sensitive_txes = false) const; + + /** + * @copydoc tx_memory_pool::get_pool_info + * @param include_sensitive_txes include private transactions + * @param max_tx_count max allowed added_txs in response + * + * @note see tx_memory_pool::get_pool_info + */ + bool get_pool_info(time_t start_time, bool include_sensitive_txes, size_t max_tx_count, std::vector>& added_txs, std::vector& remaining_added_txids, std::vector& removed_txs, bool& incremental) const; + + /** * @copydoc tx_memory_pool::get_transactions * @param include_sensitive_txes include private transactions * @@ -1035,6 +1059,13 @@ namespace cryptonote */ bool relay_txpool_transactions(); + /** + * @brief sends notification of txpool events to subscribers + * + * @return true on success, false otherwise + */ + bool notify_txpool_event(const epee::span tx_blobs, epee::span tx_hashes, epee::span txs, const std::vector &just_broadcasted) const; + /** * @brief checks DNS versions * diff --git a/src/cryptonote_core/cryptonote_tx_utils.cpp b/src/cryptonote_core/cryptonote_tx_utils.cpp index 1d2024a0521..60b399bb5a2 100644 --- a/src/cryptonote_core/cryptonote_tx_utils.cpp +++ b/src/cryptonote_core/cryptonote_tx_utils.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2022, The Monero Project +// Copyright (c) 2014-2023, The Monero Project // // All rights reserved. // @@ -203,7 +203,7 @@ namespace cryptonote return addr.m_view_public_key; } //--------------------------------------------------------------- - bool construct_tx_with_tx_key(const account_keys& sender_account_keys, const std::unordered_map& subaddresses, std::vector& sources, std::vector& destinations, const boost::optional& change_addr, const std::vector &extra, transaction& tx, uint64_t unlock_time, const crypto::secret_key &tx_key, const std::vector &additional_tx_keys, bool rct, const rct::RCTConfig &rct_config, rct::multisig_out *msout, bool shuffle_outs, bool use_view_tags) + bool construct_tx_with_tx_key(const account_keys& sender_account_keys, const std::unordered_map& subaddresses, std::vector& sources, std::vector& destinations, const boost::optional& change_addr, const std::vector &extra, transaction& tx, uint64_t unlock_time, const crypto::secret_key &tx_key, const std::vector &additional_tx_keys, bool rct, const rct::RCTConfig &rct_config, bool shuffle_outs, bool use_view_tags) { hw::device &hwdev = sender_account_keys.get_device(); @@ -216,10 +216,6 @@ namespace cryptonote std::vector amount_keys; tx.set_null(); amount_keys.clear(); - if (msout) - { - msout->c.clear(); - } tx.version = rct ? 2 : 1; tx.unlock_time = unlock_time; @@ -333,8 +329,8 @@ namespace cryptonote return false; } - //check that derivated key is equal with real output key (if non multisig) - if(!msout && !(in_ephemeral.pub == src_entr.outputs[src_entr.real_output].second.dest) ) + //check that derivated key is equal with real output key + if(!(in_ephemeral.pub == src_entr.outputs[src_entr.real_output].second.dest) ) { LOG_ERROR("derived public key mismatch with output public key at index " << idx << ", real out " << src_entr.real_output << "! "<< ENDL << "derived_key:" << string_tools::pod_to_hex(in_ephemeral.pub) << ENDL << "real output_public_key:" @@ -347,7 +343,7 @@ namespace cryptonote //put key image into tx input txin_to_key input_to_key; input_to_key.amount = src_entr.amount; - input_to_key.k_image = msout ? rct::rct2ki(src_entr.multisig_kLRki.ki) : img; + input_to_key.k_image = img; //fill outputs array and use relative offsets for(const tx_source_entry::output_entry& out_entry: src_entr.outputs) @@ -441,6 +437,8 @@ namespace cryptonote if (!sort_tx_extra(tx.extra, tx.extra)) return false; + CHECK_AND_ASSERT_MES(tx.extra.size() <= MAX_TX_EXTRA_SIZE, false, "TX extra size (" << tx.extra.size() << ") is greater than max allowed (" << MAX_TX_EXTRA_SIZE << ")"); + //check money if(summary_outs_money > summary_inputs_money ) { @@ -529,7 +527,6 @@ namespace cryptonote rct::keyV destinations; std::vector inamounts, outamounts; std::vector index; - std::vector kLRki; for (size_t i = 0; i < sources.size(); ++i) { rct::ctkey ctkey; @@ -543,10 +540,6 @@ namespace cryptonote memwipe(&ctkey, sizeof(rct::ctkey)); // inPk: (public key, commitment) // will be done when filling in mixRing - if (msout) - { - kLRki.push_back(sources[i].multisig_kLRki); - } } for (size_t i = 0; i < tx.vout.size(); ++i) { @@ -598,9 +591,9 @@ namespace cryptonote get_transaction_prefix_hash(tx, tx_prefix_hash, hwdev); rct::ctkeyV outSk; if (use_simple_rct) - tx.rct_signatures = rct::genRctSimple(rct::hash2rct(tx_prefix_hash), inSk, destinations, inamounts, outamounts, amount_in - amount_out, mixRing, amount_keys, msout ? &kLRki : NULL, msout, index, outSk, rct_config, hwdev); + tx.rct_signatures = rct::genRctSimple(rct::hash2rct(tx_prefix_hash), inSk, destinations, inamounts, outamounts, amount_in - amount_out, mixRing, amount_keys, index, outSk, rct_config, hwdev); else - tx.rct_signatures = rct::genRct(rct::hash2rct(tx_prefix_hash), inSk, destinations, outamounts, mixRing, amount_keys, msout ? &kLRki[0] : NULL, msout, sources[0].real_output, outSk, rct_config, hwdev); // same index assumption + tx.rct_signatures = rct::genRct(rct::hash2rct(tx_prefix_hash), inSk, destinations, outamounts, mixRing, amount_keys, sources[0].real_output, outSk, rct_config, hwdev); // same index assumption memwipe(inSk.data(), inSk.size() * sizeof(rct::ctkey)); CHECK_AND_ASSERT_MES(tx.vout.size() == outSk.size(), false, "outSk size does not match vout"); @@ -613,7 +606,7 @@ namespace cryptonote return true; } //--------------------------------------------------------------- - bool construct_tx_and_get_tx_key(const account_keys& sender_account_keys, const std::unordered_map& subaddresses, std::vector& sources, std::vector& destinations, const boost::optional& change_addr, const std::vector &extra, transaction& tx, uint64_t unlock_time, crypto::secret_key &tx_key, std::vector &additional_tx_keys, bool rct, const rct::RCTConfig &rct_config, rct::multisig_out *msout, bool use_view_tags) + bool construct_tx_and_get_tx_key(const account_keys& sender_account_keys, const std::unordered_map& subaddresses, std::vector& sources, std::vector& destinations, const boost::optional& change_addr, const std::vector &extra, transaction& tx, uint64_t unlock_time, crypto::secret_key &tx_key, std::vector &additional_tx_keys, bool rct, const rct::RCTConfig &rct_config, bool use_view_tags) { hw::device &hwdev = sender_account_keys.get_device(); hwdev.open_tx(tx_key); @@ -634,7 +627,7 @@ namespace cryptonote } bool shuffle_outs = true; - bool r = construct_tx_with_tx_key(sender_account_keys, subaddresses, sources, destinations, change_addr, extra, tx, unlock_time, tx_key, additional_tx_keys, rct, rct_config, msout, shuffle_outs, use_view_tags); + bool r = construct_tx_with_tx_key(sender_account_keys, subaddresses, sources, destinations, change_addr, extra, tx, unlock_time, tx_key, additional_tx_keys, rct, rct_config, shuffle_outs, use_view_tags); hwdev.close_tx(); return r; } catch(...) { @@ -650,7 +643,7 @@ namespace cryptonote crypto::secret_key tx_key; std::vector additional_tx_keys; std::vector destinations_copy = destinations; - return construct_tx_and_get_tx_key(sender_account_keys, subaddresses, sources, destinations_copy, change_addr, extra, tx, unlock_time, tx_key, additional_tx_keys, false, { rct::RangeProofBorromean, 0}, NULL, false); + return construct_tx_and_get_tx_key(sender_account_keys, subaddresses, sources, destinations_copy, change_addr, extra, tx, unlock_time, tx_key, additional_tx_keys, false, { rct::RangeProofBorromean, 0}); } //--------------------------------------------------------------- bool generate_genesis_block( @@ -678,10 +671,10 @@ namespace cryptonote return true; } //--------------------------------------------------------------- - void get_altblock_longhash(const block& b, crypto::hash& res, const uint64_t main_height, const uint64_t height, const uint64_t seed_height, const crypto::hash& seed_hash) + void get_altblock_longhash(const block& b, crypto::hash& res, const crypto::hash& seed_hash) { blobdata bd = get_block_hashing_blob(b); - rx_slow_hash(main_height, seed_height, seed_hash.data, bd.data(), bd.size(), res.data, 0, 1); + rx_slow_hash(seed_hash.data, bd.data(), bd.size(), res.data); } bool get_block_longhash(const Blockchain *pbc, const blobdata& bd, crypto::hash& res, const uint64_t height, const int major_version, const crypto::hash *seed_hash, const int miners) @@ -695,20 +688,16 @@ namespace cryptonote } if (major_version >= RX_BLOCK_VERSION) { - uint64_t seed_height, main_height; crypto::hash hash; if (pbc != NULL) { - seed_height = rx_seedheight(height); + const uint64_t seed_height = rx_seedheight(height); hash = seed_hash ? *seed_hash : pbc->get_pending_block_id_by_height(seed_height); - main_height = pbc->get_current_blockchain_height(); } else { memset(&hash, 0, sizeof(hash)); // only happens when generating genesis block - seed_height = 0; - main_height = 0; } - rx_slow_hash(main_height, seed_height, hash.data, bd.data(), bd.size(), res.data, seed_hash ? 0 : miners, !!seed_hash); + rx_slow_hash(hash.data, bd.data(), bd.size(), res.data); } else { const int pow_variant = major_version >= 7 ? major_version - 6 : 0; crypto::cn_slow_hash(bd.data(), bd.size(), res, pow_variant, height); @@ -722,20 +711,10 @@ namespace cryptonote return get_block_longhash(pbc, bd, res, height, b.major_version, seed_hash, miners); } - bool get_block_longhash(const Blockchain *pbc, const block& b, crypto::hash& res, const uint64_t height, const int miners) - { - return get_block_longhash(pbc, b, res, height, NULL, miners); - } - - crypto::hash get_block_longhash(const Blockchain *pbc, const block& b, const uint64_t height, const int miners) + crypto::hash get_block_longhash(const Blockchain *pbc, const block& b, const uint64_t height, const crypto::hash *seed_hash, const int miners) { crypto::hash p = crypto::null_hash; - get_block_longhash(pbc, b, p, height, miners); + get_block_longhash(pbc, b, p, height, seed_hash, miners); return p; } - - void get_block_longhash_reorg(const uint64_t split_height) - { - rx_reorg(split_height); - } } diff --git a/src/cryptonote_core/cryptonote_tx_utils.h b/src/cryptonote_core/cryptonote_tx_utils.h index f4ffb98ff09..2ee93fd8f52 100644 --- a/src/cryptonote_core/cryptonote_tx_utils.h +++ b/src/cryptonote_core/cryptonote_tx_utils.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2022, The Monero Project +// Copyright (c) 2014-2023, The Monero Project // // All rights reserved. // @@ -119,8 +119,8 @@ namespace cryptonote //--------------------------------------------------------------- crypto::public_key get_destination_view_key_pub(const std::vector &destinations, const boost::optional& change_addr); bool construct_tx(const account_keys& sender_account_keys, std::vector &sources, const std::vector& destinations, const boost::optional& change_addr, const std::vector &extra, transaction& tx, uint64_t unlock_time); - bool construct_tx_with_tx_key(const account_keys& sender_account_keys, const std::unordered_map& subaddresses, std::vector& sources, std::vector& destinations, const boost::optional& change_addr, const std::vector &extra, transaction& tx, uint64_t unlock_time, const crypto::secret_key &tx_key, const std::vector &additional_tx_keys, bool rct = false, const rct::RCTConfig &rct_config = { rct::RangeProofBorromean, 0 }, rct::multisig_out *msout = NULL, bool shuffle_outs = true, bool use_view_tags = false); - bool construct_tx_and_get_tx_key(const account_keys& sender_account_keys, const std::unordered_map& subaddresses, std::vector& sources, std::vector& destinations, const boost::optional& change_addr, const std::vector &extra, transaction& tx, uint64_t unlock_time, crypto::secret_key &tx_key, std::vector &additional_tx_keys, bool rct = false, const rct::RCTConfig &rct_config = { rct::RangeProofBorromean, 0 }, rct::multisig_out *msout = NULL, bool use_view_tags = false); + bool construct_tx_with_tx_key(const account_keys& sender_account_keys, const std::unordered_map& subaddresses, std::vector& sources, std::vector& destinations, const boost::optional& change_addr, const std::vector &extra, transaction& tx, uint64_t unlock_time, const crypto::secret_key &tx_key, const std::vector &additional_tx_keys, bool rct = false, const rct::RCTConfig &rct_config = { rct::RangeProofBorromean, 0 }, bool shuffle_outs = true, bool use_view_tags = false); + bool construct_tx_and_get_tx_key(const account_keys& sender_account_keys, const std::unordered_map& subaddresses, std::vector& sources, std::vector& destinations, const boost::optional& change_addr, const std::vector &extra, transaction& tx, uint64_t unlock_time, crypto::secret_key &tx_key, std::vector &additional_tx_keys, bool rct = false, const rct::RCTConfig &rct_config = { rct::RangeProofBorromean, 0 }, bool use_view_tags = false); bool generate_output_ephemeral_keys(const size_t tx_version, const cryptonote::account_keys &sender_account_keys, const crypto::public_key &txkey_pub, const crypto::secret_key &tx_key, const cryptonote::tx_destination_entry &dst_entr, const boost::optional &change_addr, const size_t output_index, const bool &need_additional_txkeys, const std::vector &additional_tx_keys, @@ -144,14 +144,10 @@ namespace cryptonote ); class Blockchain; - bool get_block_longhash(const Blockchain *pb, const blobdata& bd, crypto::hash& res, const uint64_t height, - const int major_version, const crypto::hash *seed_hash, const int miners); - bool get_block_longhash(const Blockchain *pb, const block& b, crypto::hash& res, const uint64_t height, const int miners); - bool get_block_longhash(const Blockchain *pb, const block& b, crypto::hash& res, const uint64_t height, const crypto::hash *seed_hash, const int miners); - void get_altblock_longhash(const block& b, crypto::hash& res, const uint64_t main_height, const uint64_t height, - const uint64_t seed_height, const crypto::hash& seed_hash); - crypto::hash get_block_longhash(const Blockchain *pb, const block& b, const uint64_t height, const int miners); - void get_block_longhash_reorg(const uint64_t split_height); + bool get_block_longhash(const Blockchain *pb, const blobdata& bd, crypto::hash& res, const uint64_t height, const int major_version, const crypto::hash *seed_hash, const int miners = 0); + bool get_block_longhash(const Blockchain *pb, const block& b, crypto::hash& res, const uint64_t height, const crypto::hash *seed_hash = nullptr, const int miners = 0); + crypto::hash get_block_longhash(const Blockchain *pb, const block& b, const uint64_t height, const crypto::hash *seed_hash = nullptr, const int miners = 0); + void get_altblock_longhash(const block& b, crypto::hash& res, const crypto::hash& seed_hash); } diff --git a/src/cryptonote_core/i_core_events.h b/src/cryptonote_core/i_core_events.h index 62919454386..cbebd9efbe4 100644 --- a/src/cryptonote_core/i_core_events.h +++ b/src/cryptonote_core/i_core_events.h @@ -1,4 +1,4 @@ -// Copyright (c) 2019-2022, The Monero Project +// Copyright (c) 2019-2023, The Monero Project // // All rights reserved. // diff --git a/src/cryptonote_core/tx_pool.cpp b/src/cryptonote_core/tx_pool.cpp index c27261860e3..d86a9f5f9c5 100644 --- a/src/cryptonote_core/tx_pool.cpp +++ b/src/cryptonote_core/tx_pool.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2022, The Monero Project +// Copyright (c) 2014-2023, The Monero Project // // All rights reserved. // @@ -40,7 +40,6 @@ #include "blockchain.h" #include "blockchain_db/locked_txn.h" #include "blockchain_db/blockchain_db.h" -#include "common/boost_serialization_helper.h" #include "int-util.h" #include "misc_language.h" #include "warnings.h" @@ -97,9 +96,9 @@ namespace cryptonote constexpr const std::chrono::seconds forward_delay_average{CRYPTONOTE_FORWARD_DELAY_AVERAGE}; // a kind of increasing backoff within min/max bounds - uint64_t get_relay_delay(time_t now, time_t received) + uint64_t get_relay_delay(time_t last_relay, time_t received) { - time_t d = (now - received + MIN_RELAY_TIME) / MIN_RELAY_TIME * MIN_RELAY_TIME; + time_t d = (last_relay - received + MIN_RELAY_TIME) / MIN_RELAY_TIME * MIN_RELAY_TIME; if (d > MAX_RELAY_TIME) d = MAX_RELAY_TIME; return d; @@ -133,6 +132,12 @@ namespace cryptonote // class code expects unsigned values throughout if (m_next_check < time_t(0)) throw std::runtime_error{"Unexpected time_t (system clock) value"}; + + m_added_txs_start_time = (time_t)0; + m_removed_txs_start_time = (time_t)0; + // We don't set these to "now" already here as we don't know how long it takes from construction + // of the pool until it "goes to work". It's safer to set when the first actual txs enter the + // corresponding lists. } //--------------------------------------------------------------------------------- bool tx_memory_pool::add_tx(transaction &tx, /*const crypto::hash& tx_prefix_hash,*/ const crypto::hash &id, const cryptonote::blobdata &blob, size_t tx_weight, tx_verification_context& tvc, relay_method tx_relay, bool relayed, uint8_t version) @@ -207,6 +212,7 @@ namespace cryptonote { tvc.m_verifivation_failed = true; tvc.m_fee_too_low = true; + tvc.m_no_drop_offense = true; return false; } @@ -219,6 +225,16 @@ namespace cryptonote return false; } + size_t tx_extra_size = tx.extra.size(); + if (!kept_by_block && tx_extra_size > MAX_TX_EXTRA_SIZE) + { + LOG_PRINT_L1("transaction tx-extra is too big: " << tx_extra_size << " bytes, the limit is: " << MAX_TX_EXTRA_SIZE); + tvc.m_verifivation_failed = true; + tvc.m_tx_extra_too_big = true; + tvc.m_no_drop_offense = true; + return false; + } + // if the transaction came from a block popped from the chain, // don't check if we have its key images as spent. // TODO: Investigate why not? @@ -281,7 +297,7 @@ namespace cryptonote return false; m_blockchain.add_txpool_tx(id, blob, meta); - m_txs_by_fee_and_receive_time.emplace(std::pair(fee / (double)(tx_weight ? tx_weight : 1), receive_time), id); + add_tx_to_transient_lists(id, fee / (double)(tx_weight ? tx_weight : 1), receive_time); lock.commit(); } catch (const std::exception &e) @@ -352,7 +368,7 @@ namespace cryptonote m_blockchain.remove_txpool_tx(id); m_blockchain.add_txpool_tx(id, blob, meta); - m_txs_by_fee_and_receive_time.emplace(std::pair(fee / (double)(tx_weight ? tx_weight : 1), receive_time), id); + add_tx_to_transient_lists(id, meta.fee / (double)(tx_weight ? tx_weight : 1), receive_time); } lock.commit(); } @@ -373,7 +389,7 @@ namespace cryptonote ++m_cookie; - MINFO("Transaction added to pool: txid " << id << " weight: " << tx_weight << " fee/byte: " << (fee / (double)(tx_weight ? tx_weight : 1))); + MINFO("Transaction added to pool: txid " << id << " weight: " << tx_weight << " fee/byte: " << (fee / (double)(tx_weight ? tx_weight : 1)) << ", count: " << m_added_txs_by_id.size()); prune(m_txpool_max_weight); @@ -402,6 +418,19 @@ namespace cryptonote m_txpool_max_weight = bytes; } //--------------------------------------------------------------------------------- + void tx_memory_pool::reduce_txpool_weight(size_t weight) + { + if (weight > m_txpool_weight) + { + MERROR("Underflow in txpool weight"); + m_txpool_weight = 0; + } + else + { + m_txpool_weight -= weight; + } + } + //--------------------------------------------------------------------------------- void tx_memory_pool::prune(size_t bytes) { CRITICAL_REGION_LOCAL(m_transactions_lock); @@ -423,8 +452,14 @@ namespace cryptonote txpool_tx_meta_t meta; if (!m_blockchain.get_txpool_tx_meta(txid, meta)) { - MERROR("Failed to find tx_meta in txpool"); - return; + static bool warned = false; + if (!warned) + { + MERROR("Failed to find tx_meta in txpool (will only print once)"); + warned = true; + } + --it; + continue; } // don't prune the kept_by_block ones, they're likely added because we're adding a block with those if (meta.kept_by_block) @@ -442,10 +477,11 @@ namespace cryptonote // remove first, in case this throws, so key images aren't removed MINFO("Pruning tx " << txid << " from txpool: weight: " << meta.weight << ", fee/byte: " << it->first.first); m_blockchain.remove_txpool_tx(txid); - m_txpool_weight -= meta.weight; + reduce_txpool_weight(meta.weight); remove_transaction_keyimages(tx, txid); MINFO("Pruned tx " << txid << " from txpool: weight: " << meta.weight << ", fee/byte: " << it->first.first); - m_txs_by_fee_and_receive_time.erase(it--); + remove_tx_from_transient_lists(it, txid, !meta.matches(relay_category::broadcasted)); + it--; changed = true; } catch (const std::exception &e) @@ -527,8 +563,7 @@ namespace cryptonote CRITICAL_REGION_LOCAL(m_transactions_lock); CRITICAL_REGION_LOCAL1(m_blockchain); - auto sorted_it = find_tx_in_sorted_container(id); - + bool sensitive = false; try { LockedTXN lock(m_blockchain.get_db()); @@ -559,10 +594,11 @@ namespace cryptonote do_not_relay = meta.do_not_relay; double_spend_seen = meta.double_spend_seen; pruned = meta.pruned; + sensitive = !meta.matches(relay_category::broadcasted); // remove first, in case this throws, so key images aren't removed m_blockchain.remove_txpool_tx(id); - m_txpool_weight -= tx_weight; + reduce_txpool_weight(tx_weight); remove_transaction_keyimages(tx, id); lock.commit(); } @@ -572,13 +608,12 @@ namespace cryptonote return false; } - if (sorted_it != m_txs_by_fee_and_receive_time.end()) - m_txs_by_fee_and_receive_time.erase(sorted_it); + remove_tx_from_transient_lists(find_tx_in_sorted_container(id), id, sensitive); ++m_cookie; return true; } //--------------------------------------------------------------------------------- - bool tx_memory_pool::get_transaction_info(const crypto::hash &txid, tx_details &td) const + bool tx_memory_pool::get_transaction_info(const crypto::hash &txid, tx_details &td, bool include_sensitive_data, bool include_blob) const { PERF_TIMER(get_transaction_info); CRITICAL_REGION_LOCAL(m_transactions_lock); @@ -590,7 +625,12 @@ namespace cryptonote txpool_tx_meta_t meta; if (!m_blockchain.get_txpool_tx_meta(txid, meta)) { - MERROR("Failed to find tx in txpool"); + LOG_PRINT_L2("Failed to find tx in txpool: " << txid); + return false; + } + if (!include_sensitive_data && !meta.matches(relay_category::broadcasted)) + { + // We don't want sensitive data && the tx is sensitive, so no need to return it return false; } cryptonote::blobdata txblob = m_blockchain.get_txpool_tx_blob(txid, relay_category::all); @@ -616,11 +656,13 @@ namespace cryptonote td.kept_by_block = meta.kept_by_block; td.last_failed_height = meta.last_failed_height; td.last_failed_id = meta.last_failed_id; - td.receive_time = meta.receive_time; - td.last_relayed_time = meta.dandelionpp_stem ? 0 : meta.last_relayed_time; + td.receive_time = include_sensitive_data ? meta.receive_time : 0; + td.last_relayed_time = (include_sensitive_data && !meta.dandelionpp_stem) ? meta.last_relayed_time : 0; td.relayed = meta.relayed; td.do_not_relay = meta.do_not_relay; td.double_spend_seen = meta.double_spend_seen; + if (include_blob) + td.tx_blob = std::move(txblob); } catch (const std::exception &e) { @@ -630,6 +672,25 @@ namespace cryptonote return true; } + //------------------------------------------------------------------ + bool tx_memory_pool::get_transactions_info(const std::vector& txids, std::vector>& txs, bool include_sensitive) const + { + CRITICAL_REGION_LOCAL(m_transactions_lock); + CRITICAL_REGION_LOCAL1(m_blockchain); + + txs.clear(); + + for (const auto &it: txids) + { + tx_details details; + bool success = get_transaction_info(it, details, include_sensitive, true/*include_blob*/); + if (success) + { + txs.push_back(std::make_pair(it, std::move(details))); + } + } + return true; + } //--------------------------------------------------------------------------------- bool tx_memory_pool::get_complement(const std::vector &hashes, std::vector &txes) const { @@ -691,15 +752,7 @@ namespace cryptonote (tx_age > CRYPTONOTE_MEMPOOL_TX_FROM_ALT_BLOCK_LIVETIME && meta.kept_by_block) ) { LOG_PRINT_L1("Tx " << txid << " removed from tx pool due to outdated, age: " << tx_age ); - auto sorted_it = find_tx_in_sorted_container(txid); - if (sorted_it == m_txs_by_fee_and_receive_time.end()) - { - LOG_PRINT_L1("Removing tx " << txid << " from tx pool, but it was not found in the sorted txs container!"); - } - else - { - m_txs_by_fee_and_receive_time.erase(sorted_it); - } + remove_tx_from_transient_lists(find_tx_in_sorted_container(txid), txid, !meta.matches(relay_category::broadcasted)); m_timed_out_transactions.insert(txid); remove.push_back(std::make_pair(txid, meta.weight)); } @@ -725,7 +778,7 @@ namespace cryptonote { // remove first, so we only remove key images if the tx removal succeeds m_blockchain.remove_txpool_tx(txid); - m_txpool_weight -= entry.second; + reduce_txpool_weight(entry.second); remove_transaction_keyimages(tx, txid); } } @@ -779,7 +832,7 @@ namespace cryptonote case relay_method::local: case relay_method::fluff: case relay_method::block: - if (now - meta.last_relayed_time <= get_relay_delay(now, meta.receive_time)) + if (now - meta.last_relayed_time <= get_relay_delay(meta.last_relayed_time, meta.receive_time)) return true; // continue to next tx break; } @@ -812,7 +865,7 @@ namespace cryptonote function is only called every ~2 minutes, so this resetting should be unnecessary, but is primarily a precaution against potential changes to the callback routines. */ - elem.second.last_relayed_time = now + get_relay_delay(now, elem.second.receive_time); + elem.second.last_relayed_time = now + get_relay_delay(elem.second.last_relayed_time, elem.second.receive_time); m_blockchain.update_txpool_tx(elem.first, elem.second); } @@ -820,8 +873,10 @@ namespace cryptonote return true; } //--------------------------------------------------------------------------------- - void tx_memory_pool::set_relayed(const epee::span hashes, const relay_method method) + void tx_memory_pool::set_relayed(const epee::span hashes, const relay_method method, std::vector &just_broadcasted) { + just_broadcasted.clear(); + crypto::random_poisson_seconds embargo_duration{dandelionpp_embargo_average}; const auto now = std::chrono::system_clock::now(); uint64_t next_relay = uint64_t{std::numeric_limits::max()}; @@ -831,12 +886,14 @@ namespace cryptonote LockedTXN lock(m_blockchain.get_db()); for (const auto& hash : hashes) { + bool was_just_broadcasted = false; try { txpool_tx_meta_t meta; if (m_blockchain.get_txpool_tx_meta(hash, meta)) { // txes can be received as "stem" or "fluff" in either order + const bool already_broadcasted = meta.matches(relay_category::broadcasted); meta.upgrade_relay_method(method); meta.relayed = true; @@ -849,6 +906,12 @@ namespace cryptonote meta.last_relayed_time = std::chrono::system_clock::to_time_t(now); m_blockchain.update_txpool_tx(hash, meta); + // wait until db update succeeds to ensure tx is visible in the pool + was_just_broadcasted = !already_broadcasted && meta.matches(relay_category::broadcasted); + + if (was_just_broadcasted) + // Make sure the tx gets re-added with an updated time + add_tx_to_transient_lists(hash, meta.fee / (double)meta.weight, std::chrono::system_clock::to_time_t(now)); } } catch (const std::exception &e) @@ -856,6 +919,7 @@ namespace cryptonote MERROR("Failed to update txpool transaction metadata: " << e.what()); // continue } + just_broadcasted.emplace_back(was_just_broadcasted); } lock.commit(); set_if_less(m_next_check, time_t(next_relay)); @@ -900,6 +964,81 @@ namespace cryptonote }, false, category); } //------------------------------------------------------------------ + bool tx_memory_pool::get_pool_info(time_t start_time, bool include_sensitive, size_t max_tx_count, std::vector>& added_txs, std::vector& remaining_added_txids, std::vector& removed_txs, bool& incremental) const + { + CRITICAL_REGION_LOCAL(m_transactions_lock); + CRITICAL_REGION_LOCAL1(m_blockchain); + + incremental = true; + if (start_time == (time_t)0) + { + // Giving no start time means give back whole pool + incremental = false; + } + else if ((m_added_txs_start_time != (time_t)0) && (m_removed_txs_start_time != (time_t)0)) + { + if ((start_time <= m_added_txs_start_time) || (start_time <= m_removed_txs_start_time)) + { + // If either of the two lists do not go back far enough it's not possible to + // deliver incremental pool info + incremental = false; + } + // The check uses "<=": We cannot be sure to have ALL txs exactly at start_time, only AFTER that time + } + else + { + // Some incremental info still missing completely + incremental = false; + } + + added_txs.clear(); + remaining_added_txids.clear(); + removed_txs.clear(); + + std::vector txids; + if (!incremental) + { + LOG_PRINT_L2("Giving back the whole pool"); + // Give back the whole pool in 'added_txs'; because calling 'get_transaction_info' right inside the + // anonymous method somehow results in an LMDB error with transactions we have to build a list of + // ids first and get the full info afterwards + get_transaction_hashes(txids, include_sensitive); + if (txids.size() > max_tx_count) + { + remaining_added_txids = std::vector(txids.begin() + max_tx_count, txids.end()); + txids.erase(txids.begin() + max_tx_count, txids.end()); + } + get_transactions_info(txids, added_txs, include_sensitive); + return true; + } + + // Give back incrementally, based on time of entry into the map + for (const auto &pit : m_added_txs_by_id) + { + if (pit.second >= start_time) + txids.push_back(pit.first); + } + get_transactions_info(txids, added_txs, include_sensitive); + if (added_txs.size() > max_tx_count) + { + remaining_added_txids.reserve(added_txs.size() - max_tx_count); + for (size_t i = max_tx_count; i < added_txs.size(); ++i) + remaining_added_txids.push_back(added_txs[i].first); + added_txs.erase(added_txs.begin() + max_tx_count, added_txs.end()); + } + + std::multimap::const_iterator rit = m_removed_txs_by_time.lower_bound(start_time); + while (rit != m_removed_txs_by_time.end()) + { + if (include_sensitive || !rit->second.sensitive) + { + removed_txs.push_back(rit->second.txid); + } + ++rit; + } + return true; + } + //------------------------------------------------------------------ void tx_memory_pool::get_transaction_backlog(std::vector& backlog, bool include_sensitive) const { CRITICAL_REGION_LOCAL(m_transactions_lock); @@ -917,26 +1056,61 @@ namespace cryptonote { CRITICAL_REGION_LOCAL(m_transactions_lock); CRITICAL_REGION_LOCAL1(m_blockchain); - const relay_category category = include_sensitive ? relay_category::all : relay_category::broadcasted; - backlog.reserve(m_blockchain.get_txpool_tx_count(include_sensitive)); - txpool_tx_meta_t tmp_meta; - m_blockchain.for_all_txpool_txes([this, &backlog, &tmp_meta](const crypto::hash &txid, const txpool_tx_meta_t &meta, const cryptonote::blobdata_ref *bd){ - transaction tx; - if (!(meta.pruned ? parse_and_validate_tx_base_from_blob(*bd, tx) : parse_and_validate_tx_from_blob(*bd, tx))) + + std::vector tmp; + uint64_t total_weight = 0; + + // First get everything from the mempool, filter it later + m_blockchain.for_all_txpool_txes([&tmp, &total_weight](const crypto::hash &txid, const txpool_tx_meta_t &meta, const cryptonote::blobdata_ref*){ + tmp.emplace_back(tx_block_template_backlog_entry{txid, meta.weight, meta.fee}); + total_weight += meta.weight; + return true; + }, false, include_sensitive ? relay_category::all : relay_category::broadcasted); + + // Limit backlog to 112.5% of current median weight. This is enough to mine a full block with the optimal block reward + const uint64_t median_weight = m_blockchain.get_current_cumulative_block_weight_median(); + const uint64_t max_backlog_weight = median_weight + (median_weight / 8); + + // If the total weight is too high, choose the best paying transactions + if (total_weight > max_backlog_weight) + std::sort(tmp.begin(), tmp.end(), [](const auto& a, const auto& b){ return a.fee * b.weight > b.fee * a.weight; }); + + backlog.clear(); + uint64_t w = 0; + + std::unordered_set k_images; + + for (const tx_block_template_backlog_entry& e : tmp) + { + try { - MERROR("Failed to parse tx from txpool"); - // continue - return true; - } - tx.set_hash(txid); + txpool_tx_meta_t meta; + if (!m_blockchain.get_txpool_tx_meta(e.id, meta)) + continue; - tmp_meta = meta; + cryptonote::blobdata txblob; + if (!m_blockchain.get_txpool_tx_blob(e.id, txblob, relay_category::all)) + continue; - if (is_transaction_ready_to_go(tmp_meta, txid, *bd, tx)) - backlog.push_back({txid, meta.weight, meta.fee}); + cryptonote::transaction tx; + if (is_transaction_ready_to_go(meta, e.id, txblob, tx)) + { + if (have_key_images(k_images, tx)) + continue; + append_key_images(k_images, tx); - return true; - }, true, category); + backlog.push_back(e); + w += e.weight; + if (w > max_backlog_weight) + break; + } + } + catch (const std::exception &e) + { + MERROR("Failed to check transaction readiness: " << e.what()); + // continue, not fatal + } + } } //------------------------------------------------------------------ void tx_memory_pool::get_transaction_stats(struct txpool_stats& stats, bool include_sensitive) const @@ -1569,6 +1743,12 @@ namespace cryptonote CRITICAL_REGION_LOCAL(m_transactions_lock); CRITICAL_REGION_LOCAL1(m_blockchain); + // Simply throw away incremental info, too difficult to update + m_added_txs_by_id.clear(); + m_added_txs_start_time = (time_t)0; + m_removed_txs_by_time.clear(); + m_removed_txs_start_time = (time_t)0; + MINFO("Validating txpool contents for v" << (unsigned)version); LockedTXN lock(m_blockchain.get_db()); @@ -1626,6 +1806,106 @@ namespace cryptonote return n_removed; } //--------------------------------------------------------------------------------- + void tx_memory_pool::add_tx_to_transient_lists(const crypto::hash& txid, double fee, time_t receive_time) + { + + time_t now = time(NULL); + const std::unordered_map::iterator it = m_added_txs_by_id.find(txid); + if (it == m_added_txs_by_id.end()) + { + m_added_txs_by_id.insert(std::make_pair(txid, now)); + } + else + { + // This tx was already added to the map earlier, probably because then it was in the "stem" + // phase of Dandelion++ and now is in the "fluff" phase i.e. got broadcasted: We have to set + // a new time for clients that are not allowed to see sensitive txs to make sure they will + // see it now if they query incrementally + it->second = now; + + auto sorted_it = find_tx_in_sorted_container(txid); + if (sorted_it == m_txs_by_fee_and_receive_time.end()) + { + MERROR("Re-adding tx " << txid << " to tx pool, but it was not found in the sorted txs container"); + } + else + { + m_txs_by_fee_and_receive_time.erase(sorted_it); + } + } + m_txs_by_fee_and_receive_time.emplace(std::pair(fee, receive_time), txid); + + // Don't check for "resurrected" txs in case of reorgs i.e. don't check in 'm_removed_txs_by_time' + // whether we have that txid there and if yes remove it; this results in possible duplicates + // where we return certain txids as deleted AND in the pool at the same time which requires + // clients to process deleted ones BEFORE processing pool txs + if (m_added_txs_start_time == (time_t)0) + { + m_added_txs_start_time = now; + } + } + //--------------------------------------------------------------------------------- + void tx_memory_pool::remove_tx_from_transient_lists(const cryptonote::sorted_tx_container::iterator& sorted_it, const crypto::hash& txid, bool sensitive) + { + if (sorted_it == m_txs_by_fee_and_receive_time.end()) + { + LOG_PRINT_L1("Removing tx " << txid << " from tx pool, but it was not found in the sorted txs container!"); + } + else + { + m_txs_by_fee_and_receive_time.erase(sorted_it); + } + + const std::unordered_map::iterator it = m_added_txs_by_id.find(txid); + if (it != m_added_txs_by_id.end()) + { + m_added_txs_by_id.erase(it); + } + else + { + MDEBUG("Removing tx " << txid << " from tx pool, but it was not found in the map of added txs"); + } + track_removed_tx(txid, sensitive); + } + //--------------------------------------------------------------------------------- + void tx_memory_pool::track_removed_tx(const crypto::hash& txid, bool sensitive) + { + time_t now = time(NULL); + m_removed_txs_by_time.insert(std::make_pair(now, removed_tx_info{txid, sensitive})); + MDEBUG("Transaction removed from pool: txid " << txid << ", total entries in removed list now " << m_removed_txs_by_time.size()); + if (m_removed_txs_start_time == (time_t)0) + { + m_removed_txs_start_time = now; + } + + // Simple system to make sure the list of removed ids does not swell to an unmanageable size: Set + // an absolute size limit plus delete entries that are x minutes old (which is ok because clients + // will sync with sensible time intervalls and should not ask for incremental info e.g. 1 hour back) + const int MAX_REMOVED = 20000; + if (m_removed_txs_by_time.size() > MAX_REMOVED) + { + auto erase_it = m_removed_txs_by_time.begin(); + std::advance(erase_it, MAX_REMOVED / 4 + 1); + m_removed_txs_by_time.erase(m_removed_txs_by_time.begin(), erase_it); + m_removed_txs_start_time = m_removed_txs_by_time.begin()->first; + MDEBUG("Erased old transactions from big removed list, leaving " << m_removed_txs_by_time.size()); + } + else + { + time_t earliest = now - (30 * 60); // 30 minutes + std::map::iterator from, to; + from = m_removed_txs_by_time.begin(); + to = m_removed_txs_by_time.lower_bound(earliest); + int distance = std::distance(from, to); + if (distance > 0) + { + m_removed_txs_by_time.erase(from, to); + m_removed_txs_start_time = earliest; + MDEBUG("Erased " << distance << " old transactions from removed list, leaving " << m_removed_txs_by_time.size()); + } + } + } + //--------------------------------------------------------------------------------- bool tx_memory_pool::init(size_t max_txpool_weight, bool mine_stem_txes) { CRITICAL_REGION_LOCAL(m_transactions_lock); @@ -1633,6 +1913,10 @@ namespace cryptonote m_txpool_max_weight = max_txpool_weight ? max_txpool_weight : DEFAULT_TXPOOL_MAX_WEIGHT; m_txs_by_fee_and_receive_time.clear(); + m_added_txs_by_id.clear(); + m_added_txs_start_time = (time_t)0; + m_removed_txs_by_time.clear(); + m_removed_txs_start_time = (time_t)0; m_spent_key_images.clear(); m_txpool_weight = 0; std::vector remove; @@ -1657,7 +1941,7 @@ namespace cryptonote MFATAL("Failed to insert key images from txpool tx"); return false; } - m_txs_by_fee_and_receive_time.emplace(std::pair(meta.fee / (double)meta.weight, meta.receive_time), txid); + add_tx_to_transient_lists(txid, meta.fee / (double)meta.weight, meta.receive_time); m_txpool_weight += meta.weight; return true; }, true, relay_category::all); diff --git a/src/cryptonote_core/tx_pool.h b/src/cryptonote_core/tx_pool.h index 62bef6c06e2..6fe2eea59ba 100644 --- a/src/cryptonote_core/tx_pool.h +++ b/src/cryptonote_core/tx_pool.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2022, The Monero Project +// Copyright (c) 2014-2023, The Monero Project // // All rights reserved. // @@ -266,7 +266,11 @@ namespace cryptonote void get_transaction_backlog(std::vector& backlog, bool include_sensitive = false) const; /** - * @brief get (hash, weight, fee) for all transactions in the pool - the minimum required information to create a block template + * @brief get (hash, weight, fee) for transactions in the pool - the minimum required information to create a block template + * + * Not all transactions in the pool will be returned for performance reasons + * If there are too many transactions in the pool, only the highest-paying transactions + * will be returned - but enough for the miner to create a full block * * @param backlog return-by-reference that data * @param include_sensitive return stempool, anonymity-pool, and unrelayed txes @@ -353,8 +357,10 @@ namespace cryptonote * * @param hashes list of tx hashes that are about to be relayed * @param tx_relay update how the tx left this node + * @param just_broadcasted true if a tx was just broadcasted + * */ - void set_relayed(epee::span hashes, relay_method tx_relay); + void set_relayed(epee::span hashes, relay_method tx_relay, std::vector &just_broadcasted); /** * @brief get the total number of transactions in the pool @@ -406,6 +412,13 @@ namespace cryptonote */ void set_txpool_max_weight(size_t bytes); + /** + * @brief reduce the cumulative txpool weight by the weight provided + * + * @param weight the weight to reduce the total txpool weight by + */ + void reduce_txpool_weight(size_t weight); + #define CURRENT_MEMPOOL_ARCHIVE_VER 11 #define CURRENT_MEMPOOL_TX_DETAILS_ARCHIVE_VER 13 @@ -415,6 +428,7 @@ namespace cryptonote struct tx_details { transaction tx; //!< the transaction + cryptonote::blobdata tx_blob; //!< the transaction's binary blob size_t blob_size; //!< the transaction's size size_t weight; //!< the transaction's weight uint64_t fee; //!< the transaction's fee amount @@ -453,13 +467,25 @@ namespace cryptonote /** * @brief get infornation about a single transaction */ - bool get_transaction_info(const crypto::hash &txid, tx_details &td) const; + bool get_transaction_info(const crypto::hash &txid, tx_details &td, bool include_sensitive_data, bool include_blob = false) const; + + /** + * @brief get information about multiple transactions + */ + bool get_transactions_info(const std::vector& txids, std::vector>& txs, bool include_sensitive_data = false) const; /** * @brief get transactions not in the passed set */ bool get_complement(const std::vector &hashes, std::vector &txes) const; + /** + * @brief get info necessary for update of pool-related info in a wallet, preferably incremental + * + * @return true on success, false on error + */ + bool get_pool_info(time_t start_time, bool include_sensitive, size_t max_tx_count, std::vector>& added_txs, std::vector& remaining_added_txids, std::vector& removed_txs, bool& incremental) const; + private: /** @@ -564,6 +590,10 @@ namespace cryptonote */ void prune(size_t bytes = 0); + void add_tx_to_transient_lists(const crypto::hash& txid, double fee, time_t receive_time); + void remove_tx_from_transient_lists(const cryptonote::sorted_tx_container::iterator& sorted_it, const crypto::hash& txid, bool sensitive); + void track_removed_tx(const crypto::hash& txid, bool sensitive); + //TODO: confirm the below comments and investigate whether or not this // is the desired behavior //! map key images to transactions which spent them @@ -596,6 +626,26 @@ namespace cryptonote std::atomic m_cookie; //!< incremented at each change + // Info when transactions entered the pool, accessible by txid + std::unordered_map m_added_txs_by_id; + + // Info at what time the pool started to track the adding of transactions + time_t m_added_txs_start_time; + + struct removed_tx_info + { + crypto::hash txid; + bool sensitive; + }; + + // Info about transactions that were removed from the pool, ordered by the time + // of deletion + std::multimap m_removed_txs_by_time; + + // Info how far back in time the list of removed tx ids currently reaches + // (it gets shorted periodically to prevent overflow) + time_t m_removed_txs_start_time; + /** * @brief get an iterator to a transaction in the sorted container * diff --git a/src/cryptonote_core/tx_sanity_check.cpp b/src/cryptonote_core/tx_sanity_check.cpp index f46ec3bbc86..99d5717cbb0 100644 --- a/src/cryptonote_core/tx_sanity_check.cpp +++ b/src/cryptonote_core/tx_sanity_check.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2019-2022, The Monero Project +// Copyright (c) 2019-2023, The Monero Project // // All rights reserved. // diff --git a/src/cryptonote_core/tx_sanity_check.h b/src/cryptonote_core/tx_sanity_check.h index 9779a8e79cb..4d7bf741c0e 100644 --- a/src/cryptonote_core/tx_sanity_check.h +++ b/src/cryptonote_core/tx_sanity_check.h @@ -1,4 +1,4 @@ -// Copyright (c) 2019-2022, The Monero Project +// Copyright (c) 2019-2023, The Monero Project // // All rights reserved. // diff --git a/src/cryptonote_core/tx_verification_utils.cpp b/src/cryptonote_core/tx_verification_utils.cpp new file mode 100644 index 00000000000..a93ef2f2536 --- /dev/null +++ b/src/cryptonote_core/tx_verification_utils.cpp @@ -0,0 +1,167 @@ +// Copyright (c) 2023, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include "cryptonote_core/blockchain.h" +#include "cryptonote_core/tx_verification_utils.h" +#include "ringct/rctSigs.h" + +#undef MONERO_DEFAULT_LOG_CATEGORY +#define MONERO_DEFAULT_LOG_CATEGORY "blockchain" + +#define VER_ASSERT(cond, msgexpr) CHECK_AND_ASSERT_MES(cond, false, msgexpr) + +using namespace cryptonote; + +// Do RCT expansion, then do post-expansion sanity checks, then do full non-semantics verification. +static bool expand_tx_and_ver_rct_non_sem(transaction& tx, const rct::ctkeyM& mix_ring) +{ + // Pruned transactions can not be expanded and verified because they are missing RCT data + VER_ASSERT(!tx.pruned, "Pruned transaction will not pass verRctNonSemanticsSimple"); + + // Calculate prefix hash + const crypto::hash tx_prefix_hash = get_transaction_prefix_hash(tx); + + // Expand mixring, tx inputs, tx key images, prefix hash message, etc into the RCT sig + const bool exp_res = Blockchain::expand_transaction_2(tx, tx_prefix_hash, mix_ring); + VER_ASSERT(exp_res, "Failed to expand rct signatures!"); + + const rct::rctSig& rv = tx.rct_signatures; + + // Check that expanded RCT mixring == input mixring + VER_ASSERT(rv.mixRing == mix_ring, "Failed to check ringct signatures: mismatched pubkeys/mixRing"); + + // Check CLSAG/MLSAG size against transaction input + const size_t n_sigs = rct::is_rct_clsag(rv.type) ? rv.p.CLSAGs.size() : rv.p.MGs.size(); + VER_ASSERT(n_sigs == tx.vin.size(), "Failed to check ringct signatures: mismatched input sigs/vin sizes"); + + // For each input, check that the key images were copied into the expanded RCT sig correctly + for (size_t n = 0; n < n_sigs; ++n) + { + const crypto::key_image& nth_vin_image = boost::get(tx.vin[n]).k_image; + + if (rct::is_rct_clsag(rv.type)) + { + const bool ki_match = 0 == memcmp(&nth_vin_image, &rv.p.CLSAGs[n].I, 32); + VER_ASSERT(ki_match, "Failed to check ringct signatures: mismatched CLSAG key image"); + } + else + { + const bool mg_nonempty = !rv.p.MGs[n].II.empty(); + VER_ASSERT(mg_nonempty, "Failed to check ringct signatures: missing MLSAG key image"); + const bool ki_match = 0 == memcmp(&nth_vin_image, &rv.p.MGs[n].II[0], 32); + VER_ASSERT(ki_match, "Failed to check ringct signatures: mismatched MLSAG key image"); + } + } + + // Mix ring data is now known to be correctly incorporated into the RCT sig inside tx. + return rct::verRctNonSemanticsSimple(rv); +} + +// Create a unique identifier for pair of tx blob + mix ring +static crypto::hash calc_tx_mixring_hash(const transaction& tx, const rct::ctkeyM& mix_ring) +{ + std::stringstream ss; + + // Start with domain seperation + ss << config::HASH_KEY_TXHASH_AND_MIXRING; + + // Then add TX hash + const crypto::hash tx_hash = get_transaction_hash(tx); + ss.write(tx_hash.data, sizeof(crypto::hash)); + + // Then serialize mix ring + binary_archive ar(ss); + ::do_serialize(ar, const_cast(mix_ring)); + + // Calculate hash of TX hash and mix ring blob + crypto::hash tx_and_mixring_hash; + get_blob_hash(ss.str(), tx_and_mixring_hash); + + return tx_and_mixring_hash; +} + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +namespace cryptonote +{ + +bool ver_rct_non_semantics_simple_cached +( + transaction& tx, + const rct::ctkeyM& mix_ring, + rct_ver_cache_t& cache, + const std::uint8_t rct_type_to_cache +) +{ + // Hello future Monero dev! If you got this assert, read the following carefully: + // + // For this version of RCT, the way we guaranteed that verification caches do not generate false + // positives (and thus possibly enabling double spends) is we take a hash of two things. One, + // we use get_transaction_hash() which gives us a (cryptographically secure) unique + // representation of all "knobs" controlled by the possibly malicious constructor of the + // transaction. Two, we take a hash of all *previously validated* blockchain data referenced by + // this transaction which is required to validate the ring signature. In our case, this is the + // mixring. Future versions of the protocol may differ in this regard, but if this assumptions + // holds true in the future, enable the verification hash by modifying the `untested_tx` + // condition below. + const bool untested_tx = tx.version > 2 || tx.rct_signatures.type > rct::RCTTypeBulletproofPlus; + VER_ASSERT(!untested_tx, "Unknown TX type. Make sure RCT cache works correctly with this type and then enable it in the code here."); + + // Don't cache older (or newer) rctSig types + // This cache only makes sense when it caches data from mempool first, + // so only "current fork version-enabled" RCT types need to be cached + if (tx.rct_signatures.type != rct_type_to_cache) + { + MDEBUG("RCT cache: tx " << get_transaction_hash(tx) << " skipped"); + return expand_tx_and_ver_rct_non_sem(tx, mix_ring); + } + + // Generate unique hash for tx+mix_ring pair + const crypto::hash tx_mixring_hash = calc_tx_mixring_hash(tx, mix_ring); + + // Search cache for successful verification of same TX + mix ring combination + if (cache.has(tx_mixring_hash)) + { + MDEBUG("RCT cache: tx " << get_transaction_hash(tx) << " hit"); + return true; + } + + // We had a cache miss, so now we must expand the mix ring and do full verification + MDEBUG("RCT cache: tx " << get_transaction_hash(tx) << " missed"); + if (!expand_tx_and_ver_rct_non_sem(tx, mix_ring)) + { + return false; + } + + // At this point, the TX RCT verified successfully, so add it to the cache and return true + cache.add(tx_mixring_hash); + + return true; +} + +} // namespace cryptonote diff --git a/src/cryptonote_core/tx_verification_utils.h b/src/cryptonote_core/tx_verification_utils.h new file mode 100644 index 00000000000..ccd401d2adb --- /dev/null +++ b/src/cryptonote_core/tx_verification_utils.h @@ -0,0 +1,78 @@ +// Copyright (c) 2023, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#pragma once + +#include "common/data_cache.h" +#include "cryptonote_basic/cryptonote_basic.h" + +namespace cryptonote +{ + +// Modifying this value should not affect consensus. You can adjust it for performance needs +static constexpr const size_t RCT_VER_CACHE_SIZE = 8192; + +using rct_ver_cache_t = ::tools::data_cache<::crypto::hash, RCT_VER_CACHE_SIZE>; + +/** + * @brief Cached version of rct::verRctNonSemanticsSimple + * + * This function will not affect how the transaction is serialized and it will never modify the + * transaction prefix. + * + * The reference to tx is mutable since the transaction's ring signatures may be expanded by + * Blockchain::expand_transaction_2. However, on cache hits, the transaction will not be + * expanded. This means that the caller does not need to call expand_transaction_2 on this + * transaction before passing it; the transaction will not successfully verify with "old" RCT data + * if the transaction has been otherwise modified since the last verification. + * + * But, if cryptonote::get_transaction_hash(tx) returns a "stale" hash, this function is not + * guaranteed to work. So make sure that the cryptonote::transaction passed has not had + * modifications to it since the last time its hash was fetched without properly invalidating the + * hashes. + * + * rct_type_to_cache can be any RCT version value as long as rct::verRctNonSemanticsSimple works for + * this RCT version, but for most applications, it doesn't make sense to not make this version + * the "current" RCT version (i.e. the version that transactions in the mempool are). + * + * @param tx transaction which contains RCT signature to verify + * @param mix_ring mixring referenced by this tx. THIS DATA MUST BE PREVIOUSLY VALIDATED + * @param cache saves tx+mixring hashes used to cache calls + * @param rct_type_to_cache Only RCT sigs with version (e.g. RCTTypeBulletproofPlus) will be cached + * @return true when verRctNonSemanticsSimple() w/ expanded tx.rct_signatures would return true + * @return false when verRctNonSemanticsSimple() w/ expanded tx.rct_signatures would return false + */ +bool ver_rct_non_semantics_simple_cached +( + transaction& tx, + const rct::ctkeyM& mix_ring, + rct_ver_cache_t& cache, + std::uint8_t rct_type_to_cache +); + +} // namespace cryptonote diff --git a/src/cryptonote_protocol/CMakeLists.txt b/src/cryptonote_protocol/CMakeLists.txt index b516e17e96f..c52eaa80288 100644 --- a/src/cryptonote_protocol/CMakeLists.txt +++ b/src/cryptonote_protocol/CMakeLists.txt @@ -1,4 +1,4 @@ -# Copyright (c) 2014-2022, The Monero Project +# Copyright (c) 2014-2023, The Monero Project # # All rights reserved. # diff --git a/src/cryptonote_protocol/block_queue.cpp b/src/cryptonote_protocol/block_queue.cpp index 4e65eafa4fe..ba22157b568 100644 --- a/src/cryptonote_protocol/block_queue.cpp +++ b/src/cryptonote_protocol/block_queue.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2022, The Monero Project +// Copyright (c) 2017-2023, The Monero Project // // All rights reserved. // diff --git a/src/cryptonote_protocol/block_queue.h b/src/cryptonote_protocol/block_queue.h index 64ff106a396..311275aa953 100644 --- a/src/cryptonote_protocol/block_queue.h +++ b/src/cryptonote_protocol/block_queue.h @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2022, The Monero Project +// Copyright (c) 2017-2023, The Monero Project // // All rights reserved. // diff --git a/src/cryptonote_protocol/cryptonote_protocol_defs.h b/src/cryptonote_protocol/cryptonote_protocol_defs.h index 0b860f1a867..23d9144a550 100644 --- a/src/cryptonote_protocol/cryptonote_protocol_defs.h +++ b/src/cryptonote_protocol/cryptonote_protocol_defs.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2022, The Monero Project +// Copyright (c) 2014-2023, The Monero Project // // All rights reserved. // diff --git a/src/cryptonote_protocol/cryptonote_protocol_handler-base.cpp b/src/cryptonote_protocol/cryptonote_protocol_handler-base.cpp index 92ee0805422..90f3c111ac5 100644 --- a/src/cryptonote_protocol/cryptonote_protocol_handler-base.cpp +++ b/src/cryptonote_protocol/cryptonote_protocol_handler-base.cpp @@ -2,7 +2,7 @@ /// @author rfree (current maintainer in monero.cc project) /// @brief This is the place to implement our handlers for protocol network actions, e.g. for ratelimit for download-requests -// Copyright (c) 2014-2022, The Monero Project +// Copyright (c) 2014-2023, The Monero Project // // All rights reserved. // diff --git a/src/cryptonote_protocol/cryptonote_protocol_handler.h b/src/cryptonote_protocol/cryptonote_protocol_handler.h index a1e4df56306..a29ee8287a0 100644 --- a/src/cryptonote_protocol/cryptonote_protocol_handler.h +++ b/src/cryptonote_protocol/cryptonote_protocol_handler.h @@ -2,7 +2,7 @@ /// @author rfree (current maintainer/user in monero.cc project - most of code is from CryptoNote) /// @brief This is the original cryptonote protocol network-events handler, modified by us -// Copyright (c) 2014-2022, The Monero Project +// Copyright (c) 2014-2023, The Monero Project // // All rights reserved. // @@ -113,12 +113,23 @@ namespace cryptonote const block_queue &get_block_queue() const { return m_block_queue; } void stop(); void on_connection_close(cryptonote_connection_context &context); - void set_max_out_peers(unsigned int max) { m_max_out_peers = max; } + void set_max_out_peers(epee::net_utils::zone zone, unsigned int max) { CRITICAL_REGION_LOCAL(m_max_out_peers_lock); m_max_out_peers[zone] = max; } + unsigned int get_max_out_peers(epee::net_utils::zone zone) const + { + CRITICAL_REGION_LOCAL(m_max_out_peers_lock); + const auto it = m_max_out_peers.find(zone); + if (it == m_max_out_peers.end()) + { + MWARNING(epee::net_utils::zone_to_string(zone) << " max out peers not set, using default"); + return P2P_DEFAULT_CONNECTIONS_COUNT; + } + return it->second; + } bool no_sync() const { return m_no_sync; } void set_no_sync(bool value) { m_no_sync = value; } std::string get_peers_overview() const; std::pair get_next_needed_pruning_stripe() const; - bool needs_new_sync_connections() const; + bool needs_new_sync_connections(epee::net_utils::zone zone) const; bool is_busy_syncing(); private: @@ -171,7 +182,8 @@ namespace cryptonote epee::math_helper::once_a_time_milliseconds<100> m_standby_checker; epee::math_helper::once_a_time_seconds<101> m_sync_search_checker; epee::math_helper::once_a_time_seconds<43> m_bad_peer_checker; - std::atomic m_max_out_peers; + std::unordered_map m_max_out_peers; + mutable epee::critical_section m_max_out_peers_lock; tools::PerformanceTimer m_sync_timer, m_add_timer; uint64_t m_last_add_end_time; uint64_t m_sync_spans_downloaded, m_sync_old_spans_downloaded, m_sync_bad_spans_downloaded; diff --git a/src/cryptonote_protocol/cryptonote_protocol_handler.inl b/src/cryptonote_protocol/cryptonote_protocol_handler.inl index 891ee109d2e..325737789d7 100644 --- a/src/cryptonote_protocol/cryptonote_protocol_handler.inl +++ b/src/cryptonote_protocol/cryptonote_protocol_handler.inl @@ -2,7 +2,7 @@ /// @author rfree (current maintainer/user in monero.cc project - most of code is from CryptoNote) /// @brief This is the original cryptonote protocol network-events handler, modified by us -// Copyright (c) 2014-2022, The Monero Project +// Copyright (c) 2014-2023, The Monero Project // // All rights reserved. // @@ -537,6 +537,10 @@ namespace cryptonote MLOG_PEER_STATE("requesting chain"); } + // load json & DNS checkpoints every 10min/hour respectively, + // and verify them with respect to what blocks we already have + CHECK_AND_ASSERT_MES(m_core.update_checkpoints(), 1, "One or more checkpoints loaded from json or dns conflicted with existing checkpoints."); + return 1; } //------------------------------------------------------------------------------------------------------------------------ @@ -819,6 +823,10 @@ namespace cryptonote post_notify(r, context); MLOG_PEER_STATE("requesting chain"); } + + // load json & DNS checkpoints every 10min/hour respectively, + // and verify them with respect to what blocks we already have + CHECK_AND_ASSERT_MES(m_core.update_checkpoints(), 1, "One or more checkpoints loaded from json or dns conflicted with existing checkpoints."); } } else @@ -1012,7 +1020,7 @@ namespace cryptonote for (auto& tx : arg.txs) { tx_verification_context tvc{}; - if (!m_core.handle_incoming_tx({tx, crypto::null_hash}, tvc, tx_relay, true)) + if (!m_core.handle_incoming_tx({tx, crypto::null_hash}, tvc, tx_relay, true) && !tvc.m_no_drop_offense) { LOG_PRINT_CCONTEXT_L1("Tx verification failed, dropping connection"); drop_connection(context, false, false); @@ -1776,33 +1784,49 @@ skip: return true; MTRACE("Checking for outgoing syncing peers..."); - unsigned n_syncing = 0, n_synced = 0; - boost::uuids::uuid last_synced_peer_id(boost::uuids::nil_uuid()); + std::unordered_map n_syncing, n_synced; + std::unordered_map last_synced_peer_id; + std::vector zones; m_p2p->for_each_connection([&](cryptonote_connection_context& context, nodetool::peerid_type peer_id, uint32_t support_flags)->bool { if (!peer_id || context.m_is_income) // only consider connected outgoing peers return true; + + const epee::net_utils::zone zone = context.m_remote_address.get_zone(); + if (n_syncing.find(zone) == n_syncing.end()) + { + n_syncing[zone] = 0; + n_synced[zone] = 0; + last_synced_peer_id[zone] = boost::uuids::nil_uuid(); + zones.push_back(zone); + } + if (context.m_state == cryptonote_connection_context::state_synchronizing) - ++n_syncing; + ++n_syncing[zone]; if (context.m_state == cryptonote_connection_context::state_normal) { - ++n_synced; + ++n_synced[zone]; if (!context.m_anchor) - last_synced_peer_id = context.m_connection_id; + last_synced_peer_id[zone] = context.m_connection_id; } return true; }); - MTRACE(n_syncing << " syncing, " << n_synced << " synced"); - // if we're at max out peers, and not enough are syncing - if (n_synced + n_syncing >= m_max_out_peers && n_syncing < P2P_DEFAULT_SYNC_SEARCH_CONNECTIONS_COUNT && last_synced_peer_id != boost::uuids::nil_uuid()) + for (const auto& zone : zones) { - if (!m_p2p->for_connection(last_synced_peer_id, [&](cryptonote_connection_context& ctx, nodetool::peerid_type peer_id, uint32_t f)->bool{ - MINFO(ctx << "dropping synced peer, " << n_syncing << " syncing, " << n_synced << " synced"); - drop_connection(ctx, false, false); - return true; - })) - MDEBUG("Failed to find peer we wanted to drop"); + const unsigned int max_out_peers = get_max_out_peers(zone); + MTRACE("[" << epee::net_utils::zone_to_string(zone) << "] " << n_syncing[zone] << " syncing, " << n_synced[zone] << " synced, " << max_out_peers << " max out peers"); + + // if we're at max out peers, and not enough are syncing, drop the last sync'd non-anchor + if (n_synced[zone] + n_syncing[zone] >= max_out_peers && n_syncing[zone] < P2P_DEFAULT_SYNC_SEARCH_CONNECTIONS_COUNT && last_synced_peer_id[zone] != boost::uuids::nil_uuid()) + { + if (!m_p2p->for_connection(last_synced_peer_id[zone], [&](cryptonote_connection_context& ctx, nodetool::peerid_type peer_id, uint32_t f)->bool{ + MINFO(ctx << "dropping synced peer, " << n_syncing[zone] << " syncing, " << n_synced[zone] << " synced, " << max_out_peers << " max out peers"); + drop_connection(ctx, false, false); + return true; + })) + MDEBUG("Failed to find peer we wanted to drop"); + } } return true; @@ -1987,11 +2011,13 @@ skip: ++n_peers_on_next_stripe; return true; }); + // TODO: investigate tallying by zone and comparing to max out peers by zone + const unsigned int max_out_peers = get_max_out_peers(epee::net_utils::zone::public_); const uint32_t distance = (peer_stripe + (1<= m_max_out_peers && n_peers_on_next_stripe == 0) || (distance > 1 && n_peers_on_next_stripe <= 2) || distance > 2) + if ((n_out_peers >= max_out_peers && n_peers_on_next_stripe == 0) || (distance > 1 && n_peers_on_next_stripe <= 2) || distance > 2) { MDEBUG(context << "we want seed " << next_stripe << ", and either " << n_out_peers << " is at max out peers (" - << m_max_out_peers << ") or distance " << distance << " from " << next_stripe << " to " << peer_stripe << + << max_out_peers << ") or distance " << distance << " from " << next_stripe << " to " << peer_stripe << " is too large and we have only " << n_peers_on_next_stripe << " peers on next seed, dropping connection to make space"); return true; } @@ -2812,11 +2838,13 @@ skip: } return true; }); - const bool use_next = (n_next > m_max_out_peers / 2 && n_subsequent <= 1) || (n_next > 2 && n_subsequent == 0); + // TODO: investigate tallying by zone and comparing to max out peers by zone + const unsigned int max_out_peers = get_max_out_peers(epee::net_utils::zone::public_); + const bool use_next = (n_next > max_out_peers / 2 && n_subsequent <= 1) || (n_next > 2 && n_subsequent == 0); const uint32_t ret_stripe = use_next ? subsequent_pruning_stripe: next_pruning_stripe; MIDEBUG(const std::string po = get_peers_overview(), "get_next_needed_pruning_stripe: want height " << want_height << " (" << want_height_from_blockchain << " from blockchain, " << want_height_from_block_queue << " from block queue), stripe " << - next_pruning_stripe << " (" << n_next << "/" << m_max_out_peers << " on it and " << n_subsequent << " on " << + next_pruning_stripe << " (" << n_next << "/" << max_out_peers << " on it and " << n_subsequent << " on " << subsequent_pruning_stripe << ", " << n_others << " others) -> " << ret_stripe << " (+" << (ret_stripe - next_pruning_stripe + (1 << CRYPTONOTE_PRUNING_LOG_STRIPES)) % (1 << CRYPTONOTE_PRUNING_LOG_STRIPES) << "), current peers " << po); @@ -2824,7 +2852,7 @@ skip: } //------------------------------------------------------------------------------------------------------------------------ template - bool t_cryptonote_protocol_handler::needs_new_sync_connections() const + bool t_cryptonote_protocol_handler::needs_new_sync_connections(epee::net_utils::zone zone) const { const uint64_t target = m_core.get_target_blockchain_height(); const uint64_t height = m_core.get_current_blockchain_height(); @@ -2832,11 +2860,11 @@ skip: return false; size_t n_out_peers = 0; m_p2p->for_each_connection([&](cryptonote_connection_context& ctx, nodetool::peerid_type peer_id, uint32_t support_flags)->bool{ - if (!ctx.m_is_income) + if (!ctx.m_is_income && ctx.m_remote_address.get_zone() == zone) ++n_out_peers; return true; }); - if (n_out_peers >= m_max_out_peers) + if (n_out_peers >= get_max_out_peers(zone)) return false; return true; } diff --git a/src/cryptonote_protocol/cryptonote_protocol_handler_common.h b/src/cryptonote_protocol/cryptonote_protocol_handler_common.h index 3b2ca5e3776..688ff2f37d9 100644 --- a/src/cryptonote_protocol/cryptonote_protocol_handler_common.h +++ b/src/cryptonote_protocol/cryptonote_protocol_handler_common.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2022, The Monero Project +// Copyright (c) 2014-2023, The Monero Project // // All rights reserved. // diff --git a/src/cryptonote_protocol/enums.h b/src/cryptonote_protocol/enums.h index aeb66ed5c5b..19cf8078cc4 100644 --- a/src/cryptonote_protocol/enums.h +++ b/src/cryptonote_protocol/enums.h @@ -1,4 +1,4 @@ -// Copyright (c) 2019-2022, The Monero Project +// Copyright (c) 2019-2023, The Monero Project // // All rights reserved. // diff --git a/src/cryptonote_protocol/fwd.h b/src/cryptonote_protocol/fwd.h index f036f41cb50..7182d9ef466 100644 --- a/src/cryptonote_protocol/fwd.h +++ b/src/cryptonote_protocol/fwd.h @@ -1,4 +1,4 @@ -// Copyright (c) 2019-2022, The Monero Project +// Copyright (c) 2019-2023, The Monero Project // // All rights reserved. // diff --git a/src/cryptonote_protocol/levin_notify.cpp b/src/cryptonote_protocol/levin_notify.cpp index 83f37015f24..5b420ec3f1c 100644 --- a/src/cryptonote_protocol/levin_notify.cpp +++ b/src/cryptonote_protocol/levin_notify.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2019-2022, The Monero Project +// Copyright (c) 2019-2023, The Monero Project // // All rights reserved. // @@ -542,6 +542,7 @@ namespace levin i_core_events* core_; std::vector txs_; boost::uuids::uuid source_; + relay_method tx_relay; //! \pre Called in `zone_->strand` void operator()() @@ -549,7 +550,7 @@ namespace levin if (!zone_ || !core_ || txs_.empty()) return; - if (!zone_->fluffing) + if (!zone_->fluffing || tx_relay == relay_method::local) { core_->on_transactions_relayed(epee::to_span(txs_), relay_method::stem); for (int tries = 2; 0 < tries; tries--) @@ -589,7 +590,7 @@ namespace levin change_channels(change_channels&&) = default; change_channels(const change_channels& source) - : zone_(source.zone_), map_(source.map_.clone()) + : zone_(source.zone_), map_(source.map_.clone()), fluffing_(source.fluffing_) {} //! \pre Called within `zone_->strand`. @@ -871,7 +872,7 @@ namespace levin { // this will change a local/forward tx to stem or fluff ... zone_->strand.dispatch( - dandelionpp_notify{zone_, core_, std::move(txs), source} + dandelionpp_notify{zone_, core_, std::move(txs), source, tx_relay} ); break; } diff --git a/src/cryptonote_protocol/levin_notify.h b/src/cryptonote_protocol/levin_notify.h index 2927eea8682..52d36efb0aa 100644 --- a/src/cryptonote_protocol/levin_notify.h +++ b/src/cryptonote_protocol/levin_notify.h @@ -1,4 +1,4 @@ -// Copyright (c) 2019-2022, The Monero Project +// Copyright (c) 2019-2023, The Monero Project // // All rights reserved. // diff --git a/src/daemon/CMakeLists.txt b/src/daemon/CMakeLists.txt index 455f72472e9..8a8d1d9a739 100644 --- a/src/daemon/CMakeLists.txt +++ b/src/daemon/CMakeLists.txt @@ -1,4 +1,4 @@ -# Copyright (c) 2014-2022, The Monero Project +# Copyright (c) 2014-2023, The Monero Project # # All rights reserved. # diff --git a/src/daemon/command_line_args.h b/src/daemon/command_line_args.h index 96fddc02dce..506d754900d 100644 --- a/src/daemon/command_line_args.h +++ b/src/daemon/command_line_args.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2022, The Monero Project +// Copyright (c) 2014-2023, The Monero Project // // All rights reserved. // diff --git a/src/daemon/command_parser_executor.cpp b/src/daemon/command_parser_executor.cpp index 20c906141f8..25b15f3b1ac 100644 --- a/src/daemon/command_parser_executor.cpp +++ b/src/daemon/command_parser_executor.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2022, The Monero Project +// Copyright (c) 2014-2023, The Monero Project // // All rights reserved. // diff --git a/src/daemon/command_parser_executor.h b/src/daemon/command_parser_executor.h index cd9e6544e1a..7224e36f60e 100644 --- a/src/daemon/command_parser_executor.h +++ b/src/daemon/command_parser_executor.h @@ -6,7 +6,7 @@ */ -// Copyright (c) 2014-2022, The Monero Project +// Copyright (c) 2014-2023, The Monero Project // // All rights reserved. // diff --git a/src/daemon/command_server.cpp b/src/daemon/command_server.cpp index fc5f1b3d702..6baaa701502 100644 --- a/src/daemon/command_server.cpp +++ b/src/daemon/command_server.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2022, The Monero Project +// Copyright (c) 2014-2023, The Monero Project // // All rights reserved. // diff --git a/src/daemon/command_server.h b/src/daemon/command_server.h index babb8e85ae9..8e3972beb5e 100644 --- a/src/daemon/command_server.h +++ b/src/daemon/command_server.h @@ -9,7 +9,7 @@ Passing RPC commands: */ -// Copyright (c) 2014-2022, The Monero Project +// Copyright (c) 2014-2023, The Monero Project // // All rights reserved. // diff --git a/src/daemon/core.h b/src/daemon/core.h index fde0d6babe5..c3a3116b103 100644 --- a/src/daemon/core.h +++ b/src/daemon/core.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2022, The Monero Project +// Copyright (c) 2014-2023, The Monero Project // // All rights reserved. // diff --git a/src/daemon/daemon.cpp b/src/daemon/daemon.cpp index 286c02b5055..97dad9357c3 100644 --- a/src/daemon/daemon.cpp +++ b/src/daemon/daemon.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2022, The Monero Project +// Copyright (c) 2014-2023, The Monero Project // // All rights reserved. // diff --git a/src/daemon/daemon.h b/src/daemon/daemon.h index 529b42d20b5..d652d9a4b8a 100644 --- a/src/daemon/daemon.h +++ b/src/daemon/daemon.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2022, The Monero Project +// Copyright (c) 2014-2023, The Monero Project // // All rights reserved. // diff --git a/src/daemon/executor.cpp b/src/daemon/executor.cpp index d67bd614121..14b0f86a3e5 100644 --- a/src/daemon/executor.cpp +++ b/src/daemon/executor.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2022, The Monero Project +// Copyright (c) 2014-2023, The Monero Project // // All rights reserved. // diff --git a/src/daemon/executor.h b/src/daemon/executor.h index d59ef22fbfc..92fc5ca467d 100644 --- a/src/daemon/executor.h +++ b/src/daemon/executor.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2022, The Monero Project +// Copyright (c) 2014-2023, The Monero Project // // All rights reserved. // diff --git a/src/daemon/main.cpp b/src/daemon/main.cpp index 73d9ebce1b3..23c313c9dc0 100644 --- a/src/daemon/main.cpp +++ b/src/daemon/main.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2022, The Monero Project +// Copyright (c) 2014-2023, The Monero Project // // All rights reserved. // @@ -83,7 +83,7 @@ uint16_t parse_public_rpc_port(const po::variables_map &vm) } uint16_t rpc_port; - if (!string_tools::get_xtype_from_string(rpc_port, rpc_port_str)) + if (!epee::string_tools::get_xtype_from_string(rpc_port, rpc_port_str)) { throw std::runtime_error("invalid RPC port " + rpc_port_str); } @@ -219,6 +219,19 @@ int main(int argc, char const * argv[]) { po::store(po::parse_config_file(config_path.string().c_str(), core_settings), vm); } + catch (const po::unknown_option &e) + { + std::string unrecognized_option = e.get_option_name(); + if (all_options.find_nothrow(unrecognized_option, false)) + { + std::cerr << "Option '" << unrecognized_option << "' is not allowed in the config file, please use it as a command line flag." << std::endl; + } + else + { + std::cerr << "Unrecognized option '" << unrecognized_option << "' in config file." << std::endl; + } + return 1; + } catch (const std::exception &e) { // log system isn't initialized yet diff --git a/src/daemon/p2p.h b/src/daemon/p2p.h index 799dc3d259a..40d7ab69559 100644 --- a/src/daemon/p2p.h +++ b/src/daemon/p2p.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2022, The Monero Project +// Copyright (c) 2014-2023, The Monero Project // // All rights reserved. // diff --git a/src/daemon/protocol.h b/src/daemon/protocol.h index 733bacfc9e8..6e960168af7 100644 --- a/src/daemon/protocol.h +++ b/src/daemon/protocol.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2022, The Monero Project +// Copyright (c) 2014-2023, The Monero Project // // All rights reserved. // diff --git a/src/daemon/rpc.h b/src/daemon/rpc.h index 7b059ebd34e..f7bdb921eae 100644 --- a/src/daemon/rpc.h +++ b/src/daemon/rpc.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2022, The Monero Project +// Copyright (c) 2014-2023, The Monero Project // // All rights reserved. // diff --git a/src/daemon/rpc_command_executor.cpp b/src/daemon/rpc_command_executor.cpp index b6364ff7702..0dcfc9d53be 100644 --- a/src/daemon/rpc_command_executor.cpp +++ b/src/daemon/rpc_command_executor.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2022, The Monero Project +// Copyright (c) 2014-2023, The Monero Project // // All rights reserved. // @@ -775,7 +775,7 @@ bool t_rpc_command_executor::print_blockchain_info(int64_t start_block_index, ui return true; } } - if (start_block_index < 0 && (uint64_t)-start_block_index >= ires.height) + if ((uint64_t)-start_block_index >= ires.height) { tools::fail_msg_writer() << "start offset is larger than blockchain height"; return true; @@ -1063,7 +1063,7 @@ bool t_rpc_command_executor::print_transaction(crypto::hash transaction_hash, cryptonote::blobdata blob; std::string source = as_hex.empty() ? pruned_as_hex + prunable_as_hex : as_hex; bool pruned = !pruned_as_hex.empty() && prunable_as_hex.empty(); - if (!string_tools::parse_hexstr_to_binbuff(source, blob)) + if (!epee::string_tools::parse_hexstr_to_binbuff(source, blob)) { tools::fail_msg_writer() << "Failed to parse tx to get json format"; } diff --git a/src/daemon/rpc_command_executor.h b/src/daemon/rpc_command_executor.h index ebd7eda8505..3d458242225 100644 --- a/src/daemon/rpc_command_executor.h +++ b/src/daemon/rpc_command_executor.h @@ -6,7 +6,7 @@ */ -// Copyright (c) 2014-2022, The Monero Project +// Copyright (c) 2014-2023, The Monero Project // // All rights reserved. // diff --git a/src/daemonizer/CMakeLists.txt b/src/daemonizer/CMakeLists.txt index 61999b3a5f9..5be9fda42be 100644 --- a/src/daemonizer/CMakeLists.txt +++ b/src/daemonizer/CMakeLists.txt @@ -1,4 +1,4 @@ -# Copyright (c) 2014-2022, The Monero Project +# Copyright (c) 2014-2023, The Monero Project # # All rights reserved. # diff --git a/src/daemonizer/daemonizer.h b/src/daemonizer/daemonizer.h index fa19c28b0a6..d313ff3f161 100644 --- a/src/daemonizer/daemonizer.h +++ b/src/daemonizer/daemonizer.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2022, The Monero Project +// Copyright (c) 2014-2023, The Monero Project // // All rights reserved. // diff --git a/src/daemonizer/posix_daemonizer.inl b/src/daemonizer/posix_daemonizer.inl index bd274103923..5defd1e692b 100644 --- a/src/daemonizer/posix_daemonizer.inl +++ b/src/daemonizer/posix_daemonizer.inl @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2022, The Monero Project +// Copyright (c) 2014-2023, The Monero Project // // All rights reserved. // diff --git a/src/daemonizer/posix_fork.h b/src/daemonizer/posix_fork.h index b1d82ff4393..49764369951 100644 --- a/src/daemonizer/posix_fork.h +++ b/src/daemonizer/posix_fork.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2022, The Monero Project +// Copyright (c) 2014-2023, The Monero Project // // All rights reserved. // diff --git a/src/daemonizer/windows_daemonizer.inl b/src/daemonizer/windows_daemonizer.inl index a0086408f5a..d3e5d66ddbd 100644 --- a/src/daemonizer/windows_daemonizer.inl +++ b/src/daemonizer/windows_daemonizer.inl @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2022, The Monero Project +// Copyright (c) 2014-2023, The Monero Project // // All rights reserved. // diff --git a/src/daemonizer/windows_service.cpp b/src/daemonizer/windows_service.cpp index 846f6c0717f..04ea62d2b93 100644 --- a/src/daemonizer/windows_service.cpp +++ b/src/daemonizer/windows_service.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2022, The Monero Project +// Copyright (c) 2014-2023, The Monero Project // // All rights reserved. // @@ -196,7 +196,7 @@ bool install_service( , 0 //, GENERIC_EXECUTE | GENERIC_READ , SERVICE_WIN32_OWN_PROCESS - , SERVICE_DEMAND_START + , SERVICE_AUTO_START , SERVICE_ERROR_NORMAL , full_command.c_str() , nullptr diff --git a/src/daemonizer/windows_service.h b/src/daemonizer/windows_service.h index a96674d19fd..55b097bf991 100644 --- a/src/daemonizer/windows_service.h +++ b/src/daemonizer/windows_service.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2022, The Monero Project +// Copyright (c) 2014-2023, The Monero Project // // All rights reserved. // diff --git a/src/daemonizer/windows_service_runner.h b/src/daemonizer/windows_service_runner.h index a8e4d3b0e29..5aa1a8c2ff5 100644 --- a/src/daemonizer/windows_service_runner.h +++ b/src/daemonizer/windows_service_runner.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2022, The Monero Project +// Copyright (c) 2014-2023, The Monero Project // // All rights reserved. // @@ -146,9 +146,6 @@ namespace windows { m_handler.run(); on_state_change_request_(SERVICE_CONTROL_STOP); - - // Ensure that the service is uninstalled - uninstall_service(m_name); } static void WINAPI on_state_change_request(DWORD control_code) diff --git a/src/debug_utilities/CMakeLists.txt b/src/debug_utilities/CMakeLists.txt index ecb0f022912..58cb713dfe1 100644 --- a/src/debug_utilities/CMakeLists.txt +++ b/src/debug_utilities/CMakeLists.txt @@ -1,4 +1,4 @@ -# Copyright (c) 2014-2022, The Monero Project +# Copyright (c) 2014-2023, The Monero Project # # All rights reserved. # diff --git a/src/debug_utilities/cn_deserialize.cpp b/src/debug_utilities/cn_deserialize.cpp index 41c397bd8b2..aa9ceec0efc 100644 --- a/src/debug_utilities/cn_deserialize.cpp +++ b/src/debug_utilities/cn_deserialize.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2022, The Monero Project +// Copyright (c) 2014-2023, The Monero Project // // All rights reserved. // diff --git a/src/debug_utilities/dns_checks.cpp b/src/debug_utilities/dns_checks.cpp index caa0421e9aa..a59a0820990 100644 --- a/src/debug_utilities/dns_checks.cpp +++ b/src/debug_utilities/dns_checks.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2019-2022, The Monero Project +// Copyright (c) 2019-2023, The Monero Project // // All rights reserved. // diff --git a/src/debug_utilities/object_sizes.cpp b/src/debug_utilities/object_sizes.cpp index 40b651ab30b..b6f9a3fbfe5 100644 --- a/src/debug_utilities/object_sizes.cpp +++ b/src/debug_utilities/object_sizes.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2022, The Monero Project +// Copyright (c) 2017-2023, The Monero Project // // All rights reserved. // diff --git a/src/device/CMakeLists.txt b/src/device/CMakeLists.txt index e4f1159b589..eef49ffeedd 100644 --- a/src/device/CMakeLists.txt +++ b/src/device/CMakeLists.txt @@ -1,4 +1,4 @@ -# Copyright (c) 2014-2022, The Monero Project +# Copyright (c) 2014-2023, The Monero Project # # All rights reserved. # diff --git a/src/device/device.cpp b/src/device/device.cpp index e6cd358b635..3d3a14fd5fa 100644 --- a/src/device/device.cpp +++ b/src/device/device.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2022, The Monero Project +// Copyright (c) 2017-2023, The Monero Project // // All rights reserved. // diff --git a/src/device/device.hpp b/src/device/device.hpp index eca91006f92..1c29018e6ce 100644 --- a/src/device/device.hpp +++ b/src/device/device.hpp @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2022, The Monero Project +// Copyright (c) 2017-2023, The Monero Project // // All rights reserved. // @@ -177,6 +177,7 @@ namespace hw { virtual bool derive_public_key(const crypto::key_derivation &derivation, const std::size_t output_index, const crypto::public_key &pub, crypto::public_key &derived_pub) = 0; virtual bool secret_key_to_public_key(const crypto::secret_key &sec, crypto::public_key &pub) = 0; virtual bool generate_key_image(const crypto::public_key &pub, const crypto::secret_key &sec, crypto::key_image &image) = 0; + virtual bool derive_view_tag(const crypto::key_derivation &derivation, const std::size_t output_index, crypto::view_tag &view_tag) = 0; // alternative prototypes available in libringct rct::key scalarmultKey(const rct::key &P, const rct::key &a) diff --git a/src/device/device_cold.hpp b/src/device/device_cold.hpp index ba4d6d8ae44..0738598b8a4 100644 --- a/src/device/device_cold.hpp +++ b/src/device/device_cold.hpp @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2022, The Monero Project +// Copyright (c) 2017-2023, The Monero Project // // All rights reserved. // diff --git a/src/device/device_default.cpp b/src/device/device_default.cpp index d70ece229c9..d938087fe1f 100644 --- a/src/device/device_default.cpp +++ b/src/device/device_default.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2022, The Monero Project +// Copyright (c) 2017-2023, The Monero Project // // All rights reserved. // diff --git a/src/device/device_default.hpp b/src/device/device_default.hpp index 7d354365201..7ee9f15b7e2 100644 --- a/src/device/device_default.hpp +++ b/src/device/device_default.hpp @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2022, The Monero Project +// Copyright (c) 2017-2023, The Monero Project // // All rights reserved. // @@ -101,7 +101,7 @@ namespace hw { bool derive_public_key(const crypto::key_derivation &derivation, const std::size_t output_index, const crypto::public_key &pub, crypto::public_key &derived_pub) override; bool secret_key_to_public_key(const crypto::secret_key &sec, crypto::public_key &pub) override; bool generate_key_image(const crypto::public_key &pub, const crypto::secret_key &sec, crypto::key_image &image) override; - bool derive_view_tag(const crypto::key_derivation &derivation, const std::size_t output_index, crypto::view_tag &view_tag); + bool derive_view_tag(const crypto::key_derivation &derivation, const std::size_t output_index, crypto::view_tag &view_tag) override; /* ======================================================================= */ diff --git a/src/device/device_io.hpp b/src/device/device_io.hpp index b333caa1375..20678568dcf 100644 --- a/src/device/device_io.hpp +++ b/src/device/device_io.hpp @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2022, The Monero Project +// Copyright (c) 2017-2023, The Monero Project // // All rights reserved. // diff --git a/src/device/device_io_hid.cpp b/src/device/device_io_hid.cpp index 3116a8713cb..472414d3bde 100644 --- a/src/device/device_io_hid.cpp +++ b/src/device/device_io_hid.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2022, The Monero Project +// Copyright (c) 2017-2023, The Monero Project // // All rights reserved. // diff --git a/src/device/device_io_hid.hpp b/src/device/device_io_hid.hpp index fd2ec851552..90d7d8553c0 100644 --- a/src/device/device_io_hid.hpp +++ b/src/device/device_io_hid.hpp @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2022, The Monero Project +// Copyright (c) 2017-2023, The Monero Project // // All rights reserved. // diff --git a/src/device/device_ledger.cpp b/src/device/device_ledger.cpp index 51e65dfa53a..27d080a1c2a 100644 --- a/src/device/device_ledger.cpp +++ b/src/device/device_ledger.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2022, The Monero Project +// Copyright (c) 2017-2023, The Monero Project // // All rights reserved. // @@ -43,6 +43,10 @@ namespace hw { #ifdef WITH_DEVICE_LEDGER + namespace { + bool apdu_verbose =true; + } + #undef MONERO_DEFAULT_LOG_CATEGORY #define MONERO_DEFAULT_LOG_CATEGORY "device.ledger" @@ -266,6 +270,7 @@ namespace hw { #define INS_DERIVE_PUBLIC_KEY 0x36 #define INS_DERIVE_SECRET_KEY 0x38 #define INS_GEN_KEY_IMAGE 0x3A + #define INS_DERIVE_VIEW_TAG 0x3B #define INS_SECRET_KEY_ADD 0x3C #define INS_SECRET_KEY_SUB 0x3E #define INS_GENERATE_KEYPAIR 0x40 @@ -525,6 +530,7 @@ namespace hw { {0x2c97, 0x0001, 0, 0xffa0}, {0x2c97, 0x0004, 0, 0xffa0}, {0x2c97, 0x0005, 0, 0xffa0}, + {0x2c97, 0x0006, 0, 0xffa0}, }; bool device_ledger::connect(void) { @@ -694,7 +700,8 @@ namespace hw { log_hexbuffer("derive_subaddress_public_key: [[IN]] pub ", pub_x.data, 32); log_hexbuffer("derive_subaddress_public_key: [[IN]] derivation", derivation_x.data, 32); log_message ("derive_subaddress_public_key: [[IN]] index ", std::to_string((int)output_index_x)); - this->controle_device->derive_subaddress_public_key(pub_x, derivation_x,output_index_x,derived_pub_x); + if (!this->controle_device->derive_subaddress_public_key(pub_x, derivation_x,output_index_x,derived_pub_x)) + return false; log_hexbuffer("derive_subaddress_public_key: [[OUT]] derived_pub", derived_pub_x.data, 32); #endif @@ -702,7 +709,8 @@ namespace hw { //If we are in TRANSACTION_PARSE, the given derivation has been retrieved uncrypted (wihtout the help //of the device), so continue that way. MDEBUG( "derive_subaddress_public_key : PARSE mode with known viewkey"); - crypto::derive_subaddress_public_key(pub, derivation, output_index,derived_pub); + if (!crypto::derive_subaddress_public_key(pub, derivation, output_index,derived_pub)) + return false; } else { AUTO_LOCK_CMD(); int offset = set_command_header_noopt(INS_DERIVE_SUBADDRESS_PUBLIC_KEY); @@ -1052,7 +1060,8 @@ namespace hw { crypto::key_derivation derivation_x; log_hexbuffer("generate_key_derivation: [[IN]] pub ", pub_x.data, 32); log_hexbuffer("generate_key_derivation: [[IN]] sec ", sec_x.data, 32); - this->controle_device->generate_key_derivation(pub_x, sec_x, derivation_x); + if (!this->controle_device->generate_key_derivation(pub_x, sec_x, derivation_x)) + return false; log_hexbuffer("generate_key_derivation: [[OUT]] derivation", derivation_x.data, 32); #endif @@ -1207,7 +1216,8 @@ namespace hw { log_hexbuffer("derive_public_key: [[IN]] derivation ", derivation_x.data, 32); log_message ("derive_public_key: [[IN]] output_index", std::to_string(output_index_x)); log_hexbuffer("derive_public_key: [[IN]] pub ", pub_x.data, 32); - this->controle_device->derive_public_key(derivation_x, output_index_x, pub_x, derived_pub_x); + if (!this->controle_device->derive_public_key(derivation_x, output_index_x, pub_x, derived_pub_x)) + return false; log_hexbuffer("derive_public_key: [[OUT]] derived_pub ", derived_pub_x.data, 32); #endif @@ -1304,6 +1314,54 @@ namespace hw { return true; } + bool device_ledger::derive_view_tag(const crypto::key_derivation &derivation, const std::size_t output_index, crypto::view_tag &view_tag){ + #ifdef DEBUG_HWDEVICE + crypto::key_derivation derivation_x; + if ((this->mode == TRANSACTION_PARSE) && has_view_key) { + derivation_x = derivation; + } else { + derivation_x = hw::ledger::decrypt(derivation); + } + const std::size_t output_index_x = output_index; + crypto::view_tag view_tag_x; + log_hexbuffer("derive_view_tag: [[IN]] derivation ", derivation_x.data, 32); + log_message ("derive_view_tag: [[IN]] output_index", std::to_string(output_index_x)); + this->controle_device->derive_view_tag(derivation_x, output_index_x, view_tag_x); + log_hexbuffer("derive_view_tag: [[OUT]] view_tag ", &view_tag_x.data, 1); + #endif + + if ((this->mode == TRANSACTION_PARSE) && has_view_key) { + //If we are in TRANSACTION_PARSE, the given derivation has been retrieved uncrypted (wihtout the help + //of the device), so continue that way. + MDEBUG( "derive_view_tag : PARSE mode with known viewkey"); + crypto::derive_view_tag(derivation, output_index, view_tag); + } else { + AUTO_LOCK_CMD(); + int offset = set_command_header_noopt(INS_DERIVE_VIEW_TAG); + //derivation + this->send_secret((unsigned char*)derivation.data, offset); + //index + this->buffer_send[offset+0] = output_index>>24; + this->buffer_send[offset+1] = output_index>>16; + this->buffer_send[offset+2] = output_index>>8; + this->buffer_send[offset+3] = output_index>>0; + offset += 4; + + this->buffer_send[4] = offset-5; + this->length_send = offset; + this->exchange(); + + //view tag + memmove(&view_tag.data, &this->buffer_recv[0], 1); + } + + #ifdef DEBUG_HWDEVICE + hw::ledger::check1("derive_view_tag", "view_tag", &view_tag_x.data, &view_tag.data); + #endif + + return true; + } + /* ======================================================================= */ /* TRANSACTION */ /* ======================================================================= */ @@ -1544,7 +1602,6 @@ namespace hw { const size_t output_index_x = output_index; const bool need_additional_txkeys_x = need_additional_txkeys; const bool use_view_tags_x = use_view_tags; - const crypto::view_tag view_tag_x = view_tag; std::vector additional_tx_keys_x; for (const auto &k: additional_tx_keys) { @@ -1554,6 +1611,7 @@ namespace hw { std::vector additional_tx_public_keys_x; std::vector amount_keys_x; crypto::public_key out_eph_public_key_x; + crypto::view_tag view_tag_x; log_message ("generate_output_ephemeral_keys: [[IN]] tx_version", std::to_string(tx_version_x)); //log_hexbuffer("generate_output_ephemeral_keys: [[IN]] sender_account_keys.view", sender_account_keys.m_sview_secret_key.data, 32); @@ -1571,11 +1629,15 @@ namespace hw { if(need_additional_txkeys_x) { log_hexbuffer("generate_output_ephemeral_keys: [[IN]] additional_tx_keys[oi]", additional_tx_keys_x[output_index].data, 32); } + log_message ("generate_output_ephemeral_keys: [[IN]] use_view_tags", std::to_string(use_view_tags_x)); this->controle_device->generate_output_ephemeral_keys(tx_version_x, sender_account_keys_x, txkey_pub_x, tx_key_x, dst_entr_x, change_addr_x, output_index_x, need_additional_txkeys_x, additional_tx_keys_x, additional_tx_public_keys_x, amount_keys_x, out_eph_public_key_x, use_view_tags_x, view_tag_x); if(need_additional_txkeys_x) { log_hexbuffer("additional_tx_public_keys_x: [[OUT]] additional_tx_public_keys_x", additional_tx_public_keys_x.back().data, 32); } + if(use_view_tags_x) { + log_hexbuffer("generate_output_ephemeral_keys: [[OUT]] view_tag", &view_tag_x.data, 1); + } log_hexbuffer("generate_output_ephemeral_keys: [[OUT]] amount_keys ", (char*)amount_keys_x.back().bytes, 32); log_hexbuffer("generate_output_ephemeral_keys: [[OUT]] out_eph_public_key ", out_eph_public_key_x.data, 32); #endif @@ -1629,6 +1691,9 @@ namespace hw { memset(&this->buffer_send[offset], 0, 32); offset += 32; } + //use_view_tags + this->buffer_send[offset] = use_view_tags; + offset++; this->buffer_send[4] = offset-5; this->length_send = offset; @@ -1659,6 +1724,14 @@ namespace hw { recv_len -= 32; } + if (use_view_tags) + { + ASSERT_X(recv_len>=1, "Not enough data from device"); + memmove(&view_tag.data, &this->buffer_recv[offset], 1); + offset++; + recv_len -= 1; + } + // add ABPkeys this->add_output_key_mapping(dst_entr.addr.m_view_public_key, dst_entr.addr.m_spend_public_key, dst_entr.is_subaddress, is_change, need_additional_txkeys, output_index, @@ -1671,6 +1744,9 @@ namespace hw { hw::ledger::check32("generate_output_ephemeral_keys", "additional_tx_key", additional_tx_public_keys_x.back().data, additional_tx_public_keys.back().data); } hw::ledger::check32("generate_output_ephemeral_keys", "out_eph_public_key", out_eph_public_key_x.data, out_eph_public_key.data); + if (use_view_tags) { + hw::ledger::check1("generate_output_ephemeral_keys", "view_tag", &view_tag_x.data, &view_tag.data); + } #endif return true; @@ -1856,7 +1932,7 @@ namespace hw { // ====== Aout, Bout, AKout, C, v, k ====== kv_offset = data_offset; - if (type==rct::RCTTypeBulletproof2 || type==rct::RCTTypeCLSAG) { + if (type==rct::RCTTypeBulletproof2 || type==rct::RCTTypeCLSAG || type==rct::RCTTypeBulletproofPlus) { C_offset = kv_offset+ (8)*outputs_size; } else { C_offset = kv_offset+ (32+32)*outputs_size; @@ -1873,7 +1949,7 @@ namespace hw { offset = set_command_header(INS_VALIDATE, 0x02, i+1); //options this->buffer_send[offset] = (i==outputs_size-1)? 0x00:0x80 ; - this->buffer_send[offset] |= (type==rct::RCTTypeBulletproof2 || type==rct::RCTTypeCLSAG)?0x02:0x00; + this->buffer_send[offset] |= (type==rct::RCTTypeBulletproof2 || type==rct::RCTTypeCLSAG || type==rct::RCTTypeBulletproofPlus)?0x02:0x00; offset += 1; //is_subaddress this->buffer_send[offset] = outKeys.is_subaddress; @@ -1894,7 +1970,7 @@ namespace hw { memmove(this->buffer_send+offset, data+C_offset,32); offset += 32; C_offset += 32; - if (type==rct::RCTTypeBulletproof2 || type==rct::RCTTypeCLSAG) { + if (type==rct::RCTTypeBulletproof2 || type==rct::RCTTypeCLSAG || type==rct::RCTTypeBulletproofPlus) { //k memset(this->buffer_send+offset, 0, 32); offset += 32; diff --git a/src/device/device_ledger.hpp b/src/device/device_ledger.hpp index 074bfaa8dcc..07127416028 100644 --- a/src/device/device_ledger.hpp +++ b/src/device/device_ledger.hpp @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2022, The Monero Project +// Copyright (c) 2017-2023, The Monero Project // // All rights reserved. // @@ -44,7 +44,7 @@ namespace hw { /* Minimal supported version */ #define MINIMAL_APP_VERSION_MAJOR 1 - #define MINIMAL_APP_VERSION_MINOR 6 + #define MINIMAL_APP_VERSION_MINOR 8 #define MINIMAL_APP_VERSION_MICRO 0 #define VERSION(M,m,u) ((M)<<16|(m)<<8|(u)) @@ -87,10 +87,6 @@ namespace hw { #define SW_PROTOCOL_NOT_SUPPORTED 0x6e00 #define SW_UNKNOWN 0x6f00 - namespace { - bool apdu_verbose =true; - } - void set_apdu_verbose(bool verbose); class ABPkeys { @@ -249,6 +245,7 @@ namespace hw { bool derive_public_key(const crypto::key_derivation &derivation, const std::size_t output_index, const crypto::public_key &pub, crypto::public_key &derived_pub) override; bool secret_key_to_public_key(const crypto::secret_key &sec, crypto::public_key &pub) override; bool generate_key_image(const crypto::public_key &pub, const crypto::secret_key &sec, crypto::key_image &image) override; + bool derive_view_tag(const crypto::key_derivation &derivation, const size_t output_index, crypto::view_tag &view_tag) override; /* ======================================================================= */ /* TRANSACTION */ diff --git a/src/device/log.cpp b/src/device/log.cpp index 9b882b784d9..1a243c831a3 100644 --- a/src/device/log.cpp +++ b/src/device/log.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2022, The Monero Project +// Copyright (c) 2017-2023, The Monero Project // // All rights reserved. // @@ -165,6 +165,10 @@ namespace hw { void check8(const std::string &msg, const std::string &info, const char *h, const char *d, bool crypted) { check(msg, info, h, d, 8, crypted); } + + void check1(const std::string &msg, const std::string &info, const char *h, const char *d, bool crypted) { + check(msg, info, h, d, 1, crypted); + } #endif } diff --git a/src/device/log.hpp b/src/device/log.hpp index 660adc63e89..92d174da115 100644 --- a/src/device/log.hpp +++ b/src/device/log.hpp @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2022, The Monero Project +// Copyright (c) 2017-2023, The Monero Project // // All rights reserved. // @@ -75,6 +75,7 @@ namespace hw { void check32(const std::string &msg, const std::string &info, const char *h, const char *d, bool crypted=false); void check8(const std::string &msg, const std::string &info, const char *h, const char *d, bool crypted=false); + void check1(const std::string &msg, const std::string &info, const char *h, const char *d, bool crypted=false); void set_check_verbose(bool verbose); #endif diff --git a/src/device_trezor/CMakeLists.txt b/src/device_trezor/CMakeLists.txt index 6688a317b01..c30fb2b1646 100644 --- a/src/device_trezor/CMakeLists.txt +++ b/src/device_trezor/CMakeLists.txt @@ -1,4 +1,4 @@ -# Copyright (c) 2014-2022, The Monero Project +# Copyright (c) 2014-2023, The Monero Project # # All rights reserved. # diff --git a/src/device_trezor/device_trezor.cpp b/src/device_trezor/device_trezor.cpp index 6f7ae9a6b09..9c8148ed630 100644 --- a/src/device_trezor/device_trezor.cpp +++ b/src/device_trezor/device_trezor.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2022, The Monero Project +// Copyright (c) 2017-2023, The Monero Project // // All rights reserved. // @@ -324,8 +324,8 @@ namespace trezor { std::vector mtds; std::vector kis; - protocol::ki::key_image_data(wallet, transfers, mtds, client_version() <= 1); - protocol::ki::generate_commitment(mtds, transfers, req, client_version() <= 1); + protocol::ki::key_image_data(wallet, transfers, mtds); + protocol::ki::generate_commitment(mtds, transfers, req); EVENT_PROGRESS(0.); this->set_msg_addr(req.get()); @@ -511,7 +511,7 @@ namespace trezor { tools::wallet2::signed_tx_set & signed_tx, hw::tx_aux_data & aux_data) { - CHECK_AND_ASSERT_THROW_MES(unsigned_tx.transfers.first == 0, "Unsuported non zero offset"); + CHECK_AND_ASSERT_THROW_MES(std::get<0>(unsigned_tx.transfers) == 0, "Unsuported non zero offset"); TREZOR_AUTO_LOCK_CMD(); require_connected(); @@ -522,7 +522,7 @@ namespace trezor { const size_t num_tx = unsigned_tx.txes.size(); m_num_transations_to_sign = num_tx; signed_tx.key_images.clear(); - signed_tx.key_images.resize(unsigned_tx.transfers.second.size()); + signed_tx.key_images.resize(std::get<2>(unsigned_tx.transfers).size()); for(size_t tx_idx = 0; tx_idx < num_tx; ++tx_idx) { std::shared_ptr signer; @@ -566,8 +566,8 @@ namespace trezor { cpend.key_images = key_images; // KI sync - for(size_t cidx=0, trans_max=unsigned_tx.transfers.second.size(); cidx < trans_max; ++cidx){ - signed_tx.key_images[cidx] = unsigned_tx.transfers.second[cidx].m_key_image; + for(size_t cidx=0, trans_max=std::get<2>(unsigned_tx.transfers).size(); cidx < trans_max; ++cidx){ + signed_tx.key_images[cidx] = std::get<2>(unsigned_tx.transfers)[cidx].m_key_image; } size_t num_sources = cdata.tx_data.sources.size(); @@ -579,9 +579,9 @@ namespace trezor { CHECK_AND_ASSERT_THROW_MES(src_idx < cdata.tx.vin.size(), "Invalid idx_mapped"); size_t idx_map_src = cdata.tx_data.selected_transfers[idx_mapped]; - CHECK_AND_ASSERT_THROW_MES(idx_map_src >= unsigned_tx.transfers.first, "Invalid offset"); + CHECK_AND_ASSERT_THROW_MES(idx_map_src >= std::get<0>(unsigned_tx.transfers), "Invalid offset"); - idx_map_src -= unsigned_tx.transfers.first; + idx_map_src -= std::get<0>(unsigned_tx.transfers); CHECK_AND_ASSERT_THROW_MES(idx_map_src < signed_tx.key_images.size(), "Invalid key image index"); const auto vini = boost::get(cdata.tx.vin[src_idx]); @@ -635,11 +635,7 @@ namespace trezor { } // Step: sort - auto perm_req = signer->step_permutation(); - if (perm_req){ - auto perm_ack = this->client_exchange(perm_req); - signer->step_permutation_ack(perm_ack); - } + signer->sort_ki(); EVENT_PROGRESS(3, 1, 1); // Step: input_vini @@ -697,13 +693,13 @@ namespace trezor { unsigned device_trezor::client_version() { auto trezor_version = get_version(); - if (trezor_version <= pack_version(2, 0, 10)){ - throw exc::TrezorException("Trezor firmware 2.0.10 and lower are not supported. Please update."); + if (trezor_version < pack_version(2, 4, 3)){ + throw exc::TrezorException("Minimal Trezor firmware version is 2.4.3. Please update."); } - unsigned client_version = 1; - if (trezor_version >= pack_version(2, 3, 1)){ - client_version = 3; + unsigned client_version = 3; + if (trezor_version >= pack_version(2, 5, 2)){ + client_version = 4; } #ifdef WITH_TREZOR_DEBUGGING @@ -739,14 +735,6 @@ namespace trezor { CHECK_AND_ASSERT_THROW_MES(init_msg, "TransactionInitRequest is empty"); CHECK_AND_ASSERT_THROW_MES(init_msg->has_tsx_data(), "TransactionInitRequest has no transaction data"); CHECK_AND_ASSERT_THROW_MES(m_features, "Device state not initialized"); // make sure the caller did not reset features - const bool nonce_required = init_msg->tsx_data().has_payment_id() && init_msg->tsx_data().payment_id().size() > 0; - - if (nonce_required && init_msg->tsx_data().payment_id().size() == 8){ - // Versions 2.0.9 and lower do not support payment ID - if (get_version() <= pack_version(2, 0, 9)) { - throw exc::TrezorException("Trezor firmware 2.0.9 and lower does not support transactions with short payment IDs or integrated addresses. Please update."); - } - } } void device_trezor::transaction_check(const protocol::tx::TData & tdata, const hw::tx_aux_data & aux_data) diff --git a/src/device_trezor/device_trezor.hpp b/src/device_trezor/device_trezor.hpp index 38aeaf6b5f0..35bb78927b4 100644 --- a/src/device_trezor/device_trezor.hpp +++ b/src/device_trezor/device_trezor.hpp @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2022, The Monero Project +// Copyright (c) 2017-2023, The Monero Project // // All rights reserved. // diff --git a/src/device_trezor/device_trezor_base.cpp b/src/device_trezor/device_trezor_base.cpp index 56f3784a75a..a8a3d9f6709 100644 --- a/src/device_trezor/device_trezor_base.cpp +++ b/src/device_trezor/device_trezor_base.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2022, The Monero Project +// Copyright (c) 2017-2023, The Monero Project // // All rights reserved. // diff --git a/src/device_trezor/device_trezor_base.hpp b/src/device_trezor/device_trezor_base.hpp index 5b692031357..3ec21e157b8 100644 --- a/src/device_trezor/device_trezor_base.hpp +++ b/src/device_trezor/device_trezor_base.hpp @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2022, The Monero Project +// Copyright (c) 2017-2023, The Monero Project // // All rights reserved. // diff --git a/src/device_trezor/trezor.hpp b/src/device_trezor/trezor.hpp index f2a352f58e3..d425b3209d3 100644 --- a/src/device_trezor/trezor.hpp +++ b/src/device_trezor/trezor.hpp @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2022, The Monero Project +// Copyright (c) 2017-2023, The Monero Project // // All rights reserved. // diff --git a/src/device_trezor/trezor/debug_link.cpp b/src/device_trezor/trezor/debug_link.cpp index 1eed0a53e86..ff7cf0ad62e 100644 --- a/src/device_trezor/trezor/debug_link.cpp +++ b/src/device_trezor/trezor/debug_link.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2022, The Monero Project +// Copyright (c) 2017-2023, The Monero Project // // All rights reserved. // diff --git a/src/device_trezor/trezor/debug_link.hpp b/src/device_trezor/trezor/debug_link.hpp index b7a25283343..4249222dc04 100644 --- a/src/device_trezor/trezor/debug_link.hpp +++ b/src/device_trezor/trezor/debug_link.hpp @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2022, The Monero Project +// Copyright (c) 2017-2023, The Monero Project // // All rights reserved. // diff --git a/src/device_trezor/trezor/exceptions.hpp b/src/device_trezor/trezor/exceptions.hpp index 818b2cb6c36..b66386c479a 100644 --- a/src/device_trezor/trezor/exceptions.hpp +++ b/src/device_trezor/trezor/exceptions.hpp @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2022, The Monero Project +// Copyright (c) 2017-2023, The Monero Project // // All rights reserved. // diff --git a/src/device_trezor/trezor/messages_map.cpp b/src/device_trezor/trezor/messages_map.cpp index 313fd682045..527fe38ba99 100644 --- a/src/device_trezor/trezor/messages_map.cpp +++ b/src/device_trezor/trezor/messages_map.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2022, The Monero Project +// Copyright (c) 2017-2023, The Monero Project // // All rights reserved. // diff --git a/src/device_trezor/trezor/messages_map.hpp b/src/device_trezor/trezor/messages_map.hpp index b1ea65e6e30..f6820524ed0 100644 --- a/src/device_trezor/trezor/messages_map.hpp +++ b/src/device_trezor/trezor/messages_map.hpp @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2022, The Monero Project +// Copyright (c) 2017-2023, The Monero Project // // All rights reserved. // diff --git a/src/device_trezor/trezor/protocol.cpp b/src/device_trezor/trezor/protocol.cpp index a400e82c7ed..49d3af0f534 100644 --- a/src/device_trezor/trezor/protocol.cpp +++ b/src/device_trezor/trezor/protocol.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2022, The Monero Project +// Copyright (c) 2017-2023, The Monero Project // // All rights reserved. // @@ -38,6 +38,7 @@ #include #include #include +#include #include "cryptonote_config.h" #include #include @@ -145,8 +146,7 @@ namespace ki { bool key_image_data(wallet_shim * wallet, const std::vector & transfers, - std::vector & res, - bool need_all_additionals) + std::vector & res) { for(auto & td : transfers){ ::crypto::public_key tx_pub_key = wallet->get_tx_pub_key_from_received_outs(td); @@ -159,11 +159,7 @@ namespace ki { cres.set_internal_output_index(td.m_internal_output_index); cres.set_sub_addr_major(td.m_subaddr_index.major); cres.set_sub_addr_minor(td.m_subaddr_index.minor); - if (need_all_additionals) { - for (auto &aux : additional_tx_pub_keys) { - cres.add_additional_tx_pub_keys(key_to_string(aux)); - } - } else if (!additional_tx_pub_keys.empty() && additional_tx_pub_keys.size() > td.m_internal_output_index) { + if (!additional_tx_pub_keys.empty() && additional_tx_pub_keys.size() > td.m_internal_output_index) { cres.add_additional_tx_pub_keys(key_to_string(additional_tx_pub_keys[td.m_internal_output_index])); } } @@ -194,8 +190,7 @@ namespace ki { void generate_commitment(std::vector & mtds, const std::vector & transfers, - std::shared_ptr & req, - bool need_subaddr_indices) + std::shared_ptr & req) { req = std::make_shared(); @@ -219,16 +214,6 @@ namespace ki { auto & st = search.first->second; st.insert(cur.m_subaddr_index.minor); } - - if (need_subaddr_indices) { - for (auto &x: sub_indices) { - auto subs = req->add_subs(); - subs->set_account(x.first); - for (auto minor : x.second) { - subs->add_minor_indices(minor); - } - } - } } void live_refresh_ack(const ::crypto::secret_key & view_key_priv, @@ -399,7 +384,7 @@ namespace tx { m_tx_idx = tx_idx; m_ct.tx_data = cur_src_tx(); m_multisig = false; - m_client_version = 1; + m_client_version = 3; } void Signer::extract_payment_id(){ @@ -474,25 +459,19 @@ namespace tx { auto & cur = src.outputs[i]; auto out = dst->add_outputs(); - if (i == src.real_output || need_ring_indices || client_version() <= 1) { + if (i == src.real_output || need_ring_indices) { out->set_idx(cur.first); } - if (i == src.real_output || need_ring_keys || client_version() <= 1) { + if (i == src.real_output || need_ring_keys) { translate_rct_key(out->mutable_key(), &(cur.second)); } } dst->set_real_out_tx_key(key_to_string(src.real_out_tx_key)); dst->set_real_output_in_tx_index(src.real_output_in_tx_index); - - if (client_version() <= 1) { - for (auto &cur : src.real_out_additional_tx_keys) { - dst->add_real_out_additional_tx_keys(key_to_string(cur)); - } - } else if (!src.real_out_additional_tx_keys.empty()) { + if (!src.real_out_additional_tx_keys.empty()) { dst->add_real_out_additional_tx_keys(key_to_string(src.real_out_additional_tx_keys.at(src.real_output_in_tx_index))); } - dst->set_amount(src.amount); dst->set_rct(src.rct); dst->set_mask(key_to_string(src.mask)); @@ -532,7 +511,7 @@ namespace tx { m_ct.tx.version = 2; m_ct.tx.unlock_time = tx.unlock_time; - m_client_version = (m_aux_data->client_version ? m_aux_data->client_version.get() : 1); + m_client_version = (m_aux_data->client_version ? m_aux_data->client_version.get() : 3); tsx_data.set_version(1); tsx_data.set_client_version(client_version()); @@ -543,18 +522,13 @@ namespace tx { tsx_data.set_monero_version(std::string(MONERO_VERSION) + "|" + MONERO_VERSION_TAG); tsx_data.set_hard_fork(m_aux_data->hard_fork ? m_aux_data->hard_fork.get() : 0); - if (client_version() <= 1){ - assign_to_repeatable(tsx_data.mutable_minor_indices(), tx.subaddr_indices.begin(), tx.subaddr_indices.end()); - } - // Rsig decision auto rsig_data = tsx_data.mutable_rsig_data(); m_ct.rsig_type = get_rsig_type(tx.rct_config, tx.splitted_dsts.size()); rsig_data->set_rsig_type(m_ct.rsig_type); - if (tx.rct_config.range_proof_type != rct::RangeProofBorromean){ - m_ct.bp_version = (m_aux_data->bp_version ? m_aux_data->bp_version.get() : 1); - rsig_data->set_bp_version((uint32_t) m_ct.bp_version); - } + CHECK_AND_ASSERT_THROW_MES(tx.rct_config.range_proof_type != rct::RangeProofBorromean, "Borromean rsig not supported"); + m_ct.bp_version = (m_aux_data->bp_version ? m_aux_data->bp_version.get() : 1); + rsig_data->set_bp_version((uint32_t) m_ct.bp_version); generate_rsig_batch_sizes(m_ct.grouping_vct, m_ct.rsig_type, tx.splitted_dsts.size()); assign_to_repeatable(rsig_data->mutable_grouping(), m_ct.grouping_vct.begin(), m_ct.grouping_vct.end()); @@ -652,22 +626,6 @@ namespace tx { }); } - std::shared_ptr Signer::step_permutation(){ - sort_ki(); - if (client_version() >= 2){ - return nullptr; - } - - auto res = std::make_shared(); - assign_to_repeatable(res->mutable_perm(), m_ct.source_permutation.begin(), m_ct.source_permutation.end()); - - return res; - } - - void Signer::step_permutation_ack(std::shared_ptr ack){ - - } - std::shared_ptr Signer::step_set_vini_input(size_t idx){ CHECK_AND_ASSERT_THROW_MES(idx < m_ct.tx_data.sources.size(), "Invalid transaction index"); CHECK_AND_ASSERT_THROW_MES(idx < m_ct.tx.vin.size(), "Invalid transaction index"); @@ -711,8 +669,10 @@ namespace tx { } void Signer::step_set_output_ack(std::shared_ptr ack){ + CHECK_AND_ASSERT_THROW_MES(is_req_bulletproof(), "Borromean rsig not supported"); cryptonote::tx_out tx_out; rct::Bulletproof bproof{}; + rct::BulletproofPlus bproof_plus{}; rct::ctkey out_pk{}; rct::ecdhTuple ecdh{}; @@ -727,7 +687,7 @@ namespace tx { rsig_buff = rsig_data.rsig(); } - if (client_version() >= 1 && rsig_data.has_mask()){ + if (rsig_data.has_mask()){ rct::key cmask{}; string_to_key(cmask, rsig_data.mask()); m_ct.rsig_gamma.emplace_back(cmask); @@ -751,22 +711,32 @@ namespace tx { memcpy(ecdh.amount.bytes, ack->ecdh_info().data(), 8); } - if (has_rsig && is_req_bulletproof() && !cn_deserialize(rsig_buff, bproof)){ - throw exc::ProtocolException("Cannot deserialize bulletproof rangesig"); - } - m_ct.tx.vout.emplace_back(tx_out); m_ct.tx_out_hmacs.push_back(ack->vouti_hmac()); m_ct.tx_out_pk.emplace_back(out_pk); m_ct.tx_out_ecdh.emplace_back(ecdh); - // ClientV0, if no rsig was generated on Trezor, do not continue. - // ClientV1+ generates BP after all masks in the current batch are generated - if (!has_rsig || (client_version() >= 1 && is_offloading())){ + rsig_v bp_obj{}; + if (has_rsig) { + bool deserialize_success; + if (is_req_bulletproof_plus()) { + deserialize_success = cn_deserialize(rsig_buff, bproof_plus); + bp_obj = bproof_plus; + } else { + deserialize_success = cn_deserialize(rsig_buff, bproof); + bp_obj = bproof; + } + if (!deserialize_success) { + throw exc::ProtocolException("Cannot deserialize bulletproof rangesig"); + } + } + + // Generates BP after all masks in the current batch are generated + if (!has_rsig || is_offloading()){ return; } - process_bproof(bproof); + process_bproof(bp_obj); m_ct.cur_batch_idx += 1; m_ct.cur_output_in_batch_idx = 0; } @@ -791,13 +761,21 @@ namespace tx { masks.push_back(m_ct.rsig_gamma[bidx]); } - auto bp = bulletproof_PROVE(amounts, masks); - auto serRsig = cn_serialize(bp); - m_ct.tx_out_rsigs.emplace_back(bp); + std::string serRsig; + if (is_req_bulletproof_plus()) { + auto bp = bulletproof_plus_PROVE(amounts, masks); + serRsig = cn_serialize(bp); + m_ct.tx_out_rsigs.emplace_back(bp); + } else { + auto bp = bulletproof_PROVE(amounts, masks); + serRsig = cn_serialize(bp); + m_ct.tx_out_rsigs.emplace_back(bp); + } + rsig_data.set_rsig(serRsig); } - void Signer::process_bproof(rct::Bulletproof & bproof){ + void Signer::process_bproof(rsig_v & bproof){ CHECK_AND_ASSERT_THROW_MES(m_ct.cur_batch_idx < m_ct.grouping_vct.size(), "Invalid batch index"); auto batch_size = m_ct.grouping_vct[m_ct.cur_batch_idx]; for (size_t i = 0; i < batch_size; ++i){ @@ -806,12 +784,22 @@ namespace tx { rct::key commitment = m_ct.tx_out_pk[bidx].mask; commitment = rct::scalarmultKey(commitment, rct::INV_EIGHT); - bproof.V.push_back(commitment); + if (is_req_bulletproof_plus()) { + boost::get(bproof).V.push_back(commitment); + } else { + boost::get(bproof).V.push_back(commitment); + } } m_ct.tx_out_rsigs.emplace_back(bproof); - if (!rct::bulletproof_VERIFY(boost::get(m_ct.tx_out_rsigs.back()))) { - throw exc::ProtocolException("Returned range signature is invalid"); + if (is_req_bulletproof_plus()) { + if (!rct::bulletproof_plus_VERIFY(boost::get(m_ct.tx_out_rsigs.back()))) { + throw exc::ProtocolException("Returned range signature is invalid"); + } + } else { + if (!rct::bulletproof_VERIFY(boost::get(m_ct.tx_out_rsigs.back()))) { + throw exc::ProtocolException("Returned range signature is invalid"); + } } } @@ -840,6 +828,7 @@ namespace tx { } void Signer::step_all_outs_set_ack(std::shared_ptr ack, hw::device &hwdev){ + CHECK_AND_ASSERT_THROW_MES(is_req_bulletproof(), "Borromean rsig not supported"); m_ct.rv = std::make_shared(); m_ct.rv->txnFee = ack->rv().txn_fee(); m_ct.rv->type = static_cast(ack->rv().rv_type()); @@ -864,24 +853,15 @@ namespace tx { // RctSig auto num_sources = m_ct.tx_data.sources.size(); - if (is_simple() || is_req_bulletproof()){ - auto dst = &m_ct.rv->pseudoOuts; - if (is_bulletproof()){ - dst = &m_ct.rv->p.pseudoOuts; - } - - dst->clear(); - for (const auto &pseudo_out : m_ct.pseudo_outs) { - dst->emplace_back(); - string_to_key(dst->back(), pseudo_out); - } - - m_ct.rv->mixRing.resize(num_sources); - } else { - m_ct.rv->mixRing.resize(m_ct.tsx_data.mixin()); - m_ct.rv->mixRing[0].resize(num_sources); + auto dst = &m_ct.rv->p.pseudoOuts; + dst->clear(); + for (const auto &pseudo_out : m_ct.pseudo_outs) { + dst->emplace_back(); + string_to_key(dst->back(), pseudo_out); } + m_ct.rv->mixRing.resize(num_sources); + CHECK_AND_ASSERT_THROW_MES(m_ct.tx_out_pk.size() == m_ct.tx_out_ecdh.size(), "Invalid vector sizes"); for(size_t i = 0; i < m_ct.tx_out_ecdh.size(); ++i){ m_ct.rv->outPk.push_back(m_ct.tx_out_pk[i]); @@ -889,10 +869,10 @@ namespace tx { } for(size_t i = 0; i < m_ct.tx_out_rsigs.size(); ++i){ - if (is_bulletproof()){ - m_ct.rv->p.bulletproofs.push_back(boost::get(m_ct.tx_out_rsigs[i])); + if (is_req_bulletproof_plus()) { + m_ct.rv->p.bulletproofs_plus.push_back(boost::get(m_ct.tx_out_rsigs[i])); } else { - m_ct.rv->p.rangeSigs.push_back(boost::get(m_ct.tx_out_rsigs[i])); + m_ct.rv->p.bulletproofs.push_back(boost::get(m_ct.tx_out_rsigs[i])); } } @@ -936,8 +916,8 @@ namespace tx { void Signer::step_sign_input_ack(std::shared_ptr ack){ m_ct.signatures.push_back(ack->signature()); - // Sync updated pseudo_outputs, client_version>=1, HF10+ - if (client_version() >= 1 && ack->has_pseudo_out()){ + // Sync updated pseudo_outputs + if (ack->has_pseudo_out()){ CHECK_AND_ASSERT_THROW_MES(m_ct.cur_input_idx < m_ct.pseudo_outs.size(), "Invalid pseudo-out index"); m_ct.pseudo_outs[m_ct.cur_input_idx] = ack->pseudo_out(); if (is_bulletproof()){ @@ -955,6 +935,8 @@ namespace tx { } void Signer::step_final_ack(std::shared_ptr ack){ + CHECK_AND_ASSERT_THROW_MES(is_clsag(), "Only CLSAGs signatures are supported"); + if (m_multisig){ auto & cout_key = ack->cout_key(); for(auto & cur : m_ct.couts){ @@ -975,47 +957,34 @@ namespace tx { m_ct.enc_keys = ack->tx_enc_keys(); // Opening the sealed signatures - if (client_version() >= 3){ - if(!ack->has_opening_key()){ - throw exc::ProtocolException("Client version 3+ requires sealed signatures"); - } + if(!ack->has_opening_key()){ + throw exc::ProtocolException("Client version 3+ requires sealed signatures"); + } - for(size_t i = 0; i < m_ct.signatures.size(); ++i){ - CHECK_AND_ASSERT_THROW_MES(m_ct.signatures[i].size() > crypto::chacha::TAG_SIZE, "Invalid signature size"); - std::string nonce = compute_sealing_key(ack->opening_key(), i, true); - std::string key = compute_sealing_key(ack->opening_key(), i, false); - size_t plen = m_ct.signatures[i].size() - crypto::chacha::TAG_SIZE; - std::unique_ptr plaintext(new uint8_t[plen]); - uint8_t * buff = plaintext.get(); - - protocol::crypto::chacha::decrypt( - m_ct.signatures[i].data(), - m_ct.signatures[i].size(), - reinterpret_cast(key.data()), - reinterpret_cast(nonce.data()), - reinterpret_cast(buff), &plen); - m_ct.signatures[i].assign(reinterpret_cast(buff), plen); - } + for(size_t i = 0; i < m_ct.signatures.size(); ++i){ + CHECK_AND_ASSERT_THROW_MES(m_ct.signatures[i].size() > crypto::chacha::TAG_SIZE, "Invalid signature size"); + std::string nonce = compute_sealing_key(ack->opening_key(), i, true); + std::string key = compute_sealing_key(ack->opening_key(), i, false); + size_t plen = m_ct.signatures[i].size() - crypto::chacha::TAG_SIZE; + std::unique_ptr plaintext(new uint8_t[plen]); + uint8_t * buff = plaintext.get(); + + protocol::crypto::chacha::decrypt( + m_ct.signatures[i].data(), + m_ct.signatures[i].size(), + reinterpret_cast(key.data()), + reinterpret_cast(nonce.data()), + reinterpret_cast(buff), &plen); + m_ct.signatures[i].assign(reinterpret_cast(buff), plen); } - if (m_ct.rv->type == rct::RCTTypeCLSAG){ - m_ct.rv->p.CLSAGs.reserve(m_ct.signatures.size()); - for (size_t i = 0; i < m_ct.signatures.size(); ++i) { - rct::clsag clsag; - if (!cn_deserialize(m_ct.signatures[i], clsag)) { - throw exc::ProtocolException("Cannot deserialize clsag[i]"); - } - m_ct.rv->p.CLSAGs.push_back(clsag); - } - } else { - m_ct.rv->p.MGs.reserve(m_ct.signatures.size()); - for (size_t i = 0; i < m_ct.signatures.size(); ++i) { - rct::mgSig mg; - if (!cn_deserialize(m_ct.signatures[i], mg)) { - throw exc::ProtocolException("Cannot deserialize mg[i]"); - } - m_ct.rv->p.MGs.push_back(mg); + m_ct.rv->p.CLSAGs.reserve(m_ct.signatures.size()); + for (size_t i = 0; i < m_ct.signatures.size(); ++i) { + rct::clsag clsag; + if (!cn_deserialize(m_ct.signatures[i], clsag)) { + throw exc::ProtocolException("Cannot deserialize clsag[i]"); } + m_ct.rv->p.CLSAGs.push_back(clsag); } m_ct.tx.rct_signatures = *(m_ct.rv); diff --git a/src/device_trezor/trezor/protocol.hpp b/src/device_trezor/trezor/protocol.hpp index 858db152007..e9d84ace8e4 100644 --- a/src/device_trezor/trezor/protocol.hpp +++ b/src/device_trezor/trezor/protocol.hpp @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2022, The Monero Project +// Copyright (c) 2017-2023, The Monero Project // // All rights reserved. // @@ -116,8 +116,7 @@ namespace ki { */ bool key_image_data(wallet_shim * wallet, const std::vector & transfers, - std::vector & res, - bool need_all_additionals=false); + std::vector & res); /** * Computes a hash over MoneroTransferDetails. Commitment used in the KI sync. @@ -129,8 +128,7 @@ namespace ki { */ void generate_commitment(std::vector & mtds, const std::vector & transfers, - std::shared_ptr & req, - bool need_subaddr_indices=false); + std::shared_ptr & req); /** * Processes Live refresh step response, parses KI, checks the signature @@ -166,7 +164,7 @@ namespace tx { ::crypto::secret_key compute_enc_key(const ::crypto::secret_key & private_view_key, const std::string & aux, const std::string & salt); std::string compute_sealing_key(const std::string & master_key, size_t idx, bool is_iv=false); - typedef boost::variant rsig_v; + typedef boost::variant rsig_v; /** * Transaction signer state holder. @@ -232,8 +230,8 @@ namespace tx { } const tools::wallet2::transfer_details & get_transfer(size_t idx) const { - CHECK_AND_ASSERT_THROW_MES(idx < m_unsigned_tx->transfers.second.size() + m_unsigned_tx->transfers.first && idx >= m_unsigned_tx->transfers.first, "Invalid transfer index"); - return m_unsigned_tx->transfers.second[idx - m_unsigned_tx->transfers.first]; + CHECK_AND_ASSERT_THROW_MES(idx < std::get<2>(m_unsigned_tx->transfers).size() + std::get<0>(m_unsigned_tx->transfers) && idx >= std::get<0>(m_unsigned_tx->transfers), "Invalid transfer index"); + return std::get<2>(m_unsigned_tx->transfers)[idx - std::get<0>(m_unsigned_tx->transfers)]; } const tools::wallet2::transfer_details & get_source_transfer(size_t idx) const { @@ -247,7 +245,7 @@ namespace tx { void compute_integrated_indices(TsxData * tsx_data); bool should_compute_bp_now() const; void compute_bproof(messages::monero::MoneroTransactionRsigData & rsig_data); - void process_bproof(rct::Bulletproof & bproof); + void process_bproof(rsig_v & bproof); void set_tx_input(MoneroTransactionSourceEntry * dst, size_t idx, bool need_ring_keys=false, bool need_ring_indices=false); public: @@ -260,8 +258,6 @@ namespace tx { void step_set_input_ack(std::shared_ptr ack); void sort_ki(); - std::shared_ptr step_permutation(); - void step_permutation_ack(std::shared_ptr ack); std::shared_ptr step_set_vini_input(size_t idx); void step_set_vini_input_ack(std::shared_ptr ack); @@ -290,11 +286,15 @@ namespace tx { return m_client_version; } - bool is_simple() const { + uint8_t get_rv_type() const { if (!m_ct.rv){ throw std::invalid_argument("RV not initialized"); } - auto tp = m_ct.rv->type; + return m_ct.rv->type; + } + + bool is_simple() const { + auto tp = get_rv_type(); return tp == rct::RCTTypeSimple; } @@ -302,12 +302,27 @@ namespace tx { return m_ct.tx_data.rct_config.range_proof_type != rct::RangeProofBorromean; } + bool is_req_clsag() const { + return is_req_bulletproof() && m_ct.tx_data.rct_config.bp_version >= 3; + } + + bool is_req_bulletproof_plus() const { + return is_req_bulletproof() && m_ct.tx_data.rct_config.bp_version == 4; // rct::genRctSimple + } + bool is_bulletproof() const { - if (!m_ct.rv){ - throw std::invalid_argument("RV not initialized"); - } - auto tp = m_ct.rv->type; - return tp == rct::RCTTypeBulletproof || tp == rct::RCTTypeBulletproof2 || tp == rct::RCTTypeCLSAG; + auto tp = get_rv_type(); + return rct::is_rct_bulletproof(tp) || rct::is_rct_bulletproof_plus(tp); + } + + bool is_bulletproof_plus() const { + auto tp = get_rv_type(); + return rct::is_rct_bulletproof_plus(tp); + } + + bool is_clsag() const { + auto tp = get_rv_type(); + return rct::is_rct_clsag(tp); } bool is_offloading() const { diff --git a/src/device_trezor/trezor/transport.cpp b/src/device_trezor/trezor/transport.cpp index 53b35a37a59..8c6b30787c3 100644 --- a/src/device_trezor/trezor/transport.cpp +++ b/src/device_trezor/trezor/transport.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2022, The Monero Project +// Copyright (c) 2017-2023, The Monero Project // // All rights reserved. // diff --git a/src/device_trezor/trezor/transport.hpp b/src/device_trezor/trezor/transport.hpp index a452724dadd..3f280b104da 100644 --- a/src/device_trezor/trezor/transport.hpp +++ b/src/device_trezor/trezor/transport.hpp @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2022, The Monero Project +// Copyright (c) 2017-2023, The Monero Project // // All rights reserved. // diff --git a/src/device_trezor/trezor/trezor_defs.hpp b/src/device_trezor/trezor/trezor_defs.hpp index 53bf2b03c10..8e7f89031b2 100644 --- a/src/device_trezor/trezor/trezor_defs.hpp +++ b/src/device_trezor/trezor/trezor_defs.hpp @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2022, The Monero Project +// Copyright (c) 2017-2023, The Monero Project // // All rights reserved. // diff --git a/src/gen_multisig/CMakeLists.txt b/src/gen_multisig/CMakeLists.txt index e8aaec62c5b..e88d21902be 100644 --- a/src/gen_multisig/CMakeLists.txt +++ b/src/gen_multisig/CMakeLists.txt @@ -1,4 +1,4 @@ -# Copyright (c) 2017-2022, The Monero Project +# Copyright (c) 2017-2023, The Monero Project # # All rights reserved. # diff --git a/src/gen_multisig/gen_multisig.cpp b/src/gen_multisig/gen_multisig.cpp index f13e74b0f83..48cf818ef92 100644 --- a/src/gen_multisig/gen_multisig.cpp +++ b/src/gen_multisig/gen_multisig.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2022, The Monero Project +// Copyright (c) 2017-2023, The Monero Project // // All rights reserved. // @@ -50,7 +50,6 @@ using namespace std; using namespace epee; using namespace cryptonote; -using boost::lexical_cast; namespace po = boost::program_options; #undef MONERO_DEFAULT_LOG_CATEGORY @@ -84,6 +83,9 @@ static bool generate_multisig(uint32_t threshold, uint32_t total, const std::str try { + if (total == 0) + throw std::runtime_error("Signer group of size 0 is not allowed."); + // create M wallets first std::vector> wallets(total); for (size_t n = 0; n < total; ++n) @@ -118,13 +120,17 @@ static bool generate_multisig(uint32_t threshold, uint32_t total, const std::str ss << " " << name << std::endl; } - //exchange keys unless exchange_multisig_keys returns no extra info - while (!kex_msgs_intermediate[0].empty()) + // exchange keys until the wallets are done + bool ready{false}; + wallets[0]->multisig(&ready); + while (!ready) { for (size_t n = 0; n < total; ++n) { kex_msgs_intermediate[n] = wallets[n]->exchange_multisig_keys(pwd_container->password(), kex_msgs_intermediate); } + + wallets[0]->multisig(&ready); } std::string address = wallets[0]->get_account().get_public_address_str(wallets[0]->nettype()); diff --git a/src/gen_ssl_cert/CMakeLists.txt b/src/gen_ssl_cert/CMakeLists.txt index efadc7c3167..e865188443c 100644 --- a/src/gen_ssl_cert/CMakeLists.txt +++ b/src/gen_ssl_cert/CMakeLists.txt @@ -1,4 +1,4 @@ -# Copyright (c) 2017-2022, The Monero Project +# Copyright (c) 2017-2023, The Monero Project # # All rights reserved. # diff --git a/src/gen_ssl_cert/gen_ssl_cert.cpp b/src/gen_ssl_cert/gen_ssl_cert.cpp index cd810ed2021..b25d9a73dec 100644 --- a/src/gen_ssl_cert/gen_ssl_cert.cpp +++ b/src/gen_ssl_cert/gen_ssl_cert.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2019-2022, The Monero Project +// Copyright (c) 2019-2023, The Monero Project // // All rights reserved. // @@ -65,29 +65,6 @@ namespace const command_line::arg_descriptor arg_prompt_for_passphrase = {"prompt-for-passphrase", gencert::tr("Prompt for a passphrase with which to encrypt the private key"), false}; } -// adapted from openssl's apps/x509.c -static std::string get_fingerprint(X509 *cert, const EVP_MD *fdig) -{ - unsigned int j; - unsigned int n; - unsigned char md[EVP_MAX_MD_SIZE]; - std::string fingerprint; - - if (!X509_digest(cert, fdig, md, &n)) - { - tools::fail_msg_writer() << tr("Failed to create fingerprint: ") << ERR_reason_error_string(ERR_get_error()); - return fingerprint; - } - fingerprint.resize(n * 3 - 1); - char *out = &fingerprint[0]; - for (j = 0; j < n; ++j) - { - snprintf(out, 3 + (j + 1 < n), "%02X%s", md[j], (j + 1 == n) ? "" : ":"); - out += 3; - } - return fingerprint; -} - int main(int argc, char* argv[]) { TRY_ENTRY(); @@ -246,7 +223,7 @@ int main(int argc, char* argv[]) tools::success_msg_writer() << tr("New certificate created:"); tools::success_msg_writer() << tr("Certificate: ") << certificate_filename; - tools::success_msg_writer() << tr("SHA-256 Fingerprint: ") << get_fingerprint(cert, EVP_sha256()); + tools::success_msg_writer() << tr("SHA-256 Fingerprint: ") << epee::net_utils::get_hr_ssl_fingerprint(cert); tools::success_msg_writer() << tr("Private key: ") << private_key_filename << " (" << (private_key_passphrase.empty() ? "unencrypted" : "encrypted") << ")"; return 0; diff --git a/src/hardforks/CMakeLists.txt b/src/hardforks/CMakeLists.txt index 81a3d694b83..20ea907c502 100644 --- a/src/hardforks/CMakeLists.txt +++ b/src/hardforks/CMakeLists.txt @@ -1,4 +1,4 @@ -# Copyright (c) 2014-2022, The Monero Project +# Copyright (c) 2014-2023, The Monero Project # # All rights reserved. # diff --git a/src/hardforks/hardforks.cpp b/src/hardforks/hardforks.cpp index 0dd0cf5f0ce..3d1d53589d6 100644 --- a/src/hardforks/hardforks.cpp +++ b/src/hardforks/hardforks.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2022, The Monero Project +// Copyright (c) 2014-2023, The Monero Project // // All rights reserved. // @@ -71,8 +71,8 @@ const hardfork_t mainnet_hard_forks[] = { { 13, 2210000, 0, 1598180817 }, { 14, 2210720, 0, 1598180818 }, - { 15, 8000000, 0, 1608223241 }, // temp so tests test with these consensus rules - { 16, 8000001, 0, 1608223242 }, // temp so tests test with these consensus rules + { 15, 2688888, 0, 1656629117 }, + { 16, 2689608, 0, 1656629118 }, }; const size_t num_mainnet_hard_forks = sizeof(mainnet_hard_forks) / sizeof(mainnet_hard_forks[0]); const uint64_t mainnet_hard_fork_version_1_till = 1009826; @@ -122,5 +122,7 @@ const hardfork_t stagenet_hard_forks[] = { { 12, 454721, 0, 1571419280 }, { 13, 675405, 0, 1598180817 }, { 14, 676125, 0, 1598180818 }, + { 15, 1151000, 0, 1656629117 }, + { 16, 1151720, 0, 1656629118 }, }; const size_t num_stagenet_hard_forks = sizeof(stagenet_hard_forks) / sizeof(stagenet_hard_forks[0]); diff --git a/src/hardforks/hardforks.h b/src/hardforks/hardforks.h index 53f14b8ebd9..fbe0d630d1e 100644 --- a/src/hardforks/hardforks.h +++ b/src/hardforks/hardforks.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2022, The Monero Project +// Copyright (c) 2014-2023, The Monero Project // // All rights reserved. // diff --git a/src/lmdb/CMakeLists.txt b/src/lmdb/CMakeLists.txt index a26c48ad505..50e08be7309 100644 --- a/src/lmdb/CMakeLists.txt +++ b/src/lmdb/CMakeLists.txt @@ -1,4 +1,4 @@ -# Copyright (c) 2014-2022, The Monero Project +# Copyright (c) 2014-2023, The Monero Project # # All rights reserved. # diff --git a/src/lmdb/database.cpp b/src/lmdb/database.cpp index 544197d5799..22569fcf083 100644 --- a/src/lmdb/database.cpp +++ b/src/lmdb/database.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2022, The Monero Project +// Copyright (c) 2014-2023, The Monero Project // All rights reserved. // // Redistribution and use in source and binary forms, with or without modification, are diff --git a/src/lmdb/database.h b/src/lmdb/database.h index 0c239065234..0f12571da03 100644 --- a/src/lmdb/database.h +++ b/src/lmdb/database.h @@ -1,4 +1,4 @@ -// Copyright (c) 2018-2022, The Monero Project +// Copyright (c) 2018-2023, The Monero Project // All rights reserved. // diff --git a/src/lmdb/error.cpp b/src/lmdb/error.cpp index 62fdb83c305..28659dddd50 100644 --- a/src/lmdb/error.cpp +++ b/src/lmdb/error.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2022, The Monero Project +// Copyright (c) 2014-2023, The Monero Project // All rights reserved. // // Redistribution and use in source and binary forms, with or without modification, are diff --git a/src/lmdb/error.h b/src/lmdb/error.h index f4134359b08..eaacc56e691 100644 --- a/src/lmdb/error.h +++ b/src/lmdb/error.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2022, The Monero Project +// Copyright (c) 2014-2023, The Monero Project // All rights reserved. // // Redistribution and use in source and binary forms, with or without modification, are diff --git a/src/lmdb/key_stream.h b/src/lmdb/key_stream.h index 11fa284dde5..1765c0e3917 100644 --- a/src/lmdb/key_stream.h +++ b/src/lmdb/key_stream.h @@ -1,4 +1,4 @@ -// Copyright (c) 2018-2022, The Monero Project +// Copyright (c) 2018-2023, The Monero Project // All rights reserved. // diff --git a/src/lmdb/table.cpp b/src/lmdb/table.cpp index 725a1a0b745..239e80ef98d 100644 --- a/src/lmdb/table.cpp +++ b/src/lmdb/table.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2018-2022, The Monero Project +// Copyright (c) 2018-2023, The Monero Project // All rights reserved. // diff --git a/src/lmdb/transaction.h b/src/lmdb/transaction.h index f358290ec63..4cb43aead33 100644 --- a/src/lmdb/transaction.h +++ b/src/lmdb/transaction.h @@ -1,4 +1,4 @@ -// Copyright (c) 2018-2022, The Monero Project +// Copyright (c) 2018-2023, The Monero Project // All rights reserved. // diff --git a/src/lmdb/util.h b/src/lmdb/util.h index c6c75bc0048..95057fe8d96 100644 --- a/src/lmdb/util.h +++ b/src/lmdb/util.h @@ -1,4 +1,4 @@ -// Copyright (c) 2018-2022, The Monero Project +// Copyright (c) 2018-2023, The Monero Project // All rights reserved. // diff --git a/src/lmdb/value_stream.cpp b/src/lmdb/value_stream.cpp index 6a1e054c16e..cccb74bc3e4 100644 --- a/src/lmdb/value_stream.cpp +++ b/src/lmdb/value_stream.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2018-2022, The Monero Project +// Copyright (c) 2018-2023, The Monero Project // All rights reserved. // diff --git a/src/lmdb/value_stream.h b/src/lmdb/value_stream.h index bd2814ef491..858a54f3e29 100644 --- a/src/lmdb/value_stream.h +++ b/src/lmdb/value_stream.h @@ -1,4 +1,4 @@ -// Copyright (c) 2018-2022, The Monero Project +// Copyright (c) 2018-2023, The Monero Project // All rights reserved. // diff --git a/src/mnemonics/CMakeLists.txt b/src/mnemonics/CMakeLists.txt index 738633ef5ac..23e63dfa115 100644 --- a/src/mnemonics/CMakeLists.txt +++ b/src/mnemonics/CMakeLists.txt @@ -1,4 +1,4 @@ -# Copyright (c) 2014-2022, The Monero Project +# Copyright (c) 2014-2023, The Monero Project # # All rights reserved. # diff --git a/src/mnemonics/chinese_simplified.h b/src/mnemonics/chinese_simplified.h index 2661b582091..79f211512d9 100644 --- a/src/mnemonics/chinese_simplified.h +++ b/src/mnemonics/chinese_simplified.h @@ -21,7 +21,7 @@ // OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION // WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. // -// Code surrounding the word list is Copyright (c) 2014-2022, The Monero Project +// Code surrounding the word list is Copyright (c) 2014-2023, The Monero Project // // All rights reserved. // diff --git a/src/mnemonics/dutch.h b/src/mnemonics/dutch.h index ace2e1b3d7d..afa856f3b01 100644 --- a/src/mnemonics/dutch.h +++ b/src/mnemonics/dutch.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2022, The Monero Project +// Copyright (c) 2014-2023, The Monero Project // // All rights reserved. // diff --git a/src/mnemonics/electrum-words.cpp b/src/mnemonics/electrum-words.cpp index b53f3acd392..69816c43c06 100644 --- a/src/mnemonics/electrum-words.cpp +++ b/src/mnemonics/electrum-words.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2022, The Monero Project +// Copyright (c) 2014-2023, The Monero Project // // All rights reserved. // diff --git a/src/mnemonics/electrum-words.h b/src/mnemonics/electrum-words.h index becd6bb4eda..ab6a4a9f106 100644 --- a/src/mnemonics/electrum-words.h +++ b/src/mnemonics/electrum-words.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2022, The Monero Project +// Copyright (c) 2014-2023, The Monero Project // // All rights reserved. // diff --git a/src/mnemonics/english.h b/src/mnemonics/english.h index 1715445d893..3abb5d3b22e 100644 --- a/src/mnemonics/english.h +++ b/src/mnemonics/english.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2022, The Monero Project +// Copyright (c) 2014-2023, The Monero Project // // All rights reserved. // diff --git a/src/mnemonics/english_old.h b/src/mnemonics/english_old.h index c609dff2d67..71ed862cbe3 100644 --- a/src/mnemonics/english_old.h +++ b/src/mnemonics/english_old.h @@ -1,6 +1,6 @@ // Word list originally created as part of the Electrum project, Copyright (C) 2014 Thomas Voegtlin // -// Copyright (c) 2014-2022, The Monero Project +// Copyright (c) 2014-2023, The Monero Project // // All rights reserved. // diff --git a/src/mnemonics/esperanto.h b/src/mnemonics/esperanto.h index 1d3437fad36..db8ccc99a07 100644 --- a/src/mnemonics/esperanto.h +++ b/src/mnemonics/esperanto.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2022, The Monero Project +// Copyright (c) 2014-2023, The Monero Project // // All rights reserved. // diff --git a/src/mnemonics/french.h b/src/mnemonics/french.h index be2f8957c3f..e39bd5902b5 100644 --- a/src/mnemonics/french.h +++ b/src/mnemonics/french.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2022, The Monero Project +// Copyright (c) 2014-2023, The Monero Project // // All rights reserved. // diff --git a/src/mnemonics/german.h b/src/mnemonics/german.h index 4f526588814..845bc729a4d 100644 --- a/src/mnemonics/german.h +++ b/src/mnemonics/german.h @@ -1,6 +1,6 @@ // Word list created by Monero contributor Shrikez // -// Copyright (c) 2014-2022, The Monero Project +// Copyright (c) 2014-2023, The Monero Project // // All rights reserved. // diff --git a/src/mnemonics/italian.h b/src/mnemonics/italian.h index dbd39984ba1..e258811d38b 100644 --- a/src/mnemonics/italian.h +++ b/src/mnemonics/italian.h @@ -1,6 +1,6 @@ // Word list created by Monero contributor Shrikez // -// Copyright (c) 2014-2022, The Monero Project +// Copyright (c) 2014-2023, The Monero Project // // All rights reserved. // diff --git a/src/mnemonics/japanese.h b/src/mnemonics/japanese.h index 5831b19957d..65d81daf18a 100644 --- a/src/mnemonics/japanese.h +++ b/src/mnemonics/japanese.h @@ -21,7 +21,7 @@ // OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION // WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. // -// Code surrounding the word list is Copyright (c) 2014-2022, The Monero Project +// Code surrounding the word list is Copyright (c) 2014-2023, The Monero Project // All rights reserved. // // Redistribution and use in source and binary forms, with or without modification, are diff --git a/src/mnemonics/language_base.h b/src/mnemonics/language_base.h index 92eb09f0da4..f6ed5bd5766 100644 --- a/src/mnemonics/language_base.h +++ b/src/mnemonics/language_base.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2022, The Monero Project +// Copyright (c) 2014-2023, The Monero Project // // All rights reserved. // diff --git a/src/mnemonics/lojban.h b/src/mnemonics/lojban.h index 68aefd5fd2b..9c127c0a081 100644 --- a/src/mnemonics/lojban.h +++ b/src/mnemonics/lojban.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2022, The Monero Project +// Copyright (c) 2014-2023, The Monero Project // // All rights reserved. // diff --git a/src/mnemonics/portuguese.h b/src/mnemonics/portuguese.h index 9ddac09bb38..f87464b270e 100644 --- a/src/mnemonics/portuguese.h +++ b/src/mnemonics/portuguese.h @@ -21,7 +21,7 @@ // OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION // WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. // -// Code surrounding the word list is Copyright (c) 2014-2022, The Monero Project +// Code surrounding the word list is Copyright (c) 2014-2023, The Monero Project // // All rights reserved. // diff --git a/src/mnemonics/russian.h b/src/mnemonics/russian.h index 8922b1ed999..23b108daa14 100644 --- a/src/mnemonics/russian.h +++ b/src/mnemonics/russian.h @@ -1,6 +1,6 @@ // Word list created by Monero contributor sammy007 // -// Copyright (c) 2014-2022, The Monero Project +// Copyright (c) 2014-2023, The Monero Project // // All rights reserved. // diff --git a/src/mnemonics/singleton.h b/src/mnemonics/singleton.h index 91faad92c23..156db9e3d29 100644 --- a/src/mnemonics/singleton.h +++ b/src/mnemonics/singleton.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2022, The Monero Project +// Copyright (c) 2014-2023, The Monero Project // // All rights reserved. // diff --git a/src/mnemonics/spanish.h b/src/mnemonics/spanish.h index 1bdb6b93412..312ac4adcf0 100644 --- a/src/mnemonics/spanish.h +++ b/src/mnemonics/spanish.h @@ -21,7 +21,7 @@ // OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION // WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. // -// Code surrounding the word list is Copyright (c) 2014-2022, The Monero Project +// Code surrounding the word list is Copyright (c) 2014-2023, The Monero Project // // All rights reserved. // diff --git a/src/multisig/CMakeLists.txt b/src/multisig/CMakeLists.txt index 294a1721fe0..62ae00e8053 100644 --- a/src/multisig/CMakeLists.txt +++ b/src/multisig/CMakeLists.txt @@ -1,4 +1,4 @@ -# Copyright (c) 2017-2022, The Monero Project +# Copyright (c) 2017-2023, The Monero Project # # All rights reserved. # @@ -30,7 +30,9 @@ set(multisig_sources multisig.cpp multisig_account.cpp multisig_account_kex_impl.cpp - multisig_kex_msg.cpp) + multisig_clsag_context.cpp + multisig_kex_msg.cpp + multisig_tx_builder_ringct.cpp) set(multisig_headers) @@ -48,6 +50,7 @@ target_link_libraries(multisig PUBLIC ringct cryptonote_basic + cryptonote_core common cncrypto PRIVATE diff --git a/src/multisig/multisig.cpp b/src/multisig/multisig.cpp index fabffdd02de..70baa7f19c9 100644 --- a/src/multisig/multisig.cpp +++ b/src/multisig/multisig.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2022, The Monero Project +// Copyright (c) 2017-2023, The Monero Project // // All rights reserved. // diff --git a/src/multisig/multisig.h b/src/multisig/multisig.h index 16dbbc544d5..2ec2f4e66dd 100644 --- a/src/multisig/multisig.h +++ b/src/multisig/multisig.h @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2022, The Monero Project +// Copyright (c) 2017-2023, The Monero Project // // All rights reserved. // diff --git a/src/multisig/multisig_account.cpp b/src/multisig/multisig_account.cpp index 9bdcf2dbc8d..4f9711b15fb 100644 --- a/src/multisig/multisig_account.cpp +++ b/src/multisig/multisig_account.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2021-2022, The Monero Project +// Copyright (c) 2021-2023, The Monero Project // // All rights reserved. // @@ -127,7 +127,7 @@ namespace multisig bool multisig_account::multisig_is_ready() const { if (main_kex_rounds_done()) - return m_kex_rounds_complete >= multisig_kex_rounds_required(m_signers.size(), m_threshold) + 1; + return m_kex_rounds_complete >= multisig_setup_rounds_required(m_signers.size(), m_threshold); else return false; } @@ -175,19 +175,20 @@ namespace multisig // only mutate account if update succeeds multisig_account temp_account{*this}; temp_account.set_multisig_config(threshold, std::move(signers)); - temp_account.kex_update_impl(expanded_msgs_rnd1); + temp_account.kex_update_impl(expanded_msgs_rnd1, false); *this = std::move(temp_account); } //---------------------------------------------------------------------------------------------------------------------- // multisig_account: EXTERNAL //---------------------------------------------------------------------------------------------------------------------- - void multisig_account::kex_update(const std::vector &expanded_msgs) + void multisig_account::kex_update(const std::vector &expanded_msgs, + const bool force_update_use_with_caution /*= false*/) { CHECK_AND_ASSERT_THROW_MES(account_is_active(), "multisig account: tried to update kex, but kex isn't initialized yet."); CHECK_AND_ASSERT_THROW_MES(!multisig_is_ready(), "multisig account: tried to update kex, but kex is already complete."); multisig_account temp_account{*this}; - temp_account.kex_update_impl(expanded_msgs); + temp_account.kex_update_impl(expanded_msgs, force_update_use_with_caution); *this = std::move(temp_account); } //---------------------------------------------------------------------------------------------------------------------- @@ -200,4 +201,11 @@ namespace multisig return num_signers - threshold + 1; } //---------------------------------------------------------------------------------------------------------------------- + // EXTERNAL + //---------------------------------------------------------------------------------------------------------------------- + std::uint32_t multisig_setup_rounds_required(const std::uint32_t num_signers, const std::uint32_t threshold) + { + return multisig_kex_rounds_required(num_signers, threshold) + 1; + } + //---------------------------------------------------------------------------------------------------------------------- } //namespace multisig diff --git a/src/multisig/multisig_account.h b/src/multisig/multisig_account.h index 7b372bbff11..0d832f24340 100644 --- a/src/multisig/multisig_account.h +++ b/src/multisig/multisig_account.h @@ -1,4 +1,4 @@ -// Copyright (c) 2021-2022, The Monero Project +// Copyright (c) 2021-2023, The Monero Project // // All rights reserved. // @@ -169,12 +169,20 @@ namespace multisig * - The main interface for multisig key exchange, this handles all the work of processing input messages, * creating new messages for new rounds, and finalizing the multisig shared public key when kex is complete. * param: expanded_msgs - kex messages corresponding to the account's 'in progress' round + * param: force_update_use_with_caution - try to force the account to update with messages from an incomplete signer set. + * - If this is the post-kex verification round, only require one input message. + * - Force updating here should only be done if we can safely assume an honest signer subgroup of size 'threshold' + * will complete the account. + * - If this is an intermediate round, only require messages from 'num signers - 1 - (round - 1)' other signers. + * - If force updating with maliciously-crafted messages, the resulting account will be invalid (either unable + * to complete signatures, or a 'hostage' to the malicious signer [i.e. can't sign without his participation]). */ - void kex_update(const std::vector &expanded_msgs); + void kex_update(const std::vector &expanded_msgs, + const bool force_update_use_with_caution = false); private: // implementation of kex_update() (non-transactional) - void kex_update_impl(const std::vector &expanded_msgs); + void kex_update_impl(const std::vector &expanded_msgs, const bool incomplete_signer_set); /** * brief: initialize_kex_update - Helper for kex_update_impl() * - Collect the local signer's shared keys to ignore in incoming messages, build the aggregate ancillary key @@ -245,4 +253,13 @@ namespace multisig * return: number of kex rounds required */ std::uint32_t multisig_kex_rounds_required(const std::uint32_t num_signers, const std::uint32_t threshold); + + /** + * brief: multisig_setup_rounds_required - The number of setup rounds required to produce an M-of-N shared key. + * - A participant must complete all kex rounds and 1 initialization round. + * param: num_signers - number of participants in multisig (N) + * param: threshold - threshold of multisig (M) + * return: number of setup rounds required + */ + std::uint32_t multisig_setup_rounds_required(const std::uint32_t num_signers, const std::uint32_t threshold); } //namespace multisig diff --git a/src/multisig/multisig_account_kex_impl.cpp b/src/multisig/multisig_account_kex_impl.cpp index be9ed9cb212..ef0acf30743 100644 --- a/src/multisig/multisig_account_kex_impl.cpp +++ b/src/multisig/multisig_account_kex_impl.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2021-2022, The Monero Project +// Copyright (c) 2021-2023, The Monero Project // // All rights reserved. // @@ -74,7 +74,7 @@ namespace multisig "Multisig threshold may not be larger than number of signers."); CHECK_AND_ASSERT_THROW_MES(threshold > 0, "Multisig threshold must be > 0."); CHECK_AND_ASSERT_THROW_MES(round > 0, "Multisig kex round must be > 0."); - CHECK_AND_ASSERT_THROW_MES(round <= multisig_kex_rounds_required(num_signers, threshold) + 1, + CHECK_AND_ASSERT_THROW_MES(round <= multisig_setup_rounds_required(num_signers, threshold), "Trying to process multisig kex for an invalid round."); } //---------------------------------------------------------------------------------------------------------------------- @@ -181,7 +181,8 @@ namespace multisig * Key aggregation via aggregation coefficients prevents key cancellation attacks. * See: https://www.getmonero.org/resources/research-lab/pubs/MRL-0009.pdf * param: final_keys - address components (public keys) obtained from other participants (not shared with local) - * param: privkeys_inout - private keys of address components known by local; each key will be multiplied by an aggregation coefficient (return by reference) + * param: privkeys_inout - private keys of address components known by local; each key will be multiplied by an aggregation + * coefficient (return by reference) * return: final multisig public spend key for the account */ //---------------------------------------------------------------------------------------------------------------------- @@ -199,7 +200,8 @@ namespace multisig for (std::size_t multisig_keys_index{0}; multisig_keys_index < privkeys_inout.size(); ++multisig_keys_index) { crypto::public_key pubkey; - CHECK_AND_ASSERT_THROW_MES(crypto::secret_key_to_public_key(privkeys_inout[multisig_keys_index], pubkey), "Failed to derive public key"); + CHECK_AND_ASSERT_THROW_MES(crypto::secret_key_to_public_key(privkeys_inout[multisig_keys_index], pubkey), + "Failed to derive public key"); own_keys_mapping[pubkey] = multisig_keys_index; @@ -307,8 +309,7 @@ namespace multisig * INTERNAL * * brief: multisig_kex_msgs_sanitize_pubkeys - Sanitize multisig kex messages. - * - Removes duplicates from msg pubkeys, ignores pubkeys equal to the local account's signing key, - * ignores messages signed by the local account, ignores keys found in input 'exclusion set', + * - Removes duplicates from msg pubkeys, ignores keys found in input 'exclusion set', * constructs map of pubkey:origins. * - Requires that all input msgs have the same round number. * @@ -316,15 +317,13 @@ namespace multisig * * - If the messages' round numbers are all '1', then only the message signing pubkey is considered * 'recommended'. Furthermore, the 'exclusion set' is ignored. - * param: own_pubkey - local account's signing key (key used to sign multisig messages) * param: expanded_msgs - set of multisig kex messages to process * param: exclude_pubkeys - pubkeys to exclude from output set * outparam: sanitized_pubkeys_out - processed pubkeys obtained from msgs, mapped to their origins * return: round number shared by all input msgs */ //---------------------------------------------------------------------------------------------------------------------- - static std::uint32_t multisig_kex_msgs_sanitize_pubkeys(const crypto::public_key &own_pubkey, - const std::vector &expanded_msgs, + static std::uint32_t multisig_kex_msgs_sanitize_pubkeys(const std::vector &expanded_msgs, const std::vector &exclude_pubkeys, multisig_keyset_map_memsafe_t &sanitized_pubkeys_out) { @@ -339,10 +338,6 @@ namespace multisig // - origins = all the signing pubkeys that recommended a given msg pubkey for (const auto &expanded_msg : expanded_msgs) { - // ignore messages from self - if (expanded_msg.get_signing_pubkey() == own_pubkey) - continue; - // in round 1, only the signing pubkey is treated as a msg pubkey if (round == 1) { @@ -355,10 +350,6 @@ namespace multisig // copy all pubkeys from message into list for (const auto &pubkey : expanded_msg.get_msg_pubkeys()) { - // ignore own pubkey - if (pubkey == own_pubkey) - continue; - // ignore pubkeys in 'ignore' set if (std::find(exclude_pubkeys.begin(), exclude_pubkeys.end(), pubkey) != exclude_pubkeys.end()) continue; @@ -375,6 +366,31 @@ namespace multisig /** * INTERNAL * + * brief: remove_key_from_mapped_sets - Remove a specified key from the mapped sets in a multisig keyset map. + * param: key_to_remove - specified key to remove + * inoutparam: keyset_inout - keyset to update + */ + //---------------------------------------------------------------------------------------------------------------------- + static void remove_key_from_mapped_sets(const crypto::public_key &key_to_remove, + multisig_keyset_map_memsafe_t &keyset_inout) + { + // remove specified key from each mapped set + for (auto keyset_it = keyset_inout.begin(); keyset_it != keyset_inout.end();) + { + // remove specified key from this set + keyset_it->second.erase(key_to_remove); + + // remove empty keyset positions or increment iterator + if (keyset_it->second.size() == 0) + keyset_it = keyset_inout.erase(keyset_it); + else + ++keyset_it; + } + } + //---------------------------------------------------------------------------------------------------------------------- + /** + * INTERNAL + * * brief: evaluate_multisig_kex_round_msgs - Evaluate pubkeys from a kex round in order to prepare for the next round. * - Sanitizes input msgs. * - Require uniqueness in: 'exclude_pubkeys'. @@ -392,6 +408,8 @@ namespace multisig * param: signers - expected participants in multisig kex * param: expanded_msgs - set of multisig kex messages to process * param: exclude_pubkeys - derivations held by the local account corresponding to round 'expected_round' + * param: incomplete_signer_set - only require the minimum number of signers to complete this round + * minimum = num_signers - (round num - 1) (including local signer) * return: fully sanitized and validated pubkey:origins map for building the account's next kex round message */ //---------------------------------------------------------------------------------------------------------------------- @@ -400,7 +418,8 @@ namespace multisig const std::uint32_t expected_round, const std::vector &signers, const std::vector &expanded_msgs, - const std::vector &exclude_pubkeys) + const std::vector &exclude_pubkeys, + const bool incomplete_signer_set) { // exclude_pubkeys should all be unique for (auto it = exclude_pubkeys.begin(); it != exclude_pubkeys.end(); ++it) @@ -410,21 +429,31 @@ namespace multisig } // sanitize input messages - multisig_keyset_map_memsafe_t pubkey_origins_map; - const std::uint32_t round = multisig_kex_msgs_sanitize_pubkeys(base_pubkey, expanded_msgs, exclude_pubkeys, pubkey_origins_map); + multisig_keyset_map_memsafe_t pubkey_origins_map; //map: [pubkey : [origins]] + const std::uint32_t round = multisig_kex_msgs_sanitize_pubkeys(expanded_msgs, exclude_pubkeys, pubkey_origins_map); CHECK_AND_ASSERT_THROW_MES(round == expected_round, "Kex messages were for round [" << round << "], but expected round is [" << expected_round << "]"); + // remove the local signer from each origins set in the sanitized pubkey map + // note: intermediate kex rounds only need keys from other signers to make progress (keys from self are useless) + remove_key_from_mapped_sets(base_pubkey, pubkey_origins_map); + // evaluate pubkeys collected - std::unordered_map> origin_pubkeys_map; + std::unordered_map> origin_pubkeys_map; //map: [origin: [pubkeys]] // 1. each pubkey should be recommended by a precise number of signers + const std::size_t num_recommendations_per_pubkey_required{ + incomplete_signer_set + ? 1 + : round + }; + for (const auto &pubkey_and_origins : pubkey_origins_map) { // expected amount = round_num // With each successive round, pubkeys are shared by incrementally larger groups, // starting at 1 in round 1 (i.e. the local multisig key to start kex with). - CHECK_AND_ASSERT_THROW_MES(pubkey_and_origins.second.size() == round, + CHECK_AND_ASSERT_THROW_MES(pubkey_and_origins.second.size() >= num_recommendations_per_pubkey_required, "A pubkey recommended by multisig kex messages had an unexpected number of recommendations."); // map (sanitized) pubkeys back to origins @@ -433,8 +462,18 @@ namespace multisig } // 2. the number of unique signers recommending pubkeys should equal the number of signers passed in (minus the local signer) - CHECK_AND_ASSERT_THROW_MES(origin_pubkeys_map.size() == signers.size() - 1, - "Number of unique other signers does not equal number of other signers that recommended pubkeys."); + // - if an incomplete set is allowed, then we need at least one signer to represent each subgroup in this round that + // doesn't include the local signer + const std::size_t num_signers_required{ + incomplete_signer_set + ? signers.size() - 1 - (round - 1) + : signers.size() - 1 + }; + + CHECK_AND_ASSERT_THROW_MES(origin_pubkeys_map.size() >= num_signers_required, + "Number of unique other signers recommending pubkeys does not equal number of required other signers " + "(kex round: " << round << ", num signers found: " << origin_pubkeys_map.size() << ", num signers required: " << + num_signers_required << ")."); // 3. each origin should recommend a precise number of pubkeys @@ -461,19 +500,20 @@ namespace multisig // other signers: (N - 2) choose (msg_round_num - 1) // - Each signer recommends keys they share with other signers. - // - In each round, a signer shares a key with 'round num - 1' other signers. - // - Since 'origins pubkey map' excludes keys shared with the local account, - // only keys shared with participants 'other than local and self' will be in the map (e.g. N - 2 signers). - // - So other signers will recommend (N - 2) choose (msg_round_num - 1) pubkeys (after removing keys shared with local). - // - Each origin should have a shared key with each group of size 'round - 1'. - // Note: Keys shared with local are ignored to facilitate kex round boosting, where one or more signers may + // - In each round, every group of size 'round num' will have a key. From a single signer's perspective, + // they will share a key with every group of size 'round num - 1' of other signers. + // - Since 'origins pubkey map' excludes keys shared with the local account, only keys shared with participants + // 'other than local and self' will be in the map (e.g. N - 2 signers). + // - Other signers will recommend (N - 2) choose (msg_round_num - 1) pubkeys (after removing keys shared with local). + // Note: Keys shared with local are filtered out to facilitate kex round boosting, where one or more signers may // have boosted the local signer (implying they didn't have access to the local signer's previous round msg). const std::uint32_t expected_recommendations_others = n_choose_k_f(signers.size() - 2, round - 1); // local: (N - 1) choose (msg_round_num - 1) const std::uint32_t expected_recommendations_self = n_choose_k_f(signers.size() - 1, round - 1); - // note: expected_recommendations_others would be 0 in the last round of 1-of-N, but we return early for that case + // note: expected_recommendations_others would be 0 in the last round of 1-of-N, but we don't call this function for + // that case CHECK_AND_ASSERT_THROW_MES(expected_recommendations_self > 0 && expected_recommendations_others > 0, "Bad num signers or round num (possibly numerical limits exceeded)."); @@ -485,7 +525,7 @@ namespace multisig for (const auto &origin_and_pubkeys : origin_pubkeys_map) { CHECK_AND_ASSERT_THROW_MES(origin_and_pubkeys.second.size() == expected_recommendations_others, - "A pubkey recommended by multisig kex messages had an unexpected number of recommendations."); + "A multisig signer recommended an unexpected number of pubkeys."); // 2 (continued). only expected signers should be recommending keys CHECK_AND_ASSERT_THROW_MES(std::find(signers.begin(), signers.end(), origin_and_pubkeys.first) != signers.end(), @@ -507,6 +547,7 @@ namespace multisig * param: expected_round - expected kex round of input messages * param: signers - expected participants in multisig kex * param: expanded_msgs - set of multisig kex messages to process + * param: incomplete_signer_set - only require the minimum amount of messages to complete this round (1 message) * return: sanitized and validated pubkey:origins map */ //---------------------------------------------------------------------------------------------------------------------- @@ -514,15 +555,20 @@ namespace multisig const crypto::public_key &base_pubkey, const std::uint32_t expected_round, const std::vector &signers, - const std::vector &expanded_msgs) + const std::vector &expanded_msgs, + const bool incomplete_signer_set) { // sanitize input messages const std::vector dummy; - multisig_keyset_map_memsafe_t pubkey_origins_map; - const std::uint32_t round = multisig_kex_msgs_sanitize_pubkeys(base_pubkey, expanded_msgs, dummy, pubkey_origins_map); + multisig_keyset_map_memsafe_t pubkey_origins_map; //map: [pubkey : [origins]] + const std::uint32_t round = multisig_kex_msgs_sanitize_pubkeys(expanded_msgs, dummy, pubkey_origins_map); CHECK_AND_ASSERT_THROW_MES(round == expected_round, "Kex messages were for round [" << round << "], but expected round is [" << expected_round << "]"); + // note: do NOT remove the local signer from the pubkey origins map, since the post-kex round can be force-updated with + // just the local signer's post-kex message (if the local signer were removed, then the post-kex message's pubkeys + // would be completely lost) + // evaluate pubkeys collected // 1) there should only be two pubkeys @@ -533,17 +579,26 @@ namespace multisig CHECK_AND_ASSERT_THROW_MES(pubkey_origins_map.begin()->second == (++(pubkey_origins_map.begin()))->second, "Multisig post-kex round messages from other signers did not all recommend the same pubkey pair."); - // 3) all signers should be present in the recommendation list + // 3) all signers should be present in the recommendation list (unless an incomplete list is permitted) auto origins = pubkey_origins_map.begin()->second; - origins.insert(base_pubkey); //add self + origins.insert(base_pubkey); //add self if missing - CHECK_AND_ASSERT_THROW_MES(origins.size() == signers.size(), - "Multisig post-kex round message origins don't line up with multisig signer set."); + const std::size_t num_signers_required{ + incomplete_signer_set + ? 1 + : signers.size() + }; + + CHECK_AND_ASSERT_THROW_MES(origins.size() >= num_signers_required, + "Multisig post-kex round message origins don't line up with multisig signer set " + "(num signers found: " << origins.size() << ", num signers required: " << num_signers_required << ")."); - for (const crypto::public_key &signer : signers) + for (const crypto::public_key &origin : origins) { - CHECK_AND_ASSERT_THROW_MES(origins.find(signer) != origins.end(), - "Could not find an expected signer in multisig post-kex round messages (all signers expected)."); + // note: if num_signers_required == signers.size(), then this test will ensure all signers are present in 'origins', + // which contains only unique pubkeys + CHECK_AND_ASSERT_THROW_MES(std::find(signers.begin(), signers.end(), origin) != signers.end(), + "An unknown origin recommended a multisig post-kex verification messsage."); } return pubkey_origins_map; @@ -564,6 +619,7 @@ namespace multisig * param: expanded_msgs - set of multisig kex messages to process * param: exclude_pubkeys - keys held by the local account corresponding to round 'current_round' * - If 'current_round' is the final round, these are the local account's shares of the final aggregate key. + * param: incomplete_signer_set - allow messages from an incomplete signer set * outparam: keys_to_origins_map_out - map between round keys and identity keys * - If in the final round, these are key shares recommended by other signers for the final aggregate key. * - Otherwise, these are the local account's DH derivations for the next round. @@ -578,6 +634,7 @@ namespace multisig const std::vector &signers, const std::vector &expanded_msgs, const std::vector &exclude_pubkeys, + const bool incomplete_signer_set, multisig_keyset_map_memsafe_t &keys_to_origins_map_out) { check_multisig_config(current_round, threshold, signers.size()); @@ -598,7 +655,8 @@ namespace multisig current_round, signers, expanded_msgs, - exclude_pubkeys); + exclude_pubkeys, + incomplete_signer_set); } else //(current_round == kex_rounds_required + 1) { @@ -606,7 +664,8 @@ namespace multisig evaluated_pubkeys = evaluate_multisig_post_kex_round_msgs(base_pubkey, current_round, signers, - expanded_msgs); + expanded_msgs, + incomplete_signer_set); } // prepare keys-to-origins map for updating the multisig account @@ -693,9 +752,9 @@ namespace multisig { // post-kex verification round: check that the multisig pubkey and common pubkey were recommended by other signers CHECK_AND_ASSERT_THROW_MES(result_keys_to_origins_map.count(m_multisig_pubkey) > 0, - "Multisig post-kex round: expected multisig pubkey wasn't found in other signers' messages."); + "Multisig post-kex round: expected multisig pubkey wasn't found in input messages."); CHECK_AND_ASSERT_THROW_MES(result_keys_to_origins_map.count(m_common_pubkey) > 0, - "Multisig post-kex round: expected common pubkey wasn't found in other signers' messages."); + "Multisig post-kex round: expected common pubkey wasn't found in input messages."); // save keys that should be recommended to other signers // - for convenience, re-recommend the post-kex verification message once an account is complete @@ -790,7 +849,8 @@ namespace multisig //---------------------------------------------------------------------------------------------------------------------- // multisig_account: INTERNAL //---------------------------------------------------------------------------------------------------------------------- - void multisig_account::kex_update_impl(const std::vector &expanded_msgs) + void multisig_account::kex_update_impl(const std::vector &expanded_msgs, + const bool incomplete_signer_set) { // check messages are for the expected kex round check_messages_round(expanded_msgs, m_kex_rounds_complete + 1); @@ -816,6 +876,7 @@ namespace multisig m_signers, expanded_msgs, exclude_pubkeys, + incomplete_signer_set, result_keys_to_origins_map); // finish account update diff --git a/src/multisig/multisig_clsag_context.cpp b/src/multisig/multisig_clsag_context.cpp new file mode 100644 index 00000000000..81e47128770 --- /dev/null +++ b/src/multisig/multisig_clsag_context.cpp @@ -0,0 +1,257 @@ +// Copyright (c) 2021-2023, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include "multisig_clsag_context.h" + +#include "int-util.h" + +#include "crypto/crypto.h" +#include "cryptonote_config.h" +#include "ringct/rctOps.h" +#include "ringct/rctTypes.h" + +#include +#include +#include + +#undef MONERO_DEFAULT_LOG_CATEGORY +#define MONERO_DEFAULT_LOG_CATEGORY "multisig" + +namespace multisig { + +namespace signing { +//---------------------------------------------------------------------------------------------------------------------- +//---------------------------------------------------------------------------------------------------------------------- +template +static rct::key string_to_key(const unsigned char (&str)[N]) { + rct::key tmp{}; + static_assert(sizeof(tmp.bytes) >= N, ""); + std::memcpy(tmp.bytes, str, N); + return tmp; +} +//---------------------------------------------------------------------------------------------------------------------- +//---------------------------------------------------------------------------------------------------------------------- +static void encode_int_to_key_le(const unsigned int i, rct::key &k_out) +{ + static_assert(sizeof(unsigned int) <= sizeof(std::uint64_t), "unsigned int max too large"); + static_assert(sizeof(std::uint64_t) <= sizeof(rct::key), ""); + std::uint64_t temp_i{SWAP64LE(i)}; + std::memcpy(k_out.bytes, &temp_i, sizeof(temp_i)); +} +//---------------------------------------------------------------------------------------------------------------------- +//---------------------------------------------------------------------------------------------------------------------- +bool CLSAG_context_t::init( + const rct::keyV& P, + const rct::keyV& C_nonzero, + const rct::key& C_offset, + const rct::key& message, + const rct::key& I, + const rct::key& D, + const unsigned int l, + const rct::keyV& s, + const std::size_t num_alpha_components +) +{ + initialized = false; + + n = P.size(); + if (n <= 0) + return false; + if (C_nonzero.size() != n) + return false; + if (s.size() != n) + return false; + if (l >= n) + return false; + + c_params.clear(); + c_params.reserve(n * 2 + 5); + b_params.clear(); + b_params.reserve(n * 3 + 2 * num_alpha_components + 7); + + c_params.push_back(string_to_key(config::HASH_KEY_CLSAG_ROUND)); + b_params.push_back(string_to_key(config::HASH_KEY_CLSAG_ROUND_MULTISIG)); + c_params.insert(c_params.end(), P.begin(), P.end()); + b_params.insert(b_params.end(), P.begin(), P.end()); + c_params.insert(c_params.end(), C_nonzero.begin(), C_nonzero.end()); + b_params.insert(b_params.end(), C_nonzero.begin(), C_nonzero.end()); + c_params.emplace_back(C_offset); + b_params.emplace_back(C_offset); + c_params.emplace_back(message); + b_params.emplace_back(message); + c_params_L_offset = c_params.size(); + b_params_L_offset = b_params.size(); + c_params.resize(c_params.size() + 1); //this is where L will be inserted later + b_params.resize(b_params.size() + num_alpha_components); //multisig aggregate public nonces for L will be inserted here later + c_params_R_offset = c_params.size(); + b_params_R_offset = b_params.size(); + c_params.resize(c_params.size() + 1); //this is where R will be inserted later + b_params.resize(b_params.size() + num_alpha_components); //multisig aggregate public nonces for R will be inserted here later + b_params.emplace_back(I); + b_params.emplace_back(D); + b_params.insert(b_params.end(), s.begin(), s.begin() + l); //fake responses before 'l' + b_params.insert(b_params.end(), s.begin() + l + 1, s.end()); //fake responses after 'l' + b_params.emplace_back(); + encode_int_to_key_le(l, b_params.back()); //real signing index 'l' + b_params.emplace_back(); + encode_int_to_key_le(num_alpha_components, b_params.back()); //number of parallel nonces + b_params.emplace_back(); + encode_int_to_key_le(n, b_params.back()); //number of ring members + + rct::keyV mu_P_params; + rct::keyV mu_C_params; + mu_P_params.reserve(n * 2 + 4); + mu_C_params.reserve(n * 2 + 4); + + mu_P_params.push_back(string_to_key(config::HASH_KEY_CLSAG_AGG_0)); + mu_C_params.push_back(string_to_key(config::HASH_KEY_CLSAG_AGG_1)); + mu_P_params.insert(mu_P_params.end(), P.begin(), P.end()); + mu_C_params.insert(mu_C_params.end(), P.begin(), P.end()); + mu_P_params.insert(mu_P_params.end(), C_nonzero.begin(), C_nonzero.end()); + mu_C_params.insert(mu_C_params.end(), C_nonzero.begin(), C_nonzero.end()); + mu_P_params.emplace_back(I); + mu_C_params.emplace_back(I); + mu_P_params.emplace_back(scalarmultKey(D, rct::INV_EIGHT)); + mu_C_params.emplace_back(mu_P_params.back()); + mu_P_params.emplace_back(C_offset); + mu_C_params.emplace_back(C_offset); + mu_P = hash_to_scalar(mu_P_params); + mu_C = hash_to_scalar(mu_C_params); + + rct::geDsmp I_precomp; + rct::geDsmp D_precomp; + rct::precomp(I_precomp.k, I); + rct::precomp(D_precomp.k, D); + rct::key wH_l; + rct::addKeys3(wH_l, mu_P, I_precomp.k, mu_C, D_precomp.k); + rct::precomp(wH_l_precomp.k, wH_l); + W_precomp.resize(n); + H_precomp.resize(n); + for (std::size_t i = 0; i < n; ++i) { + rct::geDsmp P_precomp; + rct::geDsmp C_precomp; + rct::key C; + rct::subKeys(C, C_nonzero[i], C_offset); + rct::precomp(P_precomp.k, P[i]); + rct::precomp(C_precomp.k, C); + rct::key W; + rct::addKeys3(W, mu_P, P_precomp.k, mu_C, C_precomp.k); + rct::precomp(W_precomp[i].k, W); + ge_p3 Hi_p3; + rct::hash_to_p3(Hi_p3, P[i]); + ge_dsm_precomp(H_precomp[i].k, &Hi_p3); + } + rct::precomp(G_precomp.k, rct::G); + this->l = l; + this->s = s; + this->num_alpha_components = num_alpha_components; + + initialized = true; + return true; +} +//---------------------------------------------------------------------------------------------------------------------- +bool CLSAG_context_t::combine_alpha_and_compute_challenge( + const rct::keyV& total_alpha_G, + const rct::keyV& total_alpha_H, + const rct::keyV& alpha, + rct::key& alpha_combined, + rct::key& c_0, + rct::key& c +) +{ + if (not initialized) + return false; + + if (num_alpha_components != total_alpha_G.size()) + return false; + if (num_alpha_components != total_alpha_H.size()) + return false; + if (num_alpha_components != alpha.size()) + return false; + + // insert aggregate public nonces for L and R components + for (std::size_t i = 0; i < num_alpha_components; ++i) { + b_params[b_params_L_offset + i] = total_alpha_G[i]; + b_params[b_params_R_offset + i] = total_alpha_H[i]; + } + + // musig2-style combination factor 'b' + const rct::key b = rct::hash_to_scalar(b_params); + + // 1) store combined public nonces in the 'L' and 'R' slots for computing the initial challenge + // - L = sum_i(b^i total_alpha_G[i]) + // - R = sum_i(b^i total_alpha_H[i]) + // 2) compute the local signer's combined private nonce + // - alpha_combined = sum_i(b^i * alpha[i]) + rct::key& L_l = c_params[c_params_L_offset]; + rct::key& R_l = c_params[c_params_R_offset]; + rct::key b_i = rct::identity(); + L_l = rct::identity(); + R_l = rct::identity(); + alpha_combined = rct::zero(); + for (std::size_t i = 0; i < num_alpha_components; ++i) { + rct::addKeys(L_l, L_l, rct::scalarmultKey(total_alpha_G[i], b_i)); + rct::addKeys(R_l, R_l, rct::scalarmultKey(total_alpha_H[i], b_i)); + sc_muladd(alpha_combined.bytes, alpha[i].bytes, b_i.bytes, alpha_combined.bytes); + sc_mul(b_i.bytes, b_i.bytes, b.bytes); + } + + // compute initial challenge from real spend components + c = rct::hash_to_scalar(c_params); + + // 1) c_0: find the CLSAG's challenge for index '0', which will be stored in the proof + // note: in the CLSAG implementation in ringct/rctSigs, c_0 is denoted 'c1' (a notation error) + // 2) c: find the final challenge for the multisig signers to respond to + for (std::size_t i = (l + 1) % n; i != l; i = (i + 1) % n) { + if (i == 0) + c_0 = c; + rct::addKeys3(c_params[c_params_L_offset], s[i], G_precomp.k, c, W_precomp[i].k); + rct::addKeys3(c_params[c_params_R_offset], s[i], H_precomp[i].k, c, wH_l_precomp.k); + c = rct::hash_to_scalar(c_params); + } + if (l == 0) + c_0 = c; + + return true; +} +//---------------------------------------------------------------------------------------------------------------------- +bool CLSAG_context_t::get_mu( + rct::key& mu_P, + rct::key& mu_C +) const +{ + if (not initialized) + return false; + mu_P = this->mu_P; + mu_C = this->mu_C; + return true; +} +//---------------------------------------------------------------------------------------------------------------------- +} //namespace signing + +} //namespace multisig diff --git a/src/multisig/multisig_clsag_context.h b/src/multisig/multisig_clsag_context.h new file mode 100644 index 00000000000..c184b83d618 --- /dev/null +++ b/src/multisig/multisig_clsag_context.h @@ -0,0 +1,137 @@ +// Copyright (c) 2021-2023, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +//// +// References +// - CLSAG (base signature scheme): https://eprint.iacr.org/2019/654 +// - MuSig2 (style for multisig signing): https://eprint.iacr.org/2020/1261 +/// + + +#pragma once + +#include "ringct/rctTypes.h" + +#include + + +namespace multisig { + +namespace signing { + +class CLSAG_context_t final { +private: + // is the CLSAG context initialized? + bool initialized; + // challenge components: c = H(domain-separator, {P}, {C}, C_offset, message, L, R) + rct::keyV c_params; + // indices in c_params where L and R will be + std::size_t c_params_L_offset; + std::size_t c_params_R_offset; + // musig2-style nonce combination factor components for multisig signing + // b = H(domain-separator, {P}, {C}, C_offset, message, {L_combined_alphas}, {R_combined_alphas}, I, D, {s_non_l}, l, k, n) + // - {P} = ring of one-time addresses + // - {C} = ring of amount commitments (1:1 with one-time addresses) + // - C_offset = pseudo-output commitment to offset all amount commitments with + // - message = message the CLSAG will sign + // - {L_combined_alphas} = set of summed-together public nonces from all multisig signers for this CLSAG's L component + // - {R_combined_alphas} = set of summed-together public nonces from all multisig signers for this CLSAG's R component + // - I = key image for one-time address at {P}[l] + // - D = auxiliary key image for the offsetted amount commitment '{C}[l] - C_offset' + // - {s_non_l} = fake responses for this proof + // - l = real signing index in {P} and '{C} - C_offset' + // - k = number of parallel nonces that each participant provides + // - n = number of ring members + rct::keyV b_params; + // indices in b_params where L and R 'alpha' components will be + std::size_t b_params_L_offset; + std::size_t b_params_R_offset; + // CLSAG 'concise' coefficients for {P} and '{C} - C_offset' + // mu_x = H(domain-separator, {P}, {C}, I, (1/8)*D, C_offset) + // - note: 'D' is stored in the form '(1/8)*D' in transaction data + rct::key mu_P; + rct::key mu_C; + // ring size + std::size_t n; + // aggregate key image: mu_P*I + mu_C*D + rct::geDsmp wH_l_precomp; + // aggregate ring members: mu_P*P_i + mu_C*(C_i - C_offset) + std::vector W_precomp; + // key image component base keys: H_p(P_i) + std::vector H_precomp; + // cache for later: generator 'G' in 'precomp' representation + rct::geDsmp G_precomp; + // real signing index in this CLSAG + std::size_t l; + // signature responses + rct::keyV s; + // number of signing nonces expected per signer + std::size_t num_alpha_components; +public: + CLSAG_context_t() : initialized{false} {} + + // prepare CLSAG challenge context + bool init( + const rct::keyV& P, + const rct::keyV& C_nonzero, + const rct::key& C_offset, + const rct::key& message, + const rct::key& I, + const rct::key& D, + const unsigned int l, + const rct::keyV& s, + const std::size_t num_alpha_components + ); + + // get the local signer's combined musig2-style private nonce and compute the CLSAG challenge + bool combine_alpha_and_compute_challenge( + // set of summed-together musig2-style public nonces from all multisig signers for this CLSAG's L component + const rct::keyV& total_alpha_G, + // set of summed-together musig2-style public nonces from all multisig signers for this CLSAG's R component + const rct::keyV& total_alpha_H, + // local signer's private musig2-style nonces + const rct::keyV& alpha, + // local signer's final private nonce, using musig2-style combination with factor 'b' + // alpha_combined = sum_i(b^i * alpha[i]) + rct::key& alpha_combined, + // CLSAG challenge to store in the proof + rct::key& c_0, + // final CLSAG challenge to respond to (need this to make multisig partial signatures) + rct::key& c + ); + + // getter for CLSAG 'concise' coefficients + bool get_mu( + rct::key& mu_P, + rct::key& mu_C + ) const; +}; + +} //namespace signing + +} //namespace multisig diff --git a/src/multisig/multisig_kex_msg.cpp b/src/multisig/multisig_kex_msg.cpp index c717e23ad03..49350948a49 100644 --- a/src/multisig/multisig_kex_msg.cpp +++ b/src/multisig/multisig_kex_msg.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2021-2022, The Monero Project +// Copyright (c) 2021-2023, The Monero Project // // All rights reserved. // diff --git a/src/multisig/multisig_kex_msg.h b/src/multisig/multisig_kex_msg.h index 833c7c8f6a3..8cb8faa7ce6 100644 --- a/src/multisig/multisig_kex_msg.h +++ b/src/multisig/multisig_kex_msg.h @@ -1,4 +1,4 @@ -// Copyright (c) 2021-2022, The Monero Project +// Copyright (c) 2021-2023, The Monero Project // // All rights reserved. // diff --git a/src/multisig/multisig_kex_msg_serialization.h b/src/multisig/multisig_kex_msg_serialization.h index e5558cdffb5..d1e07ea8ae5 100644 --- a/src/multisig/multisig_kex_msg_serialization.h +++ b/src/multisig/multisig_kex_msg_serialization.h @@ -1,4 +1,4 @@ -// Copyright (c) 2021-2022, The Monero Project +// Copyright (c) 2021-2023, The Monero Project // // All rights reserved. // diff --git a/src/multisig/multisig_tx_builder_ringct.cpp b/src/multisig/multisig_tx_builder_ringct.cpp new file mode 100644 index 00000000000..8643a8af40a --- /dev/null +++ b/src/multisig/multisig_tx_builder_ringct.cpp @@ -0,0 +1,1059 @@ +// Copyright (c) 2021-2023, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include "multisig_tx_builder_ringct.h" + +#include "int-util.h" +#include "memwipe.h" + +#include "cryptonote_basic/cryptonote_basic.h" +#include "cryptonote_basic/account.h" +#include "cryptonote_basic/cryptonote_format_utils.h" +#include "cryptonote_config.h" +#include "cryptonote_core/cryptonote_tx_utils.h" +#include "device/device.hpp" +#include "multisig_clsag_context.h" +#include "ringct/bulletproofs.h" +#include "ringct/bulletproofs_plus.h" +#include "ringct/rctSigs.h" + +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +#undef MONERO_DEFAULT_LOG_CATEGORY +#define MONERO_DEFAULT_LOG_CATEGORY "multisig" + +namespace multisig { + +namespace signing { +//---------------------------------------------------------------------------------------------------------------------- +//---------------------------------------------------------------------------------------------------------------------- +bool view_tag_required(const int bp_version) +{ + // view tags were introduced at the same time as BP+, so they are needed after BP+ (v4 and later) + if (bp_version <= 3) + return false; + else + return true; +} +//---------------------------------------------------------------------------------------------------------------------- +//---------------------------------------------------------------------------------------------------------------------- +static void sort_sources( + std::vector& sources +) +{ + std::sort(sources.begin(), sources.end(), [](const auto& lhs, const auto& rhs){ + const rct::key& ki0 = lhs.multisig_kLRki.ki; + const rct::key& ki1 = rhs.multisig_kLRki.ki; + return memcmp(&ki0, &ki1, sizeof(rct::key)) > 0; + }); +} +//---------------------------------------------------------------------------------------------------------------------- +//---------------------------------------------------------------------------------------------------------------------- +static bool compute_keys_for_sources( + const cryptonote::account_keys& account_keys, + const std::vector& sources, + const std::uint32_t subaddr_account, + const std::set& subaddr_minor_indices, + rct::keyV& input_secret_keys +) +{ + const std::size_t num_sources = sources.size(); + hw::device& hwdev = account_keys.get_device(); + std::unordered_map subaddresses; + for (const std::uint32_t minor_index: subaddr_minor_indices) { + subaddresses[hwdev.get_subaddress_spend_public_key( + account_keys, + {subaddr_account, minor_index} + )] = {subaddr_account, minor_index}; + } + input_secret_keys.resize(num_sources); + for (std::size_t i = 0; i < num_sources; ++i) { + const auto& src = sources[i]; + crypto::key_image tmp_key_image; + cryptonote::keypair tmp_keys; + if (src.real_output >= src.outputs.size()) + return false; + if (not cryptonote::generate_key_image_helper( + account_keys, + subaddresses, + rct::rct2pk(src.outputs[src.real_output].second.dest), + src.real_out_tx_key, + src.real_out_additional_tx_keys, + src.real_output_in_tx_index, + tmp_keys, + tmp_key_image, + hwdev + )) { + return false; + } + input_secret_keys[i] = rct::sk2rct(tmp_keys.sec); + } + return true; +} +//---------------------------------------------------------------------------------------------------------------------- +//---------------------------------------------------------------------------------------------------------------------- +static void shuffle_destinations( + std::vector& destinations +) +{ + std::shuffle(destinations.begin(), destinations.end(), crypto::random_device{}); +} +//---------------------------------------------------------------------------------------------------------------------- +//---------------------------------------------------------------------------------------------------------------------- +static bool set_tx_extra( + const cryptonote::account_keys& account_keys, + const std::vector& destinations, + const cryptonote::tx_destination_entry& change, + const crypto::secret_key& tx_secret_key, + const crypto::public_key& tx_public_key, + const std::vector& tx_aux_public_keys, + const std::vector& extra, + cryptonote::transaction& tx +) +{ + hw::device &hwdev = account_keys.get_device(); + tx.extra = extra; + // if we have a stealth payment id, find it and encrypt it with the tx key now + std::vector tx_extra_fields; + if (cryptonote::parse_tx_extra(tx.extra, tx_extra_fields)) + { + bool add_dummy_payment_id = true; + cryptonote::tx_extra_nonce extra_nonce; + if (cryptonote::find_tx_extra_field_by_type(tx_extra_fields, extra_nonce)) + { + crypto::hash payment_id = crypto::null_hash; + crypto::hash8 payment_id8 = crypto::null_hash8; + if (cryptonote::get_encrypted_payment_id_from_tx_extra_nonce(extra_nonce.nonce, payment_id8)) + { + LOG_PRINT_L2("Encrypting payment id " << payment_id8); + crypto::public_key view_key_pub = cryptonote::get_destination_view_key_pub(destinations, change.addr); + if (view_key_pub == crypto::null_pkey) + { + // valid combinations: + // - 1 output with encrypted payment ID, dummy change output (0 amount) + // - 0 outputs, 1 change output with encrypted payment ID + // - 1 output with encrypted payment ID, 1 change output + LOG_ERROR("Destinations have to have exactly one output to support encrypted payment ids"); + return false; + } + + if (!hwdev.encrypt_payment_id(payment_id8, view_key_pub, tx_secret_key)) + { + LOG_ERROR("Failed to encrypt payment id"); + return false; + } + + std::string extra_nonce_updated; + cryptonote::set_encrypted_payment_id_to_tx_extra_nonce(extra_nonce_updated, payment_id8); + cryptonote::remove_field_from_tx_extra(tx.extra, typeid(cryptonote::tx_extra_nonce)); + if (!cryptonote::add_extra_nonce_to_tx_extra(tx.extra, extra_nonce_updated)) + { + LOG_ERROR("Failed to add encrypted payment id to tx extra"); + return false; + } + LOG_PRINT_L1("Encrypted payment ID: " << payment_id8); + add_dummy_payment_id = false; + } + else if (cryptonote::get_payment_id_from_tx_extra_nonce(extra_nonce.nonce, payment_id)) + { + add_dummy_payment_id = false; + } + } + + // we don't add one if we've got more than the usual 1 destination plus change + if (destinations.size() > 2) + add_dummy_payment_id = false; + + if (add_dummy_payment_id) + { + // if we have neither long nor short payment id, add a dummy short one, + // this should end up being the vast majority of txes as time goes on + std::string extra_nonce_updated; + crypto::hash8 payment_id8 = crypto::null_hash8; + crypto::public_key view_key_pub = cryptonote::get_destination_view_key_pub(destinations, change.addr); + if (view_key_pub == crypto::null_pkey) + { + LOG_ERROR("Failed to get key to encrypt dummy payment id with"); + } + else + { + hwdev.encrypt_payment_id(payment_id8, view_key_pub, tx_secret_key); + cryptonote::set_encrypted_payment_id_to_tx_extra_nonce(extra_nonce_updated, payment_id8); + if (!cryptonote::add_extra_nonce_to_tx_extra(tx.extra, extra_nonce_updated)) + { + LOG_ERROR("Failed to add dummy encrypted payment id to tx extra"); + // continue anyway + } + } + } + } + else + { + MWARNING("Failed to parse tx extra"); + tx_extra_fields.clear(); + } + + cryptonote::remove_field_from_tx_extra(tx.extra, typeid(cryptonote::tx_extra_pub_key)); + cryptonote::add_tx_pub_key_to_extra(tx.extra, tx_public_key); + cryptonote::remove_field_from_tx_extra(tx.extra, typeid(cryptonote::tx_extra_additional_pub_keys)); + LOG_PRINT_L2("tx pubkey: " << tx_public_key); + if (tx_aux_public_keys.size()) + { + LOG_PRINT_L2("additional tx pubkeys: "); + for (size_t i = 0; i < tx_aux_public_keys.size(); ++i) + LOG_PRINT_L2(tx_aux_public_keys[i]); + cryptonote::add_additional_tx_pub_keys_to_extra(tx.extra, tx_aux_public_keys); + } + if (not cryptonote::sort_tx_extra(tx.extra, tx.extra)) + return false; + return true; +} +//---------------------------------------------------------------------------------------------------------------------- +//---------------------------------------------------------------------------------------------------------------------- +static void make_tx_secret_key_seed(const crypto::secret_key& tx_secret_key_entropy, + const std::vector& sources, + crypto::secret_key& tx_secret_key_seed) +{ + // seed = H(H("domain separator"), entropy, {KI}) + static const std::string domain_separator{config::HASH_KEY_MULTISIG_TX_PRIVKEYS_SEED}; + + rct::keyV hash_context; + hash_context.reserve(2 + sources.size()); + auto hash_context_wiper = epee::misc_utils::create_scope_leave_handler([&]{ + memwipe(hash_context.data(), hash_context.size()); + }); + hash_context.emplace_back(); + rct::cn_fast_hash(hash_context.back(), domain_separator.data(), domain_separator.size()); //domain sep + hash_context.emplace_back(rct::sk2rct(tx_secret_key_entropy)); //entropy + + for (const cryptonote::tx_source_entry& source : sources) + hash_context.emplace_back(source.multisig_kLRki.ki); //{KI} + + // set the seed + tx_secret_key_seed = rct::rct2sk(rct::cn_fast_hash(hash_context)); +} +//---------------------------------------------------------------------------------------------------------------------- +//---------------------------------------------------------------------------------------------------------------------- +static void make_tx_secret_keys(const crypto::secret_key& tx_secret_key_seed, + const std::size_t num_tx_keys, + std::vector& tx_secret_keys) +{ + // make tx secret keys as a hash chain of the seed + // h1 = H_n(seed || H("domain separator")) + // h2 = H_n(seed || h1) + // h3 = H_n(seed || h2) + // ... + static const std::string domain_separator{config::HASH_KEY_MULTISIG_TX_PRIVKEYS}; + + rct::keyV hash_context; + hash_context.resize(2); + auto hash_context_wiper = epee::misc_utils::create_scope_leave_handler([&]{ + memwipe(hash_context.data(), hash_context.size()); + }); + hash_context[0] = rct::sk2rct(tx_secret_key_seed); + rct::cn_fast_hash(hash_context[1], domain_separator.data(), domain_separator.size()); + + tx_secret_keys.clear(); + tx_secret_keys.resize(num_tx_keys); + + for (crypto::secret_key& tx_secret_key : tx_secret_keys) + { + // advance the hash chain + hash_context[1] = rct::hash_to_scalar(hash_context); + + // set this key + tx_secret_key = rct::rct2sk(hash_context[1]); + } +} +//---------------------------------------------------------------------------------------------------------------------- +//---------------------------------------------------------------------------------------------------------------------- +static bool collect_tx_secret_keys(const std::vector& tx_secret_keys, + crypto::secret_key& tx_secret_key, + std::vector& tx_aux_secret_keys) +{ + if (tx_secret_keys.size() == 0) + return false; + + tx_secret_key = tx_secret_keys[0]; + tx_aux_secret_keys.clear(); + tx_aux_secret_keys.reserve(tx_secret_keys.size() - 1); + for (std::size_t tx_key_index{1}; tx_key_index < tx_secret_keys.size(); ++tx_key_index) + tx_aux_secret_keys.emplace_back(tx_secret_keys[tx_key_index]); + + return true; +} +//---------------------------------------------------------------------------------------------------------------------- +//---------------------------------------------------------------------------------------------------------------------- +static bool compute_keys_for_destinations( + const cryptonote::account_keys& account_keys, + const std::uint32_t subaddr_account, + const std::vector& destinations, + const cryptonote::tx_destination_entry& change, + const std::vector& extra, + const bool use_view_tags, + const bool reconstruction, + const crypto::secret_key& tx_secret_key_seed, + crypto::secret_key& tx_secret_key, + std::vector& tx_aux_secret_keys, + rct::keyV& output_public_keys, + rct::keyV& output_amount_secret_keys, + std::vector& view_tags, + cryptonote::transaction& unsigned_tx +) +{ + hw::device &hwdev = account_keys.get_device(); + + // check non-zero change amount case + if (change.amount > 0) + { + // the change output must be directed to the local account + if (change.addr != hwdev.get_subaddress(account_keys, {subaddr_account})) + return false; + + // expect the change destination to be in the destination set + if (std::find_if(destinations.begin(), destinations.end(), + [&change](const auto &destination) -> bool + { + return destination.addr == change.addr; + }) == destinations.end()) + return false; + } + + // collect non-change recipients into normal/subaddress buckets + std::unordered_set unique_subbaddr_recipients; + std::unordered_set unique_std_recipients; + for(const auto& dst_entr: destinations) { + if (dst_entr.addr == change.addr) + continue; + if (dst_entr.is_subaddress) + unique_subbaddr_recipients.insert(dst_entr.addr); + else + unique_std_recipients.insert(dst_entr.addr); + } + + // figure out how many tx secret keys are needed + // - tx aux keys: add if there are > 1 non-change recipients, with at least one to a subaddress + const std::size_t num_destinations = destinations.size(); + const bool need_tx_aux_keys = unique_subbaddr_recipients.size() + bool(unique_std_recipients.size()) > 1; + + const std::size_t num_tx_keys = 1 + (need_tx_aux_keys ? num_destinations : 0); + + // make tx secret keys + std::vector all_tx_secret_keys; + make_tx_secret_keys(tx_secret_key_seed, num_tx_keys, all_tx_secret_keys); + + // split up tx secret keys + crypto::secret_key tx_secret_key_temp; + std::vector tx_aux_secret_keys_temp; + if (not collect_tx_secret_keys(all_tx_secret_keys, tx_secret_key_temp, tx_aux_secret_keys_temp)) + return false; + + if (reconstruction) + { + // when reconstructing, the tx secret keys should be reproducible from input seed + if (!(tx_secret_key == tx_secret_key_temp)) + return false; + if (!(tx_aux_secret_keys == tx_aux_secret_keys_temp)) + return false; + } + else + { + tx_secret_key = tx_secret_key_temp; + tx_aux_secret_keys = std::move(tx_aux_secret_keys_temp); + } + + // tx pub key: R + crypto::public_key tx_public_key; + if (unique_std_recipients.empty() && unique_subbaddr_recipients.size() == 1) { + // if there is exactly 1 non-change recipient, and it's to a subaddress, then the tx pubkey = r*Ksi_nonchange_recipient + tx_public_key = rct::rct2pk( + hwdev.scalarmultKey( + rct::pk2rct(unique_subbaddr_recipients.begin()->m_spend_public_key), + rct::sk2rct(tx_secret_key) + )); + } + else { + // otherwise, the tx pub key = r*G + // - if there are > 1 non-change recipients, with at least one to a subaddress, then the tx pubkey is not used + // (additional tx keys will be used instead) + // - if all non-change recipients are to normal addresses, then the tx pubkey will be used by all recipients + // (including change recipient, even if change is to a subaddress) + tx_public_key = rct::rct2pk(hwdev.scalarmultBase(rct::sk2rct(tx_secret_key))); + } + + // additional tx pubkeys: R_t + output_public_keys.resize(num_destinations); + view_tags.resize(num_destinations); + std::vector tx_aux_public_keys; + crypto::public_key temp_output_public_key; + + for (std::size_t i = 0; i < num_destinations; ++i) { + if (not hwdev.generate_output_ephemeral_keys( + unsigned_tx.version, + account_keys, + tx_public_key, + tx_secret_key, + destinations[i], + change.addr, + i, + need_tx_aux_keys, + tx_aux_secret_keys, + tx_aux_public_keys, + output_amount_secret_keys, + temp_output_public_key, + use_view_tags, + view_tags[i] //unused variable if use_view_tags is not set + )) { + return false; + } + output_public_keys[i] = rct::pk2rct(temp_output_public_key); + } + + if (num_destinations != output_amount_secret_keys.size()) + return false; + + CHECK_AND_ASSERT_MES( + tx_aux_public_keys.size() == tx_aux_secret_keys.size(), + false, + "Internal error creating additional public keys" + ); + + if (not set_tx_extra(account_keys, destinations, change, tx_secret_key, tx_public_key, tx_aux_public_keys, extra, unsigned_tx)) + return false; + + return true; +} +//---------------------------------------------------------------------------------------------------------------------- +//---------------------------------------------------------------------------------------------------------------------- +static void set_tx_inputs( + const std::vector& sources, + cryptonote::transaction& unsigned_tx +) +{ + const std::size_t num_sources = sources.size(); + unsigned_tx.vin.resize(num_sources); + for (std::size_t i = 0; i < num_sources; ++i) { + std::vector offsets; + offsets.reserve(sources[i].outputs.size()); + for (const auto& e: sources[i].outputs) + offsets.emplace_back(e.first); + unsigned_tx.vin[i] = cryptonote::txin_to_key{ + .amount = 0, + .key_offsets = cryptonote::absolute_output_offsets_to_relative(offsets), + .k_image = rct::rct2ki(sources[i].multisig_kLRki.ki), + }; + } +} +//---------------------------------------------------------------------------------------------------------------------- +//---------------------------------------------------------------------------------------------------------------------- +static bool onetime_addresses_are_unique(const rct::keyV& output_public_keys) +{ + for (auto addr_it = output_public_keys.begin(); addr_it != output_public_keys.end(); ++addr_it) + { + if (std::find(output_public_keys.begin(), addr_it, *addr_it) != addr_it) + return false; + } + + return true; +} +//---------------------------------------------------------------------------------------------------------------------- +//---------------------------------------------------------------------------------------------------------------------- +static bool set_tx_outputs(const rct::keyV& output_public_keys, cryptonote::transaction& unsigned_tx) +{ + // sanity check: all onetime addresses should be unique + if (not onetime_addresses_are_unique(output_public_keys)) + return false; + + // set the tx outputs + const std::size_t num_destinations = output_public_keys.size(); + unsigned_tx.vout.resize(num_destinations); + for (std::size_t i = 0; i < num_destinations; ++i) + cryptonote::set_tx_out(0, rct::rct2pk(output_public_keys[i]), false, crypto::view_tag{}, unsigned_tx.vout[i]); + + return true; +} +//---------------------------------------------------------------------------------------------------------------------- +//---------------------------------------------------------------------------------------------------------------------- +static bool set_tx_outputs_with_view_tags( + const rct::keyV& output_public_keys, + const std::vector& view_tags, + cryptonote::transaction& unsigned_tx +) +{ + // sanity check: all onetime addresses should be unique + if (not onetime_addresses_are_unique(output_public_keys)) + return false; + + // set the tx outputs (with view tags) + const std::size_t num_destinations = output_public_keys.size(); + CHECK_AND_ASSERT_MES(view_tags.size() == num_destinations, false, + "multisig signing protocol: internal error, view tag size mismatch."); + unsigned_tx.vout.resize(num_destinations); + for (std::size_t i = 0; i < num_destinations; ++i) + cryptonote::set_tx_out(0, rct::rct2pk(output_public_keys[i]), true, view_tags[i], unsigned_tx.vout[i]); + + return true; +} +//---------------------------------------------------------------------------------------------------------------------- +//---------------------------------------------------------------------------------------------------------------------- +static void make_new_range_proofs(const int bp_version, + const std::vector& output_amounts, + const rct::keyV& output_amount_masks, + rct::rctSigPrunable& sigs) +{ + sigs.bulletproofs.clear(); + sigs.bulletproofs_plus.clear(); + + if (bp_version == 3) + sigs.bulletproofs.push_back(rct::bulletproof_PROVE(output_amounts, output_amount_masks)); + else if (bp_version == 4) + sigs.bulletproofs_plus.push_back(rct::bulletproof_plus_PROVE(output_amounts, output_amount_masks)); +} +//---------------------------------------------------------------------------------------------------------------------- +//---------------------------------------------------------------------------------------------------------------------- +static bool try_reconstruct_range_proofs(const int bp_version, + const rct::rctSigPrunable& original_sigs, + const std::size_t num_destinations, + const rct::ctkeyV& output_public_keys, + rct::rctSigPrunable& reconstructed_sigs) +{ + auto try_reconstruct_range_proofs = + [&](const auto &original_range_proofs, auto &new_range_proofs) -> bool + { + if (original_range_proofs.size() != 1) + return false; + + new_range_proofs = original_range_proofs; + new_range_proofs[0].V.resize(num_destinations); + for (std::size_t i = 0; i < num_destinations; ++i) + new_range_proofs[0].V[i] = rct::scalarmultKey(output_public_keys[i].mask, rct::INV_EIGHT); + + return true; + }; + + if (bp_version == 3) + { + if (not try_reconstruct_range_proofs(original_sigs.bulletproofs, reconstructed_sigs.bulletproofs)) + return false; + return rct::bulletproof_VERIFY(reconstructed_sigs.bulletproofs); + } + else if (bp_version == 4) + { + if (not try_reconstruct_range_proofs(original_sigs.bulletproofs_plus, reconstructed_sigs.bulletproofs_plus)) + return false; + return rct::bulletproof_plus_VERIFY(reconstructed_sigs.bulletproofs_plus); + } + + return false; +} +//---------------------------------------------------------------------------------------------------------------------- +//---------------------------------------------------------------------------------------------------------------------- +static bool set_tx_rct_signatures( + const std::uint64_t fee, + const std::vector& sources, + const std::vector& destinations, + const rct::keyV& input_secret_keys, + const rct::keyV& output_public_keys, + const rct::keyV& output_amount_secret_keys, + const rct::RCTConfig& rct_config, + const bool reconstruction, + cryptonote::transaction& unsigned_tx, + std::vector& CLSAG_contexts, + rct::keyV& cached_w +) +{ + if (rct_config.bp_version != 3 && + rct_config.bp_version != 4) + return false; + if (rct_config.range_proof_type != rct::RangeProofPaddedBulletproof) + return false; + + const std::size_t num_destinations = destinations.size(); + const std::size_t num_sources = sources.size(); + + // rct_signatures component of tx + rct::rctSig rv{}; + + // set misc. fields + if (rct_config.bp_version == 3) + rv.type = rct::RCTTypeCLSAG; + else if (rct_config.bp_version == 4) + rv.type = rct::RCTTypeBulletproofPlus; + else + return false; + rv.txnFee = fee; + rv.message = rct::hash2rct(cryptonote::get_transaction_prefix_hash(unsigned_tx)); + + // define outputs + std::vector output_amounts(num_destinations); + rct::keyV output_amount_masks(num_destinations); + rv.ecdhInfo.resize(num_destinations); + rv.outPk.resize(num_destinations); + for (std::size_t i = 0; i < num_destinations; ++i) { + rv.outPk[i].dest = output_public_keys[i]; + output_amounts[i] = destinations[i].amount; + output_amount_masks[i] = genCommitmentMask(output_amount_secret_keys[i]); + rv.ecdhInfo[i].amount = rct::d2h(output_amounts[i]); + rct::addKeys2( + rv.outPk[i].mask, + output_amount_masks[i], + rv.ecdhInfo[i].amount, + rct::H + ); + rct::ecdhEncode(rv.ecdhInfo[i], output_amount_secret_keys[i], true); + } + + // output range proofs + if (not reconstruction) { + make_new_range_proofs(rct_config.bp_version, output_amounts, output_amount_masks, rv.p); + } + else { + if (not try_reconstruct_range_proofs(rct_config.bp_version, + unsigned_tx.rct_signatures.p, + num_destinations, + rv.outPk, + rv.p)) + return false; + } + + // prepare rings for input CLSAGs + rv.mixRing.resize(num_sources); + for (std::size_t i = 0; i < num_sources; ++i) { + const std::size_t ring_size = sources[i].outputs.size(); + rv.mixRing[i].resize(ring_size); + for (std::size_t j = 0; j < ring_size; ++j) { + rv.mixRing[i][j].dest = sources[i].outputs[j].second.dest; + rv.mixRing[i][j].mask = sources[i].outputs[j].second.mask; + } + } + + // make pseudo-output commitments + rct::keyV a; //pseudo-output commitment blinding factors + auto a_wiper = epee::misc_utils::create_scope_leave_handler([&]{ + memwipe(static_cast(a.data()), a.size() * sizeof(rct::key)); + }); + if (not reconstruction) { + a.resize(num_sources); + rv.p.pseudoOuts.resize(num_sources); + a[num_sources - 1] = rct::zero(); + for (std::size_t i = 0; i < num_destinations; ++i) { + sc_add( + a[num_sources - 1].bytes, + a[num_sources - 1].bytes, + output_amount_masks[i].bytes + ); + } + for (std::size_t i = 0; i < num_sources - 1; ++i) { + rct::skGen(a[i]); + sc_sub( + a[num_sources - 1].bytes, + a[num_sources - 1].bytes, + a[i].bytes + ); + rct::genC(rv.p.pseudoOuts[i], a[i], sources[i].amount); + } + rct::genC( + rv.p.pseudoOuts[num_sources - 1], + a[num_sources - 1], + sources[num_sources - 1].amount + ); + } + // check balance if reconstructing the tx + else { + rv.p.pseudoOuts = unsigned_tx.rct_signatures.p.pseudoOuts; + if (num_sources != rv.p.pseudoOuts.size()) + return false; + rct::key balance_accumulator = rct::scalarmultH(rct::d2h(fee)); + for (const auto& e: rv.outPk) + rct::addKeys(balance_accumulator, balance_accumulator, e.mask); + for (const auto& pseudoOut: rv.p.pseudoOuts) + rct::subKeys(balance_accumulator, balance_accumulator, pseudoOut); + if (not (balance_accumulator == rct::identity())) + return false; + } + + // prepare input CLSAGs for signing + const rct::key message = get_pre_mlsag_hash(rv, hw::get_device("default")); + + rv.p.CLSAGs.resize(num_sources); + if (reconstruction) { + if (num_sources != unsigned_tx.rct_signatures.p.CLSAGs.size()) + return false; + } + + CLSAG_contexts.resize(num_sources); + if (not reconstruction) + cached_w.resize(num_sources); + + for (std::size_t i = 0; i < num_sources; ++i) { + const std::size_t ring_size = rv.mixRing[i].size(); + const rct::key& I = sources[i].multisig_kLRki.ki; + const std::size_t l = sources[i].real_output; + if (l >= ring_size) + return false; + rct::keyV& s = rv.p.CLSAGs[i].s; + const rct::key& C_offset = rv.p.pseudoOuts[i]; + rct::keyV P(ring_size); + rct::keyV C_nonzero(ring_size); + + if (not reconstruction) { + s.resize(ring_size); + for (std::size_t j = 0; j < ring_size; ++j) { + if (j != l) + s[j] = rct::skGen(); //make fake responses + } + } + else { + if (ring_size != unsigned_tx.rct_signatures.p.CLSAGs[i].s.size()) + return false; + s = unsigned_tx.rct_signatures.p.CLSAGs[i].s; + } + + for (std::size_t j = 0; j < ring_size; ++j) { + P[j] = rv.mixRing[i][j].dest; + C_nonzero[j] = rv.mixRing[i][j].mask; + } + + rct::key D; + rct::key z; + auto z_wiper = epee::misc_utils::create_scope_leave_handler([&]{ + memwipe(static_cast(&z), sizeof(rct::key)); + }); + if (not reconstruction) { + sc_sub(z.bytes, sources[i].mask.bytes, a[i].bytes); //commitment to zero privkey + ge_p3 H_p3; + rct::hash_to_p3(H_p3, rv.mixRing[i][l].dest); + rct::key H_l; + ge_p3_tobytes(H_l.bytes, &H_p3); + D = rct::scalarmultKey(H_l, z); //auxilliary key image (for commitment to zero) + rv.p.CLSAGs[i].D = rct::scalarmultKey(D, rct::INV_EIGHT); + rv.p.CLSAGs[i].I = I; + } + else { + rv.p.CLSAGs[i].D = unsigned_tx.rct_signatures.p.CLSAGs[i].D; + rv.p.CLSAGs[i].I = I; + D = rct::scalarmultKey(rv.p.CLSAGs[i].D, rct::EIGHT); + } + + if (not CLSAG_contexts[i].init(P, C_nonzero, C_offset, message, I, D, l, s, kAlphaComponents)) + return false; + + if (not reconstruction) { + rct::key mu_P; + rct::key mu_C; + if (not CLSAG_contexts[i].get_mu(mu_P, mu_C)) + return false; + sc_mul(cached_w[i].bytes, mu_P.bytes, input_secret_keys[i].bytes); + sc_muladd(cached_w[i].bytes, mu_C.bytes, z.bytes, cached_w[i].bytes); + } + } + unsigned_tx.rct_signatures = std::move(rv); + return true; +} +//---------------------------------------------------------------------------------------------------------------------- +//---------------------------------------------------------------------------------------------------------------------- +static bool compute_tx_fee( + const std::vector& sources, + const std::vector& destinations, + std::uint64_t& fee +) +{ + boost::multiprecision::uint128_t in_amount = 0; + for (const auto& src: sources) + in_amount += src.amount; + + boost::multiprecision::uint128_t out_amount = 0; + for (const auto& dst: destinations) + out_amount += dst.amount; + + if (out_amount > in_amount) + return false; + + if (in_amount - out_amount > std::numeric_limits::max()) + return false; + + fee = static_cast(in_amount - out_amount); + return true; +} +//---------------------------------------------------------------------------------------------------------------------- +tx_builder_ringct_t::tx_builder_ringct_t(): initialized(false) {} +//---------------------------------------------------------------------------------------------------------------------- +tx_builder_ringct_t::~tx_builder_ringct_t() +{ + memwipe(static_cast(cached_w.data()), cached_w.size() * sizeof(rct::key)); +} +//---------------------------------------------------------------------------------------------------------------------- +bool tx_builder_ringct_t::init( + const cryptonote::account_keys& account_keys, + const std::vector& extra, + const std::uint64_t unlock_time, + const std::uint32_t subaddr_account, + const std::set& subaddr_minor_indices, + std::vector& sources, + std::vector& destinations, + const cryptonote::tx_destination_entry& change, + const rct::RCTConfig& rct_config, + const bool use_rct, + const bool reconstruction, + crypto::secret_key& tx_secret_key, + std::vector& tx_aux_secret_keys, + crypto::secret_key& tx_secret_key_entropy, + cryptonote::transaction& unsigned_tx +) +{ + initialized = false; + this->reconstruction = reconstruction; + if (not use_rct) + return false; + if (sources.empty()) + return false; + + if (not reconstruction) + unsigned_tx.set_null(); + + std::uint64_t fee; + if (not compute_tx_fee(sources, destinations, fee)) + return false; + + // decide if view tags are needed + const bool use_view_tags{view_tag_required(rct_config.bp_version)}; + + // misc. fields + unsigned_tx.version = 2; //rct = 2 + unsigned_tx.unlock_time = unlock_time; + + // sort inputs + sort_sources(sources); + + // prepare tx secret key seed (must be AFTER sorting sources) + // - deriving the seed from sources plus entropy ensures uniqueness for every new tx attempt + // - the goal is that two multisig txs added to the chain will never have outputs with the same onetime addresses, + // which would burn funds (embedding the inputs' key images guarantees this) + // - it is acceptable if two tx attempts use the same input set and entropy (only a malicious tx proposer will do + // that, but all it can accomplish is leaking information about the recipients - which a malicious proposer can + // easily do outside the signing ritual anyway) + if (not reconstruction) + tx_secret_key_entropy = rct::rct2sk(rct::skGen()); + + // expect not null (note: wallet serialization code may set this to null if handling an old partial tx) + if (tx_secret_key_entropy == crypto::null_skey) + return false; + + crypto::secret_key tx_secret_key_seed; + make_tx_secret_key_seed(tx_secret_key_entropy, sources, tx_secret_key_seed); + + // get secret keys for signing input CLSAGs (multisig: or for the initial partial signature) + rct::keyV input_secret_keys; + auto input_secret_keys_wiper = epee::misc_utils::create_scope_leave_handler([&]{ + memwipe(static_cast(input_secret_keys.data()), input_secret_keys.size() * sizeof(rct::key)); + }); + if (not compute_keys_for_sources(account_keys, sources, subaddr_account, subaddr_minor_indices, input_secret_keys)) + return false; + + // randomize output order + if (not reconstruction) + shuffle_destinations(destinations); + + // prepare outputs + rct::keyV output_public_keys; + rct::keyV output_amount_secret_keys; + std::vector view_tags; + auto output_amount_secret_keys_wiper = epee::misc_utils::create_scope_leave_handler([&]{ + memwipe(static_cast(output_amount_secret_keys.data()), output_amount_secret_keys.size() * sizeof(rct::key)); + }); + if (not compute_keys_for_destinations(account_keys, + subaddr_account, + destinations, + change, + extra, + use_view_tags, + reconstruction, + tx_secret_key_seed, + tx_secret_key, + tx_aux_secret_keys, + output_public_keys, + output_amount_secret_keys, + view_tags, + unsigned_tx)) + return false; + + // add inputs to tx + set_tx_inputs(sources, unsigned_tx); + + // add output one-time addresses to tx + bool set_tx_outputs_result{false}; + if (use_view_tags) + set_tx_outputs_result = set_tx_outputs_with_view_tags(output_public_keys, view_tags, unsigned_tx); + else + set_tx_outputs_result = set_tx_outputs(output_public_keys, unsigned_tx); + + if (not set_tx_outputs_result) + return false; + + // prepare input signatures + if (not set_tx_rct_signatures(fee, sources, destinations, input_secret_keys, output_public_keys, output_amount_secret_keys, + rct_config, reconstruction, unsigned_tx, CLSAG_contexts, cached_w)) + return false; + + initialized = true; + return true; +} +//---------------------------------------------------------------------------------------------------------------------- +bool tx_builder_ringct_t::first_partial_sign( + const std::size_t source, + const rct::keyV& total_alpha_G, + const rct::keyV& total_alpha_H, + const rct::keyV& alpha, + rct::key& c_0, + rct::key& s +) +{ + if (not initialized or reconstruction) + return false; + const std::size_t num_sources = CLSAG_contexts.size(); + if (source >= num_sources) + return false; + rct::key c; + rct::key alpha_combined; + auto alpha_combined_wiper = epee::misc_utils::create_scope_leave_handler([&]{ + memwipe(static_cast(&alpha_combined), sizeof(rct::key)); + }); + if (not CLSAG_contexts[source].combine_alpha_and_compute_challenge( + total_alpha_G, + total_alpha_H, + alpha, + alpha_combined, + c_0, + c + )) { + return false; + } + + // initial partial response: + // s = alpha_combined_local - challenge*[mu_P*(local keys and sender-receiver secret and subaddress material) + + // mu_C*(commitment-to-zero secret)] + sc_mulsub(s.bytes, c.bytes, cached_w[source].bytes, alpha_combined.bytes); + return true; +} +//---------------------------------------------------------------------------------------------------------------------- +bool tx_builder_ringct_t::next_partial_sign( + const rct::keyM& total_alpha_G, + const rct::keyM& total_alpha_H, + const rct::keyM& alpha, + const rct::key& x, + rct::keyV& c_0, + rct::keyV& s +) +{ + if (not initialized or not reconstruction) + return false; + const std::size_t num_sources = CLSAG_contexts.size(); + if (num_sources != total_alpha_G.size()) + return false; + if (num_sources != total_alpha_H.size()) + return false; + if (num_sources != alpha.size()) + return false; + if (num_sources != c_0.size()) + return false; + if (num_sources != s.size()) + return false; + for (std::size_t i = 0; i < num_sources; ++i) { + rct::key c; + rct::key alpha_combined; + auto alpha_combined_wiper = epee::misc_utils::create_scope_leave_handler([&]{ + memwipe(static_cast(&alpha_combined), sizeof(rct::key)); + }); + if (not CLSAG_contexts[i].combine_alpha_and_compute_challenge( + total_alpha_G[i], + total_alpha_H[i], + alpha[i], + alpha_combined, + c_0[i], + c + )) { + return false; + } + rct::key mu_P; + rct::key mu_C; + if (not CLSAG_contexts[i].get_mu(mu_P, mu_C)) + return false; + rct::key w; + auto w_wiper = epee::misc_utils::create_scope_leave_handler([&]{ + memwipe(static_cast(&w), sizeof(rct::key)); + }); + sc_mul(w.bytes, mu_P.bytes, x.bytes); + + // include local signer's response: + // s += alpha_combined_local - challenge*[mu_P*(local keys)] + sc_add(s[i].bytes, s[i].bytes, alpha_combined.bytes); + sc_mulsub(s[i].bytes, c.bytes, w.bytes, s[i].bytes); + } + return true; +} +//---------------------------------------------------------------------------------------------------------------------- +bool tx_builder_ringct_t::finalize_tx( + const std::vector& sources, + const rct::keyV& c_0, + const rct::keyV& s, + cryptonote::transaction& unsigned_tx +) +{ + // checks + const std::size_t num_sources = sources.size(); + if (num_sources != unsigned_tx.rct_signatures.p.CLSAGs.size()) + return false; + if (num_sources != c_0.size()) + return false; + if (num_sources != s.size()) + return false; + + // finalize tx signatures + for (std::size_t i = 0; i < num_sources; ++i) { + const std::size_t ring_size = unsigned_tx.rct_signatures.p.CLSAGs[i].s.size(); + if (sources[i].real_output >= ring_size) + return false; + unsigned_tx.rct_signatures.p.CLSAGs[i].s[sources[i].real_output] = s[i]; + unsigned_tx.rct_signatures.p.CLSAGs[i].c1 = c_0[i]; + } + + return true; +} +//---------------------------------------------------------------------------------------------------------------------- +} //namespace signing + +} //namespace multisig diff --git a/src/multisig/multisig_tx_builder_ringct.h b/src/multisig/multisig_tx_builder_ringct.h new file mode 100644 index 00000000000..f1bd24e73dc --- /dev/null +++ b/src/multisig/multisig_tx_builder_ringct.h @@ -0,0 +1,120 @@ +// Copyright (c) 2021-2023, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +#pragma once + +#include "ringct/rctTypes.h" + +#include +#include + +namespace cryptonote { + +class transaction; +struct tx_source_entry; +struct tx_destination_entry; +struct account_keys; + +} + +namespace multisig { + +namespace signing { + +class CLSAG_context_t; + +// number of parallel signing nonces to use per signer (2 nonces as in musig2 and FROST) +constexpr std::size_t kAlphaComponents = 2; + +class tx_builder_ringct_t final { +private: + // the tx builder has been initialized + bool initialized; + // the tx builder is 'reconstructing' a tx that has already been created using this object + bool reconstruction; + // cached: mu_P*(local keys and sender-receiver secret and subaddress material) + mu_C*(commitment-to-zero secret) + // - these are only used for the initial building of a tx (not reconstructions) + rct::keyV cached_w; + // contexts for making CLSAG challenges with multisig nonces + std::vector CLSAG_contexts; +public: + tx_builder_ringct_t(); + ~tx_builder_ringct_t(); + + // prepare an unsigned transaction (and get tx privkeys for outputs) + bool init( + const cryptonote::account_keys& account_keys, + const std::vector& extra, + const std::uint64_t unlock_time, + const std::uint32_t subaddr_account, + const std::set& subaddr_minor_indices, + std::vector& sources, + std::vector& destinations, + const cryptonote::tx_destination_entry& change, + const rct::RCTConfig& rct_config, + const bool use_rct, + const bool reconstruction, + crypto::secret_key& tx_secret_key, + std::vector& tx_aux_secret_keys, + crypto::secret_key& tx_secret_key_entropy, + cryptonote::transaction& unsigned_tx + ); + + // get the first partial signature for the specified input ('source') + bool first_partial_sign( + const std::size_t source, + const rct::keyV& total_alpha_G, + const rct::keyV& total_alpha_H, + const rct::keyV& alpha, + rct::key& c_0, + rct::key& s + ); + + // get intermediate partial signatures for all the inputs + bool next_partial_sign( + const rct::keyM& total_alpha_G, + const rct::keyM& total_alpha_H, + const rct::keyM& alpha, + const rct::key& x, + rct::keyV& c_0, + rct::keyV& s + ); + + // finalize an unsigned transaction (add challenges and real responses to incomplete CLSAG signatures) + static bool finalize_tx( + const std::vector& sources, + const rct::keyV& c_0, + const rct::keyV& s, + cryptonote::transaction& unsigned_tx + ); +}; + +} //namespace signing + +} //namespace multisig diff --git a/src/net/CMakeLists.txt b/src/net/CMakeLists.txt index a4d31eedf7b..29807f50e56 100644 --- a/src/net/CMakeLists.txt +++ b/src/net/CMakeLists.txt @@ -1,4 +1,4 @@ -# Copyright (c) 2018-2022, The Monero Project +# Copyright (c) 2018-2023, The Monero Project # # All rights reserved. diff --git a/src/net/dandelionpp.cpp b/src/net/dandelionpp.cpp index e75fb27533d..b384270862e 100644 --- a/src/net/dandelionpp.cpp +++ b/src/net/dandelionpp.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2019-2022, The Monero Project +// Copyright (c) 2019-2023, The Monero Project // // All rights reserved. // diff --git a/src/net/dandelionpp.h b/src/net/dandelionpp.h index e344bc7ceff..24e1fc62593 100644 --- a/src/net/dandelionpp.h +++ b/src/net/dandelionpp.h @@ -1,4 +1,4 @@ -// Copyright (c) 2019-2022, The Monero Project +// Copyright (c) 2019-2023, The Monero Project // // All rights reserved. // diff --git a/src/net/error.cpp b/src/net/error.cpp index 254db7ae163..0f3de6a6cdb 100644 --- a/src/net/error.cpp +++ b/src/net/error.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2018-2022, The Monero Project +// Copyright (c) 2018-2023, The Monero Project // // All rights reserved. diff --git a/src/net/error.h b/src/net/error.h index 969eefc4149..3bce227ee97 100644 --- a/src/net/error.h +++ b/src/net/error.h @@ -1,4 +1,4 @@ -// Copyright (c) 2018-2022, The Monero Project +// Copyright (c) 2018-2023, The Monero Project // // All rights reserved. diff --git a/src/net/fwd.h b/src/net/fwd.h index b105da1154a..871c134320e 100644 --- a/src/net/fwd.h +++ b/src/net/fwd.h @@ -1,4 +1,4 @@ -// Copyright (c) 2018-2022, The Monero Project +// Copyright (c) 2018-2023, The Monero Project // // All rights reserved. diff --git a/src/net/http.cpp b/src/net/http.cpp index c0ed3d43043..8eaf5b4b414 100644 --- a/src/net/http.cpp +++ b/src/net/http.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2020-2022, The Monero Project +// Copyright (c) 2020-2023, The Monero Project // // All rights reserved. diff --git a/src/net/http.h b/src/net/http.h index bbb8ee9847b..72bd3f66d3f 100644 --- a/src/net/http.h +++ b/src/net/http.h @@ -1,4 +1,4 @@ -// Copyright (c) 2020-2022, The Monero Project +// Copyright (c) 2020-2023, The Monero Project // // All rights reserved. diff --git a/src/net/i2p_address.cpp b/src/net/i2p_address.cpp index b0194a52562..a151e420bad 100644 --- a/src/net/i2p_address.cpp +++ b/src/net/i2p_address.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2019-2022, The Monero Project +// Copyright (c) 2019-2023, The Monero Project // // All rights reserved. // diff --git a/src/net/i2p_address.h b/src/net/i2p_address.h index 7f6ef1b3f22..4d342e67d64 100644 --- a/src/net/i2p_address.h +++ b/src/net/i2p_address.h @@ -1,4 +1,4 @@ -// Copyright (c) 2019-2022, The Monero Project +// Copyright (c) 2019-2023, The Monero Project // // All rights reserved. // diff --git a/src/net/parse.cpp b/src/net/parse.cpp index 1df6175b45d..71d0d7b26ab 100644 --- a/src/net/parse.cpp +++ b/src/net/parse.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2018-2022, The Monero Project +// Copyright (c) 2018-2023, The Monero Project // // All rights reserved. @@ -38,7 +38,7 @@ namespace net { void get_network_address_host_and_port(const std::string& address, std::string& host, std::string& port) { - // require ipv6 address format "[addr:addr:addr:...:addr]:port" + // If IPv6 address format with port "[addr:addr:addr:...:addr]:port" if (address.find(']') != std::string::npos) { host = address.substr(1, address.rfind(']') - 1); @@ -47,6 +47,12 @@ namespace net port = address.substr(address.rfind(':') + 1); } } + // Else if IPv6 address format without port e.g. "addr:addr:addr:...:addr" + else if (std::count(address.begin(), address.end(), ':') >= 2) + { + host = address; + } + // Else IPv4, Tor, I2P address or hostname else { host = address.substr(0, address.rfind(':')); diff --git a/src/net/parse.h b/src/net/parse.h index 648076d7bb6..695049268aa 100644 --- a/src/net/parse.h +++ b/src/net/parse.h @@ -1,4 +1,4 @@ -// Copyright (c) 2018-2022, The Monero Project +// Copyright (c) 2018-2023, The Monero Project // // All rights reserved. @@ -38,6 +38,16 @@ namespace net { + /*! + * \brief Takes a valid address string (IP, Tor, I2P, or DNS name) and splits it into host and port + * + * The host of an IPv6 addresses in the format "[x:x:..:x]:port" will have the braces stripped. + * For example, when the address is "[ffff::2023]", host will be set to "ffff::2023". + * + * \param address The address string one wants to split + * \param[out] host The host part of the address string. Is always set. + * \param[out] port The port part of the address string. Is only set when address string contains a port. + */ void get_network_address_host_and_port(const std::string& address, std::string& host, std::string& port); /*! diff --git a/src/net/resolve.cpp b/src/net/resolve.cpp index f4881ffe5b5..c4c8e8d08dc 100644 --- a/src/net/resolve.cpp +++ b/src/net/resolve.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2020-2022, The Monero Project +// Copyright (c) 2020-2023, The Monero Project // // All rights reserved. diff --git a/src/net/resolve.h b/src/net/resolve.h index 39b1d1830fb..afbe5d8796d 100644 --- a/src/net/resolve.h +++ b/src/net/resolve.h @@ -1,4 +1,4 @@ -// Copyright (c) 2020-2022, The Monero Project +// Copyright (c) 2020-2023, The Monero Project // // All rights reserved. diff --git a/src/net/socks.cpp b/src/net/socks.cpp index 97ef4a672bc..cc8e0be6d28 100644 --- a/src/net/socks.cpp +++ b/src/net/socks.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2018-2022, The Monero Project +// Copyright (c) 2018-2023, The Monero Project // // All rights reserved. // diff --git a/src/net/socks.h b/src/net/socks.h index af67d4abe2e..920496c6eed 100644 --- a/src/net/socks.h +++ b/src/net/socks.h @@ -1,4 +1,4 @@ -// Copyright (c) 2018-2022, The Monero Project +// Copyright (c) 2018-2023, The Monero Project // // All rights reserved. // diff --git a/src/net/socks_connect.cpp b/src/net/socks_connect.cpp index 5317564de90..f2456967403 100644 --- a/src/net/socks_connect.cpp +++ b/src/net/socks_connect.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2019-2022, The Monero Project +// Copyright (c) 2019-2023, The Monero Project // // All rights reserved. // diff --git a/src/net/socks_connect.h b/src/net/socks_connect.h index 587e3cd3cc8..e036d9c310e 100644 --- a/src/net/socks_connect.h +++ b/src/net/socks_connect.h @@ -1,4 +1,4 @@ -// Copyright (c) 2019-2022, The Monero Project +// Copyright (c) 2019-2023, The Monero Project // // All rights reserved. // diff --git a/src/net/tor_address.cpp b/src/net/tor_address.cpp index ac36dbffd97..53b73a839d5 100644 --- a/src/net/tor_address.cpp +++ b/src/net/tor_address.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2018-2022, The Monero Project +// Copyright (c) 2018-2023, The Monero Project // // All rights reserved. diff --git a/src/net/tor_address.h b/src/net/tor_address.h index 5b036f0d46b..3dd320b5d1f 100644 --- a/src/net/tor_address.h +++ b/src/net/tor_address.h @@ -1,4 +1,4 @@ -// Copyright (c) 2018-2022, The Monero Project +// Copyright (c) 2018-2023, The Monero Project // // All rights reserved. diff --git a/src/net/zmq.cpp b/src/net/zmq.cpp index 2b3ca837698..f57cbdac622 100644 --- a/src/net/zmq.cpp +++ b/src/net/zmq.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2019-2022, The Monero Project +// Copyright (c) 2019-2023, The Monero Project // // All rights reserved. // diff --git a/src/net/zmq.h b/src/net/zmq.h index 18bb80c8bbd..2b60432d72b 100644 --- a/src/net/zmq.h +++ b/src/net/zmq.h @@ -1,4 +1,4 @@ -// Copyright (c) 2019-2022, The Monero Project +// Copyright (c) 2019-2023, The Monero Project // // All rights reserved. // diff --git a/src/p2p/CMakeLists.txt b/src/p2p/CMakeLists.txt index af58d2bb023..bd2345979cd 100644 --- a/src/p2p/CMakeLists.txt +++ b/src/p2p/CMakeLists.txt @@ -1,4 +1,4 @@ -# Copyright (c) 2014-2022, The Monero Project +# Copyright (c) 2014-2023, The Monero Project # # All rights reserved. # diff --git a/src/p2p/net_node.cpp b/src/p2p/net_node.cpp index f9803fd813f..13ff06c8b18 100644 --- a/src/p2p/net_node.cpp +++ b/src/p2p/net_node.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2022, The Monero Project +// Copyright (c) 2014-2023, The Monero Project // // All rights reserved. // diff --git a/src/p2p/net_node.h b/src/p2p/net_node.h index 98d8ecfffec..cf2d212b3b8 100644 --- a/src/p2p/net_node.h +++ b/src/p2p/net_node.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2022, The Monero Project +// Copyright (c) 2014-2023, The Monero Project // // All rights reserved. // diff --git a/src/p2p/net_node.inl b/src/p2p/net_node.inl index a3bc3bf24de..4f77ce83449 100644 --- a/src/p2p/net_node.inl +++ b/src/p2p/net_node.inl @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2022, The Monero Project +// Copyright (c) 2014-2023, The Monero Project // // All rights reserved. // @@ -51,7 +51,6 @@ #include "common/dns_utils.h" #include "common/pruning.h" #include "net/error.h" -#include "net/net_helper.h" #include "math_helper.h" #include "misc_log_ex.h" #include "p2p_protocol_defs.h" @@ -67,8 +66,6 @@ #undef MONERO_DEFAULT_LOG_CATEGORY #define MONERO_DEFAULT_LOG_CATEGORY "net.p2p" -#define NET_MAKE_IP(b1,b2,b3,b4) ((LPARAM)(((DWORD)(b1)<<24)+((DWORD)(b2)<<16)+((DWORD)(b3)<<8)+((DWORD)(b4)))) - #define MIN_WANTED_SEED_NODES 12 static inline boost::asio::ip::address_v4 make_address_v4_from_v6(const boost::asio::ip::address_v6& a) @@ -247,7 +244,23 @@ namespace nodetool if (it == m_blocked_hosts.end()) { m_blocked_hosts[host_str] = limit; - added = true; + + // if the host was already blocked due to being in a blocked subnet, let it be silent + bool matches_blocked_subnet = false; + if (addr.get_type_id() == epee::net_utils::address_type::ipv4) + { + auto ipv4_address = addr.template as(); + for (auto jt = m_blocked_subnets.begin(); jt != m_blocked_subnets.end(); ++jt) + { + if (jt->first.matches(ipv4_address)) + { + matches_blocked_subnet = true; + break; + } + } + } + if (!matches_blocked_subnet) + added = true; } else if (it->second < limit || !add_only) it->second = limit; @@ -317,6 +330,7 @@ namespace nodetool limit = std::numeric_limits::max(); else limit = now + seconds; + const bool added = m_blocked_subnets.find(subnet) == m_blocked_subnets.end(); m_blocked_subnets[subnet] = limit; // drop any connection to that subnet. This should only have to look into @@ -349,7 +363,10 @@ namespace nodetool conns.clear(); } - MCLOG_CYAN(el::Level::Info, "global", "Subnet " << subnet.host_str() << " blocked."); + if (added) + MCLOG_CYAN(el::Level::Info, "global", "Subnet " << subnet.host_str() << " blocked."); + else + MINFO("Subnet " << subnet.host_str() << " blocked."); return true; } //----------------------------------------------------------------------------------- @@ -538,7 +555,7 @@ namespace nodetool if ( !set_max_out_peers(public_zone, command_line::get_arg(vm, arg_out_peers) ) ) return false; else - m_payload_handler.set_max_out_peers(public_zone.m_config.m_net_config.max_out_connection_count); + m_payload_handler.set_max_out_peers(epee::net_utils::zone::public_, public_zone.m_config.m_net_config.max_out_connection_count); if ( !set_max_in_peers(public_zone, command_line::get_arg(vm, arg_in_peers) ) ) @@ -575,6 +592,8 @@ namespace nodetool if (!set_max_out_peers(zone, proxy.max_connections)) return false; + else + m_payload_handler.set_max_out_peers(proxy.zone, proxy.max_connections); epee::byte_slice this_noise = nullptr; if (proxy.noise) @@ -643,20 +662,10 @@ namespace nodetool { using namespace boost::asio; - std::string host = addr; + // Split addr string into host string and port string + std::string host; std::string port = std::to_string(default_port); - size_t colon_pos = addr.find_last_of(':'); - size_t dot_pos = addr.find_last_of('.'); - size_t square_brace_pos = addr.find('['); - - // IPv6 will have colons regardless. IPv6 and IPv4 address:port will have a colon but also either a . or a [ - // as IPv6 addresses specified as address:port are to be specified as "[addr:addr:...:addr]:port" - // One may also specify an IPv6 address as simply "[addr:addr:...:addr]" without the port; in that case - // the square braces will be stripped here. - if ((std::string::npos != colon_pos && std::string::npos != dot_pos) || std::string::npos != square_brace_pos) - { - net::get_network_address_host_and_port(addr, host, port); - } + net::get_network_address_host_and_port(addr, host, port); MINFO("Resolving node address: host=" << host << ", port=" << port); io_service io_srv; @@ -693,34 +702,32 @@ namespace nodetool std::set full_addrs; if (m_nettype == cryptonote::TESTNET) { - full_addrs.insert("212.83.175.67:28080"); - full_addrs.insert("212.83.172.165:28080"); full_addrs.insert("176.9.0.187:28080"); full_addrs.insert("88.99.173.38:28080"); full_addrs.insert("51.79.173.165:28080"); + full_addrs.insert("192.99.8.110:28080"); + full_addrs.insert("37.187.74.171:28080"); } else if (m_nettype == cryptonote::STAGENET) { - full_addrs.insert("162.210.173.150:38080"); full_addrs.insert("176.9.0.187:38080"); full_addrs.insert("88.99.173.38:38080"); full_addrs.insert("51.79.173.165:38080"); + full_addrs.insert("192.99.8.110:38080"); + full_addrs.insert("37.187.74.171:38080"); } else if (m_nettype == cryptonote::FAKECHAIN) { } else { - full_addrs.insert("212.83.175.67:18080"); - full_addrs.insert("212.83.172.165:18080"); full_addrs.insert("176.9.0.187:18080"); full_addrs.insert("88.198.163.90:18080"); - full_addrs.insert("95.217.25.101:18080"); - full_addrs.insert("136.244.105.131:18080"); - full_addrs.insert("104.238.221.81:18080"); full_addrs.insert("66.85.74.134:18080"); full_addrs.insert("88.99.173.38:18080"); full_addrs.insert("51.79.173.165:18080"); + full_addrs.insert("192.99.8.110:18080"); + full_addrs.insert("37.187.74.171:18080"); } return full_addrs; } @@ -855,6 +862,8 @@ namespace nodetool "4pixvbejrvihnkxmduo2agsnmc3rrulrqc7s3cbwwrep6h6hrzsibeqd.onion:18083", "zbjkbsxc5munw3qusl7j2hpcmikhqocdf4pqhnhtpzw5nt5jrmofptid.onion:18083", "qz43zul2x56jexzoqgkx2trzwcfnr6l3hbtfcfx54g4r3eahy3bssjyd.onion:18083", + "plowsof3t5hogddwabaeiyrno25efmzfxyro2vligremt7sxpsclfaid.onion:18083", + "plowsoffjexmxalw73tkjmf422gq6575fc7vicuu4javzn2ynnte6tyd.onion:18083", }; } return {}; @@ -863,7 +872,9 @@ namespace nodetool { return { "s3l6ke4ed3df466khuebb4poienoingwof7oxtbo6j4n56sghe3a.b32.i2p:18080", - "sel36x6fibfzujwvt4hf5gxolz6kd3jpvbjqg6o3ud2xtionyl2q.b32.i2p:18080" + "sel36x6fibfzujwvt4hf5gxolz6kd3jpvbjqg6o3ud2xtionyl2q.b32.i2p:18080", + "uqj3aphckqtjsitz7kxx5flqpwjlq5ppr3chazfued7xucv3nheq.b32.i2p:18080", + "vdmnehdjkpkg57nthgnjfuaqgku673r5bpbqg56ix6fyqoywgqrq.b32.i2p:18080", }; } return {}; @@ -2411,7 +2422,7 @@ namespace nodetool return false; } return true; - }); + }, "0.0.0.0", m_ssl_support); if(!r) { LOG_WARNING_CC(context, "Failed to call connect_async, network error."); @@ -2462,8 +2473,12 @@ namespace nodetool const epee::net_utils::zone zone_type = context.m_remote_address.get_zone(); network_zone& zone = m_network_zones.at(zone_type); + //will add self to peerlist if in same zone as outgoing later in this function + const bool outgoing_to_same_zone = !context.m_is_income && zone.m_our_address.get_zone() == zone_type; + const uint32_t max_peerlist_size = P2P_DEFAULT_PEERS_IN_HANDSHAKE - (outgoing_to_same_zone ? 1 : 0); + std::vector local_peerlist_new; - zone.m_peerlist.get_peerlist_head(local_peerlist_new, true, P2P_DEFAULT_PEERS_IN_HANDSHAKE); + zone.m_peerlist.get_peerlist_head(local_peerlist_new, true, max_peerlist_size); //only include out peers we did not already send rsp.local_peerlist_new.reserve(local_peerlist_new.size()); @@ -2483,7 +2498,7 @@ namespace nodetool etc., because someone could give faulty addresses over Tor/I2P to get the real peer with that identity banned/blacklisted. */ - if(!context.m_is_income && zone.m_our_address.get_zone() == zone_type) + if(outgoing_to_same_zone) rsp.local_peerlist_new.push_back(peerlist_entry{zone.m_our_address, zone.m_config.m_peer_id, std::time(nullptr)}); LOG_DEBUG_CC(context, "COMMAND_TIMED_SYNC"); @@ -2758,7 +2773,7 @@ namespace nodetool public_zone->second.m_config.m_net_config.max_out_connection_count = count; if(current > count) public_zone->second.m_net_server.get_config_object().del_out_connections(current - count); - m_payload_handler.set_max_out_peers(count); + m_payload_handler.set_max_out_peers(epee::net_utils::zone::public_, count); } } @@ -2887,10 +2902,12 @@ namespace nodetool { if (m_offline) return true; if (!m_exclusive_peers.empty()) return true; - if (m_payload_handler.needs_new_sync_connections()) return true; for (auto& zone : m_network_zones) { + if (m_payload_handler.needs_new_sync_connections(zone.first)) + continue; + if (zone.second.m_net_server.is_stop_signal_sent()) return false; diff --git a/src/p2p/net_node_common.h b/src/p2p/net_node_common.h index ef4552f4474..66ef40f1ed4 100644 --- a/src/p2p/net_node_common.h +++ b/src/p2p/net_node_common.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2022, The Monero Project +// Copyright (c) 2014-2023, The Monero Project // // All rights reserved. // diff --git a/src/p2p/net_peerlist.cpp b/src/p2p/net_peerlist.cpp index 3e132c91fae..11cf235a572 100644 --- a/src/p2p/net_peerlist.cpp +++ b/src/p2p/net_peerlist.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2018-2022, The Monero Project +// Copyright (c) 2018-2023, The Monero Project // // All rights reserved. diff --git a/src/p2p/net_peerlist.h b/src/p2p/net_peerlist.h index dc480462d35..c2f7005980f 100644 --- a/src/p2p/net_peerlist.h +++ b/src/p2p/net_peerlist.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2022, The Monero Project +// Copyright (c) 2014-2023, The Monero Project // // All rights reserved. // diff --git a/src/p2p/net_peerlist_boost_serialization.h b/src/p2p/net_peerlist_boost_serialization.h index 7d8bd1480a7..7f4aecce371 100644 --- a/src/p2p/net_peerlist_boost_serialization.h +++ b/src/p2p/net_peerlist_boost_serialization.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2022, The Monero Project +// Copyright (c) 2014-2023, The Monero Project // // All rights reserved. // diff --git a/src/p2p/p2p_protocol_defs.h b/src/p2p/p2p_protocol_defs.h index 8b9cd0f0905..18b853d3529 100644 --- a/src/p2p/p2p_protocol_defs.h +++ b/src/p2p/p2p_protocol_defs.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2022, The Monero Project +// Copyright (c) 2014-2023, The Monero Project // // All rights reserved. // diff --git a/src/platform/mingw/alloca.h b/src/platform/mingw/alloca.h index 10a9db198a0..8ed1c65145a 100644 --- a/src/platform/mingw/alloca.h +++ b/src/platform/mingw/alloca.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2022, The Monero Project +// Copyright (c) 2014-2023, The Monero Project // // All rights reserved. // diff --git a/src/platform/msc/alloca.h b/src/platform/msc/alloca.h index 3b197bf6d0e..8c4701b78e0 100644 --- a/src/platform/msc/alloca.h +++ b/src/platform/msc/alloca.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2022, The Monero Project +// Copyright (c) 2014-2023, The Monero Project // // All rights reserved. // diff --git a/src/platform/msc/inline_c.h b/src/platform/msc/inline_c.h index 36d3d0026ab..72bad922370 100644 --- a/src/platform/msc/inline_c.h +++ b/src/platform/msc/inline_c.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2022, The Monero Project +// Copyright (c) 2014-2023, The Monero Project // // All rights reserved. // diff --git a/src/platform/msc/stdbool.h b/src/platform/msc/stdbool.h index d4932e8f8d0..1bfd78f5707 100644 --- a/src/platform/msc/stdbool.h +++ b/src/platform/msc/stdbool.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2022, The Monero Project +// Copyright (c) 2014-2023, The Monero Project // // All rights reserved. // diff --git a/src/platform/msc/sys/param.h b/src/platform/msc/sys/param.h index 61002e1a165..1f1950ac60e 100644 --- a/src/platform/msc/sys/param.h +++ b/src/platform/msc/sys/param.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2022, The Monero Project +// Copyright (c) 2014-2023, The Monero Project // // All rights reserved. // diff --git a/src/ringct/CMakeLists.txt b/src/ringct/CMakeLists.txt index d415e640937..1f3d1913f1b 100644 --- a/src/ringct/CMakeLists.txt +++ b/src/ringct/CMakeLists.txt @@ -1,4 +1,4 @@ -# Copyright (c) 2016-2022, The Monero Project +# Copyright (c) 2016-2023, The Monero Project # # All rights reserved. # diff --git a/src/ringct/bulletproofs.cc b/src/ringct/bulletproofs.cc index ba7ab2c0d19..2cfaf18c028 100644 --- a/src/ringct/bulletproofs.cc +++ b/src/ringct/bulletproofs.cc @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2022, The Monero Project +// Copyright (c) 2017-2023, The Monero Project // // All rights reserved. // diff --git a/src/ringct/bulletproofs.h b/src/ringct/bulletproofs.h index c3fe6f8b972..7ff47e8a0db 100644 --- a/src/ringct/bulletproofs.h +++ b/src/ringct/bulletproofs.h @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2022, The Monero Project +// Copyright (c) 2017-2023, The Monero Project // // All rights reserved. // diff --git a/src/ringct/bulletproofs_plus.cc b/src/ringct/bulletproofs_plus.cc index 77b8000641f..0619811cb96 100644 --- a/src/ringct/bulletproofs_plus.cc +++ b/src/ringct/bulletproofs_plus.cc @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2022, The Monero Project +// Copyright (c) 2017-2023, The Monero Project // // All rights reserved. // diff --git a/src/ringct/bulletproofs_plus.h b/src/ringct/bulletproofs_plus.h index 861c54f4fcc..80a8de8fb3e 100644 --- a/src/ringct/bulletproofs_plus.h +++ b/src/ringct/bulletproofs_plus.h @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2022, The Monero Project +// Copyright (c) 2017-2023, The Monero Project // // All rights reserved. // diff --git a/src/ringct/multiexp.cc b/src/ringct/multiexp.cc index 67814682a79..5e241f2cc35 100644 --- a/src/ringct/multiexp.cc +++ b/src/ringct/multiexp.cc @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2022, The Monero Project +// Copyright (c) 2017-2023, The Monero Project // // All rights reserved. @@ -57,28 +57,28 @@ extern "C" // 1 1 52.8 70.4 70.2 // Pippenger: -// 1 2 3 4 5 6 7 8 9 bestN -// 2 555 598 621 804 1038 1733 2486 5020 8304 1 -// 4 783 747 800 1006 1428 2132 3285 5185 9806 2 -// 8 1174 1071 1095 1286 1640 2398 3869 6378 12080 2 -// 16 2279 1874 1745 1739 2144 2831 4209 6964 12007 4 -// 32 3910 3706 2588 2477 2782 3467 4856 7489 12618 4 -// 64 7184 5429 4710 4368 4010 4672 6027 8559 13684 5 -// 128 14097 10574 8452 7297 6841 6718 8615 10580 15641 6 -// 256 27715 20800 16000 13550 11875 11400 11505 14090 18460 6 -// 512 55100 41250 31740 26570 22030 19830 20760 21380 25215 6 -// 1024 111520 79000 61080 49720 43080 38320 37600 35040 36750 8 -// 2048 219480 162680 122120 102080 83760 70360 66600 63920 66160 8 -// 4096 453320 323080 247240 210200 180040 150240 132440 114920 110560 9 - -// 2 4 8 16 32 64 128 256 512 1024 2048 4096 -// Bos Coster 858 994 1316 1949 3183 5512 9865 17830 33485 63160 124280 246320 -// Straus 226 341 548 980 1870 3538 7039 14490 29020 57200 118640 233640 -// Straus/cached 226 315 485 785 1514 2858 5753 11065 22970 45120 98880 194840 -// Pippenger 555 747 1071 1739 2477 4010 6718 11400 19830 35040 63920 110560 - -// Best/cached Straus Straus Straus Straus Straus Straus Straus Straus Pip Pip Pip Pip -// Best/uncached Straus Straus Straus Straus Straus Straus Pip Pip Pip Pip Pip Pip +// 1 2 3 4 5 6 7 8 9 bestN +// 2 555 598 621 804 1038 1733 2486 5020 8304 1 +// 4 783 747 800 1006 1428 2132 3285 5185 9806 2 +// 8 1174 1071 1095 1286 1640 2398 3869 6378 12080 2 +// 16 2279 1874 1745 1739 2144 2831 4209 6964 12007 4 +// 32 3910 3706 2588 2477 2782 3467 4856 7489 12618 4 +// 64 7184 5429 4710 4368 4010 4672 6027 8559 13684 5 +// 128 14097 10574 8452 7297 6841 6718 8615 10580 15641 6 +// 256 27715 20800 16000 13550 11875 11400 11505 14090 18460 6 +// 512 55100 41250 31740 26570 22030 19830 20760 21380 25215 6 +// 1024 111520 79000 61080 49720 43080 38320 37600 35040 36750 8 +// 2048 219480 162680 122120 102080 83760 70360 66600 63920 66160 8 +// 4096 453320 323080 247240 210200 180040 150240 132440 114920 110560 9 + +// 2 4 8 16 32 64 128 256 512 1024 2048 4096 +// Bos Coster 858 994 1316 1949 3183 5512 9865 17830 33485 63160 124280 246320 +// Straus 226 341 548 980 1870 3538 7039 14490 29020 57200 118640 233640 +// Straus/cached 226 315 485 785 1514 2858 5753 11065 22970 45120 98880 194840 +// Pippenger 555 747 1071 1739 2477 4010 6718 11400 19830 35040 63920 110560 + +// Best/cached Straus Straus Straus Straus Straus Straus Straus Straus Pip Pip Pip Pip +// Best/uncached Straus Straus Straus Straus Straus Straus Pip Pip Pip Pip Pip Pip // New timings: // Pippenger: @@ -443,7 +443,7 @@ size_t straus_get_cache_size(const std::shared_ptr &cache) return sz; } -rct::key straus(const std::vector &data, const std::shared_ptr &cache, size_t STEP) +ge_p3 straus_p3(const std::vector &data, const std::shared_ptr &cache, size_t STEP) { CHECK_AND_ASSERT_THROW_MES(cache == NULL || cache->size >= data.size(), "Cache is too small"); MULTIEXP_PERF(PERF_TIMER_UNIT(straus, 1000000)); @@ -554,7 +554,13 @@ rct::key straus(const std::vector &data, const std::shared_ptr &data, const std::shared_ptr &cache, size_t STEP) +{ rct::key res; + const ge_p3 res_p3 = straus_p3(data, cache, STEP); ge_p3_tobytes(res.bytes, &res_p3); return res; } @@ -571,14 +577,6 @@ size_t get_pippenger_c(size_t N) return 9; } -struct pippenger_cached_data -{ - size_t size; - ge_cached *cached; - pippenger_cached_data(): size(0), cached(NULL) {} - ~pippenger_cached_data() { aligned_free(cached); } -}; - std::shared_ptr pippenger_init_cache(const std::vector &data, size_t start_offset, size_t N) { MULTIEXP_PERF(PERF_TIMER_START_UNIT(pippenger_init_cache, 1000000)); @@ -586,13 +584,11 @@ std::shared_ptr pippenger_init_cache(const std::vector cache(new pippenger_cached_data()); + std::shared_ptr cache = std::make_shared(); - cache->size = N; - cache->cached = (ge_cached*)aligned_realloc(cache->cached, N * sizeof(ge_cached), 4096); - CHECK_AND_ASSERT_THROW_MES(cache->cached, "Out of memory"); + cache->resize(N); for (size_t i = 0; i < N; ++i) - ge_p3_to_cached(&cache->cached[i], &data[i+start_offset].point); + ge_p3_to_cached(&(*cache)[i], &data[i+start_offset].point); MULTIEXP_PERF(PERF_TIMER_STOP(pippenger_init_cache)); return cache; @@ -600,14 +596,14 @@ std::shared_ptr pippenger_init_cache(const std::vector &cache) { - return cache->size * sizeof(*cache->cached); + return cache->size() * sizeof(ge_cached); } -rct::key pippenger(const std::vector &data, const std::shared_ptr &cache, size_t cache_size, size_t c) +ge_p3 pippenger_p3(const std::vector &data, const std::shared_ptr &cache, size_t cache_size, size_t c) { if (cache != NULL && cache_size == 0) - cache_size = cache->size; - CHECK_AND_ASSERT_THROW_MES(cache == NULL || cache_size <= cache->size, "Cache is too small"); + cache_size = cache->size(); + CHECK_AND_ASSERT_THROW_MES(cache == NULL || cache_size <= cache->size(), "Cache is too small"); if (c == 0) c = get_pippenger_c(data.size()); CHECK_AND_ASSERT_THROW_MES(c <= 9, "c is too large"); @@ -661,9 +657,9 @@ rct::key pippenger(const std::vector &data, const std::shared_ptr< if (buckets_init[bucket]) { if (i < cache_size) - add(buckets[bucket], local_cache->cached[i]); + add(buckets[bucket], (*local_cache)[i]); else - add(buckets[bucket], local_cache_2->cached[i - cache_size]); + add(buckets[bucket], (*local_cache_2)[i - cache_size]); } else { @@ -700,8 +696,14 @@ rct::key pippenger(const std::vector &data, const std::shared_ptr< } } + return result; +} + +rct::key pippenger(const std::vector &data, const std::shared_ptr &cache, const size_t cache_size, const size_t c) +{ rct::key res; - ge_p3_tobytes(res.bytes, &result); + const ge_p3 result_p3 = pippenger_p3(data, cache, cache_size, c); + ge_p3_tobytes(res.bytes, &result_p3); return res; } diff --git a/src/ringct/multiexp.h b/src/ringct/multiexp.h index 788516c0b7d..ae10fcc8eda 100644 --- a/src/ringct/multiexp.h +++ b/src/ringct/multiexp.h @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2022, The Monero Project +// Copyright (c) 2017-2023, The Monero Project // // All rights reserved. @@ -35,10 +35,16 @@ #define MULTIEXP_H #include +extern "C" +{ +#include "crypto/crypto-ops.h" +} #include "crypto/crypto.h" #include "rctTypes.h" #include "misc_log_ex.h" +#include + namespace rct { @@ -55,17 +61,19 @@ struct MultiexpData { }; struct straus_cached_data; -struct pippenger_cached_data; +using pippenger_cached_data = std::vector>; rct::key bos_coster_heap_conv(std::vector data); rct::key bos_coster_heap_conv_robust(std::vector data); std::shared_ptr straus_init_cache(const std::vector &data, size_t N =0); size_t straus_get_cache_size(const std::shared_ptr &cache); +ge_p3 straus_p3(const std::vector &data, const std::shared_ptr &cache = NULL, size_t STEP = 0); rct::key straus(const std::vector &data, const std::shared_ptr &cache = NULL, size_t STEP = 0); std::shared_ptr pippenger_init_cache(const std::vector &data, size_t start_offset = 0, size_t N =0); size_t pippenger_get_cache_size(const std::shared_ptr &cache); size_t get_pippenger_c(size_t N); -rct::key pippenger(const std::vector &data, const std::shared_ptr &cache = NULL, size_t cache_size = 0, size_t c = 0); +ge_p3 pippenger_p3(const std::vector &data, const std::shared_ptr &cache = NULL, size_t cache_size = 0, size_t c = 0); +rct::key pippenger(const std::vector &data, const std::shared_ptr &cache = NULL, const size_t cache_size = 0, const size_t c = 0); } diff --git a/src/ringct/rctCryptoOps.c b/src/ringct/rctCryptoOps.c index a2bb68665c4..b4fcc64bd8b 100644 --- a/src/ringct/rctCryptoOps.c +++ b/src/ringct/rctCryptoOps.c @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2022, The Monero Project +// Copyright (c) 2014-2023, The Monero Project // // All rights reserved. // diff --git a/src/ringct/rctCryptoOps.h b/src/ringct/rctCryptoOps.h index 2d6e13bb708..ec5708818d3 100644 --- a/src/ringct/rctCryptoOps.h +++ b/src/ringct/rctCryptoOps.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2022, The Monero Project +// Copyright (c) 2014-2023, The Monero Project // // All rights reserved. // diff --git a/src/ringct/rctOps.cpp b/src/ringct/rctOps.cpp index 245a3f4779c..56dd53b2c31 100644 --- a/src/ringct/rctOps.cpp +++ b/src/ringct/rctOps.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2016, Monero Research Labs +// Copyright (c) 2016-2023, Monero Research Labs // // Author: Shen Noether // @@ -671,7 +671,7 @@ namespace rct { //Elliptic Curve Diffie Helman: encodes and decodes the amount b and mask a // where C= aG + bH - static key ecdhHash(const key &k) + key genAmountEncodingFactor(const key &k) { char data[38]; rct::key hash; @@ -700,7 +700,7 @@ namespace rct { if (v2) { unmasked.mask = zero(); - xor8(unmasked.amount, ecdhHash(sharedSec)); + xor8(unmasked.amount, genAmountEncodingFactor(sharedSec)); } else { @@ -715,7 +715,7 @@ namespace rct { if (v2) { masked.mask = genCommitmentMask(sharedSec); - xor8(masked.amount, ecdhHash(sharedSec)); + xor8(masked.amount, genAmountEncodingFactor(sharedSec)); } else { diff --git a/src/ringct/rctOps.h b/src/ringct/rctOps.h index 679ed14413e..eaab998cb0c 100644 --- a/src/ringct/rctOps.h +++ b/src/ringct/rctOps.h @@ -1,5 +1,5 @@ //#define DBG -// Copyright (c) 2016, Monero Research Labs +// Copyright (c) 2016-2023, Monero Research Labs // // Author: Shen Noether // @@ -184,6 +184,7 @@ namespace rct { //Elliptic Curve Diffie Helman: encodes and decodes the amount b and mask a // where C= aG + bH + key genAmountEncodingFactor(const key &k); key genCommitmentMask(const key &sk); void ecdhEncode(ecdhTuple & unmasked, const key & sharedSec, bool v2); void ecdhDecode(ecdhTuple & masked, const key & sharedSec, bool v2); diff --git a/src/ringct/rctSigs.cpp b/src/ringct/rctSigs.cpp index bd67778eca8..717bc27a3da 100644 --- a/src/ringct/rctSigs.cpp +++ b/src/ringct/rctSigs.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2016, Monero Research Labs +// Copyright (c) 2016-2023, Monero Research Labs // // Author: Shen Noether // @@ -238,14 +238,12 @@ namespace rct { // P[l] == p*G // C[l] == z*G // C[i] == C_nonzero[i] - C_offset (for hashing purposes) for all i - clsag CLSAG_Gen(const key &message, const keyV & P, const key & p, const keyV & C, const key & z, const keyV & C_nonzero, const key & C_offset, const unsigned int l, const multisig_kLRki *kLRki, key *mscout, key *mspout, hw::device &hwdev) { + clsag CLSAG_Gen(const key &message, const keyV & P, const key & p, const keyV & C, const key & z, const keyV & C_nonzero, const key & C_offset, const unsigned int l, hw::device &hwdev) { clsag sig; size_t n = P.size(); // ring size CHECK_AND_ASSERT_THROW_MES(n == C.size(), "Signing and commitment key vector sizes must match!"); CHECK_AND_ASSERT_THROW_MES(n == C_nonzero.size(), "Signing and commitment key vector sizes must match!"); CHECK_AND_ASSERT_THROW_MES(l < n, "Signing index out of range!"); - CHECK_AND_ASSERT_THROW_MES((kLRki && mscout) || (!kLRki && !mscout), "Only one of kLRki/mscout is present"); - CHECK_AND_ASSERT_THROW_MES((mscout && mspout) || !kLRki, "Multisig pointers are not all present"); // Key images ge_p3 H_p3; @@ -260,16 +258,7 @@ namespace rct { key aG; key aH; - // Multisig - if (kLRki) - { - sig.I = kLRki->ki; - scalarmultKey(D,H,z); - } - else - { - hwdev.clsag_prepare(p,z,sig.I,D,H,a,aG,aH); - } + hwdev.clsag_prepare(p,z,sig.I,D,H,a,aG,aH); geDsmp I_precomp; geDsmp D_precomp; @@ -317,18 +306,9 @@ namespace rct { c_to_hash[2*n+1] = C_offset; c_to_hash[2*n+2] = message; - // Multisig data is present - if (kLRki) - { - a = kLRki->k; - c_to_hash[2*n+3] = kLRki->L; - c_to_hash[2*n+4] = kLRki->R; - } - else - { - c_to_hash[2*n+3] = aG; - c_to_hash[2*n+4] = aH; - } + c_to_hash[2*n+3] = aG; + c_to_hash[2*n+4] = aH; + hwdev.clsag_hash(c_to_hash,c); size_t i; @@ -380,16 +360,11 @@ namespace rct { hwdev.clsag_sign(c,a,p,z,mu_P,mu_C,sig.s[l]); memwipe(&a, sizeof(key)); - if (mscout) - *mscout = c; - if (mspout) - *mspout = mu_P; - return sig; } clsag CLSAG_Gen(const key &message, const keyV & P, const key & p, const keyV & C, const key & z, const keyV & C_nonzero, const key & C_offset, const unsigned int l) { - return CLSAG_Gen(message, P, p, C, z, C_nonzero, C_offset, l, NULL, NULL, NULL, hw::get_device("default")); + return CLSAG_Gen(message, P, p, C, z, C_nonzero, C_offset, l, hw::get_device("default")); } // MLSAG signatures @@ -397,7 +372,7 @@ namespace rct { // This generalization allows for some dimensions not to require linkability; // this is used in practice for commitment data within signatures // Note that using more than one linkable dimension is not recommended. - mgSig MLSAG_Gen(const key &message, const keyM & pk, const keyV & xx, const multisig_kLRki *kLRki, key *mscout, const unsigned int index, size_t dsRows, hw::device &hwdev) { + mgSig MLSAG_Gen(const key &message, const keyM & pk, const keyV & xx, const unsigned int index, size_t dsRows, hw::device &hwdev) { mgSig rv; size_t cols = pk.size(); CHECK_AND_ASSERT_THROW_MES(cols >= 2, "Error! What is c if cols = 1!"); @@ -409,8 +384,6 @@ namespace rct { } CHECK_AND_ASSERT_THROW_MES(xx.size() == rows, "Bad xx size"); CHECK_AND_ASSERT_THROW_MES(dsRows <= rows, "Bad dsRows size"); - CHECK_AND_ASSERT_THROW_MES((kLRki && mscout) || (!kLRki && !mscout), "Only one of kLRki/mscout is present"); - CHECK_AND_ASSERT_THROW_MES(!kLRki || dsRows == 1, "Multisig requires exactly 1 dsRows"); size_t i = 0, j = 0, ii = 0; key c, c_old, L, R, Hi; @@ -428,20 +401,11 @@ namespace rct { DP("here1"); for (i = 0; i < dsRows; i++) { toHash[3 * i + 1] = pk[index][i]; - if (kLRki) { - // multisig - alpha[i] = kLRki->k; - toHash[3 * i + 2] = kLRki->L; - toHash[3 * i + 3] = kLRki->R; - rv.II[i] = kLRki->ki; - } - else { - hash_to_p3(Hi_p3, pk[index][i]); - ge_p3_tobytes(Hi.bytes, &Hi_p3); - hwdev.mlsag_prepare(Hi, xx[i], alpha[i] , aG[i] , aHP[i] , rv.II[i]); - toHash[3 * i + 2] = aG[i]; - toHash[3 * i + 3] = aHP[i]; - } + hash_to_p3(Hi_p3, pk[index][i]); + ge_p3_tobytes(Hi.bytes, &Hi_p3); + hwdev.mlsag_prepare(Hi, xx[i], alpha[i] , aG[i] , aHP[i] , rv.II[i]); + toHash[3 * i + 2] = aG[i]; + toHash[3 * i + 3] = aHP[i]; precomp(Ip[i].k, rv.II[i]); } size_t ndsRows = 3 * dsRows; //non Double Spendable Rows (see identity chains paper) @@ -485,8 +449,6 @@ namespace rct { } } hwdev.mlsag_sign(c, xx, alpha, rows, dsRows, rv.ss[index]); - if (mscout) - *mscout = c; return rv; } @@ -722,7 +684,7 @@ namespace rct { // this shows that sum inputs = sum outputs //Ver: // verifies the above sig is created corretly - mgSig proveRctMG(const key &message, const ctkeyM & pubs, const ctkeyV & inSk, const ctkeyV &outSk, const ctkeyV & outPk, const multisig_kLRki *kLRki, key *mscout, unsigned int index, const key &txnFeeKey, hw::device &hwdev) { + mgSig proveRctMG(const key &message, const ctkeyM & pubs, const ctkeyV & inSk, const ctkeyV &outSk, const ctkeyV & outPk, unsigned int index, const key &txnFeeKey, hw::device &hwdev) { //setup vars size_t cols = pubs.size(); CHECK_AND_ASSERT_THROW_MES(cols >= 1, "Empty pubs"); @@ -733,7 +695,6 @@ namespace rct { } CHECK_AND_ASSERT_THROW_MES(inSk.size() == rows, "Bad inSk size"); CHECK_AND_ASSERT_THROW_MES(outSk.size() == outPk.size(), "Bad outSk/outPk size"); - CHECK_AND_ASSERT_THROW_MES((kLRki && mscout) || (!kLRki && !mscout), "Only one of kLRki/mscout is present"); keyV sk(rows + 1); keyV tmp(rows + 1); @@ -766,7 +727,7 @@ namespace rct { for (size_t j = 0; j < outPk.size(); j++) { sc_sub(sk[rows].bytes, sk[rows].bytes, outSk[j].mask.bytes); //subtract output masks in last row.. } - mgSig result = MLSAG_Gen(message, M, sk, kLRki, mscout, index, rows, hwdev); + mgSig result = MLSAG_Gen(message, M, sk, index, rows, hwdev); memwipe(sk.data(), sk.size() * sizeof(key)); return result; } @@ -779,12 +740,11 @@ namespace rct { // inSk is x, a_in corresponding to signing index // a_out, Cout is for the output commitment // index is the signing index.. - mgSig proveRctMGSimple(const key &message, const ctkeyV & pubs, const ctkey & inSk, const key &a , const key &Cout, const multisig_kLRki *kLRki, key *mscout, unsigned int index, hw::device &hwdev) { + mgSig proveRctMGSimple(const key &message, const ctkeyV & pubs, const ctkey & inSk, const key &a , const key &Cout, unsigned int index, hw::device &hwdev) { //setup vars size_t rows = 1; size_t cols = pubs.size(); CHECK_AND_ASSERT_THROW_MES(cols >= 1, "Empty pubs"); - CHECK_AND_ASSERT_THROW_MES((kLRki && mscout) || (!kLRki && !mscout), "Only one of kLRki/mscout is present"); keyV tmp(rows + 1); keyV sk(rows + 1); size_t i; @@ -796,17 +756,16 @@ namespace rct { M[i][0] = pubs[i].dest; subKeys(M[i][1], pubs[i].mask, Cout); } - mgSig result = MLSAG_Gen(message, M, sk, kLRki, mscout, index, rows, hwdev); + mgSig result = MLSAG_Gen(message, M, sk, index, rows, hwdev); memwipe(sk.data(), sk.size() * sizeof(key)); return result; } - clsag proveRctCLSAGSimple(const key &message, const ctkeyV &pubs, const ctkey &inSk, const key &a, const key &Cout, const multisig_kLRki *kLRki, key *mscout, key *mspout, unsigned int index, hw::device &hwdev) { + clsag proveRctCLSAGSimple(const key &message, const ctkeyV &pubs, const ctkey &inSk, const key &a, const key &Cout, unsigned int index, hw::device &hwdev) { //setup vars size_t rows = 1; size_t cols = pubs.size(); CHECK_AND_ASSERT_THROW_MES(cols >= 1, "Empty pubs"); - CHECK_AND_ASSERT_THROW_MES((kLRki && mscout) || (!kLRki && !mscout), "Only one of kLRki/mscout is present"); keyV tmp(rows + 1); keyV sk(rows + 1); keyM M(cols, tmp); @@ -826,7 +785,7 @@ namespace rct { sk[0] = copy(inSk.dest); sc_sub(sk[1].bytes, inSk.mask.bytes, a.bytes); - clsag result = CLSAG_Gen(message, P, sk[0], C, sk[1], C_nonzero, Cout, index, kLRki, mscout, mspout, hwdev); + clsag result = CLSAG_Gen(message, P, sk[0], C, sk[1], C_nonzero, Cout, index, hwdev); memwipe(sk.data(), sk.size() * sizeof(key)); return result; } @@ -1084,14 +1043,13 @@ namespace rct { // must know the destination private key to find the correct amount, else will return a random number // Note: For txn fees, the last index in the amounts vector should contain that // Thus the amounts vector will be "one" longer than the destinations vectort - rctSig genRct(const key &message, const ctkeyV & inSk, const keyV & destinations, const vector & amounts, const ctkeyM &mixRing, const keyV &amount_keys, const multisig_kLRki *kLRki, multisig_out *msout, unsigned int index, ctkeyV &outSk, const RCTConfig &rct_config, hw::device &hwdev) { + rctSig genRct(const key &message, const ctkeyV & inSk, const keyV & destinations, const vector & amounts, const ctkeyM &mixRing, const keyV &amount_keys, unsigned int index, ctkeyV &outSk, const RCTConfig &rct_config, hw::device &hwdev) { CHECK_AND_ASSERT_THROW_MES(amounts.size() == destinations.size() || amounts.size() == destinations.size() + 1, "Different number of amounts/destinations"); CHECK_AND_ASSERT_THROW_MES(amount_keys.size() == destinations.size(), "Different number of amount_keys/destinations"); CHECK_AND_ASSERT_THROW_MES(index < mixRing.size(), "Bad index into mixRing"); for (size_t n = 0; n < mixRing.size(); ++n) { CHECK_AND_ASSERT_THROW_MES(mixRing[n].size() == inSk.size(), "Bad mixRing size"); } - CHECK_AND_ASSERT_THROW_MES((kLRki && msout) || (!kLRki && !msout), "Only one of kLRki/msout is present"); CHECK_AND_ASSERT_THROW_MES(inSk.size() < 2, "genRct is not suitable for 2+ rings"); rctSig rv; @@ -1130,23 +1088,21 @@ namespace rct { key txnFeeKey = scalarmultH(d2h(rv.txnFee)); rv.mixRing = mixRing; - if (msout) - msout->c.resize(1); - rv.p.MGs.push_back(proveRctMG(get_pre_mlsag_hash(rv, hwdev), rv.mixRing, inSk, outSk, rv.outPk, kLRki, msout ? &msout->c[0] : NULL, index, txnFeeKey,hwdev)); + rv.p.MGs.push_back(proveRctMG(get_pre_mlsag_hash(rv, hwdev), rv.mixRing, inSk, outSk, rv.outPk, index, txnFeeKey,hwdev)); return rv; } - rctSig genRct(const key &message, const ctkeyV & inSk, const ctkeyV & inPk, const keyV & destinations, const vector & amounts, const keyV &amount_keys, const multisig_kLRki *kLRki, multisig_out *msout, const int mixin, const RCTConfig &rct_config, hw::device &hwdev) { + rctSig genRct(const key &message, const ctkeyV & inSk, const ctkeyV & inPk, const keyV & destinations, const vector & amounts, const keyV &amount_keys, const int mixin, const RCTConfig &rct_config, hw::device &hwdev) { unsigned int index; ctkeyM mixRing; ctkeyV outSk; tie(mixRing, index) = populateFromBlockchain(inPk, mixin); - return genRct(message, inSk, destinations, amounts, mixRing, amount_keys, kLRki, msout, index, outSk, rct_config, hwdev); + return genRct(message, inSk, destinations, amounts, mixRing, amount_keys, index, outSk, rct_config, hwdev); } //RCT simple //for post-rct only - rctSig genRctSimple(const key &message, const ctkeyV & inSk, const keyV & destinations, const vector &inamounts, const vector &outamounts, xmr_amount txnFee, const ctkeyM & mixRing, const keyV &amount_keys, const std::vector *kLRki, multisig_out *msout, const std::vector & index, ctkeyV &outSk, const RCTConfig &rct_config, hw::device &hwdev) { + rctSig genRctSimple(const key &message, const ctkeyV & inSk, const keyV & destinations, const vector &inamounts, const vector &outamounts, xmr_amount txnFee, const ctkeyM & mixRing, const keyV &amount_keys, const std::vector & index, ctkeyV &outSk, const RCTConfig &rct_config, hw::device &hwdev) { const bool bulletproof_or_plus = rct_config.range_proof_type > RangeProofBorromean; CHECK_AND_ASSERT_THROW_MES(inamounts.size() > 0, "Empty inamounts"); CHECK_AND_ASSERT_THROW_MES(inamounts.size() == inSk.size(), "Different number of inamounts/inSk"); @@ -1157,10 +1113,6 @@ namespace rct { for (size_t n = 0; n < mixRing.size(); ++n) { CHECK_AND_ASSERT_THROW_MES(index[n] < mixRing[n].size(), "Bad index into mixRing"); } - CHECK_AND_ASSERT_THROW_MES((kLRki && msout) || (!kLRki && !msout), "Only one of kLRki/msout is present"); - if (kLRki && msout) { - CHECK_AND_ASSERT_THROW_MES(kLRki->size() == inamounts.size(), "Mismatched kLRki/inamounts sizes"); - } rctSig rv; if (bulletproof_or_plus) @@ -1322,11 +1274,7 @@ namespace rct { DP(pseudoOuts[i]); key full_message = get_pre_mlsag_hash(rv,hwdev); - if (msout) - { - msout->c.resize(inamounts.size()); - msout->mu_p.resize(is_rct_clsag(rv.type) ? inamounts.size() : 0); - } + for (i = 0 ; i < inamounts.size(); i++) { if (is_rct_clsag(rv.type)) @@ -1334,17 +1282,17 @@ namespace rct { if (hwdev.get_mode() == hw::device::TRANSACTION_CREATE_FAKE) rv.p.CLSAGs[i] = make_dummy_clsag(rv.mixRing[i].size()); else - rv.p.CLSAGs[i] = proveRctCLSAGSimple(full_message, rv.mixRing[i], inSk[i], a[i], pseudoOuts[i], kLRki ? &(*kLRki)[i]: NULL, msout ? &msout->c[i] : NULL, msout ? &msout->mu_p[i] : NULL, index[i], hwdev); + rv.p.CLSAGs[i] = proveRctCLSAGSimple(full_message, rv.mixRing[i], inSk[i], a[i], pseudoOuts[i], index[i], hwdev); } else { - rv.p.MGs[i] = proveRctMGSimple(full_message, rv.mixRing[i], inSk[i], a[i], pseudoOuts[i], kLRki ? &(*kLRki)[i]: NULL, msout ? &msout->c[i] : NULL, index[i], hwdev); + rv.p.MGs[i] = proveRctMGSimple(full_message, rv.mixRing[i], inSk[i], a[i], pseudoOuts[i], index[i], hwdev); } } return rv; } - rctSig genRctSimple(const key &message, const ctkeyV & inSk, const ctkeyV & inPk, const keyV & destinations, const vector &inamounts, const vector &outamounts, const keyV &amount_keys, const std::vector *kLRki, multisig_out *msout, xmr_amount txnFee, unsigned int mixin, const RCTConfig &rct_config, hw::device &hwdev) { + rctSig genRctSimple(const key &message, const ctkeyV & inSk, const ctkeyV & inPk, const keyV & destinations, const vector &inamounts, const vector &outamounts, const keyV &amount_keys, xmr_amount txnFee, unsigned int mixin, const RCTConfig &rct_config, hw::device &hwdev) { std::vector index; index.resize(inPk.size()); ctkeyM mixRing; @@ -1354,7 +1302,7 @@ namespace rct { mixRing[i].resize(mixin+1); index[i] = populateFromBlockchainSimple(mixRing[i], inPk[i], mixin); } - return genRctSimple(message, inSk, destinations, inamounts, outamounts, txnFee, mixRing, amount_keys, kLRki, msout, index, outSk, rct_config, hwdev); + return genRctSimple(message, inSk, destinations, inamounts, outamounts, txnFee, mixRing, amount_keys, index, outSk, rct_config, hwdev); } //RingCT protocol @@ -1385,7 +1333,7 @@ namespace rct { try { if (semantics) { - tools::threadpool& tpool = tools::threadpool::getInstance(); + tools::threadpool& tpool = tools::threadpool::getInstanceForCompute(); tools::threadpool::waiter waiter(tpool); std::deque results(rv.outPk.size(), false); DP("range proofs verified?"); @@ -1435,7 +1383,7 @@ namespace rct { { PERF_TIMER(verRctSemanticsSimple); - tools::threadpool& tpool = tools::threadpool::getInstance(); + tools::threadpool& tpool = tools::threadpool::getInstanceForCompute(); tools::threadpool::waiter waiter(tpool); std::deque results; std::vector bp_proofs; @@ -1588,7 +1536,7 @@ namespace rct { const size_t threads = std::max(rv.outPk.size(), rv.mixRing.size()); std::deque results(threads); - tools::threadpool& tpool = tools::threadpool::getInstance(); + tools::threadpool& tpool = tools::threadpool::getInstanceForCompute(); tools::threadpool::waiter waiter(tpool); const keyV &pseudoOuts = bulletproof || bulletproof_plus ? rv.p.pseudoOuts : rv.pseudoOuts; @@ -1700,60 +1648,4 @@ namespace rct { key mask; return decodeRctSimple(rv, sk, i, mask, hwdev); } - - bool signMultisigMLSAG(rctSig &rv, const std::vector &indices, const keyV &k, const multisig_out &msout, const key &secret_key) { - CHECK_AND_ASSERT_MES(rv.type == RCTTypeFull || rv.type == RCTTypeSimple || rv.type == RCTTypeBulletproof || rv.type == RCTTypeBulletproof2, - false, "unsupported rct type"); - CHECK_AND_ASSERT_MES(!is_rct_clsag(rv.type), false, "CLSAG signature type in MLSAG signature function"); - CHECK_AND_ASSERT_MES(indices.size() == k.size(), false, "Mismatched k/indices sizes"); - CHECK_AND_ASSERT_MES(k.size() == rv.p.MGs.size(), false, "Mismatched k/MGs size"); - CHECK_AND_ASSERT_MES(k.size() == msout.c.size(), false, "Mismatched k/msout.c size"); - CHECK_AND_ASSERT_MES(rv.p.CLSAGs.empty(), false, "CLSAGs not empty for MLSAGs"); - if (rv.type == RCTTypeFull) - { - CHECK_AND_ASSERT_MES(rv.p.MGs.size() == 1, false, "MGs not a single element"); - } - for (size_t n = 0; n < indices.size(); ++n) { - CHECK_AND_ASSERT_MES(indices[n] < rv.p.MGs[n].ss.size(), false, "Index out of range"); - CHECK_AND_ASSERT_MES(!rv.p.MGs[n].ss[indices[n]].empty(), false, "empty ss line"); - } - - // MLSAG: each player contributes a share to the secret-index ss: k - cc*secret_key_share - // cc: msout.c[n], secret_key_share: secret_key - for (size_t n = 0; n < indices.size(); ++n) { - rct::key diff; - sc_mulsub(diff.bytes, msout.c[n].bytes, secret_key.bytes, k[n].bytes); - sc_add(rv.p.MGs[n].ss[indices[n]][0].bytes, rv.p.MGs[n].ss[indices[n]][0].bytes, diff.bytes); - } - return true; - } - - bool signMultisigCLSAG(rctSig &rv, const std::vector &indices, const keyV &k, const multisig_out &msout, const key &secret_key) { - CHECK_AND_ASSERT_MES(is_rct_clsag(rv.type), false, "unsupported rct type"); - CHECK_AND_ASSERT_MES(indices.size() == k.size(), false, "Mismatched k/indices sizes"); - CHECK_AND_ASSERT_MES(k.size() == rv.p.CLSAGs.size(), false, "Mismatched k/CLSAGs size"); - CHECK_AND_ASSERT_MES(k.size() == msout.c.size(), false, "Mismatched k/msout.c size"); - CHECK_AND_ASSERT_MES(rv.p.MGs.empty(), false, "MGs not empty for CLSAGs"); - CHECK_AND_ASSERT_MES(msout.c.size() == msout.mu_p.size(), false, "Bad mu_p size"); - for (size_t n = 0; n < indices.size(); ++n) { - CHECK_AND_ASSERT_MES(indices[n] < rv.p.CLSAGs[n].s.size(), false, "Index out of range"); - } - - // CLSAG: each player contributes a share to the secret-index ss: k - cc*mu_p*secret_key_share - // cc: msout.c[n], mu_p, msout.mu_p[n], secret_key_share: secret_key - for (size_t n = 0; n < indices.size(); ++n) { - rct::key diff, sk; - sc_mul(sk.bytes, msout.mu_p[n].bytes, secret_key.bytes); - sc_mulsub(diff.bytes, msout.c[n].bytes, sk.bytes, k[n].bytes); - sc_add(rv.p.CLSAGs[n].s[indices[n]].bytes, rv.p.CLSAGs[n].s[indices[n]].bytes, diff.bytes); - } - return true; - } - - bool signMultisig(rctSig &rv, const std::vector &indices, const keyV &k, const multisig_out &msout, const key &secret_key) { - if (is_rct_clsag(rv.type)) - return signMultisigCLSAG(rv, indices, k, msout, secret_key); - else - return signMultisigMLSAG(rv, indices, k, msout, secret_key); - } } diff --git a/src/ringct/rctSigs.h b/src/ringct/rctSigs.h index a0346b34ea2..5c4773eb917 100644 --- a/src/ringct/rctSigs.h +++ b/src/ringct/rctSigs.h @@ -1,4 +1,4 @@ -// Copyright (c) 2016, Monero Research Labs +// Copyright (c) 2016-2023, Monero Research Labs // // Author: Shen Noether // @@ -74,12 +74,12 @@ namespace rct { // Gen creates a signature which proves that for some column in the keymatrix "pk" // the signer knows a secret key for each row in that column // Ver verifies that the MG sig was created correctly - mgSig MLSAG_Gen(const key &message, const keyM & pk, const keyV & xx, const multisig_kLRki *kLRki, key *mscout, const unsigned int index, size_t dsRows, hw::device &hwdev); + mgSig MLSAG_Gen(const key &message, const keyM & pk, const keyV & xx, const unsigned int index, size_t dsRows, hw::device &hwdev); bool MLSAG_Ver(const key &message, const keyM &pk, const mgSig &sig, size_t dsRows); - clsag CLSAG_Gen(const key &message, const keyV & P, const key & p, const keyV & C, const key & z, const keyV & C_nonzero, const key & C_offset, const unsigned int l, const multisig_kLRki *kLRki, key *mscout, key *mspout, hw::device &hwdev); + clsag CLSAG_Gen(const key &message, const keyV & P, const key & p, const keyV & C, const key & z, const keyV & C_nonzero, const key & C_offset, const unsigned int l, hw::device &hwdev); clsag CLSAG_Gen(const key &message, const keyV & P, const key & p, const keyV & C, const key & z, const keyV & C_nonzero, const key & C_offset, const unsigned int l); - clsag proveRctCLSAGSimple(const key &, const ctkeyV &, const ctkey &, const key &, const key &, const multisig_kLRki *, key *, key *, unsigned int, hw::device &); + clsag proveRctCLSAGSimple(const key &, const ctkeyV &, const ctkey &, const key &, const key &, unsigned int, hw::device &); bool verRctCLSAGSimple(const key &, const clsag &, const ctkeyV &, const key &); //proveRange and verRange @@ -100,8 +100,8 @@ namespace rct { // this shows that sum inputs = sum outputs //Ver: // verifies the above sig is created corretly - mgSig proveRctMG(const ctkeyM & pubs, const ctkeyV & inSk, const keyV &outMasks, const ctkeyV & outPk, const multisig_kLRki *kLRki, key *mscout, unsigned int index, const key &txnFee, const key &message, hw::device &hwdev); - mgSig proveRctMGSimple(const key & message, const ctkeyV & pubs, const ctkey & inSk, const key &a , const key &Cout, const multisig_kLRki *kLRki, key *mscout, unsigned int index, hw::device &hwdev); + mgSig proveRctMG(const ctkeyM & pubs, const ctkeyV & inSk, const keyV &outMasks, const ctkeyV & outPk, unsigned int index, const key &txnFee, const key &message, hw::device &hwdev); + mgSig proveRctMGSimple(const key & message, const ctkeyV & pubs, const ctkey & inSk, const key &a , const key &Cout, unsigned int index, hw::device &hwdev); bool verRctMG(const mgSig &mg, const ctkeyM & pubs, const ctkeyV & outPk, const key &txnFee, const key &message); bool verRctMGSimple(const key &message, const mgSig &mg, const ctkeyV & pubs, const key & C); @@ -123,10 +123,10 @@ namespace rct { //decodeRct: (c.f. https://eprint.iacr.org/2015/1098 section 5.1.1) // uses the attached ecdh info to find the amounts represented by each output commitment // must know the destination private key to find the correct amount, else will return a random number - rctSig genRct(const key &message, const ctkeyV & inSk, const keyV & destinations, const std::vector & amounts, const ctkeyM &mixRing, const keyV &amount_keys, const multisig_kLRki *kLRki, multisig_out *msout, unsigned int index, ctkeyV &outSk, const RCTConfig &rct_config, hw::device &hwdev); - rctSig genRct(const key &message, const ctkeyV & inSk, const ctkeyV & inPk, const keyV & destinations, const std::vector & amounts, const keyV &amount_keys, const multisig_kLRki *kLRki, multisig_out *msout, const int mixin, const RCTConfig &rct_config, hw::device &hwdev); - rctSig genRctSimple(const key & message, const ctkeyV & inSk, const ctkeyV & inPk, const keyV & destinations, const std::vector & inamounts, const std::vector & outamounts, const keyV &amount_keys, const std::vector *kLRki, multisig_out *msout, xmr_amount txnFee, unsigned int mixin, const RCTConfig &rct_config, hw::device &hwdev); - rctSig genRctSimple(const key & message, const ctkeyV & inSk, const keyV & destinations, const std::vector & inamounts, const std::vector & outamounts, xmr_amount txnFee, const ctkeyM & mixRing, const keyV &amount_keys, const std::vector *kLRki, multisig_out *msout, const std::vector & index, ctkeyV &outSk, const RCTConfig &rct_config, hw::device &hwdev); + rctSig genRct(const key &message, const ctkeyV & inSk, const keyV & destinations, const std::vector & amounts, const ctkeyM &mixRing, const keyV &amount_keys, unsigned int index, ctkeyV &outSk, const RCTConfig &rct_config, hw::device &hwdev); + rctSig genRct(const key &message, const ctkeyV & inSk, const ctkeyV & inPk, const keyV & destinations, const std::vector & amounts, const keyV &amount_keys, const int mixin, const RCTConfig &rct_config, hw::device &hwdev); + rctSig genRctSimple(const key & message, const ctkeyV & inSk, const ctkeyV & inPk, const keyV & destinations, const std::vector & inamounts, const std::vector & outamounts, const keyV &amount_keys, xmr_amount txnFee, unsigned int mixin, const RCTConfig &rct_config, hw::device &hwdev); + rctSig genRctSimple(const key & message, const ctkeyV & inSk, const keyV & destinations, const std::vector & inamounts, const std::vector & outamounts, xmr_amount txnFee, const ctkeyM & mixRing, const keyV &amount_keys, const std::vector & index, ctkeyV &outSk, const RCTConfig &rct_config, hw::device &hwdev); bool verRct(const rctSig & rv, bool semantics); static inline bool verRct(const rctSig & rv) { return verRct(rv, true) && verRct(rv, false); } bool verRctSemanticsSimple(const rctSig & rv); @@ -138,7 +138,6 @@ namespace rct { xmr_amount decodeRctSimple(const rctSig & rv, const key & sk, unsigned int i, key & mask, hw::device &hwdev); xmr_amount decodeRctSimple(const rctSig & rv, const key & sk, unsigned int i, hw::device &hwdev); key get_pre_mlsag_hash(const rctSig &rv, hw::device &hwdev); - bool signMultisig(rctSig &rv, const std::vector &indices, const keyV &k, const multisig_out &msout, const key &secret_key); } #endif /* RCTSIGS_H */ diff --git a/src/ringct/rctTypes.cpp b/src/ringct/rctTypes.cpp index c22b0524f56..4ac75b78b8a 100644 --- a/src/ringct/rctTypes.cpp +++ b/src/ringct/rctTypes.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2016, Monero Research Labs +// Copyright (c) 2016-2023, Monero Research Labs // // Author: Shen Noether // diff --git a/src/ringct/rctTypes.h b/src/ringct/rctTypes.h index 59ed4d6a6ac..380ca2f532d 100644 --- a/src/ringct/rctTypes.h +++ b/src/ringct/rctTypes.h @@ -1,4 +1,4 @@ -// Copyright (c) 2016, Monero Research Labs +// Copyright (c) 2016-2023, Monero Research Labs // // Author: Shen Noether // @@ -97,6 +97,14 @@ namespace rct { struct ctkey { key dest; key mask; //C here if public + + bool operator==(const ctkey &other) const { + return (dest == other.dest) && (mask == other.mask); + } + + bool operator!=(const ctkey &other) const { + return !(*this == other); + } }; typedef std::vector ctkeyV; typedef std::vector ctkeyM; diff --git a/src/rpc/CMakeLists.txt b/src/rpc/CMakeLists.txt index edfc7006781..fe80d3083e3 100644 --- a/src/rpc/CMakeLists.txt +++ b/src/rpc/CMakeLists.txt @@ -1,4 +1,4 @@ -# Copyright (c) 2014-2022, The Monero Project +# Copyright (c) 2014-2023, The Monero Project # # All rights reserved. # diff --git a/src/rpc/bootstrap_node_selector.cpp b/src/rpc/bootstrap_node_selector.cpp index a1cad3e59d5..d224c5c53ab 100644 --- a/src/rpc/bootstrap_node_selector.cpp +++ b/src/rpc/bootstrap_node_selector.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2020-2022, The Monero Project +// Copyright (c) 2020-2023, The Monero Project // // All rights reserved. diff --git a/src/rpc/bootstrap_node_selector.h b/src/rpc/bootstrap_node_selector.h index 616c180fc97..3e52c3538a2 100644 --- a/src/rpc/bootstrap_node_selector.h +++ b/src/rpc/bootstrap_node_selector.h @@ -1,4 +1,4 @@ -// Copyright (c) 2020-2022, The Monero Project +// Copyright (c) 2020-2023, The Monero Project // // All rights reserved. diff --git a/src/rpc/core_rpc_server.cpp b/src/rpc/core_rpc_server.cpp index 0fe28465f8b..0adf0b65e8f 100644 --- a/src/rpc/core_rpc_server.cpp +++ b/src/rpc/core_rpc_server.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2022, The Monero Project +// Copyright (c) 2014-2023, The Monero Project // // All rights reserved. // @@ -352,6 +352,7 @@ namespace cryptonote const auto ssl_base_path = (boost::filesystem::path{data_dir} / "rpc_ssl").string(); const bool ssl_cert_file_exists = boost::filesystem::exists(ssl_base_path + ".crt"); const bool ssl_pkey_file_exists = boost::filesystem::exists(ssl_base_path + ".key"); + const bool ssl_fp_file_exists = boost::filesystem::exists(ssl_base_path + ".fingerprint"); if (store_ssl_key) { // .key files are often given different read permissions as their corresponding .crt files. @@ -361,13 +362,39 @@ namespace cryptonote MFATAL("Certificate (.crt) and private key (.key) files must both exist or both not exist at path: " << ssl_base_path); return false; } + else if (!ssl_cert_file_exists && ssl_fp_file_exists) // only fingerprint file is present + { + MFATAL("Fingerprint file is present while certificate (.crt) and private key (.key) files are not at path: " << ssl_base_path); + return false; + } else if (ssl_cert_file_exists) { // and ssl_pkey_file_exists // load key from previous run, password prompted by OpenSSL store_ssl_key = false; rpc_config->ssl_options.auth = epee::net_utils::ssl_authentication_t{ssl_base_path + ".key", ssl_base_path + ".crt"}; + + // Since the .fingerprint file was added afterwards, sometimes the other 2 are present, and .fingerprint isn't + // In that case, generate the .fingerprint file from the existing .crt file + if (!ssl_fp_file_exists) + { + try + { + std::string fingerprint = epee::net_utils::get_hr_ssl_fingerprint_from_file(ssl_base_path + ".crt"); + if (!epee::file_io_utils::save_string_to_file(ssl_base_path + ".fingerprint", fingerprint)) + { + MWARNING("Could not save SSL fingerprint to file '" << ssl_base_path << ".fingerprint'"); + } + const auto fp_perms = boost::filesystem::owner_read | boost::filesystem::group_read | boost::filesystem::others_read; + boost::filesystem::permissions(ssl_base_path + ".fingerprint", fp_perms); + } + catch (const std::exception& e) + { + // Do nothing. The fingerprint file is helpful, but not at all necessary. + MWARNING("While trying to store SSL fingerprint file, got error (ignoring): " << e.what()); + } + } } - } + } // if (store_ssl_key) auto rng = [](size_t len, uint8_t *ptr){ return crypto::rand(len, ptr); }; const bool inited = epee::http_server_impl_base::init( @@ -492,7 +519,6 @@ namespace cryptonote } CHECK_PAYMENT_MIN1(req, res, COST_PER_GET_INFO, false); - db_rtxn_guard rtxn_guard(&m_core.get_blockchain_storage().get_db()); const bool restricted = m_restricted && ctx; @@ -598,90 +624,166 @@ namespace cryptonote } CHECK_PAYMENT(req, res, 1); - db_rtxn_guard rtxn_guard(&m_core.get_blockchain_storage().get_db()); - // quick check for noop - if (!req.block_ids.empty()) + res.daemon_time = (uint64_t)time(NULL); + // Always set daemon time, and set it early rather than late, as delivering some incremental pool + // info twice because of slightly overlapping time intervals is no problem, whereas producing gaps + // and never delivering something is + + bool get_blocks = false; + bool get_pool = false; + switch (req.requested_info) { - uint64_t last_block_height; - crypto::hash last_block_hash; - m_core.get_blockchain_top(last_block_height, last_block_hash); - if (last_block_hash == req.block_ids.front()) - { - res.start_height = 0; - res.current_height = last_block_height + 1; - res.status = CORE_RPC_STATUS_OK; + case COMMAND_RPC_GET_BLOCKS_FAST::BLOCKS_ONLY: + // Compatibility value 0: Clients that do not set 'requested_info' want blocks, and only blocks + get_blocks = true; + break; + case COMMAND_RPC_GET_BLOCKS_FAST::BLOCKS_AND_POOL: + get_blocks = true; + get_pool = true; + break; + case COMMAND_RPC_GET_BLOCKS_FAST::POOL_ONLY: + get_pool = true; + break; + default: + res.status = "Failed, wrong requested info"; return true; - } } - size_t max_blocks = COMMAND_RPC_GET_BLOCKS_FAST_MAX_BLOCK_COUNT; - if (m_rpc_payment) + res.pool_info_extent = COMMAND_RPC_GET_BLOCKS_FAST::NONE; + + if (get_pool) { - max_blocks = res.credits / COST_PER_BLOCK; - if (max_blocks > COMMAND_RPC_GET_BLOCKS_FAST_MAX_BLOCK_COUNT) - max_blocks = COMMAND_RPC_GET_BLOCKS_FAST_MAX_BLOCK_COUNT; - if (max_blocks == 0) + const bool restricted = m_restricted && ctx; + const bool request_has_rpc_origin = ctx != NULL; + const bool allow_sensitive = !request_has_rpc_origin || !restricted; + const size_t max_tx_count = restricted ? RESTRICTED_TRANSACTIONS_COUNT : std::numeric_limits::max(); + + bool incremental; + std::vector> added_pool_txs; + bool success = m_core.get_pool_info((time_t)req.pool_info_since, allow_sensitive, max_tx_count, added_pool_txs, res.remaining_added_pool_txids, res.removed_pool_txids, incremental); + if (success) { - res.status = CORE_RPC_STATUS_PAYMENT_REQUIRED; + res.added_pool_txs.clear(); + if (m_rpc_payment) + { + CHECK_PAYMENT_SAME_TS(req, res, added_pool_txs.size() * COST_PER_TX + (res.remaining_added_pool_txids.size() + res.removed_pool_txids.size()) * COST_PER_POOL_HASH); + } + for (const auto &added_pool_tx: added_pool_txs) + { + COMMAND_RPC_GET_BLOCKS_FAST::pool_tx_info info; + info.tx_hash = added_pool_tx.first; + std::stringstream oss; + binary_archive ar(oss); + bool r = req.prune + ? const_cast(added_pool_tx.second.tx).serialize_base(ar) + : ::serialization::serialize(ar, const_cast(added_pool_tx.second.tx)); + if (!r) + { + res.status = "Failed to serialize transaction"; + return true; + } + info.tx_blob = oss.str(); + info.double_spend_seen = added_pool_tx.second.double_spend_seen; + res.added_pool_txs.push_back(std::move(info)); + } + } + if (success) + { + res.pool_info_extent = incremental ? COMMAND_RPC_GET_BLOCKS_FAST::INCREMENTAL : COMMAND_RPC_GET_BLOCKS_FAST::FULL; + } + else + { + res.status = "Failed to get pool info"; return true; } } - std::vector, std::vector > > > bs; - if(!m_core.find_blockchain_supplement(req.start_height, req.block_ids, bs, res.current_height, res.start_height, req.prune, !req.no_miner_tx, max_blocks, COMMAND_RPC_GET_BLOCKS_FAST_MAX_TX_COUNT)) - { - res.status = "Failed"; - add_host_fail(ctx); - return true; - } - - CHECK_PAYMENT_SAME_TS(req, res, bs.size() * COST_PER_BLOCK); - - size_t size = 0, ntxes = 0; - res.blocks.reserve(bs.size()); - res.output_indices.reserve(bs.size()); - for(auto& bd: bs) + if (get_blocks) { - res.blocks.resize(res.blocks.size()+1); - res.blocks.back().pruned = req.prune; - res.blocks.back().block = bd.first.first; - size += bd.first.first.size(); - res.output_indices.push_back(COMMAND_RPC_GET_BLOCKS_FAST::block_output_indices()); - ntxes += bd.second.size(); - res.output_indices.back().indices.reserve(1 + bd.second.size()); - if (req.no_miner_tx) - res.output_indices.back().indices.push_back(COMMAND_RPC_GET_BLOCKS_FAST::tx_output_indices()); - res.blocks.back().txs.reserve(bd.second.size()); - for (std::vector>::iterator i = bd.second.begin(); i != bd.second.end(); ++i) + // quick check for noop + if (!req.block_ids.empty()) { - res.blocks.back().txs.push_back({std::move(i->second), crypto::null_hash}); - i->second.clear(); - i->second.shrink_to_fit(); - size += res.blocks.back().txs.back().blob.size(); + uint64_t last_block_height; + crypto::hash last_block_hash; + m_core.get_blockchain_top(last_block_height, last_block_hash); + if (last_block_hash == req.block_ids.front()) + { + res.start_height = 0; + res.current_height = last_block_height + 1; + res.status = CORE_RPC_STATUS_OK; + return true; + } } - const size_t n_txes_to_lookup = bd.second.size() + (req.no_miner_tx ? 0 : 1); - if (n_txes_to_lookup > 0) + size_t max_blocks = COMMAND_RPC_GET_BLOCKS_FAST_MAX_BLOCK_COUNT; + if (m_rpc_payment) { - std::vector> indices; - bool r = m_core.get_tx_outputs_gindexs(req.no_miner_tx ? bd.second.front().first : bd.first.second, n_txes_to_lookup, indices); - if (!r) + max_blocks = res.credits / COST_PER_BLOCK; + if (max_blocks > COMMAND_RPC_GET_BLOCKS_FAST_MAX_BLOCK_COUNT) + max_blocks = COMMAND_RPC_GET_BLOCKS_FAST_MAX_BLOCK_COUNT; + if (max_blocks == 0) { - res.status = "Failed"; + res.status = CORE_RPC_STATUS_PAYMENT_REQUIRED; return true; } - if (indices.size() != n_txes_to_lookup || res.output_indices.back().indices.size() != (req.no_miner_tx ? 1 : 0)) + } + + std::vector, std::vector > > > bs; + if(!m_core.find_blockchain_supplement(req.start_height, req.block_ids, bs, res.current_height, res.start_height, req.prune, !req.no_miner_tx, max_blocks, COMMAND_RPC_GET_BLOCKS_FAST_MAX_TX_COUNT)) + { + res.status = "Failed"; + add_host_fail(ctx); + return true; + } + + CHECK_PAYMENT_SAME_TS(req, res, bs.size() * COST_PER_BLOCK); + + size_t size = 0, ntxes = 0; + res.blocks.reserve(bs.size()); + res.output_indices.reserve(bs.size()); + for(auto& bd: bs) + { + res.blocks.resize(res.blocks.size()+1); + res.blocks.back().pruned = req.prune; + res.blocks.back().block = bd.first.first; + size += bd.first.first.size(); + res.output_indices.push_back(COMMAND_RPC_GET_BLOCKS_FAST::block_output_indices()); + ntxes += bd.second.size(); + res.output_indices.back().indices.reserve(1 + bd.second.size()); + if (req.no_miner_tx) + res.output_indices.back().indices.push_back(COMMAND_RPC_GET_BLOCKS_FAST::tx_output_indices()); + res.blocks.back().txs.reserve(bd.second.size()); + for (std::vector>::iterator i = bd.second.begin(); i != bd.second.end(); ++i) + { + res.blocks.back().txs.push_back({std::move(i->second), crypto::null_hash}); + i->second.clear(); + i->second.shrink_to_fit(); + size += res.blocks.back().txs.back().blob.size(); + } + + const size_t n_txes_to_lookup = bd.second.size() + (req.no_miner_tx ? 0 : 1); + if (n_txes_to_lookup > 0) { - res.status = "Failed"; - return true; + std::vector> indices; + bool r = m_core.get_tx_outputs_gindexs(req.no_miner_tx ? bd.second.front().first : bd.first.second, n_txes_to_lookup, indices); + if (!r) + { + res.status = "Failed"; + return true; + } + if (indices.size() != n_txes_to_lookup || res.output_indices.back().indices.size() != (req.no_miner_tx ? 1 : 0)) + { + res.status = "Failed"; + return true; + } + for (size_t i = 0; i < indices.size(); ++i) + res.output_indices.back().indices.push_back({std::move(indices[i])}); } - for (size_t i = 0; i < indices.size(); ++i) - res.output_indices.back().indices.push_back({std::move(indices[i])}); } + MDEBUG("on_get_blocks: " << bs.size() << " blocks, " << ntxes << " txes, size " << size); } - MDEBUG("on_get_blocks: " << bs.size() << " blocks, " << ntxes << " txes, size " << size); res.status = CORE_RPC_STATUS_OK; return true; } @@ -730,7 +832,6 @@ namespace cryptonote res.blocks.clear(); res.blocks.reserve(req.heights.size()); CHECK_PAYMENT_MIN1(req, res, req.heights.size() * COST_PER_BLOCK, false); - db_rtxn_guard rtxn_guard(&m_core.get_blockchain_storage().get_db()); for (uint64_t height : req.heights) { block blk; @@ -922,17 +1023,16 @@ namespace cryptonote // try the pool for any missing txes size_t found_in_pool = 0; std::unordered_set pool_tx_hashes; - std::unordered_map per_tx_pool_tx_info; + std::unordered_map per_tx_pool_tx_details; if (!missed_txs.empty()) { - std::vector pool_tx_info; - std::vector pool_key_image_info; - bool r = m_core.get_pool_transactions_and_spent_keys_info(pool_tx_info, pool_key_image_info, !request_has_rpc_origin || !restricted); + std::vector> pool_txs; + bool r = m_core.get_pool_transactions_info(missed_txs, pool_txs, !request_has_rpc_origin || !restricted); if(r) { // sort to match original request std::vector> sorted_txs; - std::vector::const_iterator i; + std::vector>::const_iterator i; unsigned txs_processed = 0; for (const crypto::hash &h: vh) { @@ -952,36 +1052,23 @@ namespace cryptonote sorted_txs.push_back(std::move(txs[txs_processed])); ++txs_processed; } - else if ((i = std::find_if(pool_tx_info.begin(), pool_tx_info.end(), [h](const tx_info &txi) { return epee::string_tools::pod_to_hex(h) == txi.id_hash; })) != pool_tx_info.end()) + else if ((i = std::find_if(pool_txs.begin(), pool_txs.end(), [h](const std::pair &pt) { return h == pt.first; })) != pool_txs.end()) { - cryptonote::transaction tx; - if (!cryptonote::parse_and_validate_tx_from_blob(i->tx_blob, tx)) - { - res.status = "Failed to parse and validate tx from blob"; - return true; - } + const tx_memory_pool::tx_details &td = i->second; std::stringstream ss; binary_archive ba(ss); - bool r = const_cast(tx).serialize_base(ba); + bool r = const_cast(td.tx).serialize_base(ba); if (!r) { res.status = "Failed to serialize transaction base"; return true; } const cryptonote::blobdata pruned = ss.str(); - const crypto::hash prunable_hash = tx.version == 1 ? crypto::null_hash : get_transaction_prunable_hash(tx); - sorted_txs.push_back(std::make_tuple(h, pruned, prunable_hash, std::string(i->tx_blob, pruned.size()))); + const crypto::hash prunable_hash = td.tx.version == 1 ? crypto::null_hash : get_transaction_prunable_hash(td.tx); + sorted_txs.push_back(std::make_tuple(h, pruned, prunable_hash, std::string(td.tx_blob, pruned.size()))); missed_txs.erase(std::find(missed_txs.begin(), missed_txs.end(), h)); pool_tx_hashes.insert(h); - const std::string hash_string = epee::string_tools::pod_to_hex(h); - for (const auto &ti: pool_tx_info) - { - if (ti.id_hash == hash_string) - { - per_tx_pool_tx_info.insert(std::make_pair(h, ti)); - break; - } - } + per_tx_pool_tx_details.insert(std::make_pair(h, td)); ++found_in_pool; } } @@ -1012,7 +1099,17 @@ namespace cryptonote CHECK_AND_ASSERT_MES(tx_hash == std::get<0>(tx), false, "mismatched tx hash"); e.tx_hash = *txhi++; e.prunable_hash = epee::string_tools::pod_to_hex(std::get<2>(tx)); - if (req.split || req.prune || std::get<3>(tx).empty()) + + // coinbase txes do not have signatures to prune, so they appear to be pruned if looking just at prunable data being empty + bool pruned = std::get<3>(tx).empty(); + if (pruned) + { + cryptonote::transaction t; + if (cryptonote::parse_and_validate_tx_base_from_blob(std::get<1>(tx), t) && is_coinbase(t)) + pruned = false; + } + + if (req.split || req.prune || pruned) { // use splitted form with pruned and prunable (filled only when prune=false and the daemon has it), leaving as_hex as empty e.pruned_as_hex = string_tools::buff_to_hex_nodelimer(std::get<1>(tx)); @@ -1077,8 +1174,8 @@ namespace cryptonote { e.block_height = e.block_timestamp = std::numeric_limits::max(); e.confirmations = 0; - auto it = per_tx_pool_tx_info.find(tx_hash); - if (it != per_tx_pool_tx_info.end()) + auto it = per_tx_pool_tx_details.find(tx_hash); + if (it != per_tx_pool_tx_details.end()) { e.double_spend_seen = it->second.double_spend_seen; e.relayed = it->second.relayed; @@ -1243,6 +1340,7 @@ namespace cryptonote { LOG_PRINT_L0("[on_send_raw_tx]: Failed to parse tx from hexbuff: " << req.tx_as_hex); res.status = "Failed"; + res.reason = "Hex decoding failed"; return true; } @@ -1278,6 +1376,8 @@ namespace cryptonote add_reason(reason, "fee too low"); if ((res.too_few_outputs = tvc.m_too_few_outputs)) add_reason(reason, "too few outputs"); + if ((res.tx_extra_too_big = tvc.m_tx_extra_too_big)) + add_reason(reason, "tx-extra too big"); const std::string punctuation = reason.empty() ? "" : ": "; if (tvc.m_verifivation_failed) { @@ -1592,7 +1692,6 @@ namespace cryptonote return r; CHECK_PAYMENT(req, res, 1); - db_rtxn_guard rtxn_guard(&m_core.get_blockchain_storage().get_db()); const bool restricted = m_restricted && ctx; const bool request_has_rpc_origin = ctx != NULL; @@ -1617,7 +1716,6 @@ namespace cryptonote return r; CHECK_PAYMENT(req, res, 1); - db_rtxn_guard rtxn_guard(&m_core.get_blockchain_storage().get_db()); const bool restricted = m_restricted && ctx; const bool request_has_rpc_origin = ctx != NULL; @@ -1720,14 +1818,11 @@ namespace cryptonote error_resp.message = "Wrong parameters, expected height"; return false; } - db_rtxn_guard rtxn_guard(&m_core.get_blockchain_storage().get_db()); uint64_t h = req[0]; - uint64_t blockchain_height = m_core.get_current_blockchain_height(); - if(blockchain_height <= h) + if(m_core.get_current_blockchain_height() <= h) { error_resp.code = CORE_RPC_ERROR_CODE_TOO_BIG_HEIGHT; - error_resp.message = std::string("Requested block height: ") + std::to_string(h) + " greater than current top block height: " + std::to_string(blockchain_height - 1); - return false; + error_resp.message = std::string("Requested block height: ") + std::to_string(h) + " greater than current top block height: " + std::to_string(m_core.get_current_blockchain_height() - 1); } res = string_tools::pod_to_hex(m_core.get_block_id_by_height(h)); return true; @@ -1877,7 +1972,6 @@ namespace cryptonote return false; } } - db_rtxn_guard rtxn_guard(&m_core.get_blockchain_storage().get_db()); crypto::hash seed_hash, next_seed_hash; if (!get_block_template(info.address, req.prev_block.empty() ? NULL : &prev_block, blob_reserve, reserved_offset, wdiff, res.height, res.expected_reward, b, res.seed_height, seed_hash, next_seed_hash, error_resp)) return false; @@ -2299,6 +2393,12 @@ namespace cryptonote return m_bootstrap_daemon->handle_result(false, {}); } + if (bootstrap_daemon_height < m_core.get_checkpoints().get_max_height()) + { + MINFO("Bootstrap daemon height is lower than the latest checkpoint"); + return m_bootstrap_daemon->handle_result(false, {}); + } + if (!m_p2p.get_payload_object().no_sync()) { uint64_t top_height = m_core.get_current_blockchain_height(); @@ -2351,7 +2451,6 @@ namespace cryptonote CHECK_CORE_READY(); CHECK_PAYMENT_MIN1(req, res, COST_PER_BLOCK_HEADER, false); - db_rtxn_guard rtxn_guard(&m_core.get_blockchain_storage().get_db()); uint64_t last_block_height; crypto::hash last_block_hash; m_core.get_blockchain_top(last_block_height, last_block_hash); @@ -2392,8 +2491,6 @@ namespace cryptonote return false; } - db_rtxn_guard rtxn_guard(&m_core.get_blockchain_storage().get_db()); - auto get = [this](const std::string &hash, bool fill_pow_hash, block_header_response &block_header, bool restricted, epee::json_rpc::error& error_resp) -> bool { crypto::hash block_hash; bool hash_parsed = parse_hash256(hash, block_hash); @@ -2453,6 +2550,13 @@ namespace cryptonote if (use_bootstrap_daemon_if_necessary(invoke_http_mode::JON_RPC, "getblockheadersrange", req, res, r)) return r; + const uint64_t bc_height = m_core.get_current_blockchain_height(); + if (req.start_height >= bc_height || req.end_height >= bc_height || req.start_height > req.end_height) + { + error_resp.code = CORE_RPC_ERROR_CODE_TOO_BIG_HEIGHT; + error_resp.message = "Invalid start/end heights."; + return false; + } const bool restricted = m_restricted && ctx; if (restricted && req.end_height - req.start_height > RESTRICTED_BLOCK_HEADER_RANGE) { @@ -2462,16 +2566,6 @@ namespace cryptonote } CHECK_PAYMENT_MIN1(req, res, (req.end_height - req.start_height + 1) * COST_PER_BLOCK_HEADER, false); - db_rtxn_guard rtxn_guard(&m_core.get_blockchain_storage().get_db()); - - const uint64_t bc_height = m_core.get_current_blockchain_height(); - if (req.start_height >= bc_height || req.end_height >= bc_height || req.start_height > req.end_height) - { - error_resp.code = CORE_RPC_ERROR_CODE_TOO_BIG_HEIGHT; - error_resp.message = "Invalid start/end heights."; - return false; - } - for (uint64_t h = req.start_height; h <= req.end_height; ++h) { crypto::hash block_hash = m_core.get_block_id_by_height(h); @@ -2516,12 +2610,10 @@ namespace cryptonote if (use_bootstrap_daemon_if_necessary(invoke_http_mode::JON_RPC, "getblockheaderbyheight", req, res, r)) return r; - db_rtxn_guard rtxn_guard(&m_core.get_blockchain_storage().get_db()); - uint64_t blockchain_height = m_core.get_current_blockchain_height(); - if(blockchain_height <= req.height) + if(m_core.get_current_blockchain_height() <= req.height) { error_resp.code = CORE_RPC_ERROR_CODE_TOO_BIG_HEIGHT; - error_resp.message = std::string("Requested block height: ") + std::to_string(req.height) + " greater than current top block height: " + std::to_string(blockchain_height - 1); + error_resp.message = std::string("Requested block height: ") + std::to_string(req.height) + " greater than current top block height: " + std::to_string(m_core.get_current_blockchain_height() - 1); return false; } CHECK_PAYMENT_MIN1(req, res, COST_PER_BLOCK_HEADER, false); @@ -2554,7 +2646,6 @@ namespace cryptonote return r; CHECK_PAYMENT_MIN1(req, res, COST_PER_BLOCK, false); - db_rtxn_guard rtxn_guard(&m_core.get_blockchain_storage().get_db()); crypto::hash block_hash; if (!req.hash.empty()) @@ -2569,11 +2660,10 @@ namespace cryptonote } else { - uint64_t blockchain_height = m_core.get_current_blockchain_height(); - if(blockchain_height <= req.height) + if(m_core.get_current_blockchain_height() <= req.height) { error_resp.code = CORE_RPC_ERROR_CODE_TOO_BIG_HEIGHT; - error_resp.message = std::string("Requested block height: ") + std::to_string(req.height) + " greater than current top block height: " + std::to_string(blockchain_height - 1); + error_resp.message = std::string("Requested block height: ") + std::to_string(req.height) + " greater than current top block height: " + std::to_string(m_core.get_current_blockchain_height() - 1); return false; } block_hash = m_core.get_block_id_by_height(req.height); @@ -2874,6 +2964,10 @@ namespace cryptonote res.version = CORE_RPC_VERSION; res.release = MONERO_VERSION_IS_RELEASE; + res.current_height = m_core.get_current_blockchain_height(); + res.target_height = m_p2p.get_payload_object().is_synchronized() ? 0 : m_core.get_target_blockchain_height(); + for (const auto &hf : m_core.get_blockchain_storage().get_hardforks()) + res.hard_forks.push_back({hf.version, hf.height}); res.status = CORE_RPC_STATUS_OK; return true; } @@ -2881,7 +2975,6 @@ namespace cryptonote bool core_rpc_server::on_get_coinbase_tx_sum(const COMMAND_RPC_GET_COINBASE_TX_SUM::request& req, COMMAND_RPC_GET_COINBASE_TX_SUM::response& res, epee::json_rpc::error& error_resp, const connection_context *ctx) { RPC_TRACKER(get_coinbase_tx_sum); - db_rtxn_guard rtxn_guard(&m_core.get_blockchain_storage().get_db()); const uint64_t bc_height = m_core.get_current_blockchain_height(); if (req.height >= bc_height || req.count > bc_height) { @@ -2923,7 +3016,6 @@ namespace cryptonote bool core_rpc_server::on_get_alternate_chains(const COMMAND_RPC_GET_ALTERNATE_CHAINS::request& req, COMMAND_RPC_GET_ALTERNATE_CHAINS::response& res, epee::json_rpc::error& error_resp, const connection_context *ctx) { RPC_TRACKER(get_alternate_chains); - db_rtxn_guard rtxn_guard(&m_core.get_blockchain_storage().get_db()); try { std::vector>> chains = m_core.get_blockchain_storage().get_alternative_chains(); @@ -3226,7 +3318,6 @@ namespace cryptonote bool r; if (use_bootstrap_daemon_if_necessary(invoke_http_mode::JON_RPC, "get_txpool_backlog", req, res, r)) return r; - db_rtxn_guard rtxn_guard(&m_core.get_blockchain_storage().get_db()); size_t n_txes = m_core.get_pool_transactions_count(); CHECK_PAYMENT_MIN1(req, res, COST_PER_TX_POOL_STATS * n_txes, false); diff --git a/src/rpc/core_rpc_server.h b/src/rpc/core_rpc_server.h index 0274f4db84b..790d5eb231e 100644 --- a/src/rpc/core_rpc_server.h +++ b/src/rpc/core_rpc_server.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2022, The Monero Project +// Copyright (c) 2014-2023, The Monero Project // // All rights reserved. // @@ -47,10 +47,6 @@ #undef MONERO_DEFAULT_LOG_CATEGORY #define MONERO_DEFAULT_LOG_CATEGORY "daemon.rpc" -// yes, epee doesn't properly use its full namespace when calling its -// functions from macros. *sigh* -using namespace epee; - namespace cryptonote { /************************************************************************/ diff --git a/src/rpc/core_rpc_server_commands_defs.h b/src/rpc/core_rpc_server_commands_defs.h index 1be2610ff30..7f1581d0c00 100644 --- a/src/rpc/core_rpc_server_commands_defs.h +++ b/src/rpc/core_rpc_server_commands_defs.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2022, The Monero Project +// Copyright (c) 2014-2023, The Monero Project // // All rights reserved. // @@ -88,7 +88,7 @@ namespace cryptonote // advance which version they will stop working with // Don't go over 32767 for any of these #define CORE_RPC_VERSION_MAJOR 3 -#define CORE_RPC_VERSION_MINOR 10 +#define CORE_RPC_VERSION_MINOR 12 #define MAKE_CORE_RPC_VERSION(major,minor) (((major)<<16)|(minor)) #define CORE_RPC_VERSION MAKE_CORE_RPC_VERSION(CORE_RPC_VERSION_MAJOR, CORE_RPC_VERSION_MINOR) @@ -162,18 +162,29 @@ namespace cryptonote struct COMMAND_RPC_GET_BLOCKS_FAST { + enum REQUESTED_INFO + { + BLOCKS_ONLY = 0, + BLOCKS_AND_POOL = 1, + POOL_ONLY = 2 + }; + struct request_t: public rpc_access_request_base { + uint8_t requested_info; std::list block_ids; //*first 10 blocks id goes sequential, next goes in pow(2,n) offset, like 2, 4, 8, 16, 32, 64 and so on, and the last one is always genesis block */ uint64_t start_height; bool prune; bool no_miner_tx; + uint64_t pool_info_since; BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE_PARENT(rpc_access_request_base) + KV_SERIALIZE_OPT(requested_info, (uint8_t)0) KV_SERIALIZE_CONTAINER_POD_AS_BLOB(block_ids) KV_SERIALIZE(start_height) KV_SERIALIZE(prune) KV_SERIALIZE_OPT(no_miner_tx, false) + KV_SERIALIZE_OPT(pool_info_since, (uint64_t)0) END_KV_SERIALIZE_MAP() }; typedef epee::misc_utils::struct_init request; @@ -196,12 +207,37 @@ namespace cryptonote END_KV_SERIALIZE_MAP() }; + struct pool_tx_info + { + crypto::hash tx_hash; + blobdata tx_blob; + bool double_spend_seen; + + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE_VAL_POD_AS_BLOB(tx_hash) + KV_SERIALIZE(tx_blob) + KV_SERIALIZE(double_spend_seen) + END_KV_SERIALIZE_MAP() + }; + + enum POOL_INFO_EXTENT + { + NONE = 0, + INCREMENTAL = 1, + FULL = 2 + }; + struct response_t: public rpc_access_response_base { std::vector blocks; uint64_t start_height; uint64_t current_height; std::vector output_indices; + uint64_t daemon_time; + uint8_t pool_info_extent; + std::vector added_pool_txs; + std::vector remaining_added_pool_txids; + std::vector removed_pool_txids; BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE_PARENT(rpc_access_response_base) @@ -209,6 +245,17 @@ namespace cryptonote KV_SERIALIZE(start_height) KV_SERIALIZE(current_height) KV_SERIALIZE(output_indices) + KV_SERIALIZE_OPT(daemon_time, (uint64_t) 0) + KV_SERIALIZE_OPT(pool_info_extent, (uint8_t) 0) + if (pool_info_extent != POOL_INFO_EXTENT::NONE) + { + KV_SERIALIZE(added_pool_txs) + KV_SERIALIZE_CONTAINER_POD_AS_BLOB(remaining_added_pool_txids) + } + if (pool_info_extent == POOL_INFO_EXTENT::INCREMENTAL) + { + KV_SERIALIZE_CONTAINER_POD_AS_BLOB(removed_pool_txids) + } END_KV_SERIALIZE_MAP() }; typedef epee::misc_utils::struct_init response; @@ -592,6 +639,7 @@ namespace cryptonote bool fee_too_low; bool too_few_outputs; bool sanity_check_failed; + bool tx_extra_too_big; BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE_PARENT(rpc_access_response_base) @@ -606,6 +654,7 @@ namespace cryptonote KV_SERIALIZE(fee_too_low) KV_SERIALIZE(too_few_outputs) KV_SERIALIZE(sanity_check_failed) + KV_SERIALIZE(tx_extra_too_big) END_KV_SERIALIZE_MAP() }; typedef epee::misc_utils::struct_init response; @@ -2123,15 +2172,34 @@ namespace cryptonote }; typedef epee::misc_utils::struct_init request; + struct hf_entry + { + uint8_t hf_version; + uint64_t height; + + bool operator==(const hf_entry& hfe) const { return hf_version == hfe.hf_version && height == hfe.height; } + + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE(hf_version) + KV_SERIALIZE(height) + END_KV_SERIALIZE_MAP() + }; + struct response_t: public rpc_response_base { uint32_t version; bool release; + uint64_t current_height; + uint64_t target_height; + std::vector hard_forks; BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE_PARENT(rpc_response_base) KV_SERIALIZE(version) KV_SERIALIZE(release) + KV_SERIALIZE_OPT(current_height, (uint64_t)0) + KV_SERIALIZE_OPT(target_height, (uint64_t)0) + KV_SERIALIZE_OPT(hard_forks, std::vector()) END_KV_SERIALIZE_MAP() }; typedef epee::misc_utils::struct_init response; diff --git a/src/rpc/core_rpc_server_error_codes.h b/src/rpc/core_rpc_server_error_codes.h index 1ba2392a9c6..2a4726a8911 100644 --- a/src/rpc/core_rpc_server_error_codes.h +++ b/src/rpc/core_rpc_server_error_codes.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2022, The Monero Project +// Copyright (c) 2014-2023, The Monero Project // // All rights reserved. // diff --git a/src/rpc/daemon_handler.cpp b/src/rpc/daemon_handler.cpp index 0466c92c2f9..52067bd4d22 100644 --- a/src/rpc/daemon_handler.cpp +++ b/src/rpc/daemon_handler.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2022, The Monero Project +// Copyright (c) 2017-2023, The Monero Project // // All rights reserved. // diff --git a/src/rpc/daemon_handler.h b/src/rpc/daemon_handler.h index 74885cf3009..11a6c28493a 100644 --- a/src/rpc/daemon_handler.h +++ b/src/rpc/daemon_handler.h @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2022, The Monero Project +// Copyright (c) 2017-2023, The Monero Project // // All rights reserved. // diff --git a/src/rpc/daemon_messages.cpp b/src/rpc/daemon_messages.cpp index a7c921ff9db..0da22f15fa4 100644 --- a/src/rpc/daemon_messages.cpp +++ b/src/rpc/daemon_messages.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2016-2022, The Monero Project +// Copyright (c) 2016-2023, The Monero Project // // All rights reserved. // diff --git a/src/rpc/daemon_messages.h b/src/rpc/daemon_messages.h index fb33f0d4a1c..d4b55f8fd4f 100644 --- a/src/rpc/daemon_messages.h +++ b/src/rpc/daemon_messages.h @@ -1,4 +1,4 @@ -// Copyright (c) 2016-2022, The Monero Project +// Copyright (c) 2016-2023, The Monero Project // // All rights reserved. // diff --git a/src/rpc/daemon_rpc_version.h b/src/rpc/daemon_rpc_version.h index 62847fd58e0..d14b85ed934 100644 --- a/src/rpc/daemon_rpc_version.h +++ b/src/rpc/daemon_rpc_version.h @@ -1,4 +1,4 @@ -// Copyright (c) 2016-2022, The Monero Project +// Copyright (c) 2016-2023, The Monero Project // // All rights reserved. // diff --git a/src/rpc/fwd.h b/src/rpc/fwd.h index cd9eacb9aeb..6227f5696dd 100644 --- a/src/rpc/fwd.h +++ b/src/rpc/fwd.h @@ -1,4 +1,4 @@ -// Copyright (c) 2019-2022, The Monero Project +// Copyright (c) 2019-2023, The Monero Project // // All rights reserved. // diff --git a/src/rpc/instanciations.cpp b/src/rpc/instanciations.cpp index 6e48d36a861..d4f5231f3c2 100644 --- a/src/rpc/instanciations.cpp +++ b/src/rpc/instanciations.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2022, The Monero Project +// Copyright (c) 2017-2023, The Monero Project // // All rights reserved. // diff --git a/src/rpc/message.cpp b/src/rpc/message.cpp index 005c40ea235..9fda96e7162 100644 --- a/src/rpc/message.cpp +++ b/src/rpc/message.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2016-2022, The Monero Project +// Copyright (c) 2016-2023, The Monero Project // // All rights reserved. // diff --git a/src/rpc/message.h b/src/rpc/message.h index 730b3ec744c..b7253e09bd0 100644 --- a/src/rpc/message.h +++ b/src/rpc/message.h @@ -1,4 +1,4 @@ -// Copyright (c) 2016-2022, The Monero Project +// Copyright (c) 2016-2023, The Monero Project // // All rights reserved. // diff --git a/src/rpc/message_data_structs.h b/src/rpc/message_data_structs.h index dd9d198ed72..542d71239a9 100644 --- a/src/rpc/message_data_structs.h +++ b/src/rpc/message_data_structs.h @@ -1,4 +1,4 @@ -// Copyright (c) 2016-2022, The Monero Project +// Copyright (c) 2016-2023, The Monero Project // // All rights reserved. // diff --git a/src/rpc/rpc_args.cpp b/src/rpc/rpc_args.cpp index 74f0879afa1..e6233fa984d 100644 --- a/src/rpc/rpc_args.cpp +++ b/src/rpc/rpc_args.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2022, The Monero Project +// Copyright (c) 2014-2023, The Monero Project // // All rights reserved. // diff --git a/src/rpc/rpc_args.h b/src/rpc/rpc_args.h index 0b0dac57a51..ac6a5d7cd4f 100644 --- a/src/rpc/rpc_args.h +++ b/src/rpc/rpc_args.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2022, The Monero Project +// Copyright (c) 2014-2023, The Monero Project // // All rights reserved. // diff --git a/src/rpc/rpc_handler.h b/src/rpc/rpc_handler.h index c84460549ec..e13d3036f65 100644 --- a/src/rpc/rpc_handler.h +++ b/src/rpc/rpc_handler.h @@ -1,4 +1,4 @@ -// Copyright (c) 2016-2022, The Monero Project +// Copyright (c) 2016-2023, The Monero Project // // All rights reserved. // diff --git a/src/rpc/rpc_payment.cpp b/src/rpc/rpc_payment.cpp index 2e05f04fbf5..71a00aca447 100644 --- a/src/rpc/rpc_payment.cpp +++ b/src/rpc/rpc_payment.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2018-2022, The Monero Project +// Copyright (c) 2018-2023, The Monero Project // // All rights reserved. // @@ -236,10 +236,8 @@ namespace cryptonote *(uint32_t*)(hashing_blob.data() + 39) = SWAP32LE(nonce); if (block.major_version >= RX_BLOCK_VERSION) { - const uint64_t seed_height = is_current ? info.seed_height : info.previous_seed_height; const crypto::hash &seed_hash = is_current ? info.seed_hash : info.previous_seed_hash; - const uint64_t height = cryptonote::get_block_height(block); - crypto::rx_slow_hash(height, seed_height, seed_hash.data, hashing_blob.data(), hashing_blob.size(), hash.data, 0, 0); + crypto::rx_slow_hash(seed_hash.data, hashing_blob.data(), hashing_blob.size(), hash.data); } else { diff --git a/src/rpc/rpc_payment.h b/src/rpc/rpc_payment.h index e93f27d24e5..4ead5a34431 100644 --- a/src/rpc/rpc_payment.h +++ b/src/rpc/rpc_payment.h @@ -1,4 +1,4 @@ -// Copyright (c) 2018-2022, The Monero Project +// Copyright (c) 2018-2023, The Monero Project // // All rights reserved. // diff --git a/src/rpc/rpc_payment_costs.h b/src/rpc/rpc_payment_costs.h index 6ba4e8064d7..36ee5ec1de9 100644 --- a/src/rpc/rpc_payment_costs.h +++ b/src/rpc/rpc_payment_costs.h @@ -1,4 +1,4 @@ -// Copyright (c) 2019-2022, The Monero Project +// Copyright (c) 2019-2023, The Monero Project // // All rights reserved. // diff --git a/src/rpc/rpc_payment_signature.cpp b/src/rpc/rpc_payment_signature.cpp index 4e4798aae05..ec7aeb7159e 100644 --- a/src/rpc/rpc_payment_signature.cpp +++ b/src/rpc/rpc_payment_signature.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2018-2022, The Monero Project +// Copyright (c) 2018-2023, The Monero Project // // All rights reserved. // diff --git a/src/rpc/rpc_payment_signature.h b/src/rpc/rpc_payment_signature.h index 55ba49e9a3b..f0539446f87 100644 --- a/src/rpc/rpc_payment_signature.h +++ b/src/rpc/rpc_payment_signature.h @@ -1,4 +1,4 @@ -// Copyright (c) 2018-2022, The Monero Project +// Copyright (c) 2018-2023, The Monero Project // // All rights reserved. // diff --git a/src/rpc/rpc_version_str.cpp b/src/rpc/rpc_version_str.cpp index 528c9241526..07756d636e5 100644 --- a/src/rpc/rpc_version_str.cpp +++ b/src/rpc/rpc_version_str.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2019-2022, The Monero Project +// Copyright (c) 2019-2023, The Monero Project // // All rights reserved. // diff --git a/src/rpc/rpc_version_str.h b/src/rpc/rpc_version_str.h index b81a76980a3..2cfa9f867a2 100644 --- a/src/rpc/rpc_version_str.h +++ b/src/rpc/rpc_version_str.h @@ -1,4 +1,4 @@ -// Copyright (c) 2019-2022, The Monero Project +// Copyright (c) 2019-2023, The Monero Project // // All rights reserved. // diff --git a/src/rpc/zmq_pub.cpp b/src/rpc/zmq_pub.cpp index 81e6de99a9c..4c620c59a7e 100644 --- a/src/rpc/zmq_pub.cpp +++ b/src/rpc/zmq_pub.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2020-2022, The Monero Project +// Copyright (c) 2020-2023, The Monero Project // // All rights reserved. diff --git a/src/rpc/zmq_pub.h b/src/rpc/zmq_pub.h index 9554f26be3d..fe60527a69c 100644 --- a/src/rpc/zmq_pub.h +++ b/src/rpc/zmq_pub.h @@ -1,4 +1,4 @@ -// Copyright (c) 2020-2022, The Monero Project +// Copyright (c) 2020-2023, The Monero Project // // All rights reserved. diff --git a/src/rpc/zmq_server.cpp b/src/rpc/zmq_server.cpp index 398a0499a1d..cb8a8bea422 100644 --- a/src/rpc/zmq_server.cpp +++ b/src/rpc/zmq_server.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2016-2022, The Monero Project +// Copyright (c) 2016-2023, The Monero Project // // All rights reserved. // diff --git a/src/rpc/zmq_server.h b/src/rpc/zmq_server.h index 2d4049baab6..b561fd58002 100644 --- a/src/rpc/zmq_server.h +++ b/src/rpc/zmq_server.h @@ -1,4 +1,4 @@ -// Copyright (c) 2016-2022, The Monero Project +// Copyright (c) 2016-2023, The Monero Project // // All rights reserved. // diff --git a/src/serialization/CMakeLists.txt b/src/serialization/CMakeLists.txt index d3b9e57ca18..07076bae2c6 100644 --- a/src/serialization/CMakeLists.txt +++ b/src/serialization/CMakeLists.txt @@ -1,4 +1,4 @@ -# Copyright (c) 2016-2022, The Monero Project +# Copyright (c) 2016-2023, The Monero Project # # All rights reserved. # diff --git a/src/serialization/binary_archive.h b/src/serialization/binary_archive.h index 07a4ec16931..b3946a84b87 100644 --- a/src/serialization/binary_archive.h +++ b/src/serialization/binary_archive.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2022, The Monero Project +// Copyright (c) 2014-2023, The Monero Project // // All rights reserved. // diff --git a/src/serialization/binary_utils.h b/src/serialization/binary_utils.h index 9b41e9529d2..247c60bebe2 100644 --- a/src/serialization/binary_utils.h +++ b/src/serialization/binary_utils.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2022, The Monero Project +// Copyright (c) 2014-2023, The Monero Project // // All rights reserved. // diff --git a/src/serialization/container.h b/src/serialization/container.h index 7b59e94082e..80e1892f2bf 100644 --- a/src/serialization/container.h +++ b/src/serialization/container.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2022, The Monero Project +// Copyright (c) 2014-2023, The Monero Project // // All rights reserved. // diff --git a/src/serialization/containers.h b/src/serialization/containers.h index dd2de829aa0..65e6359b2f4 100644 --- a/src/serialization/containers.h +++ b/src/serialization/containers.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2022, The Monero Project +// Copyright (c) 2014-2023, The Monero Project // // All rights reserved. // diff --git a/src/serialization/crypto.h b/src/serialization/crypto.h index c2a4846eec6..896d00583bc 100644 --- a/src/serialization/crypto.h +++ b/src/serialization/crypto.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2022, The Monero Project +// Copyright (c) 2014-2023, The Monero Project // // All rights reserved. // diff --git a/src/serialization/debug_archive.h b/src/serialization/debug_archive.h index 479ed0cac16..ef3bb4345d9 100644 --- a/src/serialization/debug_archive.h +++ b/src/serialization/debug_archive.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2022, The Monero Project +// Copyright (c) 2014-2023, The Monero Project // // All rights reserved. // diff --git a/src/serialization/difficulty_type.h b/src/serialization/difficulty_type.h index b13693c26c7..37761839bf4 100644 --- a/src/serialization/difficulty_type.h +++ b/src/serialization/difficulty_type.h @@ -1,4 +1,4 @@ -// Copyright (c) 2019-2022, The Monero Project +// Copyright (c) 2019-2023, The Monero Project // // All rights reserved. // diff --git a/src/serialization/json_archive.h b/src/serialization/json_archive.h index 8c4486d05df..194ddaee9a5 100644 --- a/src/serialization/json_archive.h +++ b/src/serialization/json_archive.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2022, The Monero Project +// Copyright (c) 2014-2023, The Monero Project // // All rights reserved. // diff --git a/src/serialization/json_object.cpp b/src/serialization/json_object.cpp index 8de5860f690..4e436c25485 100644 --- a/src/serialization/json_object.cpp +++ b/src/serialization/json_object.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2016-2022, The Monero Project +// Copyright (c) 2016-2023, The Monero Project // // All rights reserved. // diff --git a/src/serialization/json_object.h b/src/serialization/json_object.h index 968707165b1..513ecfd344f 100644 --- a/src/serialization/json_object.h +++ b/src/serialization/json_object.h @@ -1,4 +1,4 @@ -// Copyright (c) 2016-2022, The Monero Project +// Copyright (c) 2016-2023, The Monero Project // // All rights reserved. // diff --git a/src/serialization/pair.h b/src/serialization/pair.h index f18260dc8ee..f21041fe82e 100644 --- a/src/serialization/pair.h +++ b/src/serialization/pair.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2022, The Monero Project +// Copyright (c) 2014-2023, The Monero Project // // All rights reserved. // diff --git a/src/serialization/serialization.h b/src/serialization/serialization.h index 381d29cfc7d..efe1270a456 100644 --- a/src/serialization/serialization.h +++ b/src/serialization/serialization.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2022, The Monero Project +// Copyright (c) 2014-2023, The Monero Project // // All rights reserved. // diff --git a/src/serialization/string.h b/src/serialization/string.h index 97692460218..a5d68eb645f 100644 --- a/src/serialization/string.h +++ b/src/serialization/string.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2022, The Monero Project +// Copyright (c) 2014-2023, The Monero Project // // All rights reserved. // diff --git a/src/serialization/tuple.h b/src/serialization/tuple.h new file mode 100644 index 00000000000..e76ca0b99a2 --- /dev/null +++ b/src/serialization/tuple.h @@ -0,0 +1,169 @@ +// Copyright (c) 2014-2023, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Parts of this file are originally copyright (c) 2012-2013 The Cryptonote developers + +#pragma once +#include +#include "serialization.h" + +namespace serialization +{ + namespace detail + { + template + bool serialize_tuple_element(Archive& ar, T& e) + { + return ::do_serialize(ar, e); + } + + template + bool serialize_tuple_element(Archive& ar, uint64_t& e) + { + ar.serialize_varint(e); + return true; + } + } +} + +template