chore(CI): Add cpp tidy linter

.github/workflows/lint_and_test_cpp.yaml

@@ -35,8 +35,8 @@ on:
       - CMakeLists.txt
       - cmake_modules/**
       - run.sh
-      - scripts/pack_server.sh
-      - scripts/pack_tools.sh
+      - build-support/pack_server.sh
+      - build-support/pack_tools.sh
       - src/**
       - thirdparty/**
 
@@ -55,7 +55,29 @@ jobs:
     steps:
       - uses: actions/checkout@v4
       - name: clang-format
-        run: ./scripts/run-clang-format.py --clang-format-executable clang-format-14 -e ./src/shell/linenoise -e ./src/shell/sds -e ./thirdparty -r .
+        run: ./build-support/run-clang-format.py --clang-format-executable clang-format-14 -e ./src/shell/linenoise -e ./src/shell/sds -e ./thirdparty -r .
+
+  cpp_clang_tidy_linter:
+    name: Tidy
+    runs-on: ubuntu-22.04
+    container:
+      image: apache/pegasus:thirdparties-bin-ubuntu2204-${{ github.base_ref }}
+    steps:
+      - name: Install Softwares
+        run: |
+          apt-get update
+          apt-get install clang-tidy -y
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - name: Rebuild thirdparty if needed
+        uses: "./.github/actions/rebuild_thirdparty_if_needed"
+      - name: clang-tidy
+        run: |
+          git config --global --add safe.directory /__w/pegasus/pegasus
+          ./run.sh build --test --compiler clang-14,clang++-14 -t debug --skip_thirdparty -c --cmake_only
+          ./build-support/clang_tidy.py --rev-range $(git log origin/${{ github.base_ref }} -n1 --format=format:"%H")
+        shell: bash
 
   cpp_clang_tidy_linter:
     name: Tidy

.github/workflows/regular-build.yml

@@ -41,7 +41,7 @@ jobs:
     steps:
       - uses: actions/checkout@v4
       - name: clang-format
-        run: ./scripts/run-clang-format.py --clang-format-executable clang-format-14 -e ./src/shell/linenoise -e ./src/shell/sds -e ./thirdparty -r .
+        run: ./build-support/run-clang-format.py --clang-format-executable clang-format-14 -e ./src/shell/linenoise -e ./src/shell/sds -e ./thirdparty -r .
 
   build_cpp:
     name: Build Cpp

.licenserc.yaml

@@ -110,8 +110,9 @@ header:
     - 'cmake_modules/FindRT.cmake'
     - 'cmake_modules/FindDL.cmake'
     # Copyright (c) 2017 Guillaume Papin
-    - 'scripts/run-clang-format.py'
+    - 'build-support/run-clang-format.py'
     # The MIT License (MIT), Copyright (c) 2015 Microsoft Corporation
+    - 'build-support/compile_thrift.py'
     - 'cmake_modules/BaseFunctions.cmake'
     - 'docs/rdsn-README.md'
     - 'idl/command.thrift'
@@ -120,7 +121,6 @@ header:
     - 'idl/metadata.thrift'
     - 'idl/meta_admin.thrift'
     - 'idl/replica_admin.thrift'
-    - 'scripts/compile_thrift.py'
     - 'scripts/learn_stat.py'
     - 'src/runtime/api_layer1.h'
     - 'src/runtime/api_task.h'

LICENSE

@@ -514,7 +514,7 @@ RESULTING FROM THE USE OF THIS SOFTWARE.
 
 --------------------------------------------------------------------------------
 
-scripts/run-clang-format.py - MIT License
+build-support/run-clang-format.py - MIT License
 
 MIT License
 

build-support/clang_tidy.py

@@ -0,0 +1,104 @@
+#!/usr/bin/env python3
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import argparse
+import collections
+import json
+import multiprocessing
+from multiprocessing.pool import ThreadPool
+import os
+import re
+import subprocess
+import sys
+import tempfile
+
+ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))
+
+BUILD_PATH = os.path.join(ROOT, "build", "latest")
+
+def run_tidy(sha="HEAD", is_rev_range=False):
+    diff_cmdline = ["git", "diff" if is_rev_range else "show", sha]
+
+    # Figure out which paths changed in the given diff.
+    changed_paths = subprocess.check_output(diff_cmdline + ["--name-only", "--pretty=format:"]).splitlines()
+    changed_paths = [p for p in changed_paths if p]
+
+    # Produce a separate diff for each file and run clang-tidy-diff on it
+    # in parallel.
+    #
+    # Note: this will incorporate any configuration from .clang-tidy.
+    def tidy_on_path(path):
+        patch_file = tempfile.NamedTemporaryFile()
+        cmd = diff_cmdline + [
+            "--src-prefix=%s/" % ROOT,
+            "--dst-prefix=%s/" % ROOT,
+            "--",
+            path]
+        subprocess.check_call(cmd, stdout=patch_file, cwd=ROOT)
+        cmdline = ["clang-tidy-diff",
+                   "-clang-tidy-binary",
+                   "clang-tidy",
+                   "-p0",
+                   "-path", BUILD_PATH,
+                   "-checks=-llvm-include-order,-modernize-concat-nested-namespaces,-cppcoreguidelines-pro-type-union-access,-cppcoreguidelines-macro-usage,-cppcoreguidelines-special-member-functions,-hicpp-special-member-functions,-modernize-use-trailing-return-type,-bugprone-easily-swappable-parameters,-google-readability-avoid-underscore-in-googletest-name,-cppcoreguidelines-avoid-c-arrays,-hicpp-avoid-c-arrays,-modernize-avoid-c-arrays,-llvm-header-guard,-cppcoreguidelines-pro-bounds-pointer-arithmetic",
+                   "-extra-arg=-language=c++",
+                   "-extra-arg=-std=c++17",
+                   "-extra-arg=-Ithirdparty/output/include"]
+        return subprocess.check_output(
+            cmdline,
+            stdin=open(patch_file.name),
+            cwd=ROOT).decode()
+    pool = ThreadPool(multiprocessing.cpu_count())
+    try:
+        return "".join(pool.imap(tidy_on_path, changed_paths))
+    except KeyboardInterrupt as ki:
+        sys.exit(1)
+    finally:
+        pool.terminate()
+        pool.join()
+
+
+def split_warnings(clang_output):
+    accumulated = ""
+    for l in clang_output.splitlines():
+        if l == "" or l == "No relevant changes found.":
+            continue
+        if l.startswith(ROOT) and re.search(r'(warning|error): ', l):
+            if accumulated:
+                yield accumulated
+            accumulated = ""
+        accumulated += l + "\n"
+    if accumulated:
+        yield accumulated
+
+
+if __name__ == "__main__":
+    # Basic setup and argument parsing.
+    parser = argparse.ArgumentParser(description="Run clang-tidy on a patch")
+    parser.add_argument("--rev-range", action="store_true",
+                        default=False,
+                        help="Whether the revision specifies the 'rev..' range")
+    parser.add_argument('rev', help="The git revision (or range of revisions) to process")
+    args = parser.parse_args()
+
+    # Run clang-tidy and parse the output.
+    clang_output = run_tidy(args.rev, args.rev_range)
+    print("Clang output")
+    print(clang_output)
+    sys.exit(None != re.match("warning: |error: ", clang_output, flags=0))

scripts/collector_table_counter_gen.py → build-support/collector_table_counter_gen.py

@@ -118,7 +118,7 @@ def generate_code_in_command_helper_header(replica_counter):
 
 
 # python3 ./collector_table_counter_gen.py counter1,counter2
-# please use `./scripts/format_files.sh` to format after generate code
+# please use `./build-support/format_files.sh` to format after generate code
 if __name__ == '__main__':
     if len(sys.argv) != 2:
         print("python3 ./collector_table_counter_gen.py {counter1,counter2..}")

java-client/README.md

@@ -60,7 +60,7 @@ mvn clean package -Dtest=TestPing
 ### Install
 
 ```
-cd scripts && bash recompile_thrift.sh && cd -
+cd build-support && bash recompile_thrift.sh && cd -
 mvn clean install -DskipTests
 ```
 

run.sh

@@ -324,8 +324,8 @@ function run_build()
     if [ ! -f "${ROOT}/src/common/serialization_helper/dsn.layer2_types.h" ]; then
         echo "Gen thrift"
         # TODO(yingchun): should be optimized
-        python3 $ROOT/scripts/compile_thrift.py
-        sh ${ROOT}/scripts/recompile_thrift.sh
+        python3 $ROOT/build-support/compile_thrift.py
+        sh ${ROOT}/build-support/recompile_thrift.sh
     fi
 
     if [ ! -d "$BUILD_DIR" ]; then
@@ -2105,19 +2105,19 @@ case $cmd in
         ;;
     pack_server)
         shift
-        PEGASUS_ROOT=$ROOT ./scripts/pack_server.sh $*
+        PEGASUS_ROOT=$ROOT ./build-support/pack_server.sh $*
         ;;
     pack_client)
         shift
-        PEGASUS_ROOT=$ROOT ./scripts/pack_client.sh $*
+        PEGASUS_ROOT=$ROOT ./build-support/pack_client.sh $*
         ;;
     pack_tools)
         shift
-        PEGASUS_ROOT=$ROOT ./scripts/pack_tools.sh $*
+        PEGASUS_ROOT=$ROOT ./build-support/pack_tools.sh $*
         ;;
     bump_version)
         shift
-        ./scripts/bump_version.sh $*
+        ./build-support/bump_version.sh $*
         ;;
     *)
         echo "ERROR: unknown command $cmd"

scala-client/README.md

@@ -68,8 +68,8 @@ NOTE: It requires the Pegasus [onebox](https://pegasus.apache.org/overview/onebo
 
 Build Java dependency at first, then build and test Scala client.
 ```
-cd ${PROJECT_ROOT}/java-client/scripts
-./recompile_thrift.sh
+cd ${PROJECT_ROOT}/build-support
+bash recompile_thrift.sh
 
 cd ${PROJECT_ROOT}/java-client
 mvn clean package -DskipTests -Dcheckstyle.skip=true

src/aio/aio_provider.h

@@ -26,7 +26,7 @@
 
 #pragma once
 
-#include <stdint.h>
+#include <cstdint>
 #include <memory>
 #include <string>
 

src/aio/aio_task.cpp

@@ -15,7 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
-#include <string.h>
+#include <cstring>
 #include <functional>
 #include <list>
 #include <memory>

src/aio/aio_task.h

@@ -26,8 +26,8 @@
 
 #pragma once
 
-#include <stddef.h>
-#include <stdint.h>
+#include <cstddef>
+#include <cstdint>
 #include <functional>
 #include <memory>
 #include <vector>

src/aio/disk_engine.h

@@ -26,8 +26,8 @@
 
 #pragma once
 
-#include <stddef.h>
-#include <stdint.h>
+#include <cstddef>
+#include <cstdint>
 #include <memory>
 
 #include "aio/aio_task.h"

src/aio/file_io.h

@@ -26,7 +26,7 @@
 
 #pragma once
 
-#include <stdint.h>
+#include <cstdint>
 #include <list>
 #include <string>
 #include <utility>

src/aio/native_linux_aio_provider.h

@@ -26,7 +26,7 @@
 
 #pragma once
 
-#include <stdint.h>
+#include <cstdint>
 #include <memory>
 #include <string>
 

src/aio/test/aio.cpp

@@ -26,7 +26,7 @@
 
 #include <fmt/core.h>
 #include <rocksdb/status.h>
-#include <string.h>
+#include <cstring>
 #include <algorithm>
 #include <cstdint>
 #include <list>

src/base/meta_store.h

@@ -21,7 +21,7 @@
 
 #include <gtest/gtest_prod.h>
 #include <rocksdb/options.h>
-#include <stdint.h>
+#include <cstdint>
 #include <string>
 
 #include "utils/error_code.h"

src/base/pegasus_key_schema.h

@@ -19,7 +19,7 @@
 
 #pragma once
 
-#include <stdint.h>
+#include <cstdint>
 #include <string>
 #include "utils/ports.h"
 #include "utils/utils.h"

src/base/pegasus_rpc_types.h

@@ -26,7 +26,7 @@
 #include "consensus_types.h"
 #include "replica_admin_types.h"
 #include <rrdb/rrdb_types.h>
-#include <rrdb/rrdb.client.h>
+#include "rrdb/rrdb.client.h"
 
 namespace pegasus {
 

src/base/pegasus_utils.cpp

@@ -17,10 +17,10 @@
  * under the License.
  */
 
-#include "pegasus_utils.h"
+#include "base/pegasus_utils.h"
 
-#include <stdio.h>
 #include <cctype>
+#include <cstdio>
 
 #include "utils/fmt_logging.h"
 

src/base/pegasus_utils.h

@@ -20,8 +20,8 @@
 #pragma once
 
 #include <rocksdb/slice.h>
-#include <stdint.h>
-#include <time.h>
+#include <cstdint>
+#include <ctime>
 #include <functional>
 #include <list>
 #include <queue>

src/base/pegasus_value_schema.h

@@ -20,8 +20,8 @@
 #pragma once
 
 #include <rocksdb/slice.h>
-#include <stdint.h>
-#include <string.h>
+#include <cstdint>
+#include <cstring>
 #include <algorithm>
 #include <array>
 #include <memory>
@@ -33,7 +33,7 @@
 #include "utils/endians.h"
 #include "utils/fmt_logging.h"
 #include <string_view>
-#include "value_field.h"
+#include "base/value_field.h"
 
 namespace pegasus {