Merge branch 'dev' into feat/stop-macro-evaluation

Author: SuperFola

.github/generate_cppcheck_report.py

@@ -0,0 +1,43 @@
+import sys
+
+UPDATED_FILES = sys.argv[1:]
+TABLE_HEADER = """| Filename | Line | Type | Description |
+| -------- | ---- | ---- | ----------- |"""
+
+with open("cppcheck.txt") as f:
+    content = f.readlines()
+
+updated = []
+anything_else = []
+
+for line in content:
+    filename, row, kind, desc = line.split(":", 3)
+    formatted = f"| {filename} | {row} | {kind.strip()} | {desc.strip()} |"
+
+    if filename == "nofile":
+        continue
+
+    if filename in UPDATED_FILES:
+        updated.append(formatted)
+    else:
+        anything_else.append(formatted)
+
+
+def make_output(data):
+    if data:
+        output = TABLE_HEADER + "\n"
+        output += "\n".join(data)
+        return output
+    return ""
+
+
+print(f"""### CppCheck report
+
+{make_output(updated)}
+
+<details>
+<summary>Report files about files you didn't modify in this PR</summary>
+
+{make_output(anything_else)}
+</details>
+""")

.github/launch-lizard.py

@@ -32,22 +32,22 @@
 
 
 def make_sorted_table_lines(lines_with_ccn):
-    output = ""
-    for line in sorted(lines_with_ccn, key=lambda e: e[1], reverse=True):
-        output += f"| {line[0]} | {line[1]} |\n"
-    return output
+    if lines_with_ccn:
+        output = TABLE_HEADERS + "\n"
+        for line in sorted(lines_with_ccn, key=lambda e: e[1], reverse=True):
+            output += f"| {line[0]} | {line[1]} |\n"
+        return output
+    return ""
 
 
 print(f"""### Lizard report
 
 Listing only functions with cyclomatic complexity >= {MAX_CCN} or NLOC >= {MAX_NLOC} or parameters >= {MAX_PARAM}.
-{TABLE_HEADERS}
 {make_sorted_table_lines(updated)}
 
 <details>
 <summary>Report about files you didn't modify in this PR</summary>
 
-{TABLE_HEADERS}
 {make_sorted_table_lines(anything_else)}
 </details>
 """)

.github/workflows/ci.yml

@@ -70,7 +70,7 @@ jobs:
         config:
           - {
             os: ubuntu-24.04, name: "Ubuntu Clang 16",
-            compiler: clang, compiler_version: 16, sanitizers: "Off",
+            compiler: clang, compiler_version: 16, sanitizers: "On",
             artifact: "ubuntu-clang-16"
           }
           - {
@@ -80,7 +80,7 @@ jobs:
           }
           - {
             os: ubuntu-24.04, name: "Ubuntu GCC 14",
-            compiler: gcc, compiler_version: 14, sanitizers: "Off",
+            compiler: gcc, compiler_version: 14, sanitizers: "On",
             artifact: "ubuntu-gcc-14"
           }
           - {
@@ -201,6 +201,7 @@ jobs:
   fuzzing:
     runs-on: ubuntu-24.04
     name: Fuzz testing
+    continue-on-error: true
 
     container:
       image: aflplusplus/aflplusplus:v4.20c
@@ -242,9 +243,31 @@ jobs:
             -- ${BUILD_FOLDER}/arkscript @@ -L ./lib
 
       - name: Summarize
+        id: summary
+        shell: bash
         run: |
-          afl-whatsup -s -d output
-          afl-showmap -C -i output -o /dev/null -- ./${BUILD_FOLDER}/arkscript @@ -L ./lib
+          echo "FUZZ_SUMMARY<<EOF" >> $GITHUB_ENV
+          afl-whatsup -s -d output >> $GITHUB_ENV
+          afl-showmap -C -i output -o /dev/null -- ./${BUILD_FOLDER}/arkscript @@ -L ./lib | sed -r "s/\x1B\[([0-9]{1,3}(;[0-9]{1,2};?)?)?[mGK]//g" | grep -v Reading | grep -v Scanning >> $GITHUB_ENV
+          echo "EOF" >> $GITHUB_ENV
+
+      - name: Find Comment
+        uses: peter-evans/find-comment@v3
+        id: fc
+        with:
+          issue-number: ${{ github.event.pull_request.number }}
+          comment-author: 'github-actions[bot]'
+          body-includes: Fuzzing report
+
+      - name: Create or update comment
+        uses: peter-evans/create-or-update-comment@v4
+        with:
+          comment-id: ${{ steps.fc.outputs.comment-id }}
+          issue-number: ${{ github.event.pull_request.number }}
+          body: |
+            ## Fuzzing report
+            ${{ env.FUZZ_SUMMARY }}
+          edit-mode: replace
 
       - name: Sort files for upload
         continue-on-error: true

.github/workflows/release.yml

@@ -26,7 +26,7 @@ jobs:
           - {
             os: ubuntu-24.04, name: "Ubuntu GCC 14",
             compiler: gcc, compiler_version: 14,
-            artifact: "ubuntu-gcc-13"
+            artifact: "ubuntu-gcc-14"
           }
           - {
             os: windows-latest, name: "Windows VS 2022",

.github/workflows/static_analysis.yml

@@ -37,11 +37,9 @@ jobs:
             -I include src \
             --enable=all --inline-suppr \
             --suppressions-list=cppcheck-suppressions.txt
-          cat cppcheck.txt | sort > cppcheck_sorted.txt
+          content=$(python .github/generate_cppcheck_report.py $(echo $(git diff --name-only -r HEAD^1 HEAD)))
           echo "CPPCHECK_REPORT<<EOF" >> $GITHUB_ENV
-          echo "| Filename | Line | Type | Description |" >> $GITHUB_ENV
-          echo "| -------- | ---- | ---- | ----------- |" >> $GITHUB_ENV
-          cat cppcheck_sorted.txt | awk -F ":" '{type=$3;out=""; for (i = 4; i <= NF; i++) {out = out " " $i}; print $1 " | " $2 " |" type " |" out}' >> $GITHUB_ENV
+          echo "$content" >> $GITHUB_ENV
           echo "EOF" >> $GITHUB_ENV
 
       - name: Find Comment
@@ -63,6 +61,5 @@ jobs:
 
             ---
 
-            ### CppCheck report
             ${{ env.CPPCHECK_REPORT }}
           edit-mode: replace

.gitignore

@@ -4,6 +4,7 @@
 
 # Personal utilities
 warnings.log
+cppcheck.txt
 
 # ArkScript
 __arkscript__/

CHANGELOG.md

@@ -53,6 +53,16 @@
 - fixed a bug when passing the wrong number of arguments to a function inside an async call was crashing the VM because the function couldn't be named
 - fixed a bug in the compiler generating invalid `fun` nodes
 - fixed a bug when generating `let`, `mut` or `set` nodes inside macros with an invalid node type
+- fixed a bug when reading invalid UTF8 codepoints in the parser caused out of bounds reads
+- fixed a bug with recursive macro, exhausting the stack space due to recursive evaluation
+- futures can be awaited again, they will return nil on all the tries
+- checking for reused argument name in macros during parsing
+- enhanced comment after node handling in macros
+- adding a hard limit on package names length (255 characters, to comply with posix limits)
+- disallow passing invalid nodes as arguments to functions and operators
+- checking for unevaluated spread inside macros
+- checking for invalid symbols when defining a function through a macro
+- added a max macro unification depth
 
 ### Removed
 - removed unused `NodeType::Closure`

CMakeLists.txt

@@ -48,18 +48,18 @@ if (CMAKE_COMPILER_IS_GNUCXX OR CMAKE_COMPILER_IS_CLANG OR APPLE)
     target_compile_options(ArkReactor
             PUBLIC
             -Wall -Wextra -pedantic -Wstrict-aliasing
-            -Wno-unknown-pragmas            # We use pragmas to disable warnings we understand.
+            -Wshadow
+            -Wconversion
+            -Werror
+            # Allow deprecation warnings to not be treated as errors
+            -Wno-error=deprecated-declarations
+            # We use pragmas to disable warnings we understand.
             # So we need to disable the warning about pragmas.
-            -Wno-unknown-warning-option     # Disable warnings about disabling warnings we have
-            # disabled.
+            -Wno-unknown-pragmas
+            # Disable warnings about disabling warnings we have disabled.
+            -Wno-unknown-warning-option
     )
 
-    if (CMAKE_COMPILER_IS_GNUCXX)
-        # The package utf8 has an issue with constant overflow.
-        # Once this is fixed remove this flag:
-        target_compile_options(ArkReactor PUBLIC -Wno-overflow)
-    endif ()
-
     if (APPLE)
         # The standard SSH libraries are depreciate on APPLE.
         # Thus they currently generate a warning that we have to ignore for now.
@@ -105,7 +105,7 @@ add_subdirectory("${ark_SOURCE_DIR}/lib/termcolor" EXCLUDE_FROM_ALL)
 target_link_libraries(ArkReactor PUBLIC termcolor)
 
 target_include_directories(ArkReactor
-        PUBLIC
+        SYSTEM PUBLIC
         "${ark_SOURCE_DIR}/lib/picosha2/"
         "${ark_SOURCE_DIR}/lib/fmt/include")
 
@@ -161,7 +161,10 @@ if (ARK_PROFILER_MIPS)
 endif ()
 
 if (ARK_BUILD_MODULES)
+    get_directory_property(old_dir_compile_options COMPILE_OPTIONS)
+    add_compile_options(-w)
     add_subdirectory(${ark_SOURCE_DIR}/lib/modules)
+    set_directory_properties(PROPERTIES COMPILE_OPTIONS "${old_dir_compile_options}")
 endif ()
 
 # TODO: consider using ctest
@@ -201,18 +204,10 @@ if (ARK_BUILD_EXE)
             ${ark_SOURCE_DIR}/lib/fmt/src/format.cc)
     add_executable(arkscript ${EXE_SOURCES})
 
-    if (MSVC)
-        # Disable warnings for lib/replxx
-        # This is a separate module we don't control so can't fix the code without help.
-        # Disable warnings when compiling this package remove when resolved.
-        #   /wd4267:    conversion from 'size_t' to 'type', possible loss of data
-        #   /wd4244:    conversion from 'type1' to 'type2', possible loss of data
-        set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /wd4267 /wd4244")
-    endif ()
-
     add_subdirectory("${ark_SOURCE_DIR}/lib/replxx" EXCLUDE_FROM_ALL)
     add_subdirectory("${ark_SOURCE_DIR}/lib/clipp" EXCLUDE_FROM_ALL)
 
+    target_include_directories(arkscript SYSTEM PUBLIC "${ark_SOURCE_DIR}/lib/clipp/include")
     target_link_libraries(arkscript PUBLIC ArkReactor replxx clipp termcolor)
     target_compile_features(arkscript PRIVATE cxx_std_20)
 

examples/quicksort.ark

@@ -31,11 +31,12 @@
 # an unsorted list to sort
 (let a [3 6 1 5 1 65 324 765 1 6 3 0 6 9 6 5 3 2 5 6 7 64 645 7 345 432 432 4 324 23])
 
+(let rep (if (>= (len sys:args) 1) (toNumber (@ sys:args 0)) 1))
+
 # a benchmarking function, to see the difference between C++ sort and ArkScript quicksort
 # obviously ArkScript will be a bit slower
 (let bench (fun (name code) {
     (mut start (time))
-    (let rep 1000)
 
     (mut i 0)
     (while (< i rep) {

include/Ark/Compiler/AST/Optimizer.hpp

@@ -2,10 +2,10 @@
  * @file Optimizer.hpp
  * @author Alexandre Plateau ([email protected])
  * @brief Optimizes a given ArkScript AST
- * @version 0.4
- * @date 2020-10-27
+ * @version 1.0
+ * @date 2024-07-09
  *
- * @copyright Copyright (c) 2020-2021
+ * @copyright Copyright (c) 2020-2024
  *
  */
 
@@ -32,8 +32,9 @@ namespace Ark::internal
         /**
          * @brief Construct a new Optimizer
          *
+         * @param debug level of debug
          */
-        explicit Optimizer(uint16_t options) noexcept;
+        explicit Optimizer(unsigned debug) noexcept;
 
         /**
          * @brief Send the AST to the optimizer, then run the different optimization strategies on it
@@ -51,7 +52,7 @@ namespace Ark::internal
 
     private:
         Node m_ast;
-        uint16_t m_options;
+        unsigned m_debug;
         std::unordered_map<std::string, unsigned> m_sym_appearances;
 
         /**
@@ -74,7 +75,7 @@ namespace Ark::internal
          * @param node
          * @param func
          */
-        void runOnGlobalScopeVars(Node& node, const std::function<void(Node&, Node&, int)>& func);
+        void runOnGlobalScopeVars(Node& node, const std::function<void(Node&, Node&, std::size_t)>& func);
 
         /**
          * @brief Count the occurrences of each symbol in the AST, recursively

include/Ark/Compiler/AST/Parser.hpp

@@ -174,13 +174,13 @@ namespace Ark::internal
 
         std::optional<Node> field()
         {
-            std::string symbol;
-            if (!name(&symbol))
+            std::string sym;
+            if (!name(&sym))
                 return std::nullopt;
 
             std::optional<Node> leaf { Node(NodeType::Field) };
             setNodePosAndFilename(leaf.value());
-            leaf->push_back(Node(NodeType::Symbol, symbol));
+            leaf->push_back(Node(NodeType::Symbol, sym));
 
             while (true)
             {

include/Ark/Compiler/AST/Predicates.hpp

@@ -114,7 +114,7 @@ namespace Ark::internal
         explicit IsChar(const char c) :
             CharPred("'" + std::string(1, c) + "'"), m_k(c)
         {}
-        explicit IsChar(const utf8_char_t c) :
+        explicit IsChar(const utf8_char_t& c) :
             CharPred(std::string(c.c_str())), m_k(c.codepoint())
         {}
         bool operator()(const utf8_char_t::codepoint_t c) const override

include/Ark/Compiler/AST/utf8_char.hpp

@@ -23,44 +23,53 @@ namespace Ark::internal
             m_codepoint(cp), m_length(len), m_repr(repr) {}
 
         // https://github.com/sheredom/utf8.h/blob/4e4d828174c35e4564c31a9e35580c299c69a063/utf8.h#L1178
-        static std::pair<std::string::iterator, utf8_char_t> at(std::string::iterator it)
+        static std::pair<std::string::iterator, utf8_char_t> at(std::string::iterator it, const std::string::iterator end)
         {
-            codepoint_t codepoint;
+            codepoint_t cp;
             length_t length;
             repr_t repr = {};
 
             if (0xf0 == (0xf8 & *it))  // 4 byte utf8 codepoint
             {
-                codepoint = (static_cast<codepoint_t>(0x07 & *it) << 18) |
+                if (it + 3 == end || it + 2 == end || it + 1 == end)
+                    return std::make_pair(end, utf8_char_t {});
+
+                cp = (static_cast<codepoint_t>(0x07 & *it) << 18) |
                     (static_cast<codepoint_t>(0x3f & *(it + 1)) << 12) |
                     (static_cast<codepoint_t>(0x3f & *(it + 2)) << 6) |
                     static_cast<codepoint_t>(0x3f & *(it + 3));
                 length = 4;
             }
             else if (0xe0 == (0xf0 & *it))  // 3 byte utf8 codepoint
             {
-                codepoint = (static_cast<codepoint_t>(0x0f & *it) << 12) |
+                if (it + 2 == end || it + 1 == end)
+                    return std::make_pair(end, utf8_char_t {});
+
+                cp = (static_cast<codepoint_t>(0x0f & *it) << 12) |
                     (static_cast<codepoint_t>(0x3f & *(it + 1)) << 6) |
                     static_cast<codepoint_t>(0x3f & *(it + 2));
                 length = 3;
             }
             else if (0xc0 == (0xe0 & *it))  // 2 byte utf8 codepoint
             {
-                codepoint = (static_cast<codepoint_t>(0x1f & *it) << 6) |
+                if (it + 1 == end)
+                    return std::make_pair(end, utf8_char_t {});
+
+                cp = (static_cast<codepoint_t>(0x1f & *it) << 6) |
                     static_cast<codepoint_t>(0x3f & *(it + 1));
                 length = 2;
             }
             else  // 1 byte utf8 codepoint otherwise
             {
-                codepoint = static_cast<unsigned char>(*it);
+                cp = static_cast<unsigned char>(*it);
                 length = 1;
             }
 
             for (length_t i = 0; i < length; ++i)
                 repr[i] = static_cast<unsigned char>(*(it + static_cast<int>(i)));
 
             return std::make_pair(it + static_cast<long>(length),
-                                  utf8_char_t(codepoint, length, repr));
+                                  utf8_char_t(cp, length, repr));
         }
 
         [[nodiscard]] bool isPrintable() const