[CI] refine_func_args_parse

ci_scripts/check_api_parameters.py

@@ -29,7 +29,10 @@ def add_path(path):
 this_dir = osp.dirname(__file__)
 # Add docs/api to PYTHONPATH
 add_path(osp.abspath(osp.join(this_dir, "..", "docs", "api")))
-from extract_api_from_docs import extract_params_desc_from_rst_file
+from extract_api_from_docs import (
+    extract_params_desc_from_rst_file,
+    gen_functions_args_str,
+)
 
 arguments = [
     # flags, dest, type, default, help
@@ -60,37 +63,11 @@ def _check_params_in_description(rstfilename, paramstr):
     params_in_title = []
     if paramstr:
         fake_func = ast.parse(f"def fake_func({paramstr}): pass")
-        # Iterate over all in_title parameters
-        num_defaults = len(fake_func.body[0].args.defaults)
-        num_args = len(fake_func.body[0].args.args)
-        # args & defaults
-        for i, arg in enumerate(fake_func.body[0].args.args):
-            if i >= num_args - num_defaults:
-                default_value = fake_func.body[0].args.defaults[
-                    i - (num_args - num_defaults)
-                ]
-                params_in_title.append(f"{arg.arg}={default_value}")
-            else:
-                params_in_title.append(arg.arg)
-        # posonlyargs
-        for arg in fake_func.body[0].args.posonlyargs:
-            params_in_title.append(arg.arg)
-        # vararg(*args)
-        if fake_func.body[0].args.vararg:
-            params_in_title.append(fake_func.body[0].args.vararg.arg)
-        # kwonlyargs & kw_defaults
-        for i, arg in enumerate(fake_func.body[0].args.kwonlyargs):
-            if (
-                i < len(fake_func.body[0].args.kw_defaults)
-                and fake_func.body[0].args.kw_defaults[i] is not None
-            ):
-                default_value = fake_func.body[0].args.kw_defaults[i]
-                params_in_title.append(f"{arg.arg}={default_value}")
-            else:
-                params_in_title.append(arg.arg)
-        # **kwargs
-        if fake_func.body[0].args.kwarg:
-            params_in_title.append(fake_func.body[0].args.kwarg.arg)
+        func_node = fake_func.body[0]
+        func_args_str = gen_functions_args_str(func_node)
+        params_in_title = func_args_str.split(", ")
+        params_in_title.remove("/")
+        params_in_title.remove("*")
 
     funcdescnode = extract_params_desc_from_rst_file(rstfilename)
     if funcdescnode:
@@ -141,11 +118,35 @@ def _check_params_in_description(rstfilename, paramstr):
 def _check_params_in_description_with_fullargspec(rstfilename, funcname):
     flag = True
     info = ""
-    funcspec = inspect.getfullargspec(eval(funcname))
+    try:
+        func = eval(funcname)
+    except NameError:
+        func = eval(funcname)
+    source = inspect.getsource(func)
+
+    class FunctionDefExtractor(ast.NodeTransformer):
+        target_name = func.__name__
+
+        def visit_FunctionDef(self, node):
+            if node.name == self.target_name:
+                node.decorator_list = []
+                node.body = [ast.Pass()]
+                return node
+            return None
+
+    tree = ast.parse(source)
+    modified_tree = FunctionDefExtractor().visit(tree)
+    modified_tree.body = [
+        node for node in modified_tree.body if node is not None
+    ]
+
+    func_node = modified_tree.body[0]
+    params_inspec = gen_functions_args_str(func_node).split(", ")
+    params_inspec.remove("/")
+    params_inspec.remove("*")
     funcdescnode = extract_params_desc_from_rst_file(rstfilename)
     if funcdescnode:
         items = funcdescnode.children[1].children[0].children
-        params_inspec = funcspec.args
         if len(items) != len(params_inspec):
             flag = False
             info = f"check_with_fullargspec failed (parammeters description): {rstfilename}"
@@ -171,10 +172,10 @@ def _check_params_in_description_with_fullargspec(rstfilename, funcname):
                         f"check failed (parammeters description): {rstfilename}, param name not found in {i} paragraph."
                     )
     else:
-        if funcspec.args:
+        if params_inspec:
             info = "params section not found in description, check it please."
             print(
-                f"check failed (parameters description not found): {rstfilename}, {funcspec.args}."
+                f"check failed (parameters description not found): {rstfilename}, {params_inspec}."
             )
             flag = False
     return flag, info

ci_scripts/ci_start.sh

@@ -83,6 +83,17 @@ if [ "${BUILD_DOC}" = "true" ] &&  [ -x /usr/local/bin/sphinx-build ] ; then
     fi
 fi
 
+git merge --no-edit upstream/${BRANCH}
+need_check_cn_doc_files=$(find_all_cn_api_files_modified_by_pr)
+echo $need_check_cn_doc_files
+
+# Check for existing stock issues.
+find_cn_rst_files() {
+    local search_dir="$SCRIPT_DIR/../docs/api/paddle"
+    find "$search_dir" -type f -name "*_cn.rst"
+}
+all_api_cn_files=$(find_cn_rst_files)
+
 check_parameters=ON
 if [ "${check_parameters}" = "OFF" ] ; then
     #echo "chinese api doc fileslist is empty, skip check."
@@ -91,7 +102,7 @@ else
     jsonfn=${OUTPUTDIR}/en/${VERSIONSTR}/gen_doc_output/api_info_all.json
     if [ -f $jsonfn ] ; then
         echo "$jsonfn exists."
-        /bin/bash ${DIR_PATH}/check_api_parameters.sh "${need_check_cn_doc_files}" ${jsonfn}
+        /bin/bash ${DIR_PATH}/check_api_parameters.sh "${all_api_cn_files}" ${jsonfn}
         if [ $? -ne 0 ];then
             exit 1
         fi
@@ -109,9 +120,6 @@ if [ $? -ne 0 ];then
     EXIT_CODE=1
 fi
 
-git merge --no-edit upstream/${BRANCH}
-need_check_cn_doc_files=$(find_all_cn_api_files_modified_by_pr)
-echo $need_check_cn_doc_files
 # 4 Chinese api docs check
 if [ "${need_check_cn_doc_files}" = "" ] ; then
     echo "chinese api doc fileslist is empty, skip check."

docs/api/gen_doc.py

@@ -362,36 +362,63 @@ def parse_module_file(mod):
 
 
 def gen_functions_args_str(node):
+    def _process_positional_args(args, params):
+        positional_args = args.posonlyargs + args.args
+        num_defaults = len(args.defaults)
+
+        total_positional = len(positional_args)
+        first_default_pos = total_positional - num_defaults
+        if args.posonlyargs:
+            for idx, arg in enumerate(args.posonlyargs):
+                if arg.arg == "self":
+                    continue
+                param = _format_arg_with_default(
+                    arg, idx, first_default_pos, args.defaults
+                )
+                params.append(param)
+            params.append("/")
+
+        for idx, arg in enumerate(args.args):
+            if arg.arg == "self":
+                continue
+            global_idx = idx + len(args.posonlyargs)
+            param = _format_arg_with_default(
+                arg, global_idx, first_default_pos, args.defaults
+            )
+            params.append(param)
+
+    def _format_arg_with_default(arg, index, first_default_pos, defaults):
+        if first_default_pos is not None and index >= first_default_pos:
+            default_index = index - first_default_pos
+            defarg_value = ast.unparse(defaults[default_index]).strip()
+            return f"{arg.arg}={defarg_value}"
+        return arg.arg
+
+    def _process_var_args(args, params):
+        if args.vararg:
+            params.append(f"*{args.vararg.arg}")
+        elif args.kwonlyargs:
+            params.append("*")
+
+    def _process_kwonly_args(args, params):
+        for idx, arg in enumerate(args.kwonlyargs):
+            if default := args.kw_defaults[idx]:
+                default_str = ast.unparse(default).strip()
+                params.append(f"{arg.arg}={default_str}")
+            else:
+                params.append(arg.arg)
+
+    def _process_kwargs(args, params):
+        if args.kwarg:
+            params.append(f"**{args.kwarg.arg}")
+
     str_args_list = []
     if isinstance(node, ast.FunctionDef):
-        # 'args', 'defaults', 'kw_defaults', 'kwarg', 'kwonlyargs', 'posonlyargs', 'vararg'
-        for arg in node.args.args:
-            if not arg.arg == "self":
-                str_args_list.append(arg.arg)
-
-        defarg_ind_start = len(str_args_list) - len(node.args.defaults)
-        for defarg_ind in range(len(node.args.defaults)):
-            if isinstance(node.args.defaults[defarg_ind], ast.Name):
-                str_args_list[defarg_ind_start + defarg_ind] += "=" + str(
-                    node.args.defaults[defarg_ind].id
-                )
-            elif isinstance(node.args.defaults[defarg_ind], ast.Constant):
-                defarg_val = str(node.args.defaults[defarg_ind].value)
-                if isinstance(node.args.defaults[defarg_ind].value, str):
-                    defarg_val = f"'{defarg_val}'"
-                str_args_list[defarg_ind_start + defarg_ind] += "=" + defarg_val
-        if node.args.vararg is not None:
-            str_args_list.append("*" + node.args.vararg.arg)
-        if len(node.args.kwonlyargs) > 0:
-            if node.args.vararg is None:
-                str_args_list.append("*")
-            for kwoarg, d in zip(node.args.kwonlyargs, node.args.kw_defaults):
-                if isinstance(d, ast.Constant):
-                    str_args_list.append(f"{kwoarg.arg}={d.value}")
-                elif isinstance(d, ast.Name):
-                    str_args_list.append(f"{kwoarg.arg}={d.id}")
-        if node.args.kwarg is not None:
-            str_args_list.append("**" + node.args.kwarg.arg)
+        func_args = node.args
+        _process_positional_args(func_args, str_args_list)
+        _process_var_args(func_args, str_args_list)
+        _process_kwonly_args(func_args, str_args_list)
+        _process_kwargs(func_args, str_args_list)
 
     return ", ".join(str_args_list)