Generate godot compat for dual build

generate compat

generate compat

Update ci.yml

Update binding_generator.py

generate compat

generate compat

lint python files

Update compat_generator.py

update docs

Update binding_generator.py

Update module_converter.py

also collect defines

Add module converter file that converts module based projects to godot_compat

Update ci.yml

update docs

Update compat_generator.py

lint python files

generate compat

generate compat

generate compat

generate compat

Update ci.yml

fix path issue when caling from outside

Update binding_generator.py

update to also take missing classes/structs

Update binding_generator.py

Generate godot compat for dual build

generate compat

generate compat

Update ci.yml

Update binding_generator.py

generate compat

generate compat

lint python files

Update compat_generator.py

update docs

Update binding_generator.py

Update module_converter.py

also collect defines

Add module converter file that converts module based projects to godot_compat

Update ci.yml

update docs

Update compat_generator.py

lint python files

generate compat

generate compat

generate compat

generate compat

Update ci.yml

fix path issue when caling from outside

Add support for build profiles.

Allow enabling or disabling specific classes (which will not be built).

Allow forwarding from `ClassDB` to `ClassDBSingleton` to support enumerations

update to also take missing classes/structs

Update binding_generator.py

Author: Ughuuu

.github/workflows/ci.yml

@@ -110,6 +110,17 @@ jobs:
         with:
           python-version: '3.x'
 
+      - name: Clone Godot
+        uses: actions/checkout@v4
+        with:
+          repository: godotengine/godot
+          path: godot
+          #ref: TODO take tag
+      
+      - name: Generate compat mappings for godot
+        run: |
+          python compat_generator.py godot
+
       - name: Android dependencies
         if: ${{ matrix.platform == 'android' }}
         uses: nttld/setup-ndk@v1

README.md

@@ -64,6 +64,21 @@ first-party `godot-cpp` extension.
 > for a list of known issues, and be sure to provide feedback on issues and PRs
 > which affect your use of this extension.
 
+## Godot and Godot Cpp Compatibility
+
+If you intend to target both building as a GDExtension and as a module using godot repo, you can generate compatibility includes that will target either GDExtension or module, based on the GODOT_MODULE_COMPAT define.
+
+If you want such a thing built, when running the build command, `scons`, make sure you have a file called `output_header_mapping.json` at root level of this repo. This file needs to have the mappings from `godot` repo. The mappings can be generated by running the compat_generator.py script.
+
+Example of how to obtain them:
+
+```
+git clone godotengine/godot
+python compat_generator.py godot
+```
+
+Then run the SConstruct build command as usual, and in the `gen/` folder you will now have a new folder, `include/godot_compat` which mirrors the `include/godot_cpp` includes, but have ifdef inside them and either include godot header or godot_cpp header.
+
 ## Contributing
 
 We greatly appreciate help in maintaining and extending this project. If you

binding_generator.py

@@ -3,6 +3,9 @@
 import json
 import re
 import shutil
+import os
+from compat_generator import map_header_files
+from header_matcher import match_headers
 from pathlib import Path
 
 
@@ -207,6 +210,7 @@ def get_file_list(api_filepath, output_dir, headers=False, sources=False, profil
 
     core_gen_folder = Path(output_dir) / "gen" / "include" / "godot_cpp" / "core"
     include_gen_folder = Path(output_dir) / "gen" / "include" / "godot_cpp"
+    include_gen_compat_folder = Path(output_dir) / "gen" / "include" / "godot_compat"
     source_gen_folder = Path(output_dir) / "gen" / "src"
 
     files.append(str((core_gen_folder / "ext_wrappers.gen.inc").as_posix()))
@@ -220,9 +224,11 @@ def get_file_list(api_filepath, output_dir, headers=False, sources=False, profil
             continue
 
         header_filename = include_gen_folder / "variant" / (camel_to_snake(builtin_class["name"]) + ".hpp")
+        header_compat_filename = include_gen_compat_folder / "variant" / (camel_to_snake(builtin_class["name"]) + ".hpp")
         source_filename = source_gen_folder / "variant" / (camel_to_snake(builtin_class["name"]) + ".cpp")
         if headers:
             files.append(str(header_filename.as_posix()))
+            files.append(str(header_compat_filename.as_posix()))
         if sources:
             files.append(str(source_filename.as_posix()))
 
@@ -232,9 +238,11 @@ def get_file_list(api_filepath, output_dir, headers=False, sources=False, profil
             engine_class["name"] = "ClassDBSingleton"
             engine_class["alias_for"] = "ClassDB"
         header_filename = include_gen_folder / "classes" / (camel_to_snake(engine_class["name"]) + ".hpp")
+        header_compat_filename = include_gen_compat_folder / "classes" / (camel_to_snake(engine_class["name"]) + ".hpp")
         source_filename = source_gen_folder / "classes" / (camel_to_snake(engine_class["name"]) + ".cpp")
         if headers:
             files.append(str(header_filename.as_posix()))
+            files.append(str(header_compat_filename.as_posix()))
         if sources and is_class_included(engine_class["name"], build_profile):
             files.append(str(source_filename.as_posix()))
 
@@ -245,8 +253,10 @@ def get_file_list(api_filepath, output_dir, headers=False, sources=False, profil
         snake_struct_name = camel_to_snake(struct_name)
 
         header_filename = include_gen_folder / "classes" / (snake_struct_name + ".hpp")
+        header_compat_filename = include_gen_compat_folder / "classes" / (snake_struct_name + ".hpp")
         if headers:
             files.append(str(header_filename.as_posix()))
+            files.append(str(header_compat_filename.as_posix()))
 
     if headers:
         for path in [
@@ -402,6 +412,7 @@ def generate_bindings(api_filepath, use_template_get_node, bits="64", precision=
     generate_builtin_bindings(api, target_dir, real_t + "_" + bits)
     generate_engine_classes_bindings(api, target_dir, use_template_get_node)
     generate_utility_functions(api, target_dir)
+    generate_compat_includes(Path(output_dir), target_dir)
 
 
 CLASS_ALIASES = {
@@ -1545,6 +1556,47 @@ def generate_engine_classes_bindings(api, output_dir, use_template_get_node):
             header_file.write("\n".join(result))
 
 
+def generate_compat_includes(output_dir: Path, target_dir: Path):
+    file_types_mapping_godot_cpp_gen = map_header_files(target_dir / "include")
+    file_types_mapping_godot_cpp = map_header_files(output_dir / "include") | file_types_mapping_godot_cpp_gen
+    godot_compat = Path("output_header_mapping_godot.json")
+    levels_to_look_back = 3
+    while not godot_compat.exists():
+        godot_compat = ".." / godot_compat
+        levels_to_look_back -= 1
+        if levels_to_look_back == 0:
+            print("Skipping godot_compat")
+            return
+    with godot_compat.open() as file:
+        mapping2 = json.load(file)
+    # Match the headers
+    file_types_mapping = match_headers(file_types_mapping_godot_cpp, mapping2)
+
+    include_gen_folder = Path(target_dir) / "include"
+    for file_godot_cpp_name, file_godot_names in file_types_mapping.items():
+        header_filename = file_godot_cpp_name.replace("godot_cpp", "godot_compat")
+        header_filepath = include_gen_folder / header_filename
+        Path(os.path.dirname(header_filepath)).mkdir(parents=True, exist_ok=True)
+        result = []
+        snake_header_name = camel_to_snake(header_filename)
+        add_header(f"{snake_header_name}.hpp", result)
+
+        header_guard = f"GODOT_COMPAT_{os.path.splitext(os.path.basename(header_filepath).upper())[0]}_HPP"
+        result.append(f"#ifndef {header_guard}")
+        result.append(f"#define {header_guard}")
+        result.append("")
+        result.append(f"#ifdef GODOT_MODULE_COMPAT")
+        for file_godot_name in file_godot_names:
+            result.append(f"#include <{file_godot_name}>")
+        result.append(f"#else")
+        result.append(f"#include <{file_godot_cpp_name}>")
+        result.append(f"#endif")
+        result.append("")
+        result.append(f"#endif // ! {header_guard}")
+        with header_filepath.open("w+", encoding="utf-8") as header_file:
+            header_file.write("\n".join(result))
+
+
 def generate_engine_class_header(class_api, used_classes, fully_used_classes, use_template_get_node):
     global singletons
     result = []
@@ -1771,12 +1823,7 @@ def generate_engine_class_header(class_api, used_classes, fully_used_classes, us
             if "is_static" in method and method["is_static"]:
                 continue
 
-            vararg = "is_vararg" in method and method["is_vararg"]
-            if vararg:
-                method_signature = "\ttemplate <typename... Args> static "
-            else:
-                method_signature = "\tstatic "
-
+            method_signature = "\tstatic "
             return_type = None
             if "return_type" in method:
                 return_type = correct_type(method["return_type"].replace("ClassDBSingleton", "ClassDB"), None, False)
@@ -1787,9 +1834,7 @@ def generate_engine_class_header(class_api, used_classes, fully_used_classes, us
                     False,
                 )
             if return_type is not None:
-                method_signature += return_type
-                if not method_signature.endswith("*"):
-                    method_signature += " "
+                method_signature += return_type + " "
             else:
                 method_signature += "void "
 

compat_generator.py

@@ -0,0 +1,63 @@
+import re
+import os
+import json
+import sys
+
+
+def parse_header_file(file_path):
+    types = {"classes": [], "structs": [], "defines": []}
+
+    with open(file_path, "r", encoding="utf-8") as file:
+        content = file.read()
+
+        # Regular expressions to match different types
+        struct_pattern = r"struct\s+[\w\s]*?([a-zA-Z_]\w*)\s*[:{]"
+        class_pattern = r"class\s+[\w\s]*?([a-zA-Z_]\w*)\s*[:{]"
+        define_pattern = r"#define\s+([a-zA-Z_]\w*)"
+
+        # Extract classes
+        types["classes"] += re.findall(class_pattern, content)
+
+        # Extract structs
+        types["structs"] += re.findall(struct_pattern, content)
+
+        # Extract defines
+        define_matches = re.findall(define_pattern, content)
+        types["defines"] += define_matches
+
+    if len(types["classes"]) == 0 and len(types["structs"]) == 0 and len(types["defines"]) == 0:
+        print(f"{file_path} missing things")
+    return types
+
+
+def map_header_files(directory):
+    file_types_mapping = {}
+
+    for root, dirs, files in os.walk(directory):
+        if "thirdparty" in dirs:
+            dirs.remove("thirdparty")
+        if "tests" in dirs:
+            dirs.remove("tests")
+        if "test" in dirs:
+            dirs.remove("test")
+        if "misc" in dirs:
+            dirs.remove("misc")
+        for file in files:
+            if file.endswith(".h") or file.endswith(".hpp"):
+                relative_path = os.path.relpath(root, directory)
+                file_path = os.path.join(root, file)
+                file_types_mapping[f"{relative_path}/{file}"] = parse_header_file(file_path)
+
+    return file_types_mapping
+
+
+if __name__ == "__main__":
+    # Get current directory
+    current_directory = os.getcwd()
+
+    if len(sys.argv) > 1:
+        current_directory = os.path.join(os.getcwd(), sys.argv[1])
+
+    file_types_mapping = map_header_files(current_directory)
+    with open("output_header_mapping.json", "w") as json_file:
+        json.dump(file_types_mapping, json_file, indent=4)

header_matcher.py

@@ -0,0 +1,31 @@
+import json
+
+
+def match_headers(mapping1, mapping2):
+    matches = {}
+    for header_file, data1 in mapping1.items():
+        for header_file2, data2 in mapping2.items():
+            # Check if classes/defines/structs in header_file1 are present in header_file2
+            if (any(class_name in data2["classes"] for class_name in data1["classes"]) or
+                any(define_name in data2["defines"] for define_name in data1["defines"]) or
+                any(define_name in data2["structs"] for define_name in data1["structs"])):
+                if header_file not in matches:
+                    matches[header_file] = []
+                matches[header_file].append(header_file2)
+    return matches
+
+
+if __name__ == "__main__":
+    # Load the two header mappings
+    with open("output_header_mapping.json", "r") as file:
+        mapping1 = json.load(file)
+
+    with open("output_header_mapping_godot.json", "r") as file:
+        mapping2 = json.load(file)
+
+    # Match the headers
+    matches = match_headers(mapping1, mapping2)
+
+    # Optionally, you can save the matches to a file
+    with open("header_matches.json", "w") as outfile:
+        json.dump(matches, outfile, indent=4)

module_converter.py

@@ -0,0 +1,49 @@
+# Using output_header_mapping.json convert all imports in specified source folder location from godot imports to godot-compat imports
+
+
+import json
+import os
+import sys
+
+from compat_generator import map_header_files
+from header_matcher import match_headers
+
+if __name__ == "__main__":
+    if len(sys.argv) > 3:
+        src_directory = os.path.join(os.getcwd(), sys.argv[1])
+        godot_directory = os.path.join(os.getcwd(), sys.argv[2])
+        godot_cpp_directory = os.path.join(os.getcwd(), sys.argv[3])
+    else:
+        raise Exception("Usage: python module_converter.py <source directory> <godot directory> <godot-cpp directory>")
+    # Load the godot mappings
+    with open(f"{godot_directory}/output_header_mapping_godot.json", "r") as file:
+        godot_mappings = json.load(file)
+    
+    # Generate mappings for godot-cpp
+    godot_cpp_mappings = map_header_files(godot_cpp_directory)
+    matches = match_headers(godot_mappings, godot_cpp_mappings)
+    # Save matches to a file
+    with open("header_matches.json", "w") as outfile:
+        json.dump(matches, outfile, indent=4)
+    src_directory = os.getcwd()
+    # Go through folder specified through all files with .cpp, .h or .hpp
+    for root, dirs, files in os.walk(src_directory):
+        for file in files:
+            if file.endswith(".cpp") or file.endswith(".h") or file.endswith(".hpp"):
+                with open(os.path.join(root, file), "r") as f:
+                    content = f.read()
+
+                # Replace imports to godot imports with godot_compat imports
+                for match in matches:
+                    generate_imports = matches[match]
+                    godot_compat_imports = ""
+                    for generate_import in generate_imports:
+                        godot_compat_import = generate_import.replace("gen/include/godot_cpp/", "godot_compat/")
+                        godot_compat_import = godot_compat_import.replace("include/godot_cpp/", "godot_compat/")
+                        godot_compat_imports += f"#include <{godot_compat_import}>\n"
+                    # Remove last 'n from imports
+                    godot_compat_imports = godot_compat_imports[:-1]
+                    content = content.replace(f"#include \"{match}\"", godot_compat_imports)
+                # Write the modified content back to the file
+                with open(os.path.join(root, file), "w") as f:
+                    f.write(content)