Generate godot compat for dual build

generate compat

generate compat

Update ci.yml

Update binding_generator.py

generate compat

generate compat

lint python files

Update compat_generator.py

update docs

Update binding_generator.py

Update module_converter.py

also collect defines

Add module converter file that converts module based projects to godot_compat

Update ci.yml

update docs

Update compat_generator.py

lint python files

generate compat

generate compat

generate compat

generate compat

Update ci.yml

fix path issue when caling from outside

.github/workflows/ci.yml

@@ -108,6 +108,17 @@ jobs:
         with:
           python-version: '3.x'
 
+      - name: Clone Godot
+        uses: actions/checkout@v4
+        with:
+          repository: godotengine/godot
+          path: godot
+          #ref: TODO take tag
+
+      - name: Generate compat mappings for godot
+        run: |
+          python compat_generator.py godot
+
       - name: Android dependencies
         if: ${{ matrix.platform == 'android' }}
         uses: nttld/setup-ndk@v1

README.md

@@ -62,6 +62,21 @@ and the [godot-cpp issue tracker](https://github.com/godotengine/godot-cpp/issue
 for a list of known issues, and be sure to provide feedback on issues and PRs
 which affect your use of this extension.
 
+## Godot and Godot Cpp Compatibility
+
+If you intend to target both building as a GDExtension and as a module using godot repo, you can generate compatibility includes that will target either GDExtension or module, based on the GODOT_MODULE_COMPAT define.
+
+If you want such a thing built, when running the build command, `scons`, make sure you have a file called `output_header_mapping.json` at root level of this repo. This file needs to have the mappings from `godot` repo. The mappings can be generated by running the compat_generator.py script.
+
+Example of how to obtain them:
+
+```
+git clone godotengine/godot
+python compat_generator.py godot
+```
+
+Then run the SConstruct build command as usual, and in the `gen/` folder you will now have a new folder, `include/godot_compat` which mirrors the `include/godot_cpp` includes, but have ifdef inside them and either include godot header or godot_cpp header.
+
 ## Contributing
 
 We greatly appreciate help in maintaining and extending this project. If you

binding_generator.py

@@ -3,6 +3,9 @@
 import json
 import re
 import shutil
+import os
+from compat_generator import map_header_files
+from header_matcher import match_headers
 from pathlib import Path
 
 
@@ -205,6 +208,7 @@ def get_file_list(api_filepath, output_dir, headers=False, sources=False):
 
     core_gen_folder = Path(output_dir) / "gen" / "include" / "godot_cpp" / "core"
     include_gen_folder = Path(output_dir) / "gen" / "include" / "godot_cpp"
+    include_gen_compat_folder = Path(output_dir) / "gen" / "include" / "godot_compat"
     source_gen_folder = Path(output_dir) / "gen" / "src"
 
     files.append(str((core_gen_folder / "ext_wrappers.gen.inc").as_posix()))
@@ -307,6 +311,7 @@ def generate_bindings(api_filepath, use_template_get_node, bits="64", precision=
     generate_builtin_bindings(api, target_dir, real_t + "_" + bits)
     generate_engine_classes_bindings(api, target_dir, use_template_get_node)
     generate_utility_functions(api, target_dir)
+    generate_compat_includes(Path(output_dir), target_dir)
 
 
 builtin_classes = []
@@ -1440,6 +1445,47 @@ def generate_engine_classes_bindings(api, output_dir, use_template_get_node):
             header_file.write("\n".join(result))
 
 
+def generate_compat_includes(output_dir: Path, target_dir: Path):
+    file_types_mapping_godot_cpp_gen = map_header_files(target_dir / "include")
+    file_types_mapping_godot_cpp = map_header_files(output_dir / "include") | file_types_mapping_godot_cpp_gen
+    godot_compat = Path("output_header_mapping_godot.json")
+    levels_to_look_back = 3
+    while not godot_compat.exists():
+        godot_compat = ".." / godot_compat
+        levels_to_look_back -= 1
+        if levels_to_look_back == 0:
+            print("Skipping godot_compat")
+            return
+    with godot_compat.open() as file:
+        mapping2 = json.load(file)
+    # Match the headers
+    file_types_mapping = match_headers(file_types_mapping_godot_cpp, mapping2)
+
+    include_gen_folder = Path(target_dir) / "include"
+    for file_godot_cpp_name, file_godot_names in file_types_mapping.items():
+        header_filename = file_godot_cpp_name.replace("godot_cpp", "godot_compat")
+        header_filepath = include_gen_folder / header_filename
+        Path(os.path.dirname(header_filepath)).mkdir(parents=True, exist_ok=True)
+        result = []
+        snake_header_name = camel_to_snake(header_filename)
+        add_header(f"{snake_header_name}.hpp", result)
+
+        header_guard = f"GODOT_COMPAT_{os.path.splitext(os.path.basename(header_filepath).upper())[0]}_HPP"
+        result.append(f"#ifndef {header_guard}")
+        result.append(f"#define {header_guard}")
+        result.append("")
+        result.append(f"#ifdef GODOT_MODULE_COMPAT")
+        for file_godot_name in file_godot_names:
+            result.append(f"#include <{file_godot_name}>")
+        result.append(f"#else")
+        result.append(f"#include <{file_godot_cpp_name}>")
+        result.append(f"#endif")
+        result.append("")
+        result.append(f"#endif // ! {header_guard}")
+        with header_filepath.open("w+", encoding="utf-8") as header_file:
+            header_file.write("\n".join(result))
+
+
 def generate_engine_class_header(class_api, used_classes, fully_used_classes, use_template_get_node):
     global singletons
     result = []

compat_generator.py

@@ -0,0 +1,63 @@
+import re
+import os
+import json
+import sys
+
+
+def parse_header_file(file_path):
+    types = {"classes": [], "structs": [], "defines": []}
+
+    with open(file_path, "r", encoding="utf-8") as file:
+        content = file.read()
+
+        # Regular expressions to match different types
+        class_pattern = r"class\s+([a-zA-Z_]\w*)\s*[:{]"
+        struct_pattern = r"struct\s+([a-zA-Z_]\w*)\s*[:{]"
+        define_pattern = r"#define\s+([a-zA-Z_]\w*)"
+
+        # Extract classes
+        types["classes"] += re.findall(class_pattern, content)
+
+        # Extract structs
+        types["structs"] += re.findall(struct_pattern, content)
+
+        # Extract defines
+        define_matches = re.findall(define_pattern, content)
+        types["defines"] += define_matches
+
+    if len(types["classes"]) == 0 and len(types["structs"]) == 0 and len(types["defines"]) == 0:
+        print(f"{file_path} missing things")
+    return types
+
+
+def map_header_files(directory):
+    file_types_mapping = {}
+
+    for root, dirs, files in os.walk(directory):
+        if "thirdparty" in dirs:
+            dirs.remove("thirdparty")
+        if "tests" in dirs:
+            dirs.remove("tests")
+        if "test" in dirs:
+            dirs.remove("test")
+        if "misc" in dirs:
+            dirs.remove("misc")
+        for file in files:
+            if file.endswith(".h") or file.endswith(".hpp"):
+                relative_path = os.path.relpath(root, directory)
+                file_path = os.path.join(root, file)
+                file_types_mapping[f"{relative_path}/{file}"] = parse_header_file(file_path)
+
+    return file_types_mapping
+
+
+if __name__ == "__main__":
+    # Get current directory
+    current_directory = os.getcwd()
+
+    if len(sys.argv) > 1:
+        current_directory = os.path.join(os.getcwd(), sys.argv[1])
+
+    file_types_mapping = map_header_files(current_directory)
+    with open("output_header_mapping.json", "w") as json_file:
+        json.dump(file_types_mapping, json_file, indent=4)

header_matcher.py

@@ -0,0 +1,31 @@
+import json
+
+
+def match_headers(mapping1, mapping2):
+    matches = {}
+    for header_file, data1 in mapping1.items():
+        for header_file2, data2 in mapping2.items():
+            # Check if classes/defines/structs in header_file1 are present in header_file2
+            if (any(class_name in data2["classes"] for class_name in data1["classes"]) or
+                any(define_name in data2["defines"] for define_name in data1["defines"]) or
+                any(define_name in data2["structs"] for define_name in data1["structs"])):
+                if header_file not in matches:
+                    matches[header_file] = []
+                matches[header_file].append(header_file2)
+    return matches
+
+
+if __name__ == "__main__":
+    # Load the two header mappings
+    with open("output_header_mapping.json", "r") as file:
+        mapping1 = json.load(file)
+
+    with open("output_header_mapping_godot.json", "r") as file:
+        mapping2 = json.load(file)
+
+    # Match the headers
+    matches = match_headers(mapping1, mapping2)
+
+    # Optionally, you can save the matches to a file
+    with open("header_matches.json", "w") as outfile:
+        json.dump(matches, outfile, indent=4)

module_converter.py

@@ -0,0 +1,46 @@
+# Using output_header_mapping.json convert all imports in specified source folder location from godot imports to godot-compat imports
+
+
+import json
+import os
+import sys
+
+from compat_generator import map_header_files
+from header_matcher import match_headers
+
+if __name__ == "__main__":
+    if len(sys.argv) > 2:
+        current_directory = os.path.join(os.getcwd(), sys.argv[1])
+        godot_cpp_directory = os.path.join(os.getcwd(), sys.argv[2])
+    # Load the godot mappings
+    with open(f"{godot_cpp_directory}/output_header_mapping.json", "r") as file:
+        godot_mappings = json.load(file)
+
+    # Generate mappings for godot-cpp
+    godot_cpp_mappings = map_header_files(godot_cpp_directory)
+    matches = match_headers(godot_mappings, godot_cpp_mappings)
+    # Save matches to a file
+    with open("header_matches.json", "w") as outfile:
+        json.dump(matches, outfile, indent=4)
+    current_directory = os.getcwd()
+    # Go through folder specified through all files with .cpp, .h or .hpp
+    for root, dirs, files in os.walk(current_directory):
+        for file in files:
+            if file.endswith(".cpp") or file.endswith(".h") or file.endswith(".hpp"):
+                with open(os.path.join(root, file), "r") as f:
+                    content = f.read()
+
+                # Replace imports to godot imports with godot_compat imports
+                for match in matches:
+                    generate_imports = matches[match]
+                    godot_compat_imports = ""
+                    for generate_import in generate_imports:
+                        godot_compat_import = generate_import.replace("gen/include/godot_cpp/", "godot_compat/")
+                        godot_compat_import = godot_compat_import.replace("include/godot_cpp/", "godot_compat/")
+                        godot_compat_imports += f"#include <{godot_compat_import}>\n"
+                    # Remove last 'n from imports
+                    godot_compat_imports = godot_compat_imports[:-1]
+                    content = content.replace(f"#include \"{match}\"", godot_compat_imports)
+                # Write the modified content back to the file
+                with open(os.path.join(root, file), "w") as f:
+                    f.write(content)