From fe2da72035af62b161aaba665862a706cfbd2401 Mon Sep 17 00:00:00 2001 From: Dragos Daian Date: Fri, 15 Mar 2024 09:57:36 +0100 Subject: [PATCH] Generate godot compat for dual build generate compat generate compat Update ci.yml Update binding_generator.py generate compat generate compat lint python files Update compat_generator.py update docs Update binding_generator.py Update module_converter.py also collect defines Add module converter file that converts module based projects to godot_compat Update ci.yml update docs Update compat_generator.py lint python files generate compat generate compat generate compat generate compat Update ci.yml fix path issue when caling from outside Update binding_generator.py update to also take missing classes/structs Update binding_generator.py Generate godot compat for dual build generate compat generate compat Update ci.yml Update binding_generator.py generate compat generate compat lint python files Update compat_generator.py update docs Update binding_generator.py Update module_converter.py also collect defines Add module converter file that converts module based projects to godot_compat Update ci.yml update docs Update compat_generator.py lint python files generate compat generate compat generate compat generate compat Update ci.yml fix path issue when caling from outside Add support for build profiles. Allow enabling or disabling specific classes (which will not be built). Allow forwarding from `ClassDB` to `ClassDBSingleton` to support enumerations update to also take missing classes/structs Update binding_generator.py update update naming of files add godot mappings. update and run output_header_mapping.json Update README.md make godot_compat work without a file generated fix the test Update binding_generator.py Update binding_generator.py --- README.md | 61 +++++++++++++++++++++++++++++++ binding_generator.py | 85 +++++++++++++++++++++++++++++++++++--------- compat_generator.py | 67 ++++++++++++++++++++++++++++++++++ header_matcher.py | 28 +++++++++++++++ tools/godotcpp.py | 8 +++++ 5 files changed, 233 insertions(+), 16 deletions(-) create mode 100644 compat_generator.py create mode 100644 header_matcher.py diff --git a/README.md b/README.md index 005c8e723..f07b02109 100644 --- a/README.md +++ b/README.md @@ -147,3 +147,64 @@ generic reusable template. Or checkout the code for the [Summator example](https://github.com/paddy-exe/GDExtensionSummator) as shown in the [official documentation](https://docs.godotengine.org/en/latest/tutorials/scripting/gdextension/gdextension_cpp_example.html). + +## Godot and Godot Cpp Compatibility + +If you intend to target both building as a GDExtension and as a module using godot repo, you can generate compatibility includes that will target either GDExtension or module, based on the GODOT_MODULE_COMPAT define. + +If you want such headers, when running the build command, `scons`, pass the `godot_repo` param with the path to the godot repository. Eg. if you have the godot repository cloned at path `../godot`, then do: + +```sh +scons godot_repo="../godot" +``` + +This will generate something like this: +``` +gen/include/godot_cpp/.. +gen/include/godot_compat/.. +``` + +Now, all you need to do is when writting your addon/module, replace includes like these: + +```cpp +#include +``` + +with + +```cpp +#include +``` + +Inside, this file will have code for both godot and godot-cpp: + +```cpp +#ifdef GODOT_MODULE_COMPAT +#include +#else +#include +#endif +``` + +### Manually generate mapping files + +The mappings can be manually generated by running the `compat_generator.py` script. + +Example of how to run `compat_generator.py`: + +```sh +git clone godotengine/godot +python compat_generator.py godot +``` + +The first argument of `compat_generator.py` is the folder where the repo is (can be godot or godot-cpp repo). If this folder is not given, the current directory is assumed. The output of this is either `output_header_mapping_godot.json` or `output_header_mapping_godot_cpp.json` + +### Manually match the mapping files + +If you want to manually match the godot mapping file with the godot-cpp one, you can do that by running: + +```sh +python header_matcher.py +``` + +This will generate the `header_matches.json` file with matches from godot and godot_cpp repo. diff --git a/binding_generator.py b/binding_generator.py index 75e56fde1..8e0ff24eb 100644 --- a/binding_generator.py +++ b/binding_generator.py @@ -3,6 +3,9 @@ import json import re import shutil +import os +from compat_generator import map_header_files +from header_matcher import match_headers from pathlib import Path @@ -197,7 +200,7 @@ def generate_virtuals(target): f.write(txt) -def get_file_list(api_filepath, output_dir, headers=False, sources=False, profile_filepath=""): +def get_file_list(api_filepath, output_dir, headers=False, sources=False, compat=False, profile_filepath=""): api = {} files = [] with open(api_filepath, encoding="utf-8") as api_file: @@ -207,6 +210,7 @@ def get_file_list(api_filepath, output_dir, headers=False, sources=False, profil core_gen_folder = Path(output_dir) / "gen" / "include" / "godot_cpp" / "core" include_gen_folder = Path(output_dir) / "gen" / "include" / "godot_cpp" + include_gen_compat_folder = Path(output_dir) / "gen" / "include" / "godot_compat" source_gen_folder = Path(output_dir) / "gen" / "src" files.append(str((core_gen_folder / "ext_wrappers.gen.inc").as_posix())) @@ -220,9 +224,12 @@ def get_file_list(api_filepath, output_dir, headers=False, sources=False, profil continue header_filename = include_gen_folder / "variant" / (camel_to_snake(builtin_class["name"]) + ".hpp") + header_compat_filename = include_gen_compat_folder / "variant" / (camel_to_snake(builtin_class["name"]) + ".hpp") source_filename = source_gen_folder / "variant" / (camel_to_snake(builtin_class["name"]) + ".cpp") if headers: files.append(str(header_filename.as_posix())) + if compat: + files.append(str(header_compat_filename.as_posix())) if sources: files.append(str(source_filename.as_posix())) @@ -232,9 +239,12 @@ def get_file_list(api_filepath, output_dir, headers=False, sources=False, profil engine_class["name"] = "ClassDBSingleton" engine_class["alias_for"] = "ClassDB" header_filename = include_gen_folder / "classes" / (camel_to_snake(engine_class["name"]) + ".hpp") + header_compat_filename = include_gen_compat_folder / "classes" / (camel_to_snake(engine_class["name"]) + ".hpp") source_filename = source_gen_folder / "classes" / (camel_to_snake(engine_class["name"]) + ".cpp") if headers: files.append(str(header_filename.as_posix())) + if compat: + files.append(str(header_compat_filename.as_posix())) if sources and is_class_included(engine_class["name"], build_profile): files.append(str(source_filename.as_posix())) @@ -245,21 +255,30 @@ def get_file_list(api_filepath, output_dir, headers=False, sources=False, profil snake_struct_name = camel_to_snake(struct_name) header_filename = include_gen_folder / "classes" / (snake_struct_name + ".hpp") + header_compat_filename = include_gen_compat_folder / "classes" / (snake_struct_name + ".hpp") if headers: files.append(str(header_filename.as_posix())) + if compat: + files.append(str(header_compat_filename.as_posix())) if headers: - for path in [ - include_gen_folder / "variant" / "builtin_types.hpp", - include_gen_folder / "variant" / "builtin_binds.hpp", - include_gen_folder / "variant" / "utility_functions.hpp", - include_gen_folder / "variant" / "variant_size.hpp", - include_gen_folder / "variant" / "builtin_vararg_methods.hpp", - include_gen_folder / "classes" / "global_constants.hpp", - include_gen_folder / "classes" / "global_constants_binds.hpp", - include_gen_folder / "core" / "version.hpp", - ]: - files.append(str(path.as_posix())) + relative_paths = [ + ["variant", "builtin_types.hpp"], + ["variant", "builtin_binds.hpp"], + ["variant", "utility_functions.hpp"], + ["variant", "variant_size.hpp"], + ["variant", "builtin_vararg_methods.hpp"], + ["classes", "global_constants.hpp"], + ["classes", "global_constants_binds.hpp"], + ["core", "version.hpp"] + ] + + for relative_path_parts in relative_paths: + full_header_path = include_gen_folder.joinpath(*relative_path_parts) + files.append(str(full_header_path.as_posix())) + if compat: + full_compat_path = include_gen_compat_folder.joinpath(*relative_path_parts) + files.append(str(full_compat_path.as_posix())) if sources: utility_functions_source_path = source_gen_folder / "variant" / "utility_functions.cpp" files.append(str(utility_functions_source_path.as_posix())) @@ -267,8 +286,8 @@ def get_file_list(api_filepath, output_dir, headers=False, sources=False, profil return files -def print_file_list(api_filepath, output_dir, headers=False, sources=False, profile_filepath=""): - print(*get_file_list(api_filepath, output_dir, headers, sources, profile_filepath), sep=";", end=None) +def print_file_list(api_filepath, output_dir, headers=False, sources=False, compat=False, profile_filepath=""): + print(*get_file_list(api_filepath, output_dir, headers, sources, compat, profile_filepath), sep=";", end=None) def parse_build_profile(profile_filepath, api): @@ -365,7 +384,7 @@ def scons_emit_files(target, source, env): if profile_filepath and not Path(profile_filepath).is_absolute(): profile_filepath = str((Path(env.Dir("#").abspath) / profile_filepath).as_posix()) - files = [env.File(f) for f in get_file_list(str(source[0]), target[0].abspath, True, True, profile_filepath)] + files = [env.File(f) for f in get_file_list(str(source[0]), target[0].abspath, True, True, env["godot_repo"] != "", profile_filepath)] env.Clean(target, files) env["godot_cpp_gen_dir"] = target[0].abspath return files, source @@ -378,11 +397,12 @@ def scons_generate_bindings(target, source, env): "32" if "32" in env["arch"] else "64", env["precision"], env["godot_cpp_gen_dir"], + env["godot_repo"], ) return None -def generate_bindings(api_filepath, use_template_get_node, bits="64", precision="single", output_dir="."): +def generate_bindings(api_filepath, use_template_get_node, bits="64", precision="single", output_dir=".", godot_repo = ""): api = None target_dir = Path(output_dir) / "gen" @@ -402,6 +422,8 @@ def generate_bindings(api_filepath, use_template_get_node, bits="64", precision= generate_builtin_bindings(api, target_dir, real_t + "_" + bits) generate_engine_classes_bindings(api, target_dir, use_template_get_node) generate_utility_functions(api, target_dir) + if godot_repo != "": + generate_compat_includes(godot_repo, target_dir) CLASS_ALIASES = { @@ -1545,6 +1567,37 @@ def generate_engine_classes_bindings(api, output_dir, use_template_get_node): header_file.write("\n".join(result)) +def generate_compat_includes(godot_repo: Path, target_dir: Path): + file_types_mapping_godot_cpp_gen = map_header_files(target_dir / "include") + file_types_mapping_godot = map_header_files(godot_repo) + # Match the headers + file_types_mapping = match_headers(file_types_mapping_godot_cpp_gen, file_types_mapping_godot) + + include_gen_folder = Path(target_dir) / "include" + for file_godot_cpp_name, file_godot_names in file_types_mapping.items(): + header_filename = file_godot_cpp_name.replace("godot_cpp", "godot_compat") + header_filepath = include_gen_folder / header_filename + Path(os.path.dirname(header_filepath)).mkdir(parents=True, exist_ok=True) + result = [] + snake_header_name = camel_to_snake(header_filename) + add_header(f"{snake_header_name}.hpp", result) + + header_guard = f"GODOT_COMPAT_{os.path.splitext(os.path.basename(header_filepath).upper())[0]}_HPP" + result.append(f"#ifndef {header_guard}") + result.append(f"#define {header_guard}") + result.append("") + result.append(f"#ifdef GODOT_MODULE_COMPAT") + for file_godot_name in file_godot_names: + result.append(f"#include <{file_godot_name}>") + result.append(f"#else") + result.append(f"#include <{file_godot_cpp_name}>") + result.append(f"#endif") + result.append("") + result.append(f"#endif // ! {header_guard}") + with header_filepath.open("w+", encoding="utf-8") as header_file: + header_file.write("\n".join(result)) + + def generate_engine_class_header(class_api, used_classes, fully_used_classes, use_template_get_node): global singletons result = [] diff --git a/compat_generator.py b/compat_generator.py new file mode 100644 index 000000000..6eacc2a8f --- /dev/null +++ b/compat_generator.py @@ -0,0 +1,67 @@ +#!/usr/bin/env python + +import re +import os +import json +import sys + + +def parse_header_file(file_path): + types = {"classes": [], "structs": [], "defines": []} + + with open(file_path, "r", encoding="utf-8") as file: + content = file.read() + + # Regular expressions to match different types + struct_pattern = r"struct\s+[\w\s]*?([a-zA-Z_]\w*)\s*[:{]" + class_pattern = r"class\s+[\w\s]*?([a-zA-Z_]\w*)\s*[:{]" + define_pattern = r"#define\s+([a-zA-Z_]\w*)" + + # Extract classes + types["classes"] += re.findall(class_pattern, content) + + # Extract structs + types["structs"] += re.findall(struct_pattern, content) + + # Extract defines + define_matches = re.findall(define_pattern, content) + types["defines"] += define_matches + + # Debug the case where no classes or structs are found + #if len(types["classes"]) == 0 and len(types["structs"]) == 0 and len(types["defines"]) == 0: + # print(f"{file_path} missing things") + return types + + +def map_header_files(directory): + file_types_mapping = {} + + for root, dirs, files in os.walk(directory): + if "thirdparty" in dirs: + dirs.remove("thirdparty") + if "tests" in dirs: + dirs.remove("tests") + if "test" in dirs: + dirs.remove("test") + if "misc" in dirs: + dirs.remove("misc") + for file in files: + if file.endswith(".h") or file.endswith(".hpp"): + relative_path = os.path.relpath(root, directory) + file_path = os.path.join(root, file) + file_types_mapping[f"{relative_path}/{file}"] = parse_header_file(file_path) + + return file_types_mapping + + +if __name__ == "__main__": + # Get current directory + current_directory = os.getcwd() + mapping_name = "" + if len(sys.argv) > 1: + mapping_name = f"_godot" + current_directory = os.path.join(os.getcwd(), sys.argv[1]) + + file_types_mapping = map_header_files(current_directory) + with open(f"output_header_mapping{mapping_name}.json", "w") as json_file: + json.dump(file_types_mapping, json_file, indent=4) diff --git a/header_matcher.py b/header_matcher.py new file mode 100644 index 000000000..fdf0cc10c --- /dev/null +++ b/header_matcher.py @@ -0,0 +1,28 @@ +import json +import os +from compat_generator import map_header_files + +def match_headers(mapping1, mapping2): + matches = {} + for header_file, data1 in mapping1.items(): + for header_file2, data2 in mapping2.items(): + # Check if classes/defines/structs in header_file1 are present in header_file2 + if header_file not in matches: + matches[header_file] = [] + if (any(class_name in data2["classes"] for class_name in data1["classes"]) or + any(define_name in data2["defines"] for define_name in data1["defines"]) or + any(define_name in data2["structs"] for define_name in data1["structs"])): + matches[header_file].append(header_file2) + return matches + + +if __name__ == "__main__": + # Load the two header mappings + with open("output_header_mapping_godot.json", "r") as file: + mapping_godot = json.load(file) + file_types_mapping_godot_cpp_gen = map_header_files(os.getcwd() / "gen" / "include") + matches = match_headers(file_types_mapping_godot_cpp_gen, mapping_godot) + + # Optionally, you can save the matches to a file + with open("header_matches.json", "w") as outfile: + json.dump(matches, outfile, indent=4) diff --git a/tools/godotcpp.py b/tools/godotcpp.py index 9ceac02b1..a43fd92a8 100644 --- a/tools/godotcpp.py +++ b/tools/godotcpp.py @@ -228,6 +228,14 @@ def options(opts, env): validator=validate_file, ) ) + opts.Add( + PathVariable( + key="godot_repo", + help="Path to a custom directory containing Godot repository. Used to generate godot_compat bindings.", + default=env.get("godot_repo", ""), + validator=validate_dir, + ) + ) opts.Add( BoolVariable( key="generate_bindings",