Scripting: Add script documentation cache to project

This PR adds a script documentation cache in the project folder.
It is loaded at alongside native documentation caches. This makes
scripts fully accessible through Search Help, including their
members, etc, right from project start, without having to compile
every single script.

editor/doc_tools.cpp

@@ -365,6 +365,14 @@ void DocTools::remove_doc(const String &p_class_name) {
 	class_list.erase(p_class_name);
 }
 
+void DocTools::remove_script_doc_by_path(const String &p_path) {
+	for (KeyValue<String, DocData::ClassDoc> &E : class_list)
+		if (E.value.is_script_doc && E.value.script_path == p_path) {
+			remove_doc(E.key);
+			return;
+		}
+}
+
 bool DocTools::has_doc(const String &p_class_name) {
 	if (p_class_name.is_empty()) {
 		return false;

editor/doc_tools.h

@@ -45,6 +45,7 @@ class DocTools {
 	void merge_from(const DocTools &p_data);
 	void add_doc(const DocData::ClassDoc &p_class_doc);
 	void remove_doc(const String &p_class_name);
+	void remove_script_doc_by_path(const String &p_path);
 	bool has_doc(const String &p_class_name);
 	enum GenerateFlags {
 		GENERATE_FLAG_SKIP_BASIC_TYPES = (1 << 0),

editor/editor_file_system.cpp

@@ -1864,6 +1864,11 @@ void EditorFileSystem::_update_script_documentation() {
 
 		if (!efd || index < 0) {
 			// The file was removed
+			DocTools *doc = EditorHelp::get_doc_data();
+			if (doc)
+				doc->remove_script_doc_by_path(path);
+			else
+				ERR_PRINT("Cannot update documentation for deleted file: EditorHelp::DocTools not initialized.");
 			continue;
 		}
 

editor/editor_help.cpp

@@ -34,8 +34,10 @@
 #include "core/core_constants.h"
 #include "core/extension/gdextension.h"
 #include "core/input/input.h"
+#include "core/io/dir_access.h"
 #include "core/object/script_language.h"
 #include "core/os/keyboard.h"
+#include "core/os/time.h"
 #include "core/string/string_builder.h"
 #include "core/version_generated.gen.h"
 #include "editor/doc_data_compressed.gen.h"
@@ -2859,6 +2861,10 @@ String EditorHelp::get_cache_full_path() {
 	return EditorPaths::get_singleton()->get_cache_dir().path_join(vformat("editor_doc_cache-%d.%d.res", VERSION_MAJOR, VERSION_MINOR));
 }
 
+String EditorHelp::get_script_doc_cache_full_path() {
+	return EditorPaths::get_singleton()->get_project_data_dir().path_join("editor_script_doc_cache.res");
+}
+
 void EditorHelp::load_xml_buffer(const uint8_t *p_buffer, int p_size) {
 	if (!ext_doc) {
 		ext_doc = memnew(DocTools);
@@ -2882,18 +2888,21 @@ void EditorHelp::remove_class(const String &p_class) {
 }
 
 void EditorHelp::_load_doc_thread(void *p_udata) {
+	bool use_script_cache = (bool)p_udata;
 	Ref<Resource> cache_res = ResourceLoader::load(get_cache_full_path());
 	if (cache_res.is_valid() && cache_res->get_meta("version_hash", "") == doc_version_hash) {
 		Array classes = cache_res->get_meta("classes", Array());
 		for (int i = 0; i < classes.size(); i++) {
 			doc->add_doc(DocData::ClassDoc::from_dict(classes[i]));
 		}
 
+		if (use_script_cache && ProjectSettings::get_singleton()->is_project_loaded())
+			_load_script_doc_cache();
 		// Extensions' docs are not cached. Generate them now (on the main thread).
 		callable_mp_static(&EditorHelp::_gen_extensions_docs).call_deferred();
 	} else {
 		// We have to go back to the main thread to start from scratch, bypassing any possibly existing cache.
-		callable_mp_static(&EditorHelp::generate_doc).call_deferred(false);
+		callable_mp_static(&EditorHelp::generate_doc).call_deferred(false, use_script_cache);
 	}
 
 	OS::get_singleton()->benchmark_end_measure("EditorHelp", vformat("Generate Documentation (Run %d)", doc_generation_count));
@@ -2923,6 +2932,10 @@ void EditorHelp::_gen_doc_thread(void *p_udata) {
 		ERR_PRINT("Cannot save editor help cache (" + get_cache_full_path() + ").");
 	}
 
+	// Load script docs after native ones are cached so native cache doesn't contain script docs.
+	bool use_script_cache = (bool)p_udata;
+	if (use_script_cache && ProjectSettings::get_singleton()->is_project_loaded())
+		_load_script_doc_cache();
 	OS::get_singleton()->benchmark_end_measure("EditorHelp", vformat("Generate Documentation (Run %d)", doc_generation_count));
 }
 
@@ -2935,7 +2948,65 @@ void EditorHelp::_gen_extensions_docs() {
 	}
 }
 
-void EditorHelp::generate_doc(bool p_use_cache) {
+void EditorHelp::_load_script_doc_cache() {
+	ERR_FAIL_COND_MSG(!ProjectSettings::get_singleton()->is_project_loaded(), "Error: cannot load script doc cache without a project.");
+	if (ResourceLoader::exists(get_script_doc_cache_full_path())) {
+		Ref<Resource> script_doc_cache_res = ResourceLoader::load(get_script_doc_cache_full_path());
+		if (script_doc_cache_res.is_valid()) {
+			Array classes = script_doc_cache_res->get_meta("classes", Array());
+			for (int i = 0; i < classes.size(); i++)
+				doc->add_doc(DocData::ClassDoc::from_dict(classes[i]));
+		}
+		return;
+	}
+
+	// Force compile all scripts, which should generate their documentation.
+	WARN_PRINT("Script documentation cache not found. Regenerating it may take a while.");
+	HashMap<String, bool> is_extension_script; // Caches ScriptServer results so it's not being queried for every file, only for every extension.
+	List<Ref<DirAccess>> dirs;
+	dirs.push_front(DirAccess::create(DirAccess::ACCESS_RESOURCES));
+	while (!dirs.is_empty()) {
+		Ref<DirAccess> dir = dirs.front()->get();
+		dirs.pop_front();
+		for (const String &file : dir->get_files()) {
+			if (file.begins_with("."))
+				continue;
+			if (!is_extension_script.has(file.get_extension()))
+				is_extension_script[file.get_extension()] = ScriptServer::get_language_for_extension(file.get_extension()) != nullptr;
+
+			if (is_extension_script[file.get_extension()]) {
+				Ref<Script> scr = ResourceLoader::load(dir->get_current_dir() + "/" + file);
+				if (scr.is_valid())
+					for (DocData::ClassDoc cd : scr->get_documentation())
+						doc->add_doc(cd);
+			}
+		}
+
+		for (const String &subdir : dir->get_directories()) {
+			if (!subdir.begins_with("."))
+				dirs.push_front(DirAccess::open(DirAccess::get_full_path(dir->get_current_dir() + "/" + subdir, DirAccess::ACCESS_RESOURCES)));
+		}
+	}
+
+	save_script_doc_cache();
+}
+
+void EditorHelp::save_script_doc_cache() {
+	Ref<Resource> cache_res;
+	cache_res.instantiate();
+	Array classes;
+	for (const KeyValue<String, DocData::ClassDoc> &E : doc->class_list)
+		if (E.value.is_script_doc)
+			classes.push_back(DocData::ClassDoc::to_dict(E.value));
+
+	cache_res->set_meta("classes", classes);
+	Error err = ResourceSaver::save(cache_res, get_script_doc_cache_full_path(), ResourceSaver::FLAG_COMPRESS);
+	if (err) {
+		ERR_PRINT("Cannot save editor help script documentation cache (" + get_script_doc_cache_full_path() + ").");
+	}
+}
+
+void EditorHelp::generate_doc(bool p_use_cache, bool p_use_script_cache) {
 	doc_generation_count++;
 	OS::get_singleton()->benchmark_begin_measure("EditorHelp", vformat("Generate Documentation (Run %d)", doc_generation_count));
 
@@ -2951,11 +3022,11 @@ void EditorHelp::generate_doc(bool p_use_cache) {
 	}
 
 	if (p_use_cache && FileAccess::exists(get_cache_full_path())) {
-		worker_thread.start(_load_doc_thread, nullptr);
+		worker_thread.start(_load_doc_thread, (void *)p_use_script_cache);
 	} else {
 		print_verbose("Regenerating editor help cache");
 		doc->generate();
-		worker_thread.start(_gen_doc_thread, nullptr);
+		worker_thread.start(_gen_doc_thread, (void *)p_use_script_cache);
 	}
 }
 

editor/editor_help.h

@@ -199,6 +199,7 @@ class EditorHelp : public VBoxContainer {
 	static void _load_doc_thread(void *p_udata);
 	static void _gen_doc_thread(void *p_udata);
 	static void _gen_extensions_docs();
+	static void _load_script_doc_cache();
 	static void _compute_doc_version_hash();
 
 	struct PropertyCompare {
@@ -218,10 +219,12 @@ class EditorHelp : public VBoxContainer {
 	static void _bind_methods();
 
 public:
-	static void generate_doc(bool p_use_cache = true);
+	static void generate_doc(bool p_use_cache = true, bool p_use_script_cache = true);
 	static DocTools *get_doc_data();
 	static void cleanup_doc();
+	static void save_script_doc_cache();
 	static String get_cache_full_path();
+	static String get_script_doc_cache_full_path();
 
 	static void load_xml_buffer(const uint8_t *p_buffer, int p_size);
 	static void remove_class(const String &p_class);

editor/editor_node.cpp

@@ -454,8 +454,9 @@ void EditorNode::_gdextensions_reloaded() {
 	// In case the developer is inspecting an object that will be changed by the reload.
 	InspectorDock::get_inspector_singleton()->update_tree();
 
-	// Regenerate documentation.
-	EditorHelp::generate_doc();
+	// Regenerate documentation without using script documentation cache since that would
+	// revert doc changes during this session.
+	EditorHelp::generate_doc(true, false);
 }
 
 void EditorNode::_select_default_main_screen_plugin() {
@@ -813,6 +814,11 @@ void EditorNode::_notification(int p_what) {
 			}
 #endif
 		} break;
+
+		case NOTIFICATION_PREDELETE: {
+			// Cannot be done in destructor since saving a resource emits EditorNode signals using memory that's being destroyed.
+			EditorHelp::save_script_doc_cache();
+		} break;
 	}
 }
 

editor/plugins/script_editor_plugin.cpp

@@ -2182,39 +2182,9 @@ void ScriptEditor::_update_script_names() {
 			sedata.push_back(sd);
 		}
 
-		Vector<String> disambiguated_script_names;
-		Vector<String> full_script_paths;
-		for (int j = 0; j < sedata.size(); j++) {
-			String name = sedata[j].name.replace("(*)", "");
-			ScriptListName script_display = (ScriptListName)(int)EDITOR_GET("text_editor/script_list/list_script_names_as");
-			switch (script_display) {
-				case DISPLAY_NAME: {
-					name = name.get_file();
-				} break;
-				case DISPLAY_DIR_AND_NAME: {
-					name = name.get_base_dir().get_file().path_join(name.get_file());
-				} break;
-				default:
-					break;
-			}
-
-			disambiguated_script_names.append(name);
-			full_script_paths.append(sedata[j].tooltip);
-		}
-
-		EditorNode::disambiguate_filenames(full_script_paths, disambiguated_script_names);
-
-		for (int j = 0; j < sedata.size(); j++) {
-			if (sedata[j].name.ends_with("(*)")) {
-				sedata.write[j].name = disambiguated_script_names[j] + "(*)";
-			} else {
-				sedata.write[j].name = disambiguated_script_names[j];
-			}
-		}
-
 		EditorHelp *eh = Object::cast_to<EditorHelp>(tab_container->get_tab_control(i));
-		if (eh) {
-			String name = eh->get_class();
+		if (eh && !eh->get_class().is_empty()) {
+			String name = eh->get_class().unquote();
 			Ref<Texture2D> icon = get_editor_theme_icon(SNAME("Help"));
 			String tooltip = vformat(TTR("%s Class Reference"), name);
 
@@ -2232,6 +2202,36 @@ void ScriptEditor::_update_script_names() {
 		}
 	}
 
+	Vector<String> disambiguated_script_names;
+	Vector<String> full_script_paths;
+	for (int j = 0; j < sedata.size(); j++) {
+		String name = sedata[j].name.replace("(*)", "");
+		ScriptListName script_display = (ScriptListName)(int)EDITOR_GET("text_editor/script_list/list_script_names_as");
+		switch (script_display) {
+			case DISPLAY_NAME: {
+				name = name.get_file();
+			} break;
+			case DISPLAY_DIR_AND_NAME: {
+				name = name.get_base_dir().get_file().path_join(name.get_file());
+			} break;
+			default:
+				break;
+		}
+
+		disambiguated_script_names.append(name);
+		full_script_paths.append(sedata[j].tooltip);
+	}
+
+	EditorNode::disambiguate_filenames(full_script_paths, disambiguated_script_names);
+
+	for (int j = 0; j < sedata.size(); j++) {
+		if (sedata[j].name.ends_with("(*)")) {
+			sedata.write[j].name = disambiguated_script_names[j] + "(*)";
+		} else {
+			sedata.write[j].name = disambiguated_script_names[j];
+		}
+	}
+
 	if (_sort_list_on_update && !sedata.is_empty()) {
 		sedata.sort();