Add custom debug shape thickness and color options to RayCast

This backports the improved RayCast debug drawing functionality
from the `master` branch.

`ArrayMesh.clear_surfaces()` was also backported from the `master`
branch and exposed because the new debug drawing code requires it.

doc/classes/ArrayMesh.xml

@@ -62,6 +62,13 @@
 				Removes all blend shapes from this [ArrayMesh].
 			</description>
 		</method>
+		<method name="clear_surfaces">
+			<return type="void">
+			</return>
+			<description>
+				Removes all surfaces from this [ArrayMesh].
+			</description>
+		</method>
 		<method name="get_blend_shape_count" qualifiers="const">
 			<return type="int">
 			</return>

doc/classes/RayCast.xml

@@ -139,6 +139,13 @@
 		<member name="collision_mask" type="int" setter="set_collision_mask" getter="get_collision_mask" default="1">
 			The ray's collision mask. Only objects in at least one collision layer enabled in the mask will be detected. See [url=https://docs.godotengine.org/en/3.3/tutorials/physics/physics_introduction.html#collision-layers-and-masks]Collision layers and masks[/url] in the documentation for more information.
 		</member>
+		<member name="debug_shape_custom_color" type="Color" setter="set_debug_shape_custom_color" getter="get_debug_shape_custom_color" default="Color( 0, 0, 0, 1 )">
+			The custom color to use to draw the shape in the editor and at run-time if [b]Visible Collision Shapes[/b] is enabled in the [b]Debug[/b] menu. This color will be highlighted at run-time if the [RayCast] is colliding with something.
+			If set to [code]Color(0.0, 0.0, 0.0)[/code] (by default), the color set in [member ProjectSettings.debug/shapes/collision/shape_color] is used.
+		</member>
+		<member name="debug_shape_thickness" type="float" setter="set_debug_shape_thickness" getter="get_debug_shape_thickness" default="2.0">
+			If set to [code]1[/code], a line is used as the debug shape. Otherwise, a truncated pyramid is drawn to represent the [RayCast]. Requires [b]Visible Collision Shapes[/b] to be enabled in the [b]Debug[/b] menu for the debug shape to be visible at run-time.
+		</member>
 		<member name="enabled" type="bool" setter="set_enabled" getter="is_enabled" default="false">
 			If [code]true[/code], collisions will be reported.
 		</member>

editor/plugins/spatial_editor_plugin.h

@@ -102,6 +102,7 @@ class EditorSpatialGizmo : public SpatialGizmo {
 
 public:
 	void add_lines(const Vector<Vector3> &p_lines, const Ref<Material> &p_material, bool p_billboard = false, const Color &p_modulate = Color(1, 1, 1));
+	void add_vertices(const Vector<Vector3> &p_vertices, const Ref<Material> &p_material, Mesh::PrimitiveType p_primitive_type, bool p_billboard = false, const Color &p_modulate = Color(1, 1, 1));
 	void add_mesh(const Ref<ArrayMesh> &p_mesh, bool p_billboard = false, const Ref<SkinReference> &p_skin_reference = Ref<SkinReference>(), const Ref<Material> &p_material = Ref<Material>());
 	void add_collision_segments(const Vector<Vector3> &p_lines);
 	void add_collision_triangles(const Ref<TriangleMesh> &p_tmesh);

editor/spatial_editor_gizmos.cpp

@@ -245,6 +245,58 @@ void EditorSpatialGizmo::add_lines(const Vector<Vector3> &p_lines, const Ref<Mat
 	instances.push_back(ins);
 }
 
+void EditorSpatialGizmo::add_vertices(const Vector<Vector3> &p_vertices, const Ref<Material> &p_material, Mesh::PrimitiveType p_primitive_type, bool p_billboard, const Color &p_modulate) {
+	if (p_vertices.empty()) {
+		return;
+	}
+
+	ERR_FAIL_COND(!spatial_node);
+	Instance ins;
+
+	Ref<ArrayMesh> mesh = memnew(ArrayMesh);
+	Array a;
+	a.resize(Mesh::ARRAY_MAX);
+
+	a[Mesh::ARRAY_VERTEX] = p_vertices;
+
+	PoolVector<Color> color;
+	color.resize(p_vertices.size());
+	{
+		PoolVector<Color>::Write w = color.write();
+		for (int i = 0; i < p_vertices.size(); i++) {
+			if (is_selected()) {
+				w[i] = Color(1, 1, 1, 0.8) * p_modulate;
+			} else {
+				w[i] = Color(1, 1, 1, 0.2) * p_modulate;
+			}
+		}
+	}
+
+	a[Mesh::ARRAY_COLOR] = color;
+
+	mesh->add_surface_from_arrays(p_primitive_type, a);
+	mesh->surface_set_material(0, p_material);
+
+	if (p_billboard) {
+		float md = 0;
+		for (int i = 0; i < p_vertices.size(); i++) {
+			md = MAX(0, p_vertices[i].length());
+		}
+		if (md) {
+			mesh->set_custom_aabb(AABB(Vector3(-md, -md, -md), Vector3(md, md, md) * 2.0));
+		}
+	}
+
+	ins.billboard = p_billboard;
+	ins.mesh = mesh;
+	if (valid) {
+		ins.create_instance(spatial_node, hidden);
+		VS::get_singleton()->instance_set_transform(ins.instance, spatial_node->get_global_transform());
+	}
+
+	instances.push_back(ins);
+}
+
 void EditorSpatialGizmo::add_unscaled_billboard(const Ref<Material> &p_material, float p_scale, const Color &p_modulate) {
 	ERR_FAIL_COND(!spatial_node);
 	Instance ins;
@@ -1848,16 +1900,15 @@ void RayCastSpatialGizmoPlugin::redraw(EditorSpatialGizmo *p_gizmo) {
 
 	p_gizmo->clear();
 
-	Vector<Vector3> lines;
+	const Ref<SpatialMaterial> material = raycast->is_enabled() ? raycast->get_debug_material() : get_material("shape_material_disabled");
 
-	lines.push_back(Vector3());
-	lines.push_back(raycast->get_cast_to());
+	p_gizmo->add_lines(raycast->get_debug_line_vertices(), material);
 
-	const Ref<SpatialMaterial> material =
-			get_material(raycast->is_enabled() ? "shape_material" : "shape_material_disabled", p_gizmo);
+	if (raycast->get_debug_shape_thickness() > 1) {
+		p_gizmo->add_vertices(raycast->get_debug_shape_vertices(), material, Mesh::PRIMITIVE_TRIANGLE_STRIP);
+	}
 
-	p_gizmo->add_lines(lines, material);
-	p_gizmo->add_collision_segments(lines);
+	p_gizmo->add_collision_segments(raycast->get_debug_line_vertices());
 }
 
 /////

scene/3d/ray_cast.cpp

@@ -37,10 +37,13 @@
 
 void RayCast::set_cast_to(const Vector3 &p_point) {
 	cast_to = p_point;
-	if (is_inside_tree() && (Engine::get_singleton()->is_editor_hint() || get_tree()->is_debugging_collisions_hint())) {
-		update_gizmo();
-	}
-	if (is_inside_tree() && get_tree()->is_debugging_collisions_hint()) {
+	update_gizmo();
+
+	if (Engine::get_singleton()->is_editor_hint()) {
+		if (is_inside_tree()) {
+			_update_debug_shape_vertices();
+		}
+	} else if (debug_shape) {
 		_update_debug_shape();
 	}
 }
@@ -143,6 +146,9 @@ bool RayCast::get_exclude_parent_body() const {
 void RayCast::_notification(int p_what) {
 	switch (p_what) {
 		case NOTIFICATION_ENTER_TREE: {
+			if (Engine::get_singleton()->is_editor_hint()) {
+				_update_debug_shape_vertices();
+			}
 			if (enabled && !Engine::get_singleton()->is_editor_hint()) {
 				set_physics_process_internal(true);
 			} else {
@@ -180,10 +186,7 @@ void RayCast::_notification(int p_what) {
 			bool prev_collision_state = collided;
 			_update_raycast_state();
 			if (prev_collision_state != collided && get_tree()->is_debugging_collisions_hint()) {
-				if (debug_material.is_valid()) {
-					Ref<SpatialMaterial> line_material = static_cast<Ref<SpatialMaterial>>(debug_material);
-					line_material->set_albedo(collided ? Color(1.0, 0, 0) : Color(1.0, 0.8, 0.6));
-				}
+				_update_debug_shape_material(true);
 			}
 
 		} break;
@@ -307,6 +310,12 @@ void RayCast::_bind_methods() {
 	ClassDB::bind_method(D_METHOD("set_collide_with_bodies", "enable"), &RayCast::set_collide_with_bodies);
 	ClassDB::bind_method(D_METHOD("is_collide_with_bodies_enabled"), &RayCast::is_collide_with_bodies_enabled);
 
+	ClassDB::bind_method(D_METHOD("set_debug_shape_custom_color", "debug_shape_custom_color"), &RayCast::set_debug_shape_custom_color);
+	ClassDB::bind_method(D_METHOD("get_debug_shape_custom_color"), &RayCast::get_debug_shape_custom_color);
+
+	ClassDB::bind_method(D_METHOD("set_debug_shape_thickness", "debug_shape_thickness"), &RayCast::set_debug_shape_thickness);
+	ClassDB::bind_method(D_METHOD("get_debug_shape_thickness"), &RayCast::get_debug_shape_thickness);
+
 	ADD_PROPERTY(PropertyInfo(Variant::BOOL, "enabled"), "set_enabled", "is_enabled");
 	ADD_PROPERTY(PropertyInfo(Variant::BOOL, "exclude_parent"), "set_exclude_parent_body", "get_exclude_parent_body");
 	ADD_PROPERTY(PropertyInfo(Variant::VECTOR3, "cast_to"), "set_cast_to", "get_cast_to");
@@ -315,27 +324,119 @@ void RayCast::_bind_methods() {
 	ADD_GROUP("Collide With", "collide_with");
 	ADD_PROPERTY(PropertyInfo(Variant::BOOL, "collide_with_areas", PROPERTY_HINT_LAYERS_3D_PHYSICS), "set_collide_with_areas", "is_collide_with_areas_enabled");
 	ADD_PROPERTY(PropertyInfo(Variant::BOOL, "collide_with_bodies", PROPERTY_HINT_LAYERS_3D_PHYSICS), "set_collide_with_bodies", "is_collide_with_bodies_enabled");
+
+	ADD_GROUP("Debug Shape", "debug_shape");
+	ADD_PROPERTY(PropertyInfo(Variant::COLOR, "debug_shape_custom_color"), "set_debug_shape_custom_color", "get_debug_shape_custom_color");
+	ADD_PROPERTY(PropertyInfo(Variant::INT, "debug_shape_thickness", PROPERTY_HINT_RANGE, "1,5"), "set_debug_shape_thickness", "get_debug_shape_thickness");
 }
 
-void RayCast::_create_debug_shape() {
-	if (!debug_material.is_valid()) {
-		debug_material = Ref<SpatialMaterial>(memnew(SpatialMaterial));
+float RayCast::get_debug_shape_thickness() const {
+	return debug_shape_thickness;
+}
+
+void RayCast::_update_debug_shape_vertices() {
+	debug_shape_vertices.clear();
+	debug_line_vertices.clear();
+
+	if (cast_to == Vector3()) {
+		return;
+	}
+
+	debug_line_vertices.push_back(Vector3());
+	debug_line_vertices.push_back(cast_to);
+
+	if (debug_shape_thickness > 1) {
+		float scale_factor = 100.0;
+		Vector3 dir = Vector3(cast_to).normalized();
+		// Draw truncated pyramid
+		Vector3 normal = (fabs(dir.x) + fabs(dir.y) > CMP_EPSILON) ? Vector3(-dir.y, dir.x, 0).normalized() : Vector3(0, -dir.z, dir.y).normalized();
+		normal *= debug_shape_thickness / scale_factor;
+		int vertices_strip_order[14] = { 4, 5, 0, 1, 2, 5, 6, 4, 7, 0, 3, 2, 7, 6 };
+		for (int v = 0; v < 14; v++) {
+			Vector3 vertex = vertices_strip_order[v] < 4 ? normal : normal / 3.0 + cast_to;
+			debug_shape_vertices.push_back(vertex.rotated(dir, Math_PI * (0.5 * (vertices_strip_order[v] % 4) + 0.25)));
+		}
+	}
+}
+
+void RayCast::set_debug_shape_thickness(const float p_debug_shape_thickness) {
+	debug_shape_thickness = p_debug_shape_thickness;
+	update_gizmo();
+
+	if (Engine::get_singleton()->is_editor_hint()) {
+		if (is_inside_tree()) {
+			_update_debug_shape_vertices();
+		}
+	} else if (debug_shape) {
+		_update_debug_shape();
+	}
+}
 
-		Ref<SpatialMaterial> line_material = static_cast<Ref<SpatialMaterial>>(debug_material);
-		line_material->set_flag(SpatialMaterial::FLAG_UNSHADED, true);
-		line_material->set_line_width(3.0);
-		line_material->set_albedo(Color(1.0, 0.8, 0.6));
+const Vector<Vector3> &RayCast::get_debug_shape_vertices() const {
+	return debug_shape_vertices;
+}
+
+const Vector<Vector3> &RayCast::get_debug_line_vertices() const {
+	return debug_line_vertices;
+}
+
+void RayCast::set_debug_shape_custom_color(const Color &p_color) {
+	debug_shape_custom_color = p_color;
+	if (debug_material.is_valid()) {
+		_update_debug_shape_material();
 	}
+}
+
+Ref<SpatialMaterial> RayCast::get_debug_material() {
+	_update_debug_shape_material();
+	return debug_material;
+}
+
+const Color &RayCast::get_debug_shape_custom_color() const {
+	return debug_shape_custom_color;
+}
+
+void RayCast::_create_debug_shape() {
+	_update_debug_shape_material();
 
 	Ref<ArrayMesh> mesh = memnew(ArrayMesh);
 
 	MeshInstance *mi = memnew(MeshInstance);
 	mi->set_mesh(mesh);
-
 	add_child(mi);
+
 	debug_shape = mi;
 }
 
+void RayCast::_update_debug_shape_material(bool p_check_collision) {
+	if (!debug_material.is_valid()) {
+		Ref<SpatialMaterial> material = memnew(SpatialMaterial);
+		debug_material = material;
+
+		material->set_flag(SpatialMaterial::FLAG_UNSHADED, true);
+		material->set_feature(SpatialMaterial::FEATURE_TRANSPARENT, true);
+	}
+
+	Color color = debug_shape_custom_color;
+	if (color == Color(0.0, 0.0, 0.0)) {
+		// Use the default debug shape color defined in the Project Settings.
+		color = get_tree()->get_debug_collisions_color();
+	}
+
+	if (p_check_collision) {
+		if ((color.get_h() < 0.055 || color.get_h() > 0.945) && color.get_s() > 0.5 && color.get_v() > 0.5) {
+			// If base color is already quite reddish, hightlight collision with green color
+			color = Color(0.0, 1.0, 0.0, color.a);
+		} else {
+			// Else, hightlight collision with red color
+			color = Color(1.0, 0, 0, color.a);
+		}
+	}
+
+	Ref<SpatialMaterial> material = static_cast<Ref<SpatialMaterial>>(debug_material);
+	material->set_albedo(color);
+}
+
 void RayCast::_update_debug_shape() {
 	if (!enabled) {
 		return;
@@ -351,27 +452,28 @@ void RayCast::_update_debug_shape() {
 		return;
 	}
 
-	Vector<Vector3> verts;
-	verts.push_back(Vector3());
-	verts.push_back(cast_to);
+	_update_debug_shape_vertices();
 
-	if (mesh->get_surface_count() == 0) {
-		Array a;
-		a.resize(Mesh::ARRAY_MAX);
-		a[Mesh::ARRAY_VERTEX] = verts;
+	mesh->clear_surfaces();
 
-		uint32_t flags = Mesh::ARRAY_FLAG_USE_DYNAMIC_UPDATE;
+	Array a;
+	a.resize(Mesh::ARRAY_MAX);
 
+	uint32_t flags = 0;
+	int surface_count = 0;
+
+	if (!debug_line_vertices.empty()) {
+		a[Mesh::ARRAY_VERTEX] = debug_line_vertices;
 		mesh->add_surface_from_arrays(Mesh::PRIMITIVE_LINES, a, Array(), flags);
-		mesh->surface_set_material(0, debug_material);
-	} else {
-		PoolByteArray byte_array;
-		int array_size = sizeof(Vector3) * verts.size();
-		byte_array.resize(array_size);
-		PoolByteArray::Write w = byte_array.write();
-		memcpy(w.ptr(), verts.ptr(), array_size);
+		mesh->surface_set_material(surface_count, debug_material);
+		++surface_count;
+	}
 
-		VS::get_singleton()->mesh_surface_update_region(mesh->get_rid(), 0, 0, byte_array);
+	if (!debug_shape_vertices.empty()) {
+		a[Mesh::ARRAY_VERTEX] = debug_shape_vertices;
+		mesh->add_surface_from_arrays(Mesh::PRIMITIVE_TRIANGLE_STRIP, a, Array(), flags);
+		mesh->surface_set_material(surface_count, debug_material);
+		++surface_count;
 	}
 }
 

scene/3d/ray_cast.h

@@ -51,9 +51,15 @@ class RayCast : public Spatial {
 
 	Node *debug_shape;
 	Ref<Material> debug_material;
+	Color debug_shape_custom_color = Color(0.0, 0.0, 0.0);
+	int debug_shape_thickness = 2;
+	Vector<Vector3> debug_shape_vertices;
+	Vector<Vector3> debug_line_vertices;
 
 	void _create_debug_shape();
 	void _update_debug_shape();
+	void _update_debug_shape_material(bool p_check_collision = false);
+	void _update_debug_shape_vertices();
 	void _clear_debug_shape();
 
 	bool collide_with_areas;
@@ -86,6 +92,17 @@ class RayCast : public Spatial {
 	void set_exclude_parent_body(bool p_exclude_parent_body);
 	bool get_exclude_parent_body() const;
 
+	const Color &get_debug_shape_custom_color() const;
+	void set_debug_shape_custom_color(const Color &p_color);
+
+	const Vector<Vector3> &get_debug_shape_vertices() const;
+	const Vector<Vector3> &get_debug_line_vertices() const;
+
+	Ref<SpatialMaterial> get_debug_material();
+
+	float get_debug_shape_thickness() const;
+	void set_debug_shape_thickness(const float p_debug_thickness);
+
 	void force_raycast_update();
 	bool is_colliding() const;
 	Object *get_collider() const;