Add save completion tracking to VoxelLodTerrain, flush stream at the end

doc/source/changelog.md

@@ -14,8 +14,11 @@ Semver is not yet in place, so each version can have breaking changes, although
 Primarily developped with Godot 4.2.
 
 - Added `ZN_SpotNoise`, exposing the same algorithm as the `SpotNoise2D` and `SpotNoise3D` nodes of graph generators
+- Saving with `save_all_modified_blocks` now automatically flushes eventual caches implemented by `VoxelStream` upon completion
 - `VoxelTool`:
     - Added `grow_sphere` as alternate way to progressively grow or shrink matter in a spherical region with smooth voxels (thanks to Piratux)
+- `VoxelLodTerrain`:
+    - `save_all_modified_blocks` now returns a completion tracker similar to `VoxelTerrain`
 - `VoxelStream`:
     - Added `flush` method to force writing to the filesystem in case the stream's implementation uses caching
 

generators/generate_block_task.cpp

@@ -154,7 +154,7 @@ void GenerateBlockTask::run_stream_saving_and_finish() {
 			// No priority data, saving doesn't need sorting.
 
 			SaveBlockDataTask *save_task = memnew(SaveBlockDataTask(
-					_volume_id, _position, _lod_index, _block_size, voxels_copy, _stream_dependency, nullptr));
+					_volume_id, _position, _lod_index, _block_size, voxels_copy, _stream_dependency, nullptr, false));
 
 			VoxelEngine::get_singleton().push_async_io_task(save_task);
 		}

generators/multipass/generate_block_multipass_cb_task.cpp

@@ -193,8 +193,8 @@ void GenerateBlockMultipassCBTask::run_stream_saving_and_finish() {
 			// No instances, generators are not designed to produce them at this stage yet.
 			// No priority data, saving doesn't need sorting.
 
-			SaveBlockDataTask *save_task = memnew(SaveBlockDataTask(
-					_volume_id, _block_position, _lod_index, _block_size, voxels_copy, _stream_dependency, nullptr));
+			SaveBlockDataTask *save_task = memnew(SaveBlockDataTask(_volume_id, _block_position, _lod_index,
+					_block_size, voxels_copy, _stream_dependency, nullptr, false));
 
 			VoxelEngine::get_singleton().push_async_io_task(save_task);
 		}

streams/save_block_data_task.cpp

@@ -15,7 +15,7 @@ std::atomic_int g_debug_save_block_tasks_count = { 0 };
 
 SaveBlockDataTask::SaveBlockDataTask(VolumeID p_volume_id, Vector3i p_block_pos, uint8_t p_lod, uint8_t p_block_size,
 		std::shared_ptr<VoxelBufferInternal> p_voxels, std::shared_ptr<StreamingDependency> p_stream_dependency,
-		std::shared_ptr<AsyncDependencyTracker> p_tracker) :
+		std::shared_ptr<AsyncDependencyTracker> p_tracker, bool flush_on_last_tracked_task) :
 		_voxels(p_voxels),
 		_position(p_block_pos),
 		_volume_id(p_volume_id),
@@ -24,14 +24,15 @@ SaveBlockDataTask::SaveBlockDataTask(VolumeID p_volume_id, Vector3i p_block_pos,
 		_save_instances(false),
 		_save_voxels(true),
 		_stream_dependency(p_stream_dependency),
-		_tracker(p_tracker) {
+		_tracker(p_tracker),
+		_flush_on_last_tracked_task(flush_on_last_tracked_task) {
 	//
 	++g_debug_save_block_tasks_count;
 }
 
 SaveBlockDataTask::SaveBlockDataTask(VolumeID p_volume_id, Vector3i p_block_pos, uint8_t p_lod, uint8_t p_block_size,
 		UniquePtr<InstanceBlockData> p_instances, std::shared_ptr<StreamingDependency> p_stream_dependency,
-		std::shared_ptr<AsyncDependencyTracker> p_tracker) :
+		std::shared_ptr<AsyncDependencyTracker> p_tracker, bool flush_on_last_tracked_task) :
 		_instances(std::move(p_instances)),
 		_position(p_block_pos),
 		_volume_id(p_volume_id),
@@ -40,7 +41,8 @@ SaveBlockDataTask::SaveBlockDataTask(VolumeID p_volume_id, Vector3i p_block_pos,
 		_save_instances(true),
 		_save_voxels(false),
 		_stream_dependency(p_stream_dependency),
-		_tracker(p_tracker) {
+		_tracker(p_tracker),
+		_flush_on_last_tracked_task(flush_on_last_tracked_task) {
 	//
 	++g_debug_save_block_tasks_count;
 }
@@ -95,6 +97,10 @@ void SaveBlockDataTask::run(zylann::ThreadedTaskContext &ctx) {
 	}
 
 	if (_tracker != nullptr) {
+		if (_flush_on_last_tracked_task && _tracker->get_remaining_count() == 1) {
+			// This was the last task in a tracked group of saving tasks, we may flush now
+			stream->flush();
+		}
 		_tracker->post_complete();
 	}
 

streams/save_block_data_task.h

@@ -17,12 +17,12 @@ class SaveBlockDataTask : public IThreadedTask {
 	// For saving voxels only
 	SaveBlockDataTask(VolumeID p_volume_id, Vector3i p_block_pos, uint8_t p_lod, uint8_t p_block_size,
 			std::shared_ptr<VoxelBufferInternal> p_voxels, std::shared_ptr<StreamingDependency> p_stream_dependency,
-			std::shared_ptr<AsyncDependencyTracker> p_tracker);
+			std::shared_ptr<AsyncDependencyTracker> p_tracker, bool flush_on_last_tracked_task);
 
 	// For saving instances only
 	SaveBlockDataTask(VolumeID p_volume_id, Vector3i p_block_pos, uint8_t p_lod, uint8_t p_block_size,
 			UniquePtr<InstanceBlockData> p_instances, std::shared_ptr<StreamingDependency> p_stream_dependency,
-			std::shared_ptr<AsyncDependencyTracker> p_tracker);
+			std::shared_ptr<AsyncDependencyTracker> p_tracker, bool flush_on_last_tracked_task);
 
 	~SaveBlockDataTask();
 
@@ -47,6 +47,7 @@ class SaveBlockDataTask : public IThreadedTask {
 	bool _has_run = false;
 	bool _save_instances = false;
 	bool _save_voxels = false;
+	bool _flush_on_last_tracked_task = false;
 	std::shared_ptr<StreamingDependency> _stream_dependency;
 	// Optional tracking, can be null
 	std::shared_ptr<AsyncDependencyTracker> _tracker;

terrain/fixed_lod/voxel_terrain.cpp

@@ -530,11 +530,20 @@ void VoxelTerrain::save_all_modified_blocks(bool with_copy, std::shared_ptr<Asyn
 	_data->consume_all_modifications(_blocks_to_save, with_copy);
 
 	if (stream.is_valid() && _instancer != nullptr && stream->supports_instance_blocks()) {
-		_instancer->save_all_modified_blocks(task_scheduler, tracker);
+		_instancer->save_all_modified_blocks(task_scheduler, tracker, true);
 	}
 
-	// And flush immediately
-	consume_block_data_save_requests(task_scheduler, tracker);
+	consume_block_data_save_requests(task_scheduler, tracker,
+			// Require all data we just gathered to be written to disk if the stream uses a cache. So if the
+			// game crashes or gets killed after all tasks are done, data won't be lost.
+			true);
+
+	if (tracker != nullptr) {
+		// Using buffered count instead of `_blocks_to_save` because it can also contain tasks from VoxelInstancer
+		tracker->set_count(task_scheduler.get_io_count());
+	}
+
+	// Schedule all tasks
 	task_scheduler.flush();
 }
 
@@ -954,8 +963,8 @@ void VoxelTerrain::send_data_load_requests() {
 	}
 }
 
-void VoxelTerrain::consume_block_data_save_requests(
-		BufferedTaskScheduler &task_scheduler, std::shared_ptr<AsyncDependencyTracker> saving_tracker) {
+void VoxelTerrain::consume_block_data_save_requests(BufferedTaskScheduler &task_scheduler,
+		std::shared_ptr<AsyncDependencyTracker> saving_tracker, bool with_flush) {
 	ZN_PROFILE_SCOPE();
 
 	// Blocks to save
@@ -964,8 +973,8 @@ void VoxelTerrain::consume_block_data_save_requests(
 		for (const VoxelData::BlockToSave &b : _blocks_to_save) {
 			ZN_PRINT_VERBOSE(format("Requesting save of block {}", b.position));
 
-			SaveBlockDataTask *task = ZN_NEW(SaveBlockDataTask(
-					_volume_id, b.position, 0, data_block_size, b.voxels, _streaming_dependency, saving_tracker));
+			SaveBlockDataTask *task = ZN_NEW(SaveBlockDataTask(_volume_id, b.position, 0, data_block_size, b.voxels,
+					_streaming_dependency, saving_tracker, with_flush));
 
 			// No priority data, saving doesn't need sorting.
 			task_scheduler.push_io_task(task);
@@ -976,11 +985,6 @@ void VoxelTerrain::consume_block_data_save_requests(
 		}
 	}
 
-	if (saving_tracker != nullptr) {
-		// Using buffered count instead of `_blocks_to_save` because it can also contain tasks from VoxelInstancer
-		saving_tracker->set_count(task_scheduler.get_io_count());
-	}
-
 	// print_line(String("Sending {0} block requests").format(varray(input.blocks_to_emerge.size())));
 	_blocks_to_save.clear();
 }
@@ -1296,7 +1300,7 @@ void VoxelTerrain::process_viewers() {
 	if (can_load_blocks) {
 		send_data_load_requests();
 		BufferedTaskScheduler &task_scheduler = BufferedTaskScheduler::get_for_current_thread();
-		consume_block_data_save_requests(task_scheduler, nullptr);
+		consume_block_data_save_requests(task_scheduler, nullptr, false);
 		task_scheduler.flush();
 	}
 

terrain/fixed_lod/voxel_terrain.h

@@ -220,8 +220,8 @@ class VoxelTerrain : public VoxelNode {
 	void save_all_modified_blocks(bool with_copy, std::shared_ptr<AsyncDependencyTracker> tracker);
 	void get_viewer_pos_and_direction(Vector3 &out_pos, Vector3 &out_direction) const;
 	void send_data_load_requests();
-	void consume_block_data_save_requests(
-			BufferedTaskScheduler &task_scheduler, std::shared_ptr<AsyncDependencyTracker> saving_tracker);
+	void consume_block_data_save_requests(BufferedTaskScheduler &task_scheduler,
+			std::shared_ptr<AsyncDependencyTracker> saving_tracker, bool with_flush);
 
 	void emit_data_block_loaded(Vector3i bpos);
 	void emit_data_block_unloaded(Vector3i bpos);

terrain/instancing/voxel_instancer.cpp

@@ -974,7 +974,7 @@ void VoxelInstancer::on_mesh_block_exit(Vector3i render_grid_position, unsigned
 				auto modified_block_it = lod.modified_blocks.find(data_grid_pos);
 				if (modified_block_it != lod.modified_blocks.end()) {
 					if (can_save) {
-						SaveBlockDataTask *task = save_block(data_grid_pos, lod_index, nullptr);
+						SaveBlockDataTask *task = save_block(data_grid_pos, lod_index, nullptr, false);
 						if (task != nullptr) {
 							tasks.push_io_task(task);
 						}
@@ -1001,7 +1001,7 @@ void VoxelInstancer::on_mesh_block_exit(Vector3i render_grid_position, unsigned
 }
 
 void VoxelInstancer::save_all_modified_blocks(
-		BufferedTaskScheduler &tasks, std::shared_ptr<AsyncDependencyTracker> tracker) {
+		BufferedTaskScheduler &tasks, std::shared_ptr<AsyncDependencyTracker> tracker, bool with_flush) {
 	ZN_DSTACK();
 
 	ZN_ASSERT_RETURN(_parent != nullptr);
@@ -1013,7 +1013,7 @@ void VoxelInstancer::save_all_modified_blocks(
 	for (unsigned int lod_index = 0; lod_index < _lods.size(); ++lod_index) {
 		Lod &lod = _lods[lod_index];
 		for (auto it = lod.modified_blocks.begin(); it != lod.modified_blocks.end(); ++it) {
-			SaveBlockDataTask *task = save_block(*it, lod_index, tracker);
+			SaveBlockDataTask *task = save_block(*it, lod_index, tracker, with_flush);
 			if (task != nullptr) {
 				tasks.push_io_task(task);
 			}
@@ -1399,7 +1399,7 @@ void VoxelInstancer::create_render_blocks(Vector3i render_grid_position, int lod
 }
 
 SaveBlockDataTask *VoxelInstancer::save_block(
-		Vector3i data_grid_pos, int lod_index, std::shared_ptr<AsyncDependencyTracker> tracker) const {
+		Vector3i data_grid_pos, int lod_index, std::shared_ptr<AsyncDependencyTracker> tracker, bool with_flush) const {
 	ZN_PROFILE_SCOPE();
 	ERR_FAIL_COND_V(_library.is_null(), nullptr);
 	ERR_FAIL_COND_V(_parent == nullptr, nullptr);
@@ -1552,8 +1552,8 @@ SaveBlockDataTask *VoxelInstancer::save_block(
 	std::shared_ptr<StreamingDependency> stream_dependency = _parent->get_streaming_dependency();
 	ZN_ASSERT(stream_dependency != nullptr);
 
-	SaveBlockDataTask *task = ZN_NEW(SaveBlockDataTask(
-			volume_id, data_grid_pos, lod_index, data_block_size, std::move(block_data), stream_dependency, tracker));
+	SaveBlockDataTask *task = ZN_NEW(SaveBlockDataTask(volume_id, data_grid_pos, lod_index, data_block_size,
+			std::move(block_data), stream_dependency, tracker, with_flush));
 
 	return task;
 }

terrain/instancing/voxel_instancer.h

@@ -69,7 +69,8 @@ class VoxelInstancer : public Node3D, public VoxelInstanceLibrary::IListener {
 
 	// Actions
 
-	void save_all_modified_blocks(BufferedTaskScheduler &tasks, std::shared_ptr<AsyncDependencyTracker> tracker);
+	void save_all_modified_blocks(
+			BufferedTaskScheduler &tasks, std::shared_ptr<AsyncDependencyTracker> tracker, bool with_flush);
 
 	// Event handlers
 
@@ -139,8 +140,8 @@ class VoxelInstancer : public Node3D, public VoxelInstanceLibrary::IListener {
 	void clear_blocks_in_layer(int layer_id);
 	void clear_layers();
 	void update_visibility();
-	SaveBlockDataTask *save_block(
-			Vector3i data_grid_pos, int lod_index, std::shared_ptr<AsyncDependencyTracker> tracker) const;
+	SaveBlockDataTask *save_block(Vector3i data_grid_pos, int lod_index,
+			std::shared_ptr<AsyncDependencyTracker> tracker, bool with_flush) const;
 
 	// Get a layer assuming it exists
 	Layer &get_layer(int id);

terrain/variable_lod/voxel_lod_terrain.cpp

@@ -31,6 +31,7 @@
 #include "../../util/thread/mutex.h"
 #include "../../util/thread/rw_lock.h"
 #include "../instancing/voxel_instancer.h"
+#include "../voxel_save_completion_tracker.h"
 #include "voxel_lod_terrain_update_task.h"
 
 namespace zylann::voxel {
@@ -2013,7 +2014,7 @@ void VoxelLodTerrain::get_meshed_block_positions_at_lod(int lod_index, std::vect
 	});
 }
 
-void VoxelLodTerrain::save_all_modified_blocks(bool with_copy) {
+void VoxelLodTerrain::save_all_modified_blocks(bool with_copy, std::shared_ptr<AsyncDependencyTracker> tracker) {
 	ZN_PROFILE_SCOPE();
 
 	// This is often called before quitting the game or forcing a global save.
@@ -2031,13 +2032,23 @@ void VoxelLodTerrain::save_all_modified_blocks(bool with_copy) {
 		_data->consume_all_modifications(blocks_to_save, with_copy);
 
 		if (_instancer != nullptr && stream->supports_instance_blocks()) {
-			_instancer->save_all_modified_blocks(task_scheduler, nullptr);
+			_instancer->save_all_modified_blocks(task_scheduler, tracker, true);
 		}
 	}
 
 	// And flush immediately
-	VoxelLodTerrainUpdateTask::send_block_save_requests(
-			_volume_id, to_span(blocks_to_save), _streaming_dependency, get_data_block_size(), task_scheduler);
+	VoxelLodTerrainUpdateTask::send_block_save_requests(_volume_id, to_span(blocks_to_save), _streaming_dependency,
+			get_data_block_size(), task_scheduler, tracker,
+			// Require all data we just gathered to be written to disk if the stream uses a cache. So if the
+			// game crashes or gets killed after all tasks are done, data won't be lost.
+			true);
+
+	if (tracker != nullptr) {
+		// Using buffered count instead of `_blocks_to_save` because it can also contain tasks from VoxelInstancer
+		tracker->set_count(task_scheduler.get_io_count());
+	}
+
+	// Schedule all tasks
 	task_scheduler.flush();
 }
 
@@ -2393,8 +2404,11 @@ void VoxelLodTerrain::get_configuration_warnings(PackedStringArray &warnings) co
 
 #endif // TOOLS_ENABLED
 
-void VoxelLodTerrain::_b_save_modified_blocks() {
-	save_all_modified_blocks(true);
+Ref<VoxelSaveCompletionTracker> VoxelLodTerrain::_b_save_modified_blocks() {
+	std::shared_ptr<AsyncDependencyTracker> tracker = make_shared_instance<AsyncDependencyTracker>();
+	save_all_modified_blocks(true, tracker);
+	ZN_ASSERT_RETURN_V(tracker != nullptr, Ref<VoxelSaveCompletionTracker>());
+	return VoxelSaveCompletionTracker::create(tracker);
 }
 
 void VoxelLodTerrain::_b_set_voxel_bounds(AABB aabb) {

terrain/variable_lod/voxel_lod_terrain.h

@@ -23,6 +23,7 @@ namespace zylann::voxel {
 class VoxelTool;
 class VoxelStream;
 class VoxelInstancer;
+class VoxelSaveCompletionTracker;
 
 class ShaderMaterialPoolVLT : public ShaderMaterialPool {
 public:
@@ -282,7 +283,7 @@ class VoxelLodTerrain : public VoxelNode {
 
 	void update_shader_material_pool_template();
 
-	void save_all_modified_blocks(bool with_copy);
+	void save_all_modified_blocks(bool with_copy, std::shared_ptr<AsyncDependencyTracker> tracker);
 
 	void process_deferred_collision_updates(uint32_t timeout_msec);
 	void process_fading_blocks(float delta);
@@ -294,7 +295,7 @@ class VoxelLodTerrain : public VoxelNode {
 
 	LocalCameraInfo get_local_camera_info() const;
 
-	void _b_save_modified_blocks();
+	Ref<VoxelSaveCompletionTracker> _b_save_modified_blocks();
 	void _b_set_voxel_bounds(AABB aabb);
 	AABB _b_get_voxel_bounds() const;
 

terrain/variable_lod/voxel_lod_terrain_update_task.cpp

@@ -293,13 +293,14 @@ static void apply_block_data_requests_as_empty(Span<const VoxelLodTerrainUpdateD
 
 static void request_voxel_block_save(VolumeID volume_id, std::shared_ptr<VoxelBufferInternal> &voxels,
 		Vector3i block_pos, int lod, std::shared_ptr<StreamingDependency> &stream_dependency,
-		unsigned int data_block_size, BufferedTaskScheduler &task_scheduler) {
+		unsigned int data_block_size, BufferedTaskScheduler &task_scheduler,
+		std::shared_ptr<AsyncDependencyTracker> tracker, bool with_flush) {
 	//
 	CRASH_COND(stream_dependency == nullptr);
 	ERR_FAIL_COND(stream_dependency->stream.is_null());
 
-	SaveBlockDataTask *task =
-			memnew(SaveBlockDataTask(volume_id, block_pos, lod, data_block_size, voxels, stream_dependency, nullptr));
+	SaveBlockDataTask *task = memnew(SaveBlockDataTask(
+			volume_id, block_pos, lod, data_block_size, voxels, stream_dependency, tracker, with_flush));
 
 	// No priority data, saving doesn't need sorting.
 
@@ -308,12 +309,13 @@ static void request_voxel_block_save(VolumeID volume_id, std::shared_ptr<VoxelBu
 
 void VoxelLodTerrainUpdateTask::send_block_save_requests(VolumeID volume_id,
 		Span<VoxelData::BlockToSave> blocks_to_save, std::shared_ptr<StreamingDependency> &stream_dependency,
-		unsigned int data_block_size, BufferedTaskScheduler &task_scheduler) {
+		unsigned int data_block_size, BufferedTaskScheduler &task_scheduler,
+		std::shared_ptr<AsyncDependencyTracker> tracker, bool with_flush) {
 	for (unsigned int i = 0; i < blocks_to_save.size(); ++i) {
 		VoxelData::BlockToSave &b = blocks_to_save[i];
 		ZN_PRINT_VERBOSE(format("Requesting save of block {} lod {}", b.position, b.lod_index));
-		request_voxel_block_save(
-				volume_id, b.voxels, b.position, b.lod_index, stream_dependency, data_block_size, task_scheduler);
+		request_voxel_block_save(volume_id, b.voxels, b.position, b.lod_index, stream_dependency, data_block_size,
+				task_scheduler, tracker, with_flush);
 	}
 }
 
@@ -655,8 +657,8 @@ void VoxelLodTerrainUpdateTask::run(ThreadedTaskContext &ctx) {
 						task_scheduler);
 			}
 
-			send_block_save_requests(
-					_volume_id, to_span(data_blocks_to_save), _streaming_dependency, data_block_size, task_scheduler);
+			send_block_save_requests(_volume_id, to_span(data_blocks_to_save), _streaming_dependency, data_block_size,
+					task_scheduler, nullptr, false);
 		}
 		data_blocks_to_load.clear();
 		data_blocks_to_save.clear();

terrain/variable_lod/voxel_lod_terrain_update_task.h

@@ -70,7 +70,7 @@ class VoxelLodTerrainUpdateTask : public IThreadedTask {
 
 	static void send_block_save_requests(VolumeID volume_id, Span<VoxelData::BlockToSave> blocks_to_save,
 			std::shared_ptr<StreamingDependency> &stream_dependency, unsigned int data_block_size,
-			BufferedTaskScheduler &task_scheduler);
+			BufferedTaskScheduler &task_scheduler, std::shared_ptr<AsyncDependencyTracker> tracker, bool with_flush);
 
 private:
 	std::shared_ptr<VoxelData> _data;