Skip to content

Commit

Permalink
Merge branch 'Compile_Using_TransferData' of https://github.com/vsg-d…
Browse files Browse the repository at this point in the history 
…ev/VulkanSceneGraph into Compile_Using_TransferData
  • Loading branch information
@robertosfield
robertosfield committed Sep 18, 2024
2 parents 1c97325 + 3798d5f commit 91765f1
Show file tree
Hide file tree
Showing 6 changed files with 88 additions and 52 deletions.
2 changes: 2 additions & 0 deletions include/vsg/app/RecordAndSubmitTask.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,9 +46,11 @@ namespace vsg

ref_ptr<TransferTask> transferTask; // data is transferred for this frame

ref_ptr<Semaphore> earlyTransferDataCompletedSemaphore;
ref_ptr<Semaphore> earlyTransferConsumerCompletedSemaphore;
bool earlyDataTransferred = false;

ref_ptr<Semaphore> lateTransferDataCompletedSemaphore;
ref_ptr<Semaphore> lateTransferConsumerCompletedSemaphore;
bool lateDataTransferred = false;

Expand Down
15 changes: 5 additions & 10 deletions include/vsg/app/TransferTask.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,9 +51,6 @@ namespace vsg

ref_ptr<Device> device;

/// advance the currentTransferBlockIndex
void advance();

void assign(const ResourceRequirements::DynamicData& dynamicData);
void assign(const BufferInfoList& bufferInfoList);
void assign(const ImageInfoList& imageInfoList);
Expand All @@ -73,33 +70,32 @@ namespace vsg
using OffsetBufferInfoMap = std::map<VkDeviceSize, ref_ptr<BufferInfo>>;
using BufferMap = std::map<ref_ptr<Buffer>, OffsetBufferInfoMap>;

// return the index of the transferBlock to use.
// relativeTransferBlockIndex of 0 is the frame currently being record, 1 is the previous frame record.
size_t index(size_t relativeTransferBlockIndex = 0) const;

mutable std::mutex _mutex;

struct TransferBlock : public Inherit<Object, TransferBlock>
{
ref_ptr<CommandBuffer> transferCommandBuffer;
ref_ptr<Fence> fence;
ref_ptr<Buffer> staging;
void* buffer_data = nullptr;
std::vector<VkBufferCopy> copyRegions;
bool waitOnFence = false;
};

struct DataToCopy
{
std::string name;
BufferMap dataMap;
std::set<ref_ptr<ImageInfo>> imageInfoSet;

uint32_t frameIndex = 0;
std::vector<ref_ptr<TransferBlock>> frames;

VkDeviceSize dataTotalRegions = 0;
VkDeviceSize dataTotalSize = 0;
VkDeviceSize imageTotalSize = 0;

uint32_t currentSemephoreCount = 0;
ref_ptr<Semaphore> transferCompleteSemaphore[2];
ref_ptr<Semaphore> transferCompleteSemaphore;
ref_ptr<Semaphore> transferConsumerCompletedSemaphore;

bool containsDataToTransfer() const { return !dataMap.empty() || !imageInfoSet.empty(); }
Expand All @@ -109,7 +105,6 @@ namespace vsg
DataToCopy _lateDataToCopy;

size_t _bufferCount;
size_t _frameCount;

TransferResult _transferData(DataToCopy& dataToCopy);

Expand Down
61 changes: 53 additions & 8 deletions src/vsg/app/RecordAndSubmitTask.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,7 @@ void RecordAndSubmitTask::advance()
{
CPU_INSTRUMENTATION_L1_NC(instrumentation, "RecordAndSubmitTask advance", COLOR_VIEWER);

//info("\nRecordAndSubmitTask::advance()");
// info("\nRecordAndSubmitTask::advance()");

if (_currentFrameIndex >= _indices.size())
{
Expand All @@ -69,8 +69,6 @@ void RecordAndSubmitTask::advance()

// pass the index for the current frame
_indices[0] = _currentFrameIndex;

if (transferTask) transferTask->advance();
}

size_t RecordAndSubmitTask::index(size_t relativeFrameIndex) const
Expand Down Expand Up @@ -100,8 +98,9 @@ VkResult RecordAndSubmitTask::submit(ref_ptr<FrameStamp> frameStamp)
if (transfer.dataTransferredSemaphore)
{
//info(" adding early transfer dataTransferredSemaphore ", transfer.dataTransferredSemaphore);
transientWaitSemaphores.push_back(transfer.dataTransferredSemaphore);
earlyDataTransferred = true;
earlyTransferDataCompletedSemaphore = transfer.dataTransferredSemaphore;
transientWaitSemaphores.push_back(transfer.dataTransferredSemaphore);
}
}
else
Expand All @@ -122,23 +121,25 @@ VkResult RecordAndSubmitTask::start()
CPU_INSTRUMENTATION_L1_NC(instrumentation, "RecordAndSubmitTask start", COLOR_RECORD);

earlyDataTransferred = false;
earlyTransferDataCompletedSemaphore.reset();
lateDataTransferred = false;
lateTransferDataCompletedSemaphore.reset();

auto current_fence = fence();
if (current_fence->hasDependencies())
{
// info("RecordAndSubmitTask::start() waiting on fence ", current_fence, ", ", current_fence->status(), ", current_fence->hasDependencies() = ", current_fence->hasDependencies());
//info("RecordAndSubmitTask::start() waiting on fence ", current_fence, ", ", current_fence->status(), ", current_fence->hasDependencies() = ", current_fence->hasDependencies());

uint64_t timeout = std::numeric_limits<uint64_t>::max();
if (VkResult result = current_fence->wait(timeout); result != VK_SUCCESS) return result;

current_fence->resetFenceAndDependencies();

// info("after RecordAndSubmitTask::start() waited on fence ", current_fence, ", ", current_fence->status(), ", current_fence->hasDependencies() = ", current_fence->hasDependencies());
//info("after RecordAndSubmitTask::start() waited on fence ", current_fence, ", ", current_fence->status(), ", current_fence->hasDependencies() = ", current_fence->hasDependencies());
}
else
{
// info("RecordAndSubmitTask::start() initial fence ", current_fence, ", ", current_fence->status(), ", current_fence->hasDependencies() = ", current_fence->hasDependencies());
//info("RecordAndSubmitTask::start() initial fence ", current_fence, ", ", current_fence->status(), ", current_fence->hasDependencies() = ", current_fence->hasDependencies());
}

return VK_SUCCESS;
Expand Down Expand Up @@ -173,8 +174,9 @@ VkResult RecordAndSubmitTask::finish(ref_ptr<RecordedCommandBuffers> recordedCom
if (transfer.dataTransferredSemaphore)
{
//info(" adding late transfer dataTransferredSemaphore ", transfer.dataTransferredSemaphore);
transientWaitSemaphores.push_back(transfer.dataTransferredSemaphore);
lateDataTransferred = true;
lateTransferDataCompletedSemaphore = transfer.dataTransferredSemaphore;
transientWaitSemaphores.push_back(transfer.dataTransferredSemaphore);
}
}
else
Expand All @@ -185,11 +187,54 @@ VkResult RecordAndSubmitTask::finish(ref_ptr<RecordedCommandBuffers> recordedCom

if (recordedCommandBuffers->empty())
{
if (!transientWaitSemaphores.empty())
{
#if 0
info("EMPTY RecordAndSubmitTask::finish() recordedCommandBuffers->empty(), transientWaitSemaphores.size() = ", transientWaitSemaphores.size());

if (earlyDataTransferred)
{
info(" earlyTransferDataCompletedSemaphore = ", earlyTransferDataCompletedSemaphore);
}
if (lateDataTransferred)
{
info(" lateTransferDataCompletedSemaphore = ", lateTransferDataCompletedSemaphore);
}
for(auto& semaphore : transientWaitSemaphores)
{
info(" semaphore = ", semaphore, ", semaphore->vk() = ", semaphore->vk());
}
#endif
if (earlyDataTransferred) transferTask->_earlyDataToCopy.transferConsumerCompletedSemaphore = earlyTransferDataCompletedSemaphore;
if (lateDataTransferred) transferTask->_lateDataToCopy.transferConsumerCompletedSemaphore = lateTransferDataCompletedSemaphore;

transientWaitSemaphores.clear();
}

// nothing to do so return early
std::this_thread::sleep_for(std::chrono::milliseconds(16)); // sleep for 1/60th of a second
return VK_SUCCESS;
}

#if 0
if (!transientWaitSemaphores.empty())
{
info("ACTIVE RecordAndSubmitTask::finish() transientWaitSemaphores.size() = ", transientWaitSemaphores.size());
if (earlyDataTransferred)
{
info(" earlyTransferConsumerCompletedSemaphore = ", earlyTransferConsumerCompletedSemaphore);
}
if (lateDataTransferred)
{
info(" lateTransferConsumerCompletedSemaphore = ", lateTransferConsumerCompletedSemaphore);
}
for(auto& semaphore : transientWaitSemaphores)
{
info(" semaphore = ", semaphore, ", semaphore->vk() = ", semaphore->vk());
}
}
#endif

// convert VSG CommandBuffer to Vulkan handles and add to the Fence's list of dependent CommandBuffers
std::vector<VkCommandBuffer> vk_commandBuffers;
std::vector<VkSemaphore> vk_waitSemaphores;
Expand Down
55 changes: 24 additions & 31 deletions src/vsg/app/TransferTask.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,8 +22,7 @@ using namespace vsg;

TransferTask::TransferTask(Device* in_device, uint32_t numBuffers) :
device(in_device),
_bufferCount(numBuffers),
_frameCount(0)
_bufferCount(numBuffers)
{
CPU_INSTRUMENTATION_L1(instrumentation);

Expand All @@ -39,21 +38,6 @@ TransferTask::TransferTask(Device* in_device, uint32_t numBuffers) :
// level = Logger::LOGGER_INFO;
}

void TransferTask::advance()
{
CPU_INSTRUMENTATION_L1(instrumentation);
std::scoped_lock<std::mutex> lock(_mutex);

++_frameCount;

log(level, "TransferTask::advance() _frameCount = ", _frameCount);
}

size_t TransferTask::index(size_t relativeTransferBlockIndex) const
{
return (_frameCount < relativeTransferBlockIndex) ? _bufferCount : ((_frameCount - relativeTransferBlockIndex) % _bufferCount);
}

bool TransferTask::containsDataToTransfer(TransferMask transferMask) const
{
std::scoped_lock<std::mutex> lock(_mutex);
Expand Down Expand Up @@ -345,11 +329,7 @@ TransferTask::TransferResult TransferTask::_transferData(DataToCopy& dataToCopy)

std::scoped_lock<std::mutex> lock(_mutex);

size_t frameIndex = index(0);
size_t previousFrameIndex = index(1);
if (frameIndex >= dataToCopy.frames.size()) return TransferResult{VK_SUCCESS, {}};

log(level, "TransferTask::_transferData( ", dataToCopy.name, " ) ", this, ", frameIndex = ", frameIndex, ", previousFrameIndex = ", previousFrameIndex);
log(level, "TransferTask::_transferData( ", dataToCopy.name, " ) ", this, ", frameIndex = ", dataToCopy.frameIndex);

//
// begin compute total data size
Expand Down Expand Up @@ -399,26 +379,35 @@ TransferTask::TransferResult TransferTask::_transferData(DataToCopy& dataToCopy)
if (totalSize == 0) return TransferResult{VK_SUCCESS, {}};

uint32_t deviceID = device->deviceID;
auto& frame = *(dataToCopy.frames[frameIndex]);
auto& frame = *(dataToCopy.frames[dataToCopy.frameIndex]);
auto& fence = frame.fence;
auto& staging = frame.staging;
auto& commandBuffer = frame.transferCommandBuffer;

uint32_t newSemaphoreIndex = 0; //(dataToCopy.currentSemephoreCount) % 2;
++dataToCopy.currentSemephoreCount;

auto& newSignalSemaphore = dataToCopy.transferCompleteSemaphore[newSemaphoreIndex];
auto& newSignalSemaphore = dataToCopy.transferCompleteSemaphore;

const auto& copyRegions = frame.copyRegions;
auto& buffer_data = frame.buffer_data;

log(level, " frameIndex = ", frameIndex);
log(level, " frameIndex = ", dataToCopy.frameIndex);
log(level, " frame = ", &frame);
log(level, " fence = ", fence);
log(level, " transferQueue = ", transferQueue);
log(level, " staging = ", staging);
log(level, " dataToCopy.transferConsumerCompletedSemaphore = ", dataToCopy.transferConsumerCompletedSemaphore, ", ", dataToCopy.transferConsumerCompletedSemaphore ? dataToCopy.transferConsumerCompletedSemaphore->vk() : VK_NULL_HANDLE);
log(level, " newSignalSemaphore = ", newSignalSemaphore, ", ", newSignalSemaphore ? newSignalSemaphore->vk() : VK_NULL_HANDLE);
log(level, " copyRegions.size() = ", copyRegions.size());

if (frame.waitOnFence && fence)
{
uint64_t timeout = std::numeric_limits<uint64_t>::max();
if (VkResult result = fence->wait(timeout); result != VK_SUCCESS) return TransferResult{result, {}};;
fence->resetFenceAndDependencies();
}
frame.waitOnFence = false;

// advance frameIndex
dataToCopy.frameIndex = (dataToCopy.frameIndex + 1) % dataToCopy.frames.size();

if (!commandBuffer)
{
auto cp = CommandPool::create(device, transferQueue->queueFamilyIndex());
Expand All @@ -436,6 +425,8 @@ TransferTask::TransferResult TransferTask::_transferData(DataToCopy& dataToCopy)
log(level, " newSignalSemaphore created ", newSignalSemaphore, ", ", newSignalSemaphore->vk());
}

if (!fence) fence = Fence::create(device);

VkResult result = VK_SUCCESS;

// allocate staging buffer if required
Expand All @@ -456,7 +447,7 @@ TransferTask::TransferResult TransferTask::_transferData(DataToCopy& dataToCopy)
buffer_data = nullptr;
result = stagingMemory->map(staging->getMemoryOffset(deviceID), staging->size, 0, &buffer_data);

log(level, " TransferTask::transferData() frameIndex = ", frameIndex, ", previousSize = ", previousSize, ", allocated staging buffer = ", staging, ", totalSize = ", totalSize, ", result = ", result);
log(level, " TransferTask::transferData() frameIndex = ", dataToCopy.frameIndex, ", previousSize = ", previousSize, ", allocated staging buffer = ", staging, ", totalSize = ", totalSize, ", result = ", result);

if (result != VK_SUCCESS) return TransferResult{VK_SUCCESS, {}};
}
Expand Down Expand Up @@ -516,7 +507,9 @@ TransferTask::TransferResult TransferTask::_transferData(DataToCopy& dataToCopy)
log(level, " TransferTask submitInfo.waitSemaphoreCount = ", submitInfo.waitSemaphoreCount);
log(level, " TransferTask submitInfo.signalSemaphoreCount = ", submitInfo.signalSemaphoreCount);

result = transferQueue->submit(submitInfo);
result = transferQueue->submit(submitInfo, fence);

frame.waitOnFence = true;

dataToCopy.transferConsumerCompletedSemaphore.reset();

Expand Down
1 change: 1 addition & 0 deletions src/vsg/app/Viewer.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -738,6 +738,7 @@ void Viewer::setupThreading()
if (transfer.dataTransferredSemaphore)
{
data->task->earlyDataTransferred = true;
data->task->earlyTransferDataCompletedSemaphore = transfer.dataTransferredSemaphore;
data->task->transientWaitSemaphores.push_back(transfer.dataTransferredSemaphore);
}
}
Expand Down
6 changes: 3 additions & 3 deletions src/vsg/platform/xcb/Xcb_Window.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -590,17 +590,17 @@ bool Xcb_Window::pollEvents(UIEvents& events)
break;
}
case (XCB_UNMAP_NOTIFY): {
//debug("xcb_unmap_notify_event_t");
//info("xcb_unmap_notify_event_t");
_windowMapped = false;
break;
}
case (XCB_MAP_NOTIFY): {
//debug("xcb_map_notify_event_t");
//info("xcb_map_notify_event_t");
_windowMapped = true;
break;
}
case (XCB_MAPPING_NOTIFY): {
//debug("xcb_mapping_notify_event_t");
//info("xcb_mapping_notify_event_t");
break;
}
case (XCB_LIST_PROPERTIES): {
Expand Down

0 comments on commit 91765f1

Please sign in to comment.