Skip to content

Commit

Permalink
feat: Performance improvements on project load (#82)
Browse files Browse the repository at this point in the history 
* chore: Excahneg save favorite symbols

* chore: Speed up pipeline loading

* feat: Thread for image preparation

* chore: Store image version to settings

* chore: Store settingd

* chore: Added some more nullptr check

* chore: Smaller fixes

* chore: Fixed linux java path

* chore: Another try to fix macos build

* chore: Tried to fix arm build

* chore: Further buld fix

* chore: Store version in database
  • Loading branch information
@joda01
joda01 authored Oct 11, 2024
1 parent 9d42103 commit a74714b
Show file tree
Hide file tree
Showing 23 changed files with 445 additions and 181 deletions.
9 changes: 5 additions & 4 deletions .github/workflows/cmake-multi-platform.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -148,15 +148,15 @@ jobs:
cd output
mkdir -p ./plugins
mkdir -p ./models
mkdir -p ./lib
mkdir -p ./java
cp -r ../imagec.app imagec.app
cp -r /Users/runner/.conan2/p/b/*/p/./plugins/* ./plugins
cp -r ${GITHUB_WORKSPACE}/resources/templates ./templates
strip ./imagec.app/Contents/MacOS/imagec
chmod +x ./imagec.app/Contents/MacOS/imagec
chmod +x ./imagec.app
cd lib
mkdir -p imagec.app/Contents/MacOS/lib
mkdir -p imagec.app/Contents/MacOS/java
cd imagec.app/Contents/MacOS/lib
cp /Users/runner/.conan2/p/b/*/p/lib/libQt6Core.6.dylib .
cp /Users/runner/.conan2/p/b/*/p/lib/libQt6Gui.6.dylib .
cp /Users/runner/.conan2/p/b/*/p/lib/libQt6Widgets.6.dylib .
Expand All @@ -165,10 +165,11 @@ jobs:
cd java
cp ${GITHUB_WORKSPACE}/resources/java/bioformats.jar .
cp ${GITHUB_WORKSPACE}/resources/java/BioFormatsWrapper.class .
mkdir -p jre_macos_arm
cd jre_macos_arm
cp -r ${GITHUB_WORKSPACE}/resources/java/jre_macos_arm.zip .
unzip jre_macos_arm.zip
rm -rf jre_macos_arm.zip
cd ..
- name: Sign Binaries
Expand Down
8 changes: 8 additions & 0 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -151,3 +151,11 @@ microsoft/onnxruntime |https://github.com/microsoft/onnxruntime | MIT

Thank's to the authors of [imagej](https://github.com/imagej/imagej2) I ported some image processing algorithms from to C++.
imageC is the follower of [evanalyzer](https://github.com/joda01/evanalyzer).



## For MacOS you have to do

chmod +x imagec.app/Contents/MacOS/imagec
xattr -dr com.apple.quarantine imagec.app
open imagec.app
71 changes: 71 additions & 0 deletions src/backend/helper/base64.hpp
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,71 @@
///
/// \file base64.hpp
/// \author Joachim Danmayr
/// \date 2024-10-11
///
/// \copyright Copyright 2019 Joachim Danmayr
/// All rights reserved! This file is subject
/// to the terms and conditions defined in file
/// LICENSE.txt, which is part of this package.
///
///

#pragma once

#include <string>
#include <vector>

namespace joda::helper {

static const std::string base64_chars =
"ABCDEFGHIJKLMNOPQRSTUVWXYZ"
"abcdefghijklmnopqrstuvwxyz"
"0123456789+/";

inline std::string base64Encode(const std::string &in)
{
std::string out;
int val = 0;
int valb = -6;
for(unsigned char c : in) {
val = (val << 8) + c;
valb += 8;
while(valb >= 0) {
out.push_back(base64_chars[(val >> valb) & 0x3F]);
valb -= 6;
}
}
if(valb > -6) {
out.push_back(base64_chars[((val << 8) >> (valb + 8)) & 0x3F]);
}
while(out.size() % 4) {
out.push_back('=');
}
return out;
}

inline std::string base64Decode(const std::string &in)
{
std::string out;
std::vector<int> T(256, -1);
for(int i = 0; i < 64; i++) {
T[base64_chars[i]] = i;
}

int val = 0;
int valb = -8;
for(unsigned char c : in) {
if(T[c] == -1) {
break; // Ignore non-base64 characters
}
val = (val << 6) + T[c];
valb += 6;
if(valb >= 0) {
out.push_back(char((val >> valb) & 0xFF));
valb -= 8;
}
}
return out;
}

} // namespace joda::helper
177 changes: 110 additions & 67 deletions src/backend/helper/database/database.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,24 +14,28 @@
#include <duckdb.h>
#include <chrono>
#include <exception>
#include <mutex>
#include <stdexcept>
#include <string>
#include "backend/artifacts/object_list/object_list.hpp"
#include "backend/enums/enums_classes.hpp"
#include "backend/enums/enums_clusters.hpp"
#include "backend/enums/enums_grouping.hpp"
#include "backend/helper/base64.hpp"
#include "backend/helper/duration_count/duration_count.h"
#include "backend/helper/file_grouper/file_grouper.hpp"
#include "backend/helper/file_grouper/file_grouper_types.hpp"
#include "backend/helper/fnv1a.hpp"
#include "backend/helper/logger/console_logger.hpp"
#include "backend/helper/reader/image_reader.hpp"
#include "backend/helper/rle/rle.hpp"
#include "backend/helper/threadpool/thread_pool.hpp"
#include "backend/helper/uuid.hpp"
#include "backend/processor/initializer/pipeline_initializer.hpp"
#include "backend/settings/analze_settings.hpp"
#include "backend/settings/project_settings/project_class.hpp"
#include "backend/settings/project_settings/project_plates.hpp"
#include "backend/settings/settings.hpp"
#include <duckdb/common/types.hpp>
#include <duckdb/common/types/string_type.hpp>
#include <duckdb/common/types/value.hpp>
Expand Down Expand Up @@ -65,6 +69,7 @@ void Database::createTables()
" experiment_id UUID,"
" job_id UUID,"
" job_name TEXT, "
" imagec_version TEXT, "
" time_started TIMESTAMP,"
" time_finished TIMESTAMP,"
" settings TEXT,"
Expand Down Expand Up @@ -306,7 +311,7 @@ void Database::insertObjects(const joda::processor::ImageContext &imgContext, co
}

auto Database::prepareImages(uint8_t plateId, enums::GroupBy groupBy, const std::string &filenameRegex,
const std::vector<std::filesystem::path> &imagePaths)
const std::vector<std::filesystem::path> &imagePaths, BS::thread_pool &globalThreadPool)
-> std::vector<std::tuple<std::filesystem::path, joda::ome::OmeInfo, uint64_t>>
{
std::vector<std::tuple<std::filesystem::path, joda::ome::OmeInfo, uint64_t>> imagesToProcess;
Expand All @@ -318,69 +323,84 @@ auto Database::prepareImages(uint8_t plateId, enums::GroupBy groupBy, const std:
auto images_groups = duckdb::Appender(*connection, "images_groups");
auto images_channels = duckdb::Appender(*connection, "images_channels");
std::set<uint16_t> addedGroups;

std::mutex insertMutex;

//
// Preparing -> Insert all images to database
//
BS::multi_future<void> prepareFuture;

for(const auto &imagePath : imagePaths) {
auto ome = joda::image::reader::ImageReader::getOmeInformation(imagePath);
uint64_t imageId = joda::helper::fnv1a(imagePath.string());

imagesToProcess.emplace_back(imagePath, ome, imageId);
auto groupInfo = grouper.getGroupForFilename(imagePath);
// Group
{
if(!addedGroups.contains(groupInfo.groupId)) {
groups.BeginRow();
groups.Append<uint16_t>(plateId); // " plate_id USMALLINT,"
groups.Append<uint16_t>(groupInfo.groupId); // " group_id USMALLINT,"
groups.Append<duckdb::string_t>(groupInfo.groupName); // " name STRING,"
groups.Append<duckdb::string_t>(""); // " notes STRING,"
groups.Append<uint32_t>(groupInfo.wellPosX); // " pos_on_plate_x UINTEGER,"
groups.Append<uint32_t>(groupInfo.wellPosY); // " pos_on_plate_y UINTEGER,"
groups.EndRow();
addedGroups.emplace(groupInfo.groupId);
auto prepareImage = [&groups, &grouper, &addedGroups, &imagesToProcess, &images, &images_groups, &images_channels, &insertMutex, plateId,
imagePath]() {
auto ome = joda::image::reader::ImageReader::getOmeInformation(imagePath);
uint64_t imageId = joda::helper::fnv1a(imagePath.string());
auto groupInfo = grouper.getGroupForFilename(imagePath);

{
std::lock_guard<std::mutex> lock(insertMutex);
imagesToProcess.emplace_back(imagePath, ome, imageId);
// Group
{
if(!addedGroups.contains(groupInfo.groupId)) {
groups.BeginRow();
groups.Append<uint16_t>(plateId); // " plate_id USMALLINT,"
groups.Append<uint16_t>(groupInfo.groupId); // " group_id USMALLINT,"
groups.Append<duckdb::string_t>(groupInfo.groupName); // " name STRING,"
groups.Append<duckdb::string_t>(""); // " notes STRING,"
groups.Append<uint32_t>(groupInfo.wellPosX); // " pos_on_plate_x UINTEGER,"
groups.Append<uint32_t>(groupInfo.wellPosY); // " pos_on_plate_y UINTEGER,"
groups.EndRow();
addedGroups.emplace(groupInfo.groupId);
}
}

// Image
{
images.BeginRow();
images.Append<uint64_t>(imageId); // " image_id UBIGINT,"
images.Append<duckdb::string_t>(imagePath.filename().string()); // " file_name TEXT,"
images.Append<duckdb::string_t>(imagePath.string()); // " original_file_path TEXT
images.Append<uint32_t>(ome.getNrOfChannels()); // " nr_of_c_stacks UINTEGER
images.Append<uint32_t>(ome.getNrOfZStack()); // " nr_of_z_stacks UINTEGER
images.Append<uint32_t>(ome.getNrOfTStack()); // " nr_of_t_stacks UINTEGER
images.Append<uint32_t>(std::get<0>(ome.getSize())); // " width UINTEGER,"
images.Append<uint32_t>(std::get<1>(ome.getSize())); // " height UINTEGER,"
images.Append<uint64_t>(0); // " validity UBIGINT,"
images.Append<bool>(false); // " processed BOOL,"
images.EndRow();
}

// Image Group
{
images_groups.BeginRow();
images_groups.Append<uint16_t>(plateId); // " plate_id USMALLINT,"
images_groups.Append<uint16_t>(groupInfo.groupId); // " group_id USMALLINT,"
images_groups.Append<uint64_t>(imageId); // " image_id UBIGINT,"
images_groups.Append<uint32_t>(groupInfo.imageIdx); // " image_group_idx UINTEGER, "
images_groups.EndRow();
}

// Image channel
{
for(const auto &[channelId, channel] : ome.getChannelInfos()) {
images_channels.BeginRow();
images_channels.Append<uint64_t>(imageId); // " image_id UBIGINT,"
images_channels.Append<uint32_t>(channelId); // " stack_c UINTEGER, "
images_channels.Append<duckdb::string_t>(channel.channelId); // " channel_id TEXT,"
images_channels.Append<duckdb::string_t>(channel.name); // " name TEXT,"
images_channels.EndRow();
}
}
}
}

// Image
{
images.BeginRow();
images.Append<uint64_t>(imageId); // " image_id UBIGINT,"
images.Append<duckdb::string_t>(imagePath.filename().string()); // " file_name TEXT,"
images.Append<duckdb::string_t>(imagePath.string()); // " original_file_path TEXT
images.Append<uint32_t>(ome.getNrOfChannels()); // " nr_of_c_stacks UINTEGER
images.Append<uint32_t>(ome.getNrOfZStack()); // " nr_of_z_stacks UINTEGER
images.Append<uint32_t>(ome.getNrOfTStack()); // " nr_of_t_stacks UINTEGER
images.Append<uint32_t>(std::get<0>(ome.getSize())); // " width UINTEGER,"
images.Append<uint32_t>(std::get<1>(ome.getSize())); // " height UINTEGER,"
images.Append<uint64_t>(0); // " validity UBIGINT,"
images.Append<bool>(false); // " processed BOOL,"
images.EndRow();
}

// Image Group
{
images_groups.BeginRow();
images_groups.Append<uint16_t>(plateId); // " plate_id USMALLINT,"
images_groups.Append<uint16_t>(groupInfo.groupId); // " group_id USMALLINT,"
images_groups.Append<uint64_t>(imageId); // " image_id UBIGINT,"
images_groups.Append<uint32_t>(groupInfo.imageIdx); // " image_group_idx UINTEGER, "
images_groups.EndRow();
}
};

// Image channel
{
for(const auto &[channelId, channel] : ome.getChannelInfos()) {
images_channels.BeginRow();
images_channels.Append<uint64_t>(imageId); // " image_id UBIGINT,"
images_channels.Append<uint32_t>(channelId); // " stack_c UINTEGER, "
images_channels.Append<duckdb::string_t>(channel.channelId); // " channel_id TEXT,"
images_channels.Append<duckdb::string_t>(channel.name); // " name TEXT,"
images_channels.EndRow();
}
}
prepareFuture.push_back(globalThreadPool.submit_task(prepareImage));
}

prepareFuture.wait();

groups.Close();
images.Close();
images_groups.Close();
Expand Down Expand Up @@ -665,7 +685,11 @@ bool Database::insertExperiment(const joda::settings::ExperimentSettings &exp)
auto Database::selectExperiment() -> AnalyzeMeta
{
joda::settings::ExperimentSettings exp;
std::chrono::system_clock::time_point timestamp;
std::chrono::system_clock::time_point timestampStart;
std::chrono::system_clock::time_point timestampFinish;
std::string settingsString;
std::string jobName;

{
std::unique_ptr<duckdb::QueryResult> result = select("SELECT experiment_id,name,notes FROM experiment");
if(result->HasError()) {
Expand All @@ -676,24 +700,43 @@ auto Database::selectExperiment() -> AnalyzeMeta
if(materializedResult->RowCount() > 0) {
exp.experimentId = materializedResult->GetValue(0, 0).GetValue<std::string>();
exp.experimentName = materializedResult->GetValue(1, 0).GetValue<std::string>();
exp.experimentName = materializedResult->GetValue(1, 0).GetValue<std::string>();
exp.notes = materializedResult->GetValue(2, 0).GetValue<std::string>();
}
}

{
std::unique_ptr<duckdb::QueryResult> resultJobs = select("SELECT time_started FROM jobs ORDER BY time_started");
std::unique_ptr<duckdb::QueryResult> resultJobs = select("SELECT time_started,time_finished,settings,job_name FROM jobs ORDER BY time_started");
if(resultJobs->HasError()) {
throw std::invalid_argument(resultJobs->GetError());
}
auto materializedResult = resultJobs->Cast<duckdb::StreamQueryResult>().Materialize();
if(materializedResult->RowCount() > 0) {
auto timestampDb = materializedResult->GetValue(0, 0).GetValue<duckdb::timestamp_t>();
time_t epochTime = duckdb::Timestamp::GetEpochSeconds(timestampDb);
timestamp = std::chrono::system_clock::from_time_t(epochTime);
{
auto timestampDb = materializedResult->GetValue(0, 0).GetValue<duckdb::timestamp_t>();
time_t epochTime = duckdb::Timestamp::GetEpochSeconds(timestampDb);
timestampStart = std::chrono::system_clock::from_time_t(epochTime);
}
{
auto timestampDb = materializedResult->GetValue(1, 0).GetValue<duckdb::timestamp_t>();
time_t epochTime = duckdb::Timestamp::GetEpochSeconds(timestampDb);
timestampFinish = std::chrono::system_clock::from_time_t(epochTime);
}

{
settingsString = helper::base64Decode(materializedResult->GetValue(2, 0).GetValue<std::string>());
}

{
jobName = materializedResult->GetValue(3, 0).GetValue<std::string>();
}
}
}

return {.experiment = exp, .timestamp = timestamp};
return {.experiment = exp,
.timestampStart = timestampStart,
.timestampFinish = timestampFinish,
.jobName = jobName,
.analyzeSettingsJsonString = settingsString};
}

///
Expand All @@ -718,13 +761,13 @@ std::string Database::insertJobAndPlates(const joda::settings::AnalyzeSettings &
duckdb::timestamp_t(std::chrono::duration_cast<std::chrono::microseconds>(std::chrono::system_clock::now().time_since_epoch()).count());
duckdb::timestamp_t nil = {};
auto prepare = connection->Prepare(
"INSERT INTO jobs (experiment_id, job_id,job_name, time_started, time_finished, settings) VALUES (?, ?, ?, ?, "
"INSERT INTO jobs (experiment_id, job_id, job_name,imagec_version, time_started, time_finished, settings) VALUES (?, ?, ?, ?, "
"?, "
"?)");

nlohmann::json json = exp;
prepare->Execute(duckdb::Value::UUID(exp.projectSettings.experimentSettings.experimentId), jobId, jobName,
duckdb::Value::TIMESTAMP(timestampStart), duckdb::Value::TIMESTAMP(nil), static_cast<std::string>(json.dump()));
prepare->Execute(duckdb::Value::UUID(exp.projectSettings.experimentSettings.experimentId), jobId, jobName, std::string(Version::getVersion()),
duckdb::Value::TIMESTAMP(timestampStart), duckdb::Value::TIMESTAMP(nil),
helper::base64Encode(settings::Settings::toString(exp)));
} catch(const std::exception &ex) {
connection->Rollback();
throw std::runtime_error(ex.what());
Expand Down
Loading

0 comments on commit a74714b

Please sign in to comment.