Update to version v4.18.0

bindings/consts.go

@@ -2,7 +2,7 @@ package bindings
 
 const CInt32Max = int(^uint32(0) >> 1)
 
-const ReindexerVersion = "v4.17.2"
+const ReindexerVersion = "v4.18.0"
 
 // public go consts from type_consts.h and reindexer_ctypes.h
 const (

changelog.md

@@ -1,3 +1,20 @@
+# Version 4.18.0 (31.10.2024) *beta*
+## Replication
+- [fix] Fixed possible origin LSN missmatch in snapshots during WAL/Force syncs
+
+## Reindexer server
+- [fix] Fixed logical operations (`or`, `not`) parsining in JSON DSL joined queries
+
+## Ported
+- [fea/fix] Ported all the fixes and features from [v3.29.0](https://github.com/Restream/reindexer/releases/tag/v3.25.0), [v3.26.0](https://github.com/Restream/reindexer/releases/tag/v3.29.0)
+
+## Face
+- [fea] Added disabled mode for selectors on NS config page
+- [fix] Fixed `shard_id` filter on the Items table after the Item editing
+- [fix] Fixed scroll position on NS menu 
+- [fix] Fixed the issue related to NS config changes saving on tabs switching
+- [fix] Fixed duplicated requests on Indexes and Items pages
+
 # Version 4.17.2 *beta* (19.09.2024)
 ## Core
 - [fix] Fixed data race in cached comparators (`joins cache` may cause incorrect comparators deletion)

cjson/creflect.go

@@ -85,7 +85,7 @@ type payloadIface struct {
 func (pl *payloadIface) ptr(field, idx, typ int) unsafe.Pointer {
 
 	if pl.p == 0 {
-		panic(fmt.Errorf("Null pointer derefernce"))
+		panic(fmt.Errorf("Null pointer dereference"))
 	}
 
 	f := &pl.t.Fields[field]
@@ -98,7 +98,7 @@ func (pl *payloadIface) ptr(field, idx, typ int) unsafe.Pointer {
 
 	if !f.IsArray {
 		if idx != 0 {
-			panic(fmt.Errorf("Trying to acces by index '%d' to non array field '%s'", idx, f.Name))
+			panic(fmt.Errorf("Trying to access by index '%d' to non array field '%s'", idx, f.Name))
 		}
 		return p
 	}
@@ -217,7 +217,7 @@ func (pl *payloadIface) getArrayLen(field int) int {
 	return int((*ArrayHeader)(p).len)
 }
 
-// get c reflect value and set to go reflect valie
+// get c reflect value and set to go reflect value
 func (pl *payloadIface) getValue(field int, idx int, v reflect.Value) {
 
 	k := v.Type().Kind()

cpp_src/CMakeLists.txt

@@ -45,7 +45,7 @@ else()
   option(LINK_RESOURCES "Link web resources as binary data" ON)
 endif()
 
-set (REINDEXER_VERSION_DEFAULT "4.17.2")
+set (REINDEXER_VERSION_DEFAULT "4.18.0")
 
 if(NOT CMAKE_BUILD_TYPE)
   set(CMAKE_BUILD_TYPE "RelWithDebInfo")
@@ -88,7 +88,7 @@ endif()
 
 if(MSVC)
   set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -wd4244 -wd4267 -wd4996 -wd4717 -MP -MD")
-  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -wd4244 -wd4267 -wd4996 -wd4717 -wd4800 -wd4396 -wd4503 -MP -MD")
+  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -wd4244 -wd4267 -wd4996 -wd4717 -wd4800 -wd4396 -wd4503 -MP -MD /bigobj")
   set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -SAFESEH:NO")
 else()
   set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wall -Wextra -Werror -Wswitch-enum")
@@ -629,8 +629,3 @@ add_custom_target(collect_coverage
   COMMAND genhtml coverage_filtered.info -o coverage_output
   COMMENT "Collecting Reindexer coverage"
 )
-
-# Configure compile options extra
-if(MSVC)
-  target_compile_options(${TARGET} PRIVATE /bigobj)
-endif()

cpp_src/client/cororeindexer.h

@@ -36,7 +36,7 @@ class CoroReindexer {
 	using ConnectionStateHandlerT = std::function<void(const Error&)>;
 
 	/// Create Reindexer database object
-	CoroReindexer(const ReindexerConfig& = ReindexerConfig());
+	explicit CoroReindexer(const ReindexerConfig& = ReindexerConfig());
 	/// Destroy Reindexer database object
 	~CoroReindexer();
 	CoroReindexer(const CoroReindexer&) = delete;
@@ -264,6 +264,7 @@ class CoroReindexer {
 	typedef CoroQueryResults QueryResultsT;
 	typedef Item ItemT;
 	typedef ReindexerConfig ConfigT;
+	typedef CoroTransaction TransactionT;
 
 private:
 	CoroReindexer(RPCClient* impl, InternalRdxContext&& ctx) : impl_(impl), ctx_(std::move(ctx)) {}

cpp_src/client/corotransaction.cc

@@ -8,8 +8,7 @@
 #include "core/keyvalue/p_string.h"
 #include "net/cproto/coroclientconnection.h"
 
-namespace reindexer {
-namespace client {
+namespace reindexer::client {
 
 CoroTransaction::~CoroTransaction() {
 	if (!IsFree()) {
@@ -226,5 +225,4 @@ CoroTransaction::Impl::Impl(CoroTransaction::Impl&&) noexcept = default;
 CoroTransaction::Impl& CoroTransaction::Impl::operator=(CoroTransaction::Impl&&) noexcept = default;
 CoroTransaction::Impl::~Impl() = default;
 
-}  // namespace client
-}  // namespace reindexer
+}  // namespace reindexer::client

cpp_src/client/corotransaction.h

@@ -7,11 +7,9 @@
 
 namespace reindexer {
 
-namespace net {
-namespace cproto {
+namespace net::cproto {
 class CoroClientConnection;
-}  // namespace cproto
-}  // namespace net
+}  // namespace net::cproto
 
 namespace client {
 
@@ -58,7 +56,7 @@ class CoroTransaction {
 	friend class RPCClient;
 	friend class ReindexerImpl;
 	friend class Transaction;
-	CoroTransaction(Error status) noexcept : i_(std::move(status)) {}
+	explicit CoroTransaction(Error status) noexcept : i_(std::move(status)) {}
 	CoroTransaction(RPCClient* rpcClient, int64_t txId, std::chrono::milliseconds requestTimeout, std::chrono::milliseconds execTimeout,
 					Namespace* ns) noexcept
 		: i_(rpcClient, txId, requestTimeout, execTimeout, ns) {}
@@ -82,18 +80,19 @@ class CoroTransaction {
 		Impl& operator=(Impl&&) noexcept;
 		~Impl();
 
-		int64_t txId_ = -1;
-		RPCClient* rpcClient_ = nullptr;
-		std::chrono::milliseconds requestTimeout_ = std::chrono::milliseconds{0};
-		std::chrono::milliseconds execTimeout_ = std::chrono::milliseconds{0};
+		int64_t txId_{-1};
+		RPCClient* rpcClient_{nullptr};
+		std::chrono::milliseconds requestTimeout_{0};
+		std::chrono::milliseconds execTimeout_{0};
 		Error status_;
 		std::unique_ptr<TagsMatcher> localTm_;
-		Namespace* ns_ = nullptr;
+		Namespace* ns_{nullptr};
 		steady_clock_w::time_point sessionTs_;
 	};
 
 	Impl i_;
 };
 
 }  // namespace client
+
 }  // namespace reindexer

cpp_src/client/rpcclient.cc

@@ -227,7 +227,6 @@ Error RPCClient::modifyItemCJSON(std::string_view nsName, Item& item, CoroQueryR
 				// Add new names missing in JSON from tm
 				newItem.impl_->addTagNamesFrom(item.impl_->tagsMatcher());
 			}
-
 			item = std::move(newItem);
 		}
 	}

cpp_src/cluster/replication/replicationthread.cc

@@ -416,15 +416,15 @@ Error ReplThread<BehaviourParamT>::nodeReplicationImpl(Node& node) {
 					const NamespaceName nsName(ns.name);
 					err = syncNamespace(node, nsName, replState);
 					if (!err.ok()) {
-						logWarn("%d:%d Namespace sync error: %s", serverId_, node.uid, err.what());
+						logWarn("%d:%d:%s Namespace sync error: %s", serverId_, node.uid, nsName, err.what());
 						if (err.code() == errNotFound) {
 							err = Error();
 							logWarn("%d:%d Expecting drop namespace record for '%s'", serverId_, node.uid, nsName);
 						} else if (err.code() == errDataHashMismatch) {
 							replState = ReplicationStateV2();
 							err = syncNamespace(node, nsName, replState);
 							if (!err.ok()) {
-								logWarn("%d:%d Namespace sync error (resync due to datahash missmatch): %s", serverId_, node.uid,
+								logWarn("%d:%d:%s Namespace sync error (resync due to datahash missmatch): %s", serverId_, node.uid, nsName,
 										err.what());
 							}
 						}
@@ -437,7 +437,7 @@ Error ReplThread<BehaviourParamT>::nodeReplicationImpl(Node& node) {
 			}
 
 			if (integralError.ok()) {
-				integralError = Error(errTimeout, "%d:%d Unable to sync namespace", serverId_, node.uid);
+				integralError = Error(errTimeout, "%d:%d:%s Unable to sync namespace", serverId_, node.uid, ns.name);
 				return;
 			}
 		});
@@ -594,12 +594,12 @@ Error ReplThread<BehaviourParamT>::syncNamespace(Node& node, const NamespaceName
 			return err;
 		}
 		if (snapshot.HasRawData()) {
-			logInfo("%d:%d:%s Snapshot has raw data, creating tmp namespace", serverId_, node.uid, nsName);
+			logInfo("%d:%d:%s Snapshot has RAW data, creating tmp namespace (performing FORCE sync)", serverId_, node.uid, nsName);
 			createTmpNamespace = true;
 		} else if (snapshot.NsVersion().Server() != requiredLsn.NsVersion().Server() ||
 				   snapshot.NsVersion().Counter() != requiredLsn.NsVersion().Counter()) {
-			logInfo("%d:%d:%s Snapshot has different ns version (%d vs %d), creating tmp namespace", serverId_, node.uid, nsName,
-					snapshot.NsVersion(), requiredLsn.NsVersion());
+			logInfo("%d:%d:%s Snapshot has different ns version (%d vs %d), creating tmp namespace (performing FORCE sync)", serverId_,
+					node.uid, nsName, snapshot.NsVersion(), requiredLsn.NsVersion());
 			createTmpNamespace = true;
 		}
 

cpp_src/cluster/replication/roleswitcher.cc

@@ -31,7 +31,7 @@ void RoleSwitcher::Run(std::vector<std::string>&& dsns, RoleSwitcher::Config&& c
 
 	nodes_.reserve(dsns.size());
 	for (auto&& dsn : dsns) {
-		nodes_.emplace_back(Node{std::move(dsn), {clientCfg}});
+		nodes_.emplace_back(Node{std::move(dsn), client::CoroReindexer{clientCfg}});
 	}
 	assert(cfg_.onRoleSwitchDone);
 

cpp_src/cmd/reindexer_tool/commandsexecutor.cc

@@ -224,7 +224,10 @@ Error CommandsExecutor<DBInterface>::fromFileImpl(std::istream& in) {
 	while (GetStatus().running && std::getline(in, line.str)) {
 		if (reindexer::checkIfStartsWith("\\upsert ", line.str) || reindexer::checkIfStartsWith("\\delete ", line.str)) {
 			try {
-				cmdCh.push(line);
+				LineData l;
+				l.lineNum = line.lineNum;
+				reindexer::deepCopy(l.str, line.str);
+				cmdCh.push(std::move(l));
 			} catch (std::exception&) {
 				break;
 			}
@@ -578,7 +581,7 @@ Error CommandsExecutor<DBInterface>::processImpl(const std::string& command) noe
 		} catch (std::exception& e) {
 			return Error(errLogic, "std::exception during command's execution: %s", e.what());
 		} catch (...) {
-			return Error(errLogic, "Unknow exception during command's execution");
+			return Error(errLogic, "Unknown exception during command's execution");
 		}
 	}
 	return Error(errParams, "Unknown command '%s'. Type '\\help' to list of available commands", token);
@@ -801,7 +804,7 @@ Error CommandsExecutor<DBInterface>::commandUpsert(const std::string& command) {
 
 	status = db().Upsert(nsName, item);
 	if (!fromFile_ && status.ok()) {
-		output_() << "Upserted successfuly: 1 items" << std::endl;
+		output_() << "Upserted successfully: 1 items" << std::endl;
 	}
 	return status;
 }
@@ -946,7 +949,7 @@ Error CommandsExecutor<DBInterface>::commandDump(const std::string& command) {
 		}
 
 		for (auto it : itemResults) {
-			if (auto err = it.Status(); !err.ok()) {
+			if (err = it.Status(); !err.ok()) {
 				return err;
 			}
 			if (cancelCtx_.IsCancelled()) {
@@ -1190,7 +1193,7 @@ Error CommandsExecutor<reindexer::client::CoroReindexer>::commandProcessDatabase
 			err = db().Status();
 		}
 		if (err.ok()) {
-			output_() << "Succesfully connected to " << currentDsn << std::endl;
+			output_() << "Successfully connected to " << currentDsn << std::endl;
 		}
 		return err;
 	} else if (subCommand == "create"sv) {
@@ -1206,7 +1209,7 @@ Error CommandsExecutor<reindexer::client::CoroReindexer>::commandProcessDatabase
 		std::vector<std::string> dbNames;
 		err = db().EnumDatabases(dbNames);
 		if (std::find(dbNames.begin(), dbNames.end(), std::string(dbName)) != dbNames.end()) {
-			output_() << "Succesfully created database '" << dbName << "'" << std::endl;
+			output_() << "Successfully created database '" << dbName << "'" << std::endl;
 		} else {
 			std::cerr << "Error on database '" << dbName << "' creation" << std::endl;
 		}

cpp_src/cmd/reindexer_tool/readme.md

@@ -23,14 +23,15 @@ reindexer_tool {OPTIONS}
 Options
   -d[DSN],      --dsn=[DSN]              DSN to 'reindexer', like 'cproto://127.0.0.1:6534/dbname', 'builtin:///var/lib/reindexer/dbname' or `ucproto://user@password:/tmp/reindexer.sock:/dbname`
   -f[FILENAME], --filename=[FILENAME]    Execute commands from file, then exit
-  -c[COMMAND],  --command=[COMMAND]      Run only single command (SQL or internal) and exit
+  -c[COMMAND],  --command=[COMMAND]      Run single command (SQL or internal) and exit
   -o[FILENAME], --output=[FILENAME]      Send query results to file
   -l[INT=1..5], --log=[INT=1..5]         Reindexer logging level
   -C[INT],      --connections=[INT]      Number of simulateonous connections to db
   -t[INT],      --threads=[INT]          Number of threads used by db connector (used only for bench)
-  --createdb                             Enable created database if missed
+  --createdb                             Creates target database if it is missing
+  -a[Application name],
+  --appname=[Application name]           Application name which will be used in login info
   --dump-mode=[DUMP_MODE]                Dump mode for sharded databases: 'full_node' (default), 'sharded_only', 'local_only'
-
 ```
 
 ## Commands
@@ -139,10 +140,13 @@ reindexer_tool --dsn cproto://127.0.0.1:6534/mydb --command '\dump' --output myd
 
 Restore database from backup file:
 ```sh
-reindexer_tool --dsn cproto://127.0.0.1:6534/mydb --filename mydb.rxdump
+reindexer_tool --dsn cproto://127.0.0.1:6534/mydb --createdb --filename mydb.rxdump
 ```
 
+Option `createdb` in example above allows to automatically create `mydb` if it does not exist. By default `reindexer_tool` requires an existing database to connect.
+
 Backup only sharded namespaces from all of the shards into single backup file:
 ```sh
 reindexer_tool --dsn cproto://127.0.0.1:6534/mydb --command '\dump' --dump-mode=sharded_only --output mydb.rxdump
 ```
+

cpp_src/cmd/reindexer_tool/reindexer_tool.cc

@@ -65,29 +65,29 @@ int main(int argc, char* argv[]) {
 		"DSN to 'reindexer'. Can be 'cproto://<ip>:<port>/<dbname>', 'builtin://<path>' or 'ucproto://<unix.socket.path>:/<dbname>'",
 		{'d', "dsn"}, "", Options::Single | Options::Global);
 #endif	// _WIN32
-	args::ValueFlag<std::string> fileName(progOptions, "FILENAME", "execute commands from file, then exit", {'f', "filename"}, "",
+	args::ValueFlag<std::string> fileName(progOptions, "FILENAME", "Execute commands from file, then exit", {'f', "filename"}, "",
 										  Options::Single | Options::Global);
-	args::ValueFlag<std::string> command(progOptions, "COMMAND", "run only single command (SQL or internal) and exit'", {'c', "command"},
-										 "", Options::Single | Options::Global);
-	args::ValueFlag<std::string> outFileName(progOptions, "FILENAME", "send query results to file", {'o', "output"}, "",
+	args::ValueFlag<std::string> command(progOptions, "COMMAND", "Run single command (SQL or internal) and exit", {'c', "command"}, "",
+										 Options::Single | Options::Global);
+	args::ValueFlag<std::string> outFileName(progOptions, "FILENAME", "Send query results to file", {'o', "output"}, "",
 											 Options::Single | Options::Global);
 	args::ValueFlag<std::string> dumpMode(progOptions, "DUMP_MODE",
-										  "dump mode for sharded databases: 'full_node' (default), 'sharded_only', 'local_only'",
+										  "Dump mode for sharded databases: 'full_node' (default), 'sharded_only', 'local_only'",
 										  {"dump-mode"}, "", Options::Single | Options::Global);
 
 	args::ValueFlag<unsigned> connThreads(progOptions, "INT=1..65535", "Number of threads(connections) used by db connector",
 										  {'t', "threads"}, 1, Options::Single | Options::Global);
 
-	args::Flag createDBF(progOptions, "", "Enable created database if missed", {"createdb"});
+	args::Flag createDBF(progOptions, "", "Creates target database if it is missing", {"createdb"});
 
 	args::Positional<std::string> dbName(progOptions, "DB name", "Name of a database to get connected to", Options::Single);
 
-	args::ActionFlag logLevel(progOptions, "INT=1..5", "reindexer logging level", {'l', "log"}, 1, &InstallLogLevel,
+	args::ActionFlag logLevel(progOptions, "INT=1..5", "Reindexer logging level", {'l', "log"}, 1, &InstallLogLevel,
 							  Options::Single | Options::Global);
 
-	args::Flag repair(progOptions, "", "Repair database", {'r', "repair"});
+	args::Flag repair(progOptions, "", "Try to repair storage", {'r', "repair"});
 
-	args::ValueFlag<std::string> appName(progOptions, "Application name", "Application name which will be used in login info",
+	args::ValueFlag<std::string> appName(progOptions, "Application name", "Application name that will be used in login info",
 										 {'a', "appname"}, "reindexer_tool", Options::Single | Options::Global);
 
 	args::GlobalOptions globals(parser, progOptions);

cpp_src/core/cjson/baseencoder.cc

@@ -1,6 +1,5 @@
 #include "baseencoder.h"
 #include <cstdlib>
-#include <limits>
 #include "cjsonbuilder.h"
 #include "cjsontools.h"
 #include "core/keyvalue/p_string.h"
@@ -48,7 +47,7 @@ void BaseEncoder<Builder>::Encode(ConstPayload& pl, Builder& builder, const h_ve
 	}
 
 	objectScalarIndexes_.reset();
-	std::fill_n(std::begin(fieldsoutcnt_), pl.NumFields(), 0);
+	std::fill(fieldsoutcnt_.begin(), fieldsoutcnt_.end(), 0);
 	builder.SetTagsMatcher(tagsMatcher_);
 	if constexpr (kWithTagsPathTracking) {
 		builder.SetTagsPath(&curTagsPath_);
@@ -175,11 +174,11 @@ bool BaseEncoder<Builder>::encode(ConstPayload* pl, Serializer& rdser, Builder&
 						[&](KeyValueType::String) { builder.Array(tagName, pl->GetArray<p_string>(tagField).subspan(cnt, count), cnt); },
 						[&](KeyValueType::Uuid) { builder.Array(tagName, pl->GetArray<Uuid>(tagField).subspan(cnt, count), cnt); },
 						[](OneOf<KeyValueType::Null, KeyValueType::Tuple, KeyValueType::Undefined, KeyValueType::Composite>) noexcept {
-							assertrx(0);
+							assertrx(false);
 							abort();
 						});
 				}
-				cnt += count;
+				cnt += int(count);
 				break;
 			}
 			case TAG_NULL: