Update to version v3.29.0

bindings/consts.go

@@ -2,7 +2,7 @@ package bindings
 
 const CInt32Max = int(^uint32(0) >> 1)
 
-const ReindexerVersion = "v3.28.2"
+const ReindexerVersion = "v3.29.0"
 
 // public go consts from type_consts.h and reindexer_ctypes.h
 const (

changelog.md

@@ -1,3 +1,28 @@
+# Version 3.29.0 (08.10.2024)
+## Core
+- [fea] Added Chinese and Japanese symbols into locale. Currently those symbols will be indexed by fulltext engine (using space-separators), but proper tokenization for the languages will be added in the separate release
+- [fea] Added [debug_rank() function](fulltext.md#debug_rank) for fulltext-indexes (it allows to get some more ranking details in the resulting docs)
+- [fix] Fixed crash in fulltext index data deletion after `incorrect encoding` error handling
+- [fix] Fixed LSN setting for deleted items
+- [fix] Fixed large `int64` convertion in SQL queries
+
+## Replication
+- [fix] Fixed possible datahash missmatch after WAL-sync in cascade replication setups
+- [fix] Fixed storage rollback from v4 with non-default `server ID` values
+
+## Reindexer server
+- [fix] Fixed GRPC queries responses in [Protobuf format](cpp_src/readme.md#protobuf)
+
+## Go connector
+- [fea] Added ["equal_positions" filters](readme.md#search-in-array-fields-with-matching-array-indexes) support into [Go DSL](dsl/dsl.go)
+- [fix] Fixed interaction between `query.Get()` and joined-queries. Now `Get()` correctly applies `limit` to the main query
+
+## Build
+- [fix] Fixed `AltLinux` `gperftools` [dependency](dependencies.sh) name
+
+## Face
+- [fix] Fixed NS config changes saving after tabs switch
+
 # Version 3.28.2 (19.09.2024)
 ## Core
 - [fix] Fixed data race in cached comparators (`joins cache` may cause incorrect comparators deletion)

cpp_src/CMakeLists.txt

@@ -44,7 +44,7 @@ else()
   option(LINK_RESOURCES "Link web resources as binary data" ON)
 endif()
 
-set(REINDEXER_VERSION_DEFAULT "3.28.2")
+set(REINDEXER_VERSION_DEFAULT "3.29.0")
 
 if(NOT CMAKE_BUILD_TYPE)
   set(CMAKE_BUILD_TYPE "RelWithDebInfo")
@@ -87,7 +87,7 @@ endif()
 
 if(MSVC)
   set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -wd4244 -wd4267 -wd4996 -wd4717 -MP -MD")
-  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -wd4244 -wd4267 -wd4996 -wd4717 -wd4800 -wd4396 -wd4503 -MP -MD")
+  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -wd4244 -wd4267 -wd4996 -wd4717 -wd4800 -wd4396 -wd4503 -MP -MD /bigobj")
   set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -SAFESEH:NO")
 else()
   set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wall -Wextra -Werror -Wswitch-enum")
@@ -619,8 +619,3 @@ add_custom_target(collect_coverage
   COMMAND genhtml coverage_filtered.info -o coverage_output
   COMMENT "Collecting Reindexer coverage"
 )
-
-# Configure compile options extra
-if(MSVC)
-  target_compile_options(${TARGET} PRIVATE /bigobj)
-endif()

cpp_src/cmd/reindexer_tool/commandsexecutor.cc

@@ -25,7 +25,6 @@ const std::string kVariableOutput = "output";
 const std::string kOutputModeJson = "json";
 const std::string kOutputModeTable = "table";
 const std::string kOutputModePretty = "pretty";
-const std::string kOutputModePrettyCollapsed = "collapsed";
 const std::string kBenchNamespace = "rxtool_bench";
 const std::string kBenchIndex = "id";
 
@@ -157,7 +156,10 @@ Error CommandsExecutor<DBInterface>::fromFileImpl(std::istream& in) {
 	while (GetStatus().running && std::getline(in, line.str)) {
 		if (reindexer::checkIfStartsWith("\\upsert ", line.str) || reindexer::checkIfStartsWith("\\delete ", line.str)) {
 			try {
-				cmdCh.push(line);
+				LineData l;
+				l.lineNum = line.lineNum;
+				reindexer::deepCopy(l.str, line.str);
+				cmdCh.push(std::move(l));
 			} catch (std::exception&) {
 				break;
 			}
@@ -506,7 +508,7 @@ Error CommandsExecutor<DBInterface>::processImpl(const std::string& command) noe
 		} catch (std::exception& e) {
 			return Error(errLogic, "std::exception during command's execution: %s", e.what());
 		} catch (...) {
-			return Error(errLogic, "Unknow exception during command's execution");
+			return Error(errLogic, "Unknown exception during command's execution");
 		}
 	}
 	return Error(errParams, "Unknown command '%s'. Type '\\help' to list of available commands", token);
@@ -683,7 +685,7 @@ Error CommandsExecutor<DBInterface>::commandUpsert(const std::string& command) {
 
 	status = db().Upsert(nsName, item);
 	if (!fromFile_ && status.ok()) {
-		output_() << "Upserted successfuly: 1 items" << std::endl;
+		output_() << "Upserted successfully: 1 items" << std::endl;
 	}
 	return status;
 }
@@ -809,7 +811,7 @@ Error CommandsExecutor<DBInterface>::commandDump(const std::string& command) {
 		}
 
 		for (auto it : itemResults) {
-			if (auto err = it.Status(); !err.ok()) {
+			if (err = it.Status(); !err.ok()) {
 				return err;
 			}
 			if (cancelCtx_.IsCancelled()) {
@@ -1103,7 +1105,7 @@ Error CommandsExecutor<reindexer::client::CoroReindexer>::commandProcessDatabase
 			err = db().Status();
 		}
 		if (err.ok()) {
-			output_() << "Succesfully connected to " << currentDsn << std::endl;
+			output_() << "Successfully connected to " << currentDsn << std::endl;
 		}
 		return err;
 	} else if (subCommand == "create"sv) {
@@ -1119,7 +1121,7 @@ Error CommandsExecutor<reindexer::client::CoroReindexer>::commandProcessDatabase
 		std::vector<std::string> dbNames;
 		err = db().EnumDatabases(dbNames);
 		if (std::find(dbNames.begin(), dbNames.end(), std::string(dbName)) != dbNames.end()) {
-			output_() << "Succesfully created database '" << dbName << "'" << std::endl;
+			output_() << "Successfully created database '" << dbName << "'" << std::endl;
 		} else {
 			std::cerr << "Error on database '" << dbName << "' creation" << std::endl;
 		}

cpp_src/core/cjson/baseencoder.cc

@@ -1,6 +1,5 @@
 #include "baseencoder.h"
 #include <cstdlib>
-#include <limits>
 #include "cjsonbuilder.h"
 #include "cjsontools.h"
 #include "core/keyvalue/p_string.h"
@@ -42,7 +41,7 @@ void BaseEncoder<Builder>::Encode(ConstPayload& pl, Builder& builder, IAdditiona
 	}
 
 	objectScalarIndexes_.reset();
-	std::fill_n(std::begin(fieldsoutcnt_), pl.NumFields(), 0);
+	std::fill(fieldsoutcnt_.begin(), fieldsoutcnt_.end(), 0);
 	builder.SetTagsMatcher(tagsMatcher_);
 	if constexpr (kWithTagsPathTracking) {
 		builder.SetTagsPath(&curTagsPath_);
@@ -167,11 +166,11 @@ bool BaseEncoder<Builder>::encode(ConstPayload* pl, Serializer& rdser, Builder&
 						[&](KeyValueType::String) { builder.Array(tagName, pl->GetArray<p_string>(tagField).subspan(cnt, count), cnt); },
 						[&](KeyValueType::Uuid) { builder.Array(tagName, pl->GetArray<Uuid>(tagField).subspan(cnt, count), cnt); },
 						[](OneOf<KeyValueType::Null, KeyValueType::Tuple, KeyValueType::Undefined, KeyValueType::Composite>) noexcept {
-							assertrx(0);
+							assertrx(false);
 							abort();
 						});
 				}
-				cnt += count;
+				cnt += int(count);
 				break;
 			}
 			case TAG_NULL:

cpp_src/core/cjson/baseencoder.h

@@ -10,7 +10,6 @@
 namespace reindexer {
 
 class TagsMatcher;
-class JsonBuilder;
 class MsgPackBuilder;
 class ProtobufBuilder;
 class CsvBuilder;
@@ -38,7 +37,7 @@ class IAdditionalDatasource {
 template <typename Builder>
 class BaseEncoder {
 public:
-	BaseEncoder(const TagsMatcher* tagsMatcher, const FieldsSet* filter = nullptr);
+	explicit BaseEncoder(const TagsMatcher* tagsMatcher, const FieldsSet* filter = nullptr);
 	void Encode(ConstPayload& pl, Builder& builder, IAdditionalDatasource<Builder>* = nullptr);
 	void Encode(std::string_view tuple, Builder& wrSer, IAdditionalDatasource<Builder>*);
 
@@ -61,9 +60,9 @@ class BaseEncoder {
 
 	std::string_view getPlTuple(ConstPayload& pl);
 
-	const TagsMatcher* tagsMatcher_;
-	std::array<int, kMaxIndexes> fieldsoutcnt_{0};
-	const FieldsSet* filter_;
+	const TagsMatcher* tagsMatcher_{nullptr};
+	std::array<int, kMaxIndexes> fieldsoutcnt_;
+	const FieldsSet* filter_{nullptr};
 	WrSerializer tmpPlTuple_;
 	TagsPath curTagsPath_;
 	IndexedTagsPathInternalT indexedTagsPath_;

cpp_src/core/cjson/cjsondecoder.cc

@@ -11,7 +11,6 @@ bool CJsonDecoder::decodeCJson(Payload& pl, Serializer& rdser, WrSerializer& wrs
 	const ctag tag = rdser.GetCTag();
 	TagType tagType = tag.Type();
 	if (tagType == TAG_END) {
-		recoder.Serialize(wrser);
 		wrser.PutCTag(kCTagEnd);
 		return false;
 	}

cpp_src/core/cjson/cjsondecoder.h

@@ -17,10 +17,8 @@ class Recoder {
 	[[nodiscard]] virtual TagType Type(TagType oldTagType) = 0;
 	virtual void Recode(Serializer&, WrSerializer&) const = 0;
 	virtual void Recode(Serializer&, Payload&, int tagName, WrSerializer&) = 0;
-	[[nodiscard]] virtual bool Match(int field) noexcept = 0;
-	[[nodiscard]] virtual bool Match(TagType, const TagsPath&) = 0;
-	virtual void Serialize(WrSerializer& wrser) = 0;
-	virtual bool Reset() = 0;
+	[[nodiscard]] virtual bool Match(int field) const noexcept = 0;
+	[[nodiscard]] virtual bool Match(const TagsPath&) const = 0;
 	virtual ~Recoder() = default;
 };
 
@@ -49,15 +47,15 @@ class CJsonDecoder {
 
 	class IndexedSkipFilter {
 	public:
-		IndexedSkipFilter(const FieldsSet& f) noexcept : f_(&f) {}
+		explicit IndexedSkipFilter(const FieldsSet& f) noexcept : f_(&f) {}
 		IndexedSkipFilter MakeCleanCopy() const noexcept { return IndexedSkipFilter(*f_); }
 		IndexedSkipFilter MakeSkipFilter() const noexcept { return IndexedSkipFilter(*f_); }
 
 		RX_ALWAYS_INLINE bool contains(int field) const noexcept { return f_->contains(field); }
 		RX_ALWAYS_INLINE bool match(const TagsPath&) const noexcept { return false; }
 
 	private:
-		const FieldsSet* f_;
+		const FieldsSet* f_{nullptr};
 	};
 
 	class RestrictingFilter {
@@ -85,8 +83,8 @@ class CJsonDecoder {
 		}
 
 	private:
-		const FieldsSet* f_;
-		bool match_;
+		const FieldsSet* f_{nullptr};
+		bool match_{false};
 	};
 
 	class DummyRecoder {
@@ -96,7 +94,6 @@ class CJsonDecoder {
 		RX_ALWAYS_INLINE bool Recode(Serializer&, Payload&, int, WrSerializer&) const noexcept { return false; }
 		RX_ALWAYS_INLINE TagType RegisterTagType(TagType tagType, int) const noexcept { return tagType; }
 		RX_ALWAYS_INLINE TagType RegisterTagType(TagType tagType, const TagsPath&) const noexcept { return tagType; }
-		RX_ALWAYS_INLINE void Serialize(WrSerializer&) const {}
 	};
 	class DefaultRecoder {
 	public:
@@ -121,14 +118,13 @@ class CJsonDecoder {
 			return needToRecode_ ? r_->Type(tagType) : tagType;
 		}
 		RX_ALWAYS_INLINE TagType RegisterTagType(TagType tagType, const TagsPath& tagsPath) {
-			needToRecode_ = r_->Match(tagType, tagsPath);
+			needToRecode_ = r_->Match(tagsPath);
 			return needToRecode_ ? r_->Type(tagType) : tagType;
 		}
-		RX_ALWAYS_INLINE void Serialize(WrSerializer& wser) const { r_->Serialize(wser); }
 
 	private:
-		Recoder* r_;
-		bool needToRecode_;
+		Recoder* r_{nullptr};
+		bool needToRecode_{false};
 	};
 	struct NamedTagOpt {};
 	struct NamelessTagOpt {};
@@ -166,7 +162,7 @@ class CJsonDecoder {
 
 	TagsMatcher& tagsMatcher_;
 	TagsPath tagsPath_;
-	int32_t arrayLevel_ = 0;
+	int32_t arrayLevel_{0};
 	ScalarIndexesSetT objectScalarIndexes_;
 	// storage for owning strings obtained from numbers
 	std::deque<std::string>& storage_;