Merge developer branch

.github/workflows/ubuntu-22.04-compile.yml

@@ -27,9 +27,19 @@ jobs:
         export CC=$(which clang)
         export CXX=$(which clang++)
         export CCACHE_DISABLE=1
+
+        git clone https://github.com/openssl/openssl openssl_1_1_1
+        cd openssl_1_1_1
+        git checkout OpenSSL_1_1_1-stable
+        ./config
+        make build_libs -j4
+
+        cd ..
+        rootPath=`pwd`
         mkdir build
         cd build
-        cmake -GNinja -DCMAKE_BUILD_TYPE=Release -DPORTABLE=1 -DTON_ARCH= -DCMAKE_CXX_FLAGS="-mavx2" ..
+
+        cmake -GNinja -DOPENSSL_FOUND=1 -DOPENSSL_INCLUDE_DIR=$rootPath/openssl_1_1_1/include -DOPENSSL_CRYPTO_LIBRARY=$rootPath/openssl_1_1_1/libcrypto.a -DCMAKE_BUILD_TYPE=Release -DPORTABLE=1 -DTON_ARCH= -DCMAKE_CXX_FLAGS="-mavx2" ..
         ninja storage-daemon storage-daemon-cli fift func tonlib tonlibjson tonlib-cli validator-engine lite-client pow-miner validator-engine-console generate-random-id json2tlo dht-server http-proxy rldp-http-proxy adnl-proxy create-state
 
     - name: Find & copy binaries

.github/workflows/ubuntu-compile.yml

@@ -30,9 +30,20 @@ jobs:
         export CC=$(which clang)
         export CXX=$(which clang++)
         export CCACHE_DISABLE=1
+
         mkdir build-${{ matrix.os }}
         cd build-${{ matrix.os }}
-        cmake -GNinja -DCMAKE_BUILD_TYPE=Release -DPORTABLE=1 -DTON_ARCH= -DCMAKE_CXX_FLAGS="-mavx2" ..
+
+        git clone https://github.com/openssl/openssl openssl_1_1_1
+        cd openssl_1_1_1
+        git checkout OpenSSL_1_1_1-stable
+        ./config
+        make build_libs -j4
+
+        cd ..
+        buildPath=`pwd`
+
+        cmake -GNinja -DOPENSSL_FOUND=1 -DOPENSSL_INCLUDE_DIR=$buildPath/openssl_1_1_1/include -DOPENSSL_CRYPTO_LIBRARY=$buildPath/openssl_1_1_1/libcrypto.a -DCMAKE_BUILD_TYPE=Release -DPORTABLE=1 -DTON_ARCH= -DCMAKE_CXX_FLAGS="-mavx2" ..
         ninja storage-daemon storage-daemon-cli fift func tonlib tonlibjson tonlib-cli validator-engine lite-client pow-miner validator-engine-console generate-random-id json2tlo dht-server http-proxy rldp-http-proxy adnl-proxy create-state create-hardfork
 
     - name: Find & copy binaries

.github/workflows/win-2019-compile.yml

@@ -48,18 +48,13 @@ jobs:
           curl  -Lo libmicrohttpd-latest-w32-bin.zip https://ftpmirror.gnu.org/libmicrohttpd/libmicrohttpd-latest-w32-bin.zip
           unzip libmicrohttpd-latest-w32-bin.zip
 
-      - name: Install pre-compiled Readline Win64
-        run: |
-          curl  -Lo readline-5.0-1-lib.zip https://github.com/neodiX42/precompiled-openssl-win64/raw/main/readline-5.0-1-lib.zip
-          unzip readline-5.0-1-lib.zip
-
       - name: Compile
         run: |
           set root=%cd%
           echo %root%
           mkdir build
           cd build
-          cmake -DREADLINE_INCLUDE_DIR=%root%\readline-5.0-1-lib\include\readline -DREADLINE_LIBRARY=%root%\readline-5.0-1-lib\lib\readline.lib -DZLIB_FOUND=1 -DMHD_FOUND=1 -DMHD_LIBRARY=%root%\libmicrohttpd-0.9.75-w32-bin\x86_64\VS2019\Release-static\libmicrohttpd.lib -DMHD_INCLUDE_DIR=%root%\libmicrohttpd-0.9.75-w32-bin\x86_64\VS2019\Release-static -DZLIB_INCLUDE_DIR=%root%\zlib -DZLIB_LIBRARY=%root%\zlib\contrib\vstudio\vc14\x64\ZlibStatReleaseWithoutAsm\zlibstat.lib -DOPENSSL_FOUND=1 -DOPENSSL_INCLUDE_DIR=%root%/openssl-1.1/x64/include -DOPENSSL_CRYPTO_LIBRARY=%root%/openssl-1.1/x64/lib/libcrypto.lib -DCMAKE_CXX_FLAGS="/DTD_WINDOWS=1 /EHsc /bigobj /W0" ..
+          cmake -DPORTABLE=1 -DZLIB_FOUND=1 -DMHD_FOUND=1 -DMHD_LIBRARY=%root%\libmicrohttpd-0.9.75-w32-bin\x86_64\VS2019\Release-static\libmicrohttpd.lib -DMHD_INCLUDE_DIR=%root%\libmicrohttpd-0.9.75-w32-bin\x86_64\VS2019\Release-static -DZLIB_INCLUDE_DIR=%root%\zlib -DZLIB_LIBRARY=%root%\zlib\contrib\vstudio\vc14\x64\ZlibStatReleaseWithoutAsm\zlibstat.lib -DOPENSSL_FOUND=1 -DOPENSSL_INCLUDE_DIR=%root%/openssl-1.1/x64/include -DOPENSSL_CRYPTO_LIBRARY=%root%/openssl-1.1/x64/lib/libcrypto.lib -DCMAKE_CXX_FLAGS="/DTD_WINDOWS=1 /EHsc /bigobj /W0" ..
           cmake --build . --target storage-daemon storage-daemon-cli blockchain-explorer fift func tonlib tonlibjson tonlib-cli validator-engine lite-client pow-miner validator-engine-console generate-random-id json2tlo dht-server http-proxy rldp-http-proxy adnl-proxy create-state create-hardfork --config Release
 
       - name: Show executables

Changelog.md

@@ -1,3 +1,12 @@
+## 2023.05 Update
+1. Archive manager optimization
+2. A series of catchain (basic consensus protocol) security improvements
+3. Update for Fift libraries and FunC: better error-handling, fixes for `catch` stack recovery
+4. A series of out message queue handling optimization (already deployed during emergency upgrades between releases)
+5. Improvement of binaries portability
+
+Besides the work of the core team, this update is based on the efforts of @aleksej-paschenko (portability improvement), [Disintar team](https://github.com/disintar/) (archive manager optimization) and [sec3-service](https://github.com/sec3-service) security auditors (funC improvements).
+
 ## 2023.04 Update
 1. CPU load optimization: previous DHT reconnect policy was too aggressive
 2. Network throughput improvements: granular control on external message broadcast, optimize celldb GC, adjust state serialization and block downloading timings, rldp2 for states and archives 

catchain/catchain-receiver-interface.h

@@ -52,6 +52,7 @@ class CatChainReceiverInterface : public td::actor::Actor {
                                           td::BufferSlice query, td::uint64 max_answer_size,
                                           td::actor::ActorId<adnl::AdnlSenderInterface> via) = 0;
   virtual void send_custom_message_data(const PublicKeyHash &dst, td::BufferSlice query) = 0;
+  virtual void on_blame_processed(td::uint32 source_id) = 0;
 
   virtual void destroy() = 0;
 

catchain/catchain-receiver-source.cpp

@@ -60,15 +60,15 @@ td::Result<std::unique_ptr<CatChainReceiverSource>> CatChainReceiverSource::crea
 
 void CatChainReceiverSourceImpl::blame(td::uint32 fork, CatChainBlockHeight height) {
   blame();
-  if (!blamed_heights_.empty()) {
-    if (blamed_heights_.size() <= fork) {
-      blamed_heights_.resize(fork + 1, 0);
-    }
-    if (blamed_heights_[fork] == 0 || blamed_heights_[fork] > height) {
-      VLOG(CATCHAIN_INFO) << this << ": blamed at " << fork << " " << height;
-      blamed_heights_[fork] = height;
-    }
+  // if (!blamed_heights_.empty()) {
+  if (blamed_heights_.size() <= fork) {
+    blamed_heights_.resize(fork + 1, 0);
+  }
+  if (blamed_heights_[fork] == 0 || blamed_heights_[fork] > height) {
+    VLOG(CATCHAIN_INFO) << this << ": blamed at " << fork << " " << height;
+    blamed_heights_[fork] = height;
   }
+  // }
 }
 
 void CatChainReceiverSourceImpl::blame() {
@@ -144,7 +144,7 @@ void CatChainReceiverSourceImpl::on_new_block(CatChainReceivedBlock *block) {
       on_found_fork_proof(create_serialize_tl_object<ton_api::catchain_block_data_fork>(block->export_tl_dep(),
                                                                                         it->second->export_tl_dep())
                               .as_slice());
-      chain_->add_prepared_event(fork_proof());
+      chain_->on_found_fork_proof(id_, fork_proof());
     }
     blame();
     return;
@@ -162,6 +162,15 @@ void CatChainReceiverSourceImpl::on_found_fork_proof(const td::Slice &proof) {
   }
 }
 
+bool CatChainReceiverSourceImpl::allow_send_block(CatChainBlockHash hash) {
+  td::uint32 count = ++block_requests_count_[hash];
+  if (count > MAX_BLOCK_REQUESTS) {
+    VLOG(CATCHAIN_INFO) << this << ": node requested block " << hash << " " << count << " times";
+    return false;
+  }
+  return true;
+}
+
 }  // namespace catchain
 
 }  // namespace ton

catchain/catchain-receiver-source.h

@@ -61,6 +61,9 @@ class CatChainReceiverSource {
   virtual td::BufferSlice fork_proof() const = 0;
   virtual bool fork_is_found() const = 0;
 
+  // One block can be sent to one node only a limited number of times to prevent DoS
+  virtual bool allow_send_block(CatChainBlockHash hash) = 0;
+
   static td::Result<std::unique_ptr<CatChainReceiverSource>> create(CatChainReceiver *chain, PublicKey pub_key,
                                                                     adnl::AdnlNodeIdShort adnl_id, td::uint32 id);
 

catchain/catchain-receiver-source.hpp

@@ -111,6 +111,8 @@ class CatChainReceiverSourceImpl : public CatChainReceiverSource {
     return chain_;
   }
 
+  bool allow_send_block(CatChainBlockHash hash) override;
+
   CatChainReceiverSourceImpl(CatChainReceiver *chain, PublicKey source, adnl::AdnlNodeIdShort adnl_id, td::uint32 id);
 
  private:
@@ -130,6 +132,11 @@ class CatChainReceiverSourceImpl : public CatChainReceiverSource {
 
   CatChainBlockHeight delivered_height_ = 0;
   CatChainBlockHeight received_height_ = 0;
+
+  std::map<CatChainBlockHash, td::uint32> block_requests_count_;
+  // One block can be sent to one node up to 5 times
+
+  static const td::uint32 MAX_BLOCK_REQUESTS = 5;
 };
 
 }  // namespace catchain