Merge #3973 #3979

3973: RCD-43 cherry picks r=karknu a=karknu

## Description

Cherry pick RCD-43 fixes from release/2.0.0.

<!--- A brief description of this PR and the problem is trying to solve -->

## Linked issue
https://iohk.myjetbrains.com/youtrack/issue/RCD-43
<!--- Put here the relevant issue from YouTrack -->



3979: [DEVOPS-1134] use text backend for conenctScripts logging config (#3856) r=disassembler a=disassembler

(cherry picked from commit b49b9de)

## Description

<!--- A brief description of this PR and the problem is trying to solve -->

## Linked issue

<!--- Put here the relevant issue from YouTrack -->



Co-authored-by: Erik de Castro Lopo <erikd@mega-nerd.com>
Co-authored-by: Karl Knutsson <karl.knutsson@iohk.io>
Co-authored-by: Kosyrev Serge <serge.kosyrev@iohk.io>
Co-authored-by: Duncan Coutts <duncan.coutts@iohk.io>
Co-authored-by: Samuel Leathers <disasm@gmail.com>

db/cardano-sl-db.cabal

@@ -141,6 +141,7 @@ library
                      , ekg-core
                      , ether >= 0.5
                      , exceptions
+                     , extra
                      , filepath
                      , formatting
                      , lens

db/src/Pos/DB/Block/Epoch.hs

@@ -47,6 +47,7 @@ import           Data.Binary (decode, encode)
 import qualified Data.ByteString.Char8 as BS
 import qualified Data.ByteString.Lazy as LBS
 import           Data.Either (partitionEithers)
+import           Data.List.Extra (chunksOf)
 import           Formatting (build, int, sformat, shown, (%))
 import           System.Directory (removeFile)
 import           System.FilePath ((</>))
@@ -351,7 +352,12 @@ consolidateOneEpoch ccp epochSlots = do
 
     -- After the check point is written, delete old blunds for the epoch we have just
     -- consolidated.
-    lift $ mapM_ deleteOldBlund sihs
+    lift $ mapM_ deleter $ chunksOf 1000 sihs
+  where
+    deleter :: ConsolidateM ctx m => [SlotIndexHash] -> m ()
+    deleter xs = do
+        mapM_ deleteOldBlund xs
+        sleepSeconds 2
 
 deleteOldBlund :: ConsolidateM ctx m => SlotIndexHash -> m ()
 deleteOldBlund (SlotIndexHash _ hh) = do
@@ -377,16 +383,18 @@ consolidateEpochBlocks fpath xs = ExceptT $ do
             (liftIO . hClose)
             (\hdl -> do
                 liftIO $ BS.hPutStr hdl epochFileHeader
-                mapM (consolidate hdl) xs
+                mapM (consolidate hdl) $ zip [0 .. ] xs
                 )
     pure $ case partitionEithers ys of
             ([], zs) -> Right $ epochIndexToOffset zs
             (e:_, _) -> Left e
   where
     consolidate
         :: ConsolidateM ctx m
-        => Handle -> SlotIndexHash -> m (Either ConsolidateError SlotIndexLength)
-    consolidate hdl  (SlotIndexHash lsi hh) = do
+        => Handle -> (Int, SlotIndexHash) -> m (Either ConsolidateError SlotIndexLength)
+    consolidate hdl  (indx, SlotIndexHash lsi hh) = do
+        when (indx `mod` 1000 == 0) $
+            sleepSeconds 2
         mblund <- getSerializedBlund hh
         case mblund of
             Nothing ->

db/src/Pos/DB/Rocks/Functions.hs

@@ -41,6 +41,7 @@ import qualified Database.RocksDB as Rocks
 import           System.Directory (createDirectoryIfMissing, doesDirectoryExist,
                      removeDirectoryRecursive)
 import           System.FilePath (takeDirectory, (</>))
+import qualified System.Info (os)
 
 import           Pos.Binary.Class (Bi, serialize')
 import           Pos.DB.BatchOp (rocksWriteBatch)
@@ -82,6 +83,7 @@ openNodeDBs recreate fp = do
     let gStatePath = fp </> "gState"
     let lrcPath = fp </> "lrc"
     let miscPath = fp </> "misc"
+
     mapM_ ensureDirectoryExists
         [ blocksDir
         , _blockDataDir
@@ -91,6 +93,11 @@ openNodeDBs recreate fp = do
         , lrcPath
         , miscPath
         ]
+
+    when (System.Info.os == "darwin") $
+        -- Prevent indexing of blocks on OSX
+        ensureEmptyFileExists (fp </> ".metadata_never_index")
+
     _blockIndexDB <- openRocksDB blocksIndexPath
     _gStateDB <- openRocksDB gStatePath
     _lrcDB <- openRocksDB lrcPath
@@ -101,6 +108,9 @@ openNodeDBs recreate fp = do
     ensureDirectoryExists :: MonadIO m => FilePath -> m ()
     ensureDirectoryExists = liftIO . createDirectoryIfMissing True
 
+    ensureEmptyFileExists :: MonadIO m => FilePath -> m ()
+    ensureEmptyFileExists file = liftIO $ withFile file AppendMode (\_ -> return ())
+
 -- | Safely close all databases from 'NodeDBs'.
 closeNodeDBs :: MonadIO m => NodeDBs -> m ()
 closeNodeDBs NodeDBs {..} =

lib/src/Pos/Diffusion/Full/Block.hs

@@ -724,6 +724,7 @@ handleStreamStart logTrace logic oq = listenerConv logTrace oq $ \__ourVerInfo n
                       lift $ traceWith logTrace (Debug, sformat ("handleStreamStart:loop MsgStart, expected MsgStreamUpdate from "%build) nodeId)
                       return ()
                   MsgUpdate u -> do
+                      lift $ OQ.clearFailureOf oq nodeId
                       lift $ traceWith logTrace (Debug, sformat ("handleStreamStart:loop new window "%shown%" from "%build) u nodeId)
                       loop nodeId conv (msuWindow u)
     loop nodeId conv window =

log-configs/connect-to-cluster.yaml

@@ -4,8 +4,19 @@
 rotation:
     logLimit: 104857600 # 100MB
     keepFiles: 100
+
 loggerTree:
-  severity: Debug+
-  files:
-    - node.pub
-    - node
+  severity: Info+
+
+  handlers:
+    - { name: "Public"
+      , filepath: "pub/node.log"
+      , logsafety: PublicLogLevel
+      , severity: Debug+
+      , backend: FileTextBE }
+    - { name: "Secret"
+      , filepath: "node"
+      , logsafety: SecretLogLevel
+      , severity: Info+
+      , backend: FileTextBE }
+

pkgs/default.nix

@@ -15006,6 +15006,7 @@ license = stdenv.lib.licenses.mit;
 , ekg-core
 , ether
 , exceptions
+, extra
 , filepath
 , formatting
 , hedgehog
@@ -15059,6 +15060,7 @@ directory
 ekg-core
 ether
 exceptions
+extra
 filepath
 formatting
 lens