Use libp2p in catalyst (#1245)

* Start of libp2p intergration in catalyst

* Refactor LibP2P use interfaces and more DI

* Update PoaNode and Tests

* Update ip/port to use multiaddress in more places

* Update unit tests

* Update more unit tests

* Add multi address in cli

* Update proto submodule

* Update PoaConsensus intergration

* Update POA Concensus

* Update get peers by repo

* Rename PeerId to Address, Add more PeerRepo signatures, Find POA nodes using PublicKey instead of PeerId

* Get POA nodes by PublicKey

* Use string instead of bytearray for publicKey

* Create custom libp2p protocol

* Backup before updating protobuf ver

* Use catalyst-protocol libp2p in peer client for testing

* Update minimum peers

* Use multi-address instead of peerid libp2p

* Add try catch to peer client

* Use peer client broadcast method

* Remove Dto

* Revert "Remove Dto"

This reverts commit f723bbe.

* Remove Dto

* Fix sync tests

* Update sync

* Add cancellation tokens and better async

* Fix POA peer repository get peer by public key

* Blacklist peers that cannot be connected to but remove the black list entry if they connect into you

* Update Catalyst LibP2P protocol intergration tests

* Add handlers to libp2p networking

* Add handlers to broadcast

* Update peer client

* Fix bug with sending signed message

* Update POA nodes config

* Remove RPC Server and Client from Project, as EthController has been discussed as a replacement

* Comment out test to test pipeline

* Add trait to CatalystProtocol intergration tests

* Update protocol and refactor

* Comment out unit tests

* Add rpc projects back to sln

* Peer info request

* Update rpc unit tests

* Comment out unit test for verify rpc

* Comment out unit test for sign rpc

* Remove channel handler in rpc

* Update ports for Catalyst libp2p protocol

* Uncomment some todo tests and update PeerId handler to use MultiAddress

* Fix FavouriteByHashAndVoterComparerTests

* Variable name refactor

* Remove inmemory sharp repo from Catalyst.Abstraction packages

* Update protocol submodule

* Revert ignore attribute on IntergrationTest for 1241

* Remove unused DI from PeerService

* Refactor catalyst node

* Use peer client instead of broadcast manager on TransactionReceivedEvent

* Remove submodule ffi

* Uncomment depricated tests

* Refactor some codacy issues

* Refactor for codacy

* Refactor more issues for codacy

* Refactor for codacy

* Refactor for codacy

* Update for some autofac issues

.gitmodules

@@ -4,9 +4,6 @@
 [submodule "submodules/Protocol"]
 	path = submodules/Protocol
 	url = https://github.com/catalyst-network/Protocol.git
-[submodule "submodules/Cryptography.FFI.Rust"]
-	path = submodules/Cryptography.FFI.Rust
-	url = https://github.com/catalyst-network/Cryptography.FFI.Rust.git
 [submodule "docs/articles"]
 	path = docs/articles
 	url = https://github.com/catalyst-network/Catalyst.Framework.wiki.git

src/Catalyst.Abstractions/Cli/Commands/ICommandContext.cs

@@ -25,6 +25,7 @@
 using Catalyst.Abstractions.IO.Transport;
 using Catalyst.Abstractions.Rpc;
 using Catalyst.Protocol.Peer;
+using MultiFormats;
 
 namespace Catalyst.Abstractions.Cli.Commands
 {
@@ -48,7 +49,7 @@ public interface ICommandContext
 
         /// <summary>Gets the peer identifier.</summary>
         /// <value>The peer identifier.</value>
-        PeerId PeerId { get; }
+        MultiAddress Address { get; }
 
         /// <summary>Gets the connected node.</summary>
         /// <param name="nodeId">The node identifier located in configuration.</param>

src/Catalyst.Abstractions/Cli/Options/IGetPeerInfoOptions.cs

@@ -34,19 +34,11 @@ public interface IGetPeerInfoOptions
         string Node { get; set; }
 
         /// <summary>
-        /// Gets or sets the ip address.
+        /// Gets or sets the address.
         /// </summary>
         /// <value>
-        /// The ip address.
+        /// The multi address.
         /// </value>
-        string IpAddress { get; set; }
-
-        /// <summary>
-        /// Gets or sets the public key.
-        /// </summary>
-        /// <value>
-        /// The public key.
-        /// </value>
-        string PublicKey { get; set; }
+        public string Address { get; set; }
     }
 }

src/Catalyst.Abstractions/Cli/Options/IPeerBlackListingOptions.cs

@@ -35,19 +35,11 @@ public interface IPeerBlackListingOptions : IOptionsBase
         bool BlackListFlag { get; set; }
 
         /// <summary>
-        /// Gets or sets the ip address.
+        /// Gets or sets the address.
         /// </summary>
         /// <value>
-        /// The ip address.
+        /// The multi address.
         /// </value>
-        string IpAddress { get; set; }
-
-        /// <summary>
-        /// Gets or sets the public key.
-        /// </summary>
-        /// <value>
-        /// The public key.
-        /// </value>
-        string PublicKey { get; set; }
+        public string Address { get; set; }
     }
 }

src/Catalyst.Abstractions/Cli/Options/IPeerReputationOptions.cs

@@ -25,8 +25,6 @@ namespace Catalyst.Abstractions.Cli.Options
 {
     public interface IPeerReputationOptions : IOptionsBase
     {
-        string IpAddress { get; set; }
-
-        string PublicKey { get; set; }
+        public string Address { get; set; }
     }
 }

src/Catalyst.Abstractions/Cli/Options/IRemovePeerOptions.cs

@@ -25,12 +25,8 @@ namespace Catalyst.Abstractions.Cli.Options
 {
     public interface IRemovePeerOptions : IOptionsBase
     {
-        /// <summary>Gets or sets the public key.</summary>
-        /// <value>The public key.</value>
-        string PublicKey { get; set; }
-
-        /// <summary>Gets or sets the ip.</summary>
-        /// <value>The ip.</value>
-        string Ip { get; set; }
+        /// <summary>Gets or sets the address.</summary>
+        /// <value>The multi address.</value>
+        public string Address { get; set; }
     }
 }

src/Catalyst.Abstractions/Dfs/BlockExchange/IBitswapService.cs

@@ -45,7 +45,7 @@ public interface IBitswapService : IService
         /// <summary>
         ///   Provides access to other peers.
         /// </summary>
-        SwarmService SwarmService { get; set; }
+        ISwarmService SwarmService { get; set; }
 
         /// <summary>
         ///   Provides access to blocks of data.

src/Catalyst.Abstractions/Dfs/IDfsService.cs

@@ -61,21 +61,21 @@ public interface IDfsService : ICoreApi, IService, IDisposable
         /// <summary>
         ///   Manages communication with other peers.
         /// </summary>
-        SwarmService SwarmService { get; }
+        ISwarmService SwarmService { get; }
 
         /// <summary>
         ///   Manages publishng and subscribing to messages.
         /// </summary>
-        PubSubService PubSubService { get; }
+        IPubSubService PubSubService { get; }
 
         /// <summary>
         ///   Exchange blocks with other peers.
         /// </summary>
         IBitswapService BitSwapService { get; }
-        
+
         /// <summary>
         ///   Finds information with a distributed hash table.
         /// </summary>
-        DhtService DhtService { get; }
+        IDhtService DhtService { get; }
     }
 }

src/Catalyst.Abstractions/FileTransfer/IFileTransferInformation.cs

@@ -26,6 +26,7 @@
 using Catalyst.Abstractions.IO.Messaging.Correlation;
 using Catalyst.Protocol.Peer;
 using DotNetty.Transport.Channels;
+using MultiFormats;
 
 namespace Catalyst.Abstractions.FileTransfer
 {
@@ -81,11 +82,11 @@ public interface IFileTransferInformation : IDisposable
 
         /// <summary>Gets or sets the recipient identifier.</summary>
         /// <value>The recipient identifier.</value>
-        PeerId RecipientId { get; set; }
+        MultiAddress Recipient { get; set; }
 
         /// <summary>Gets or sets the peer identifier.</summary>
         /// <value>The peer identifier.</value>
-        PeerId PeerId { get; set; }
+        MultiAddress Address { get; set; }
 
         /// <summary>The cancellation token</summary>
         CancellationToken CancellationToken { get; set; }

src/Catalyst.Abstractions/Hashing/HashExtensions.cs

@@ -22,6 +22,8 @@
 #endregion
 
 using System.Buffers;
+using System.Text;
+using DotNetty.Common.Utilities;
 using Google.Protobuf;
 using MultiFormats;
 
@@ -62,6 +64,37 @@ public static MultiHash ComputeMultiHash(this IHashProvider provider, IMessage m
             }
         }
 
+        public static MultiHash ComputeMultiHash(this IHashProvider provider, string message, byte[] suffix)
+        {
+            return ComputeMultiHash(provider, Encoding.UTF8.GetBytes(message), suffix);
+        }
+
+        public static MultiHash ComputeMultiHash(this IHashProvider provider, byte[] message, byte[] suffix)
+        {
+            var calculateSize = message.Length;
+
+            var required = calculateSize + suffix.Length;
+            var array = ArrayPool<byte>.Shared.Rent(required);
+
+            try
+            {
+                message.CopyTo(array, 0);
+                suffix.CopyTo(array, calculateSize);
+
+                var result = provider.ComputeMultiHash(array, 0, required);
+                return result;
+            }
+            finally
+            {
+                ArrayPool<byte>.Shared.Return(array);
+            }
+        }
+
+        public static MultiHash ComputeMultiHash(this IHashProvider provider, MultiAddress address, byte[] suffix)
+        {
+            return ComputeMultiHash(provider, address.ToArray(), suffix);
+        }
+
         /// <summary>
         /// Serializes the <paramref name="message"/> returns the hash of it.
         /// </summary>

src/Catalyst.Abstractions/IO/Handlers/IInboundMessageHandler.cs

@@ -0,0 +1,27 @@
+#region LICENSE
+
+/**
+* Copyright (c) 2019 Catalyst Network
+*
+* This file is part of Catalyst.Node <https://github.com/catalyst-network/Catalyst.Node>
+*
+* Catalyst.Node is free software: you can redistribute it and/or modify
+* it under the terms of the GNU General Public License as published by
+* the Free Software Foundation, either version 2 of the License, or
+* (at your option) any later version.
+*
+* Catalyst.Node is distributed in the hope that it will be useful,
+* but WITHOUT ANY WARRANTY; without even the implied warranty of
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+* GNU General Public License for more details.
+*
+* You should have received a copy of the GNU General Public License
+* along with Catalyst.Node. If not, see <https://www.gnu.org/licenses/>.
+*/
+
+#endregion
+
+namespace Catalyst.Abstractions.IO.Handlers
+{
+    public interface IInboundMessageHandler : IMessageHandler { }
+}

src/Catalyst.Abstractions/IO/Handlers/IMessageHandler.cs

@@ -0,0 +1,33 @@
+#region LICENSE
+
+/**
+* Copyright (c) 2019 Catalyst Network
+*
+* This file is part of Catalyst.Node <https://github.com/catalyst-network/Catalyst.Node>
+*
+* Catalyst.Node is free software: you can redistribute it and/or modify
+* it under the terms of the GNU General Public License as published by
+* the Free Software Foundation, either version 2 of the License, or
+* (at your option) any later version.
+*
+* Catalyst.Node is distributed in the hope that it will be useful,
+* but WITHOUT ANY WARRANTY; without even the implied warranty of
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+* GNU General Public License for more details.
+*
+* You should have received a copy of the GNU General Public License
+* along with Catalyst.Node. If not, see <https://www.gnu.org/licenses/>.
+*/
+
+#endregion
+
+using Catalyst.Protocol.Wire;
+using System.Threading.Tasks;
+
+namespace Catalyst.Abstractions.IO.Handlers
+{
+    public interface IMessageHandler
+    {
+        Task<bool> ProcessAsync(ProtocolMessage message);
+    }
+}

src/Catalyst.Abstractions/IO/Handlers/IOutboundMessageHandler.cs

@@ -0,0 +1,27 @@
+#region LICENSE
+
+/**
+* Copyright (c) 2019 Catalyst Network
+*
+* This file is part of Catalyst.Node <https://github.com/catalyst-network/Catalyst.Node>
+*
+* Catalyst.Node is free software: you can redistribute it and/or modify
+* it under the terms of the GNU General Public License as published by
+* the Free Software Foundation, either version 2 of the License, or
+* (at your option) any later version.
+*
+* Catalyst.Node is distributed in the hope that it will be useful,
+* but WITHOUT ANY WARRANTY; without even the implied warranty of
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+* GNU General Public License for more details.
+*
+* You should have received a copy of the GNU General Public License
+* along with Catalyst.Node. If not, see <https://www.gnu.org/licenses/>.
+*/
+
+#endregion
+
+namespace Catalyst.Abstractions.IO.Handlers
+{
+    public interface IOutboundMessageHandler : IMessageHandler { }
+}

src/Catalyst.Abstractions/IO/Messaging/Correlation/ICacheEvictionEvent.cs

@@ -23,12 +23,13 @@
 
 using Catalyst.Protocol.Peer;
 using Google.Protobuf;
+using MultiFormats;
 
 namespace Catalyst.Abstractions.IO.Messaging.Correlation
 {
     public interface ICacheEvictionEvent<out T> where T : IMessage
     {
         T EvictedContent { get; }
-        PeerId PeerId { get; }
+        MultiAddress Address { get; }
     }
 }

src/Catalyst.Abstractions/IO/Messaging/Correlation/ICorrelatableMessage.cs

@@ -24,13 +24,14 @@
 using System;
 using Catalyst.Protocol.Peer;
 using Google.Protobuf;
+using MultiFormats;
 
 namespace Catalyst.Abstractions.IO.Messaging.Correlation
 {
     public interface ICorrelatableMessage<T> where T : IMessage 
     {
         T Content { get; set; }
-        PeerId Recipient { get; set; }
+        MultiAddress Recipient { get; set; }
         DateTimeOffset SentAt { get; set; }
     }
 }

src/Catalyst.Abstractions/IO/Messaging/Dto/IMessageDto.cs

@@ -22,16 +22,16 @@
 #endregion
 
 using Catalyst.Abstractions.IO.Messaging.Correlation;
-using Catalyst.Protocol.Peer;
 using DotNetty.Transport.Channels;
 using Google.Protobuf;
+using MultiFormats;
 
 namespace Catalyst.Abstractions.IO.Messaging.Dto
 {
     public interface IMessageDto<out T> : IAddressedEnvelope<T> where T : IMessage
     {
         ICorrelationId CorrelationId { get; }
-        PeerId RecipientPeerIdentifier { get; }
-        PeerId SenderPeerIdentifier { get; }
+        MultiAddress RecipientAddress { get; }
+        MultiAddress SenderAddress { get; }
     }
 }

src/Catalyst.Abstractions/IO/Observers/IRpcResponseObserver.cs

@@ -25,11 +25,12 @@
 using Catalyst.Protocol.Peer;
 using DotNetty.Transport.Channels;
 using Google.Protobuf;
+using MultiFormats;
 
 namespace Catalyst.Abstractions.IO.Observers
 {
     public interface IRpcResponseObserver : IResponseMessageObserver
     {
-        void HandleResponseObserver(IMessage messageDto, IChannelHandlerContext channelHandlerContext, PeerId senderPeerIdentifier, ICorrelationId correlationId);
+        void HandleResponseObserver(IMessage messageDto, IChannelHandlerContext channelHandlerContext, MultiAddress senderentifier, ICorrelationId correlationId);
     }
 }

src/Catalyst.Abstractions/IO/Transport/Channels/IChannelFactory.cs

@@ -25,14 +25,14 @@
 using System.Security.Cryptography.X509Certificates;
 using System.Threading.Tasks;
 using Catalyst.Abstractions.IO.EventLoop;
+using MultiFormats;
 
 namespace Catalyst.Abstractions.IO.Transport.Channels
 {
     public interface IChannelFactory
     {
         Task<IObservableChannel> BuildChannelAsync(IEventLoopGroupFactory eventLoopGroupFactory,
-            IPAddress targetAddress,
-            int targetPort,
+            MultiAddress address,
             X509Certificate2 certificate = null);
     }
 }

src/Catalyst.Abstractions/Keystore/IKeyApi.cs

@@ -131,9 +131,8 @@ Task<IKey> CreateAsync(string name,
         Task<IKey> ImportAsync(string name, string pem, char[] password, CancellationToken cancel = default);
 
         Task<IKey> GetKeyAsync(string self);
-        Task<string> GetIpfsPublicKeyAsync(string name, CancellationToken cancel = default);
+        Task<string> GetDfsPublicKeyAsync(string name, CancellationToken cancel = default);
 
-        Task<AsymmetricKeyParameter> GetPublicKeyAsync(string self);
         Task<AsymmetricKeyParameter> GetPrivateKeyAsync(string self);
 
         Task<byte[]> CreateProtectedDataAsync(string keyName,