[neox] Merge master (#1788)

* Allow call from native contract (#1700)

* Fix MethodCallback (#1723)

* Fix #1714 (#1715)

* Add base64 SYSCALLs (#1717)

* Neo.VM.3.0.0-CI00230 (#1725)

* Allow to iterate buffer (#1726)

* Buffer iterator

* Rename

* Remove using

Co-authored-by: Erik Zhang <erik@neo.org>

* Update StorageContext.cs (#1728)

StorageContext needs to be public so it can be accessed by Neo Debugger

Co-authored-by: Erik Zhang <erik@neo.org>

* Fix return value check (#1730)

Co-authored-by: Shargon <shargon@gmail.com>

* Fix ContractEventDescriptor (#1733)

* fix enumerator (#1744)

Co-authored-by: Tommo-L <luchuan@neo.org>

* Change json fields to all lower case for consistency (#1736)

* Replace DataCache.Find by DataCache.Seek (#1740)

* replace DataCache.Find by DataCache.Seek

* fix order

* optimize

* Update ByteArrayComparer.cs

* Update ByteArrayComparer.cs

* Update DataCache.cs

* fix comments

* fix comments

* fix comments

* Update DataCache.cs

* Update DataCache.cs

* Reorder methods

Co-authored-by: Tommo-L <luchuan@neo.org>
Co-authored-by: erikzhang <erik@neo.org>
Co-authored-by: Shargon <shargon@gmail.com>

* Add MaxVerificationGas (#1745)

Co-authored-by: Shargon <shargon@gmail.com>

* Create KeyBuilder (#1748)

* Speed up the initialization of ApplicationEngine. (#1749)

* Change nef checksum to double SHA256 (#1751)

* Change to double SHA256

* Clean code

* Revert change

* Clean code

* Fix checksum

* Update NefFile.cs

* Fix compile

* Clean code

Clean code
Fix checksum
Update NefFile.cs
Fix compile

* Optimize

* Optimize

* Fix

Co-authored-by: erikzhang <erik@neo.org>

* Check witnesses on isStandard (#1754)

* Check witness on isStandard

* Add IsDeployed

* Add IsPayable

* Fix UT

* format

* Add coverage

* Remove IsPayable, IsDeployed

* Move NEP10 to manifest (#1729)

* NEP10 abi

* To lower case

* Move to Manifest

* Clean code

* Update ContractManifest.cs

* Update ContractManifest.cs

* Fix native contracts

Co-authored-by: Erik Zhang <erik@neo.org>

* Capture fault Exception (#1761)

* Capture faultException

* Update Wallet error

* Remove flag check

* Clean code

* Sender from signers (#1752)

* Sender from signers

* Remove co-

* Move signers outside attributes

* Fix UT and remove Signers class

* Fix UT

* Add FeeOnly scope

* Remove orderBy

* Remove _signersCache

* Fix Signers

* Fix WitnessScope

* Fix Sender

* Update TransactionAttributeType.cs

* Update Wallet.cs

* Fix Wallet

* Rename

* Update Wallet.cs

* Update Wallet.cs

* Partial UT fix

* More UT fixes

* Fix Sender's WitnessScope

* Fix Wallet

* Fix UT

* Explicit FeeOnly for DeployNativeContracts

* Same order as serialization

* Test FeeOnly

* dotnet format

* format

Co-authored-by: Erik Zhang <erik@neo.org>

* IApplicationEngineProvider (#1758)

* Remove the lock from SendersFeeMonitor and rename it to TransactionVerificationContext (#1756)

* Add EffectiveVoterTurnout (#1762)

* Remove AllowedTriggers from SYSCALLs (#1755)

* fix validatorscount (#1770)

Co-authored-by: Tommo-L <luchuan@neo.org>

* impl SeekInternal

Co-authored-by: Erik Zhang <erik@neo.org>
Co-authored-by: Shargon <shargon@gmail.com>
Co-authored-by: Harry Pierson <harrypierson@hotmail.com>
Co-authored-by: Luchuan <luchuan@ngd.neo.org>
Co-authored-by: Tommo-L <luchuan@neo.org>
Co-authored-by: joeqian <qianzhuo@ngd.neo.org>

src/neo/Consensus/ConsensusContext.cs

@@ -41,7 +41,7 @@ internal class ConsensusContext : IDisposable, ISerializable
         /// <summary>
         /// Store all verified unsorted transactions' senders' fee currently in the consensus context.
         /// </summary>
-        public SendersFeeMonitor SendersFeeMonitor = new SendersFeeMonitor();
+        public TransactionVerificationContext VerificationContext = new TransactionVerificationContext();
 
         public SnapshotView Snapshot { get; private set; }
         private KeyPair keyPair;
@@ -130,18 +130,18 @@ public void Deserialize(BinaryReader reader)
             ViewNumber = reader.ReadByte();
             TransactionHashes = reader.ReadSerializableArray<UInt256>();
             Transaction[] transactions = reader.ReadSerializableArray<Transaction>(Block.MaxTransactionsPerBlock);
-            PreparationPayloads = reader.ReadNullableArray<ConsensusPayload>(ProtocolSettings.Default.MaxValidatorsCount);
-            CommitPayloads = reader.ReadNullableArray<ConsensusPayload>(ProtocolSettings.Default.MaxValidatorsCount);
-            ChangeViewPayloads = reader.ReadNullableArray<ConsensusPayload>(ProtocolSettings.Default.MaxValidatorsCount);
-            LastChangeViewPayloads = reader.ReadNullableArray<ConsensusPayload>(ProtocolSettings.Default.MaxValidatorsCount);
+            PreparationPayloads = reader.ReadNullableArray<ConsensusPayload>(ProtocolSettings.Default.ValidatorsCount);
+            CommitPayloads = reader.ReadNullableArray<ConsensusPayload>(ProtocolSettings.Default.ValidatorsCount);
+            ChangeViewPayloads = reader.ReadNullableArray<ConsensusPayload>(ProtocolSettings.Default.ValidatorsCount);
+            LastChangeViewPayloads = reader.ReadNullableArray<ConsensusPayload>(ProtocolSettings.Default.ValidatorsCount);
             if (TransactionHashes.Length == 0 && !RequestSentOrReceived)
                 TransactionHashes = null;
             Transactions = transactions.Length == 0 && !RequestSentOrReceived ? null : transactions.ToDictionary(p => p.Hash);
-            SendersFeeMonitor = new SendersFeeMonitor();
+            VerificationContext = new TransactionVerificationContext();
             if (Transactions != null)
             {
                 foreach (Transaction tx in Transactions.Values)
-                    SendersFeeMonitor.AddSenderFee(tx);
+                    VerificationContext.AddTransaction(tx);
             }
         }
 
@@ -301,7 +301,7 @@ internal void EnsureMaxBlockLimitation(IEnumerable<Transaction> txs)
             txs = txs.Take((int)maxTransactionsPerBlock);
             List<UInt256> hashes = new List<UInt256>();
             Transactions = new Dictionary<UInt256, Transaction>();
-            SendersFeeMonitor = new SendersFeeMonitor();
+            VerificationContext = new TransactionVerificationContext();
 
             // Expected block size
             var blockSize = GetExpectedBlockSizeWithoutTransactions(txs.Count());
@@ -320,7 +320,7 @@ internal void EnsureMaxBlockLimitation(IEnumerable<Transaction> txs)
 
                 hashes.Add(tx.Hash);
                 Transactions.Add(tx.Hash, tx);
-                SendersFeeMonitor.AddSenderFee(tx);
+                VerificationContext.AddTransaction(tx);
             }
 
             TransactionHashes = hashes.ToArray();

src/neo/Consensus/ConsensusService.cs

@@ -64,20 +64,24 @@ internal ConsensusService(IActorRef localNode, IActorRef taskManager, IActorRef
 
         private bool AddTransaction(Transaction tx, bool verify)
         {
-            if (verify && tx.Verify(context.Snapshot, context.SendersFeeMonitor.GetSenderFee(tx.Sender)) != VerifyResult.Succeed)
+            if (verify)
             {
-                Log($"Invalid transaction: {tx.Hash}{Environment.NewLine}{tx.ToArray().ToHexString()}", LogLevel.Warning);
-                RequestChangeView(ChangeViewReason.TxInvalid);
-                return false;
-            }
-            if (!NativeContract.Policy.CheckPolicy(tx, context.Snapshot))
-            {
-                Log($"reject tx: {tx.Hash}{Environment.NewLine}{tx.ToArray().ToHexString()}", LogLevel.Warning);
-                RequestChangeView(ChangeViewReason.TxRejectedByPolicy);
-                return false;
+                VerifyResult result = tx.Verify(context.Snapshot, context.VerificationContext);
+                if (result == VerifyResult.PolicyFail)
+                {
+                    Log($"reject tx: {tx.Hash}{Environment.NewLine}{tx.ToArray().ToHexString()}", LogLevel.Warning);
+                    RequestChangeView(ChangeViewReason.TxRejectedByPolicy);
+                    return false;
+                }
+                else if (result != VerifyResult.Succeed)
+                {
+                    Log($"Invalid transaction: {tx.Hash}{Environment.NewLine}{tx.ToArray().ToHexString()}", LogLevel.Warning);
+                    RequestChangeView(ChangeViewReason.TxInvalid);
+                    return false;
+                }
             }
             context.Transactions[tx.Hash] = tx;
-            context.SendersFeeMonitor.AddSenderFee(tx);
+            context.VerificationContext.AddTransaction(tx);
             return CheckPrepareResponse();
         }
 
@@ -443,7 +447,7 @@ private void OnPrepareRequestReceived(ConsensusPayload payload, PrepareRequest m
             context.Block.ConsensusData.Nonce = message.Nonce;
             context.TransactionHashes = message.TransactionHashes;
             context.Transactions = new Dictionary<UInt256, Transaction>();
-            context.SendersFeeMonitor = new SendersFeeMonitor();
+            context.VerificationContext = new TransactionVerificationContext();
             for (int i = 0; i < context.PreparationPayloads.Length; i++)
                 if (context.PreparationPayloads[i] != null)
                     if (!context.PreparationPayloads[i].GetDeserializedMessage<PrepareResponse>().PreparationHash.Equals(payload.Hash))

src/neo/Consensus/RecoveryMessage.cs

@@ -31,7 +31,7 @@ public RecoveryMessage() : base(ConsensusMessageType.RecoveryMessage)
         public override void Deserialize(BinaryReader reader)
         {
             base.Deserialize(reader);
-            ChangeViewMessages = reader.ReadSerializableArray<ChangeViewPayloadCompact>(ProtocolSettings.Default.MaxValidatorsCount).ToDictionary(p => (int)p.ValidatorIndex);
+            ChangeViewMessages = reader.ReadSerializableArray<ChangeViewPayloadCompact>(ProtocolSettings.Default.ValidatorsCount).ToDictionary(p => (int)p.ValidatorIndex);
             if (reader.ReadBoolean())
                 PrepareRequestMessage = reader.ReadSerializable<PrepareRequest>();
             else
@@ -41,8 +41,8 @@ public override void Deserialize(BinaryReader reader)
                     PreparationHash = new UInt256(reader.ReadFixedBytes(preparationHashSize));
             }
 
-            PreparationMessages = reader.ReadSerializableArray<PreparationPayloadCompact>(ProtocolSettings.Default.MaxValidatorsCount).ToDictionary(p => (int)p.ValidatorIndex);
-            CommitMessages = reader.ReadSerializableArray<CommitPayloadCompact>(ProtocolSettings.Default.MaxValidatorsCount).ToDictionary(p => (int)p.ValidatorIndex);
+            PreparationMessages = reader.ReadSerializableArray<PreparationPayloadCompact>(ProtocolSettings.Default.ValidatorsCount).ToDictionary(p => (int)p.ValidatorIndex);
+            CommitMessages = reader.ReadSerializableArray<CommitPayloadCompact>(ProtocolSettings.Default.ValidatorsCount).ToDictionary(p => (int)p.ValidatorIndex);
         }
 
         internal ConsensusPayload[] GetChangeViewPayloads(ConsensusContext context, ConsensusPayload payload)

src/neo/IO/ByteArrayComparer.cs

@@ -1,13 +1,30 @@
 using System;
 using System.Collections.Generic;
+using System.Runtime.CompilerServices;
 
 namespace Neo.IO
 {
     internal class ByteArrayComparer : IComparer<byte[]>
     {
-        public static readonly ByteArrayComparer Default = new ByteArrayComparer();
+        public static readonly ByteArrayComparer Default = new ByteArrayComparer(1);
+        public static readonly ByteArrayComparer Reverse = new ByteArrayComparer(-1);
+
+        private readonly int direction;
+
+        private ByteArrayComparer(int direction)
+        {
+            this.direction = direction;
+        }
 
         public int Compare(byte[] x, byte[] y)
+        {
+            return direction > 0
+                ? CompareInternal(x, y)
+                : -CompareInternal(x, y);
+        }
+
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        private static int CompareInternal(byte[] x, byte[] y)
         {
             int length = Math.Min(x.Length, y.Length);
             for (int i = 0; i < length; i++)

src/neo/IO/Caching/CloneCache.cs

@@ -24,15 +24,15 @@ protected override void DeleteInternal(TKey key)
             innerCache.Delete(key);
         }
 
-        protected override IEnumerable<(TKey, TValue)> FindInternal(byte[] key_prefix)
+        protected override TValue GetInternal(TKey key)
         {
-            foreach (var (key, value) in innerCache.Find(key_prefix))
-                yield return (key, value.Clone());
+            return innerCache[key].Clone();
         }
 
-        protected override TValue GetInternal(TKey key)
+        protected override IEnumerable<(TKey, TValue)> SeekInternal(byte[] keyOrPreifx, SeekDirection direction)
         {
-            return innerCache[key].Clone();
+            foreach (var (key, value) in innerCache.Seek(keyOrPreifx, direction))
+                yield return (key, value.Clone());
         }
 
         protected override TValue TryGetInternal(TKey key)

src/neo/IO/Caching/DataCache.cs

@@ -151,57 +151,18 @@ public void Delete(TKey key)
         /// <returns>Entries found with the desired prefix</returns>
         public IEnumerable<(TKey Key, TValue Value)> Find(byte[] key_prefix = null)
         {
-            IEnumerable<(byte[], TKey, TValue)> cached;
-            HashSet<TKey> cachedKeySet;
-            lock (dictionary)
-            {
-                cached = dictionary
-                    .Where(p => p.Value.State != TrackState.Deleted && (key_prefix == null || p.Key.ToArray().AsSpan().StartsWith(key_prefix)))
-                    .Select(p =>
-                    (
-                        KeyBytes: p.Key.ToArray(),
-                        p.Key,
-                        p.Value.Item
-                    ))
-                    .OrderBy(p => p.KeyBytes, ByteArrayComparer.Default)
-                    .ToArray();
-                cachedKeySet = new HashSet<TKey>(dictionary.Keys);
-            }
-            var uncached = FindInternal(key_prefix ?? Array.Empty<byte>())
-                .Where(p => !cachedKeySet.Contains(p.Key))
-                .Select(p =>
-                (
-                    KeyBytes: p.Key.ToArray(),
-                    p.Key,
-                    p.Value
-                ));
-            using (var e1 = cached.GetEnumerator())
-            using (var e2 = uncached.GetEnumerator())
-            {
-                (byte[] KeyBytes, TKey Key, TValue Item) i1, i2;
-                bool c1 = e1.MoveNext();
-                bool c2 = e2.MoveNext();
-                i1 = c1 ? e1.Current : default;
-                i2 = c2 ? e2.Current : default;
-                while (c1 || c2)
-                {
-                    if (!c2 || (c1 && ByteArrayComparer.Default.Compare(i1.KeyBytes, i2.KeyBytes) < 0))
-                    {
-                        yield return (i1.Key, i1.Item);
-                        c1 = e1.MoveNext();
-                        i1 = c1 ? e1.Current : default;
-                    }
-                    else
-                    {
-                        yield return (i2.Key, i2.Item);
-                        c2 = e2.MoveNext();
-                        i2 = c2 ? e2.Current : default;
-                    }
-                }
-            }
+            foreach (var (key, value) in Seek(key_prefix, SeekDirection.Forward))
+                if (key.ToArray().AsSpan().StartsWith(key_prefix))
+                    yield return (key, value);
         }
 
-        protected abstract IEnumerable<(TKey Key, TValue Value)> FindInternal(byte[] key_prefix);
+        public IEnumerable<(TKey Key, TValue Value)> FindRange(TKey start, TKey end)
+        {
+            var endKey = end.ToArray();
+            foreach (var (key, value) in Seek(start.ToArray(), SeekDirection.Forward))
+                if (ByteArrayComparer.Default.Compare(key.ToArray(), endKey) < 0)
+                    yield return (key, value);
+        }
 
         public IEnumerable<Trackable> GetChangeSet()
         {
@@ -299,6 +260,67 @@ public TValue GetOrAdd(TKey key, Func<TValue> factory)
             }
         }
 
+        /// <summary>
+        /// Seek to the entry with specific key
+        /// </summary>
+        /// <param name="keyOrPrefix">The key to be sought</param>
+        /// <param name="direction">The direction of seek</param>
+        /// <returns>An enumerator containing all the entries after seeking.</returns>
+        public IEnumerable<(TKey Key, TValue Value)> Seek(byte[] keyOrPrefix = null, SeekDirection direction = SeekDirection.Forward)
+        {
+            IEnumerable<(byte[], TKey, TValue)> cached;
+            HashSet<TKey> cachedKeySet;
+            ByteArrayComparer comparer = direction == SeekDirection.Forward ? ByteArrayComparer.Default : ByteArrayComparer.Reverse;
+            lock (dictionary)
+            {
+                cached = dictionary
+                    .Where(p => p.Value.State != TrackState.Deleted && (keyOrPrefix == null || comparer.Compare(p.Key.ToArray(), keyOrPrefix) >= 0))
+                    .Select(p =>
+                    (
+                        KeyBytes: p.Key.ToArray(),
+                        p.Key,
+                        p.Value.Item
+                    ))
+                    .OrderBy(p => p.KeyBytes, comparer)
+                    .ToArray();
+                cachedKeySet = new HashSet<TKey>(dictionary.Keys);
+            }
+            var uncached = SeekInternal(keyOrPrefix ?? Array.Empty<byte>(), direction)
+                .Where(p => !cachedKeySet.Contains(p.Key))
+                .Select(p =>
+                (
+                    KeyBytes: p.Key.ToArray(),
+                    p.Key,
+                    p.Value
+                ));
+            using (var e1 = cached.GetEnumerator())
+            using (var e2 = uncached.GetEnumerator())
+            {
+                (byte[] KeyBytes, TKey Key, TValue Item) i1, i2;
+                bool c1 = e1.MoveNext();
+                bool c2 = e2.MoveNext();
+                i1 = c1 ? e1.Current : default;
+                i2 = c2 ? e2.Current : default;
+                while (c1 || c2)
+                {
+                    if (!c2 || (c1 && comparer.Compare(i1.KeyBytes, i2.KeyBytes) < 0))
+                    {
+                        yield return (i1.Key, i1.Item);
+                        c1 = e1.MoveNext();
+                        i1 = c1 ? e1.Current : default;
+                    }
+                    else
+                    {
+                        yield return (i2.Key, i2.Item);
+                        c2 = e2.MoveNext();
+                        i2 = c2 ? e2.Current : default;
+                    }
+                }
+            }
+        }
+
+        protected abstract IEnumerable<(TKey Key, TValue Value)> SeekInternal(byte[] keyOrPrefix, SeekDirection direction);
+
         public TValue TryGet(TKey key)
         {
             lock (dictionary)

src/neo/IO/Caching/SeekDirection.cs

@@ -0,0 +1,8 @@
+namespace Neo.IO.Caching
+{
+    public enum SeekDirection : sbyte
+    {
+        Forward = 1,
+        Backward = -1
+    }
+}

src/neo/Ledger/Blockchain.cs

@@ -32,8 +32,8 @@ public class RelayResult { public IInventory Inventory; public VerifyResult Resu
         public const uint DecrementInterval = 2000000;
         public static readonly uint[] GenerationAmount = { 6, 5, 4, 3, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 };
         public static readonly TimeSpan TimePerBlock = TimeSpan.FromMilliseconds(MillisecondsPerBlock);
-        public static readonly ECPoint[] StandbyCommittee = ProtocolSettings.Default.StandbyCommittee.Take(ProtocolSettings.Default.MaxCommitteeMembersCount).Select(p => ECPoint.DecodePoint(p.HexToBytes(), ECCurve.Secp256r1)).ToArray();
-        public static readonly ECPoint[] StandbyValidators = StandbyCommittee.Take(ProtocolSettings.Default.MaxValidatorsCount).ToArray();
+        public static readonly ECPoint[] StandbyCommittee = ProtocolSettings.Default.StandbyCommittee.Select(p => ECPoint.DecodePoint(p.HexToBytes(), ECCurve.Secp256r1)).ToArray();
+        public static readonly ECPoint[] StandbyValidators = StandbyCommittee[0..ProtocolSettings.Default.ValidatorsCount];
 
         public static readonly Block GenesisBlock = new Block
         {
@@ -166,8 +166,15 @@ private static Transaction DeployNativeContracts()
             {
                 Version = 0,
                 Script = script,
-                Sender = (new[] { (byte)OpCode.PUSH1 }).ToScriptHash(),
                 SystemFee = 0,
+                Signers = new[]
+                {
+                    new Signer
+                    {
+                        Account = (new[] { (byte)OpCode.PUSH1 }).ToScriptHash(),
+                        Scopes = WitnessScope.FeeOnly
+                    }
+                },
                 Attributes = Array.Empty<TransactionAttribute>(),
                 Witnesses = new[]
                 {
@@ -283,15 +290,10 @@ private void OnFillMemoryPool(IEnumerable<Transaction> transactions)
             {
                 if (View.ContainsTransaction(tx.Hash))
                     continue;
-                if (!NativeContract.Policy.CheckPolicy(tx, currentSnapshot))
-                    continue;
                 // First remove the tx if it is unverified in the pool.
                 MemPool.TryRemoveUnVerified(tx.Hash, out _);
-                // Verify the the transaction
-                if (tx.Verify(currentSnapshot, MemPool.SendersFeeMonitor.GetSenderFee(tx.Sender)) != VerifyResult.Succeed)
-                    continue;
                 // Add to the memory pool
-                MemPool.TryAdd(tx.Hash, tx);
+                MemPool.TryAdd(tx, currentSnapshot);
             }
             // Transactions originally in the pool will automatically be reverified based on their priority.
 
@@ -355,11 +357,7 @@ private VerifyResult OnNewInventory(IInventory inventory)
         private VerifyResult OnNewTransaction(Transaction transaction)
         {
             if (ContainsTransaction(transaction.Hash)) return VerifyResult.AlreadyExists;
-            if (!MemPool.CanTransactionFitInPool(transaction)) return VerifyResult.OutOfMemory;
-            VerifyResult reason = transaction.Verify(currentSnapshot, MemPool.SendersFeeMonitor.GetSenderFee(transaction.Sender));
-            if (reason != VerifyResult.Succeed) return reason;
-            if (!MemPool.TryAdd(transaction.Hash, transaction)) return VerifyResult.OutOfMemory;
-            return VerifyResult.Succeed;
+            return MemPool.TryAdd(transaction, currentSnapshot);
         }
 
         protected override void OnReceive(object message)
@@ -404,7 +402,7 @@ private void Persist(Block block)
                 snapshot.PersistingBlock = block;
                 if (block.Index > 0)
                 {
-                    using (ApplicationEngine engine = new ApplicationEngine(TriggerType.System, null, snapshot, 0, true))
+                    using (ApplicationEngine engine = ApplicationEngine.Create(TriggerType.System, null, snapshot, 0, true))
                     {
                         engine.LoadScript(onPersistNativeContractScript);
                         if (engine.Execute() != VMState.HALT) throw new InvalidOperationException();
@@ -427,7 +425,7 @@ private void Persist(Block block)
                     clonedSnapshot.Transactions.Add(tx.Hash, state);
                     clonedSnapshot.Transactions.Commit();
 
-                    using (ApplicationEngine engine = new ApplicationEngine(TriggerType.Application, tx, clonedSnapshot, tx.SystemFee))
+                    using (ApplicationEngine engine = ApplicationEngine.Create(TriggerType.Application, tx, clonedSnapshot, tx.SystemFee))
                     {
                         engine.LoadScript(tx.Script);
                         state.VMState = engine.Execute();