[move-lang] Table extension.

This adds tables as an extension to Move, using the new VM extension mechanism.

Tables (and other future extensions) are suggested to live in the `language/extensions` tree. They come with a Move library and a Rust implementation of the table extension data, including changeset representation, and a native function table.

The CLI and unit testing framework has been extended to support extensions. This is controlled by a compile-time feature flag `table-extension`. The mechanism is generic enough to also support future extensions.

A fairly complete test suite has been copied over from the EVM project.  However, while functionality should be covered, Move unit tests do not support testing of change set generation and remote table resolver access, so this aspect stays untested. For this integration tests are needed.

Closes: #150

Cargo.toml

@@ -10,6 +10,7 @@ members = [
     "language/evm/exec-utils",
     "language/evm/move-to-yul",
     "language/extensions/async/move-async-vm",
+    "language/extensions/move-table-extension",
     "language/move-analyzer",
     "language/move-binary-format",
     "language/move-binary-format/serializer-tests",

language/extensions/README.md

@@ -0,0 +1,5 @@
+This tree contains extensions to the Move core language and runtime. Those extensions are not enabled by
+default but can be integrated into a Move-based environment following the instructions found for each individual
+extension.
+
+Some extensions may eventually become part of the core language after some period of stabilization.

language/extensions/move-table-extension/Cargo.toml

@@ -0,0 +1,35 @@
+[package]
+name = "move-table-extension"
+version = "0.1.0"
+authors = ["Diem Association <opensource@diem.com>"]
+description = "Wrapper for the Move VM which coordinates multiple extensions"
+repository = "https://github.com/diem/move"
+license = "Apache-2.0"
+edition = "2018"
+publish = false
+
+[dependencies]
+anyhow = "1.0.52"
+downcast-rs = "1.2.0"
+walkdir = "2.3.1"
+itertools = "0.10.0"
+smallvec = "1.6.1"
+bcs = "0.1.2"
+sha3 = "0.9.1"
+once_cell = "1.7.2"
+move-command-line-common = { path = "../../move-command-line-common" }
+move-core-types = { path = "../../move-core/types" }
+move-compiler = { path = "../../move-compiler" }
+move-vm-types = { path = "../../move-vm/types" }
+move-vm-runtime = { path = "../../move-vm/runtime", features = ["debugging"] }
+move-binary-format = { path = "../../move-binary-format" }
+workspace-hack = { version = "0.1", path = "../../../crates/workspace-hack" }
+
+[dev-dependencies]
+move-stdlib = { path = "../../move-stdlib", features = ["testing"] }
+move-unit-test = { path = "../../tools/move-unit-test", features = ["table-extension"] }
+tempfile = "3.2.0"
+#dir-diff = "0.3.2"
+#file_diff = "1.0.0"
+move-cli = { path = "../../tools/move-cli" }
+move-package = { path = "../../tools/move-package" }

language/extensions/move-table-extension/Move.toml

@@ -0,0 +1,14 @@
+[package]
+name = "MoveTableExtension"
+version = "1.0.0"
+
+[addresses]
+Std = "_"
+Extensions = "_"
+
+[dev-addresses]
+Std = "0x1"
+Extensions = "0x2"
+
+[dependencies]
+MoveStdlib = { local = "../../move-stdlib" }

language/extensions/move-table-extension/README.md

@@ -0,0 +1,36 @@
+This crate contains an extension to the Move language with large-scale storage tables.
+
+In order to use this extension with the Move CLI and package system, you need to compile with
+`feature = ["table-extension"]`.
+
+In order to use this extension in your adapter, you do something as follows:
+
+```rust
+use move_core_types::account_address::AccountAddress;
+use move_stdlib::natives;
+use move_table_extension::NativeTableContext;
+use move_vm_runtime::move_vm::MoveVM;
+use move_vm_runtime::native_functions::NativeContextExtensions;
+
+fn run() {
+    let resource_resolver = unimplemented!(); // a resource resolver the adapter provides
+    let txn_hash = unimplemented!(); // a unique hash for table creation for this transaction
+    let table_resolver = unimplemented!(); // a remote table resover the adapter provides
+    let std_addr = unimplemented!(); // address where to deploy the std lib
+    let extension_addr = unimplemented!(); // address where to deploy the table extension
+
+    let mut extensions = NativeContextExtensions::default();
+    extensions.add(NativeTableContext::new(txn_hash, table_resolver));
+    let mut natives = move_stdlib::natives::all_natives(std_addr);
+    natives.append(&mut move_table_extension::table_natives(extension_addr));
+    let vm = MoveVM::new(natives);
+
+    let session = vm.new_session_with_extensions(resource_resolver, extensions);
+    let result = session.execute_function(..)?;
+    let (change_set, events, extensions) = session.finish_with_extensions()?;
+    let table_change_set = extensions.get::<NativeTableContext>().into_change_set();
+
+    // Do something with the table change set
+    // ...
+}
+```

language/extensions/move-table-extension/sources/Table.move

@@ -0,0 +1,94 @@
+/// Type of large-scale storage tables.
+module Extensions::Table {
+    /// Type of tables
+    struct Table<phantom K, phantom V> has store {
+        handle: u128
+    }
+
+    /// Create a new Table.
+    public fun new<K, V: store>(): Table<K, V> {
+        Table{handle: new_table_handle()}
+    }
+
+    /// Destroy a table. The table must be empty to succeed.
+    public fun destroy_empty<K, V>(table: Table<K, V>) {
+        destroy_empty_box<K, V, Box<V>>(&table);
+        drop_unchecked_box<K, V, Box<V>>(table)
+    }
+
+    /// Add a new entry to the table. Aborts if an entry for this
+    /// key already exists. The entry itself is not stored in the
+    /// table, and cannot be discovered from it.
+    public fun add<K, V>(table: &mut Table<K, V>, key: &K, val: V) {
+        add_box<K, V, Box<V>>(table, key, Box{val})
+    }
+
+    /// Acquire an immutable reference to the value which `key` maps to.
+    /// Aborts if there is no entry for `key`.
+    public fun borrow<K, V>(table: &Table<K, V>, key: &K): &V {
+        &borrow_box<K, V, Box<V>>(table, key).val
+    }
+
+    /// Acquire a mutable reference to the value which `key` maps to.
+    /// Aborts if there is no entry for `key`.
+    public fun borrow_mut<K, V>(table: &mut Table<K, V>, key: &K): &mut V {
+        &mut borrow_box_mut<K, V, Box<V>>(table, key).val
+    }
+
+    /// Returns the length of the table, i.e. the number of entries.
+    public fun length<K, V>(table: &Table<K, V>): u64 {
+        length_box<K, V, Box<V>>(table)
+    }
+
+    /// Returns true if this table is empty.
+    public fun empty<K, V>(table: &Table<K, V>): bool {
+        length(table) == 0
+    }
+
+    /// Acquire a mutable reference to the value which `key` maps to.
+    /// Insert the pair (`key`, `default`) first if there is no entry for `key`.
+    public fun borrow_mut_with_default<K, V: drop>(table: &mut Table<K, V>, key: &K, default: V): &mut V {
+        if (!contains(table, key)) {
+            add(table, key, default)
+        };
+        borrow_mut(table, key)
+    }
+
+    /// Remove from `table` and return the value which `key` maps to.
+    /// Aborts if there is no entry for `key`.
+    public fun remove<K, V>(table: &mut Table<K, V>, key: &K): V {
+        let Box{val} = remove_box<K, V, Box<V>>(table, key);
+        val
+    }
+
+    /// Returns true iff `table` contains an entry for `key`.
+    public fun contains<K, V>(table: &Table<K, V>, key: &K): bool {
+        contains_box<K, V, Box<V>>(table, key)
+    }
+
+    #[test_only]
+    /// Testing only: allows to drop a table even if it is not empty.
+    public fun drop_unchecked<K, V>(table: Table<K, V>) {
+        drop_unchecked_box<K, V, Box<V>>(table)
+    }
+
+    // ======================================================================================================
+    // Internal API
+
+    /// Wrapper for values. Required for making values appear as resources in the implementation.
+    struct Box<V> has key, drop, store {
+        val: V
+    }
+
+    // Primitives which take as an additional type parameter `Box<V>`, so the implementation
+    // can use this to determine serialization layout.
+    native fun new_table_handle(): u128;
+    native fun add_box<K, V, B>(table: &mut Table<K, V>, key: &K, val: Box<V>);
+    native fun borrow_box<K, V, B>(table: &Table<K, V>, key: &K): &Box<V>;
+    native fun borrow_box_mut<K, V, B>(table: &mut Table<K, V>, key: &K): &mut Box<V>;
+    native fun length_box<K, V, B>(table: &Table<K, V>): u64;
+    native fun contains_box<K, V, B>(table: &Table<K, V>, key: &K): bool;
+    native fun remove_box<K, V, B>(table: &mut Table<K, V>, key: &K): Box<V>;
+    native fun destroy_empty_box<K, V, B>(table: &Table<K, V>);
+    native fun drop_unchecked_box<K, V, B>(table: Table<K, V>);
+}