Skip to content
This repository has been archived by the owner on Aug 2, 2022. It is now read-only.

kv support cleos #9536

Merged
merged 3 commits into from
Sep 30, 2020
Merged

kv support cleos #9536

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
ce0dfc6
kv support cleos
kimjh2005 Sep 30, 2020
000d642
change help
kimjh2005 Sep 30, 2020
066b18a
fix for comments
kimjh2005 Sep 30, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions libraries/chain/include/eosio/chain/abi_def.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -116,6 +116,14 @@ struct kv_table_def {
type_name type; // the name of the struct_def
primary_index_def primary_index; // primary index field
map<index_name, secondary_index_def> secondary_indices; // secondary indices fields

string get_index_type(const string& index_name)const {
if( index_name == primary_index.name.to_string() ) return primary_index.type;
for( const auto& kv : secondary_indices ) {
if( index_name == kv.first.to_string() ) return kv.second.type;
}
return {};
}
};

struct clause_pair {
Expand Down
197 changes: 177 additions & 20 deletions plugins/chain_plugin/chain_plugin.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@

#include <boost/signals2/connection.hpp>
#include <boost/algorithm/string.hpp>
#include <boost/algorithm/hex.hpp>
#include <boost/lexical_cast.hpp>

#include <fc/io/json.hpp>
Expand Down Expand Up @@ -1911,33 +1912,172 @@ read_only::get_table_rows_result read_only::get_table_rows( const read_only::get
#pragma GCC diagnostic pop
}

void read_only::convert_key(const string& index_type, const string& encode_type, const string& index_value, vector<char>& bin)const {
constexpr int hex_base = 16;
constexpr int dec_base = 10;

try {
// converts arbitrary hex strings to bytes ex) "FFFEFD" to {255, 254, 253}
if( encode_type == "bytes" ) {
if( bin.capacity() < index_value.size() ) {
bin.reserve(index_value.size());
}
string bytes_data = boost::algorithm::unhex(index_value);
auto bytes_datasize = bytes_data.size();
kimjh2005 marked this conversation as resolved.
Show resolved Hide resolved
for( char c : bytes_data ) {
bin.push_back(c);
}
kimjh2005 marked this conversation as resolved.
Show resolved Hide resolved
return;
}

if( encode_type == "string" ) {
if( bin.capacity() < index_value.size() ) {
bin.reserve(index_value.size() + 2); // two zeros at the end
kimjh2005 marked this conversation as resolved.
Show resolved Hide resolved
}
convert_to_key(index_value, bin);
return;
}

if ( index_type == "name" ) {
name nm(index_value);
convert_to_key(nm.to_uint64_t(), bin);
kimjh2005 marked this conversation as resolved.
Show resolved Hide resolved
return;
}

size_t pos = 0;
if( encode_type == "hex" ) {
if( index_type == "sha256" || index_type == "i256" ) {
checksum256_type sha{index_value};
const char* sha_data = sha.data();
for( int i = 0; i < sha.data_size(); ++i ) {
kimjh2005 marked this conversation as resolved.
Show resolved Hide resolved
bin[i] = sha_data[i];
}
} else if( index_type == "ripemd160" ) {
checksum160_type ripem160{index_value};
const char* ripem160_data = ripem160.data();
for( int i = 0; i < ripem160.data_size(); ++i ) {
bin[i] = ripem160_data[i];
}
} else if( index_type == "uint64" ) {
uint64_t u64 = std::stoul(index_value, &pos, hex_base);
if( pos == 0 ) throw std::invalid_argument("invalid index_value " + index_value);
convert_to_key(u64, bin);
} else if( index_type == "uint32" ) {
uint32_t u32 = std::stoul(index_value, &pos, hex_base);
if( pos == 0 ) throw std::invalid_argument("invalid index_value " + index_value);
convert_to_key(u32, bin);
} else if( index_type == "uint16" ) {
uint16_t u16 = std::stoul(index_value, &pos, hex_base);
if( pos == 0 ) throw std::invalid_argument("invalid index_value " + index_value);
convert_to_key(u16, bin);
} else if( index_type == "uint8" ) {
uint8_t u8 = std::stoul(index_value, &pos, hex_base);
if( pos == 0 ) throw std::invalid_argument("invalid index_value " + index_value);
convert_to_key(u8, bin);
} else if( index_type == "int64" ) {
int64_t i64 = std::stol(index_value, &pos, hex_base);
if( pos == 0 ) throw std::invalid_argument("invalid index_value " + index_value);
convert_to_key(i64, bin);
} else if( index_type == "int32" ) {
int32_t i32 = std::stoul(index_value, &pos, hex_base);
if( pos == 0 ) throw std::invalid_argument("invalid index_value " + index_value);
convert_to_key(i32, bin);
} else if( index_type == "int16" ) {
int16_t i16 = std::stoul(index_value, &pos, hex_base);
if( pos == 0 ) throw std::invalid_argument("invalid index_value " + index_value);
convert_to_key(i16, bin);
} else if( index_type == "int8" ) {
int8_t i8 = std::stoul(index_value, &pos, hex_base);
if( pos == 0 ) throw std::invalid_argument("invalid index_value " + index_value);
convert_to_key(i8, bin);
} else {
EOS_ASSERT(false, chain::contract_table_query_exception, "Unsupported index type/encode_type: ${t}/${e} ", ("t", index_type)("e", encode_type));
}
} else if( encode_type == "dec" ) {
if( index_type == "float32" ) {
float d = convert_to_type<float>( index_value, "index_value" );
convert_to_key(d, bin);
} else if( index_type == "float64" ) {
double d = convert_to_type<double>( index_value, "index_value" );
convert_to_key(d, bin);
} else if( index_type == "uint64" ) {
uint64_t u64 = std::stoul(index_value, &pos, dec_base);
if( pos == 0 ) throw std::invalid_argument("invalid index_value " + index_value);
convert_to_key(u64, bin);
} else if( index_type == "uint32" ) {
uint32_t u32 = std::stoul(index_value, &pos, dec_base);
if( pos == 0 ) throw std::invalid_argument("invalid index_value " + index_value);
convert_to_key(u32, bin);
} else if( index_type == "uint16" ) {
uint16_t u16 = std::stoul(index_value, &pos, dec_base);
if( pos == 0 ) throw std::invalid_argument("invalid index_value " + index_value);
convert_to_key(u16, bin);
} else if( index_type == "uint8" ) {
uint8_t u8 = std::stoul(index_value, &pos, dec_base);
if( pos == 0 ) throw std::invalid_argument("invalid index_value " + index_value);
convert_to_key(u8, bin);
} else if( index_type == "int64" ) {
int64_t i64 = std::stol(index_value, &pos, dec_base);
if( pos == 0 ) throw std::invalid_argument("invalid index_value " + index_value);
convert_to_key(i64, bin);
} else if( index_type == "int32" ) {
int32_t i32 = std::stoul(index_value, &pos, dec_base);
if( pos == 0 ) throw std::invalid_argument("invalid index_value " + index_value);
convert_to_key(i32, bin);
} else if( index_type == "int16" ) {
int16_t i16 = std::stoul(index_value, &pos, dec_base);
if( pos == 0 ) throw std::invalid_argument("invalid index_value " + index_value);
convert_to_key(i16, bin);
} else if( index_type == "int8" ) {
int8_t i8 = std::stoul(index_value, &pos, dec_base);
if( pos == 0 ) throw std::invalid_argument("invalid index_value " + index_value);
convert_to_key(i8, bin);
} else {
EOS_ASSERT(false, chain::contract_table_query_exception, "Unsupported index type/encode_type: ${t}/${e} ", ("t", index_type)("e", encode_type));
}
}
} catch( const std::invalid_argument& e) {
EOS_ASSERT(false, chain::contract_table_query_exception, "Unsupported index type/encode_type: ${t}/${e} {$v} ", ("t", index_type)("e", encode_type)("v", index_value));
kimjh2005 marked this conversation as resolved.
Show resolved Hide resolved
} catch( const std::out_of_range& e ) {
EOS_ASSERT(false, chain::contract_table_query_exception, "Unsupported index type/encode_type: ${t}/${e} {$v} ", ("t", index_type)("e", encode_type)("v", index_value));
} catch( boost::bad_lexical_cast& e ) {
EOS_ASSERT(false, chain::contract_table_query_exception, "Unsupported index type/encode_type: ${t}/${e} {$v} ", ("t", index_type)("e", encode_type)("v", index_value));
} catch( boost::exception& e ) {
EOS_ASSERT(false, chain::contract_table_query_exception, "Unsupported index type/encode_type: ${t}/${e} {$v} ", ("t", index_type)("e", encode_type)("v", index_value));
}
}

// prefix 17bytes: status(1 byte) + table_name(8bytes) + index_name(8 bytes)
void read_only::make_prefix(eosio::name table_name, eosio::name index_name, uint8_t status, vector<char>& prefix)const {
EOS_ASSERT(prefix.size() == 2 * sizeof(uint64_t) + 1, chain::contract_table_query_exception, "Invalid prefix");

prefix[0] = static_cast<char>(status);
vector<char> bin;
vector<char> bin;
bin.reserve(sizeof(uint64_t));
convert_to_key(table_name.to_uint64_t(), bin);
for(int i = 0; i < sizeof(uint64_t); ++i) {
for( int i = 0; i < sizeof(uint64_t); ++i ) {
prefix[i + 1] = bin[i];
}

bin.clear();
convert_to_key(index_name.to_uint64_t(), bin);
int offset = sizeof(uint64_t) + 1;
for(int i = 0; i < sizeof(uint64_t); ++i) {
for( int i = 0; i < sizeof(uint64_t ); ++i) {
prefix[offset + i] = bin[i];
}
}

read_only::get_table_rows_result read_only::get_kv_table_rows( const read_only::get_kv_table_rows_params& p)const {
const string &index_value = *p.index_value;
const string &lower_bound = *p.lower_bound;
const string &upper_bound = *p.upper_bound;

name database_id = chain::kvram_id;
auto& gp = db.get_global_properties();
auto& kv_config = gp.kv_configuration;
const chain::kv_database_config& limits = kv_config.kvram;

auto&database = db.db();
auto& database = db.db();
auto kv_context = chain::create_kv_chainbase_context(const_cast<chainbase::database&>(database), database_id, chain::name{0}, {}, limits);
kimjh2005 marked this conversation as resolved.
Show resolved Hide resolved
const abi_def abi = eosio::chain_apis::get_abi( db, p.code );

Expand All @@ -1960,6 +2100,9 @@ read_only::get_table_rows_result read_only::get_kv_table_rows_context( const rea
bool is_sec_idx = (kv_tbl_def.secondary_indices.find(p.index_name) != kv_tbl_def.secondary_indices.end());
EOS_ASSERT(is_primary_idx || is_sec_idx, chain::contract_table_query_exception, "Unknown kv index: ${t} ${i}", ("t", p.table)("i", p.index_name));

int offset = 0;

string index_type = kv_tbl_def.get_index_type(p.index_name.to_string());
// Is point query of ranged query?
bool point_query = p.index_value.has_value() && !p.index_value.value().empty();

Expand All @@ -1976,36 +2119,41 @@ read_only::get_table_rows_result read_only::get_kv_table_rows_context( const rea

auto cur_time = fc::time_point::now();
auto end_time = cur_time + fc::microseconds(1000 * 10);

auto wait_time = end_time - cur_time;
///////////////////////////////////////////////////////////
// point query
///////////////////////////////////////////////////////////
if( point_query ) {
const string &index_value = *p.index_value;
vector<char> iv;
convert_key(index_type, p.encode_type, index_value, iv);
vector<char> key;
key.resize(prefix.size() + index_value.size());
key.resize(prefix.size() + iv.size());
memcpy(key.data(), prefix.data(), prefix.size());
memcpy(key.data() + prefix.size(), index_value.data(), index_value.size());
memcpy(key.data() + prefix.size(), iv.data(), iv.size());

fc::variant row_var;
auto success = kv_context.kv_get(p.code.to_uint64_t(), key.data(), key.size(), value_size);
if( success ) {
vector<char> value;
value.resize(value_size);
auto return_size = kv_context.kv_get_data(0, value.data(), value_size);
auto return_size = kv_context.kv_get_data(offset, value.data(), value_size);
EOS_ASSERT(return_size == value_size, chain::contract_table_query_exception, "point query value size mismatch: ${s1} ${s2}", ("s1", return_size)("s2", value_size));

fc::variant row_var;
if( p.json ) {
try {
row_var = abis.binary_to_variant( p.table.to_string(), value, abi_serializer::create_yield_function( wait_time ), shorten_abi_errors );
} catch (fc::exception &e) {
row_var = fc::variant( value );
} catch ( fc::exception &e ) {
row_var = fc::variant(value);
}
} else {
row_var = fc::variant( value );
row_var = fc::variant(value);
}
result.rows.emplace_back( std::move(row_var) );
}

return result;
}

Expand All @@ -2018,15 +2166,19 @@ read_only::get_table_rows_result read_only::get_kv_table_rows_context( const rea

vector<char> lb_key;
const string &lower_bound = *p.lower_bound;
lb_key.resize(prefix.size() + lower_bound.size());
vector<char> lv;
convert_key(index_type, p.encode_type, lower_bound, lv);
lb_key.resize(prefix.size() + lv.size());
memcpy(lb_key.data(), prefix.data(), prefix.size());
memcpy(lb_key.data() + prefix.size(), lower_bound.data(), lower_bound.size());
memcpy(lb_key.data() + prefix.size(), lv.data(), lv.size());

vector<char> ub_key;
const string &upper_bound = *p.upper_bound;
ub_key.resize(prefix.size() + upper_bound.size());
vector<char> uv;
convert_key(index_type, p.encode_type, upper_bound, uv);
ub_key.resize(prefix.size() + uv.size());
memcpy(ub_key.data(), prefix.data(), prefix.size());
memcpy(ub_key.data() + prefix.size(), upper_bound.data(), upper_bound.size());
memcpy(ub_key.data() + prefix.size(), uv.data(), uv.size());

// Iterate the range
vector<char> row_key;
Expand All @@ -2045,15 +2197,16 @@ read_only::get_table_rows_result read_only::get_kv_table_rows_context( const rea
// Find lower bound iterator
auto status_lb = lb_itr->kv_it_lower_bound(lb_key.data(), lb_key.size(), &lb_key_size, &lb_value_size);
EOS_ASSERT(status_lb != chain::kv_it_stat::iterator_erased, chain::contract_table_query_exception, "Invalid lower bound iterator in ${t} ${i}", ("t", p.table)("i", p.index_name));
// Find upper bound iterator
auto status_ub = ub_itr->kv_it_lower_bound(ub_key.data(), ub_key.size(), &ub_key_size, &ub_value_size);
EOS_ASSERT(status_ub != chain::kv_it_stat::iterator_erased, chain::contract_table_query_exception, "Invalid upper bound iterator in ${t} ${i}", ("t", p.table)("i", p.index_name));

lb_key.resize(lb_key_size);
status_lb = lb_itr->kv_it_key(0, lb_key.data(), lb_key_size, lb_key_actual_size);

// Find upper bound iterator
auto status_ub = ub_itr->kv_it_lower_bound(ub_key.data(), ub_key.size(), &ub_key_size, &ub_value_size);
EOS_ASSERT(status_ub != chain::kv_it_stat::iterator_erased, chain::contract_table_query_exception, "Invalid upper bound iterator in ${t} ${i}", ("t", p.table)("i", p.index_name));
ub_key.resize(ub_key_size);
status_ub = ub_itr->kv_it_key(0, ub_key.data(), ub_key_size, ub_key_actual_size);

kv_it_stat status;
// iterate the range
auto walk_table_row_range = [&]( auto &itr, auto &end_itr, bool reverse ) {
uint32_t actual_size = 0;
Expand All @@ -2070,23 +2223,27 @@ read_only::get_table_rows_result read_only::get_kv_table_rows_context( const rea
auto cmp = itr->kv_it_key_compare(end_key.data(), end_key.size());
if( reverse ) cmp *= -1;

fc::variant row_var;
kimjh2005 marked this conversation as resolved.
Show resolved Hide resolved
unsigned int count = 0;
for( count = 0; cur_time <= end_time && count < p.limit && cmp < 0; cur_time = fc::time_point::now() ) {
row_value.clear();
row_value.resize(val_size);
auto status = itr->kv_it_value(0, row_value.data(), val_size, actual_size);
status = itr->kv_it_value(offset, row_value.data(), val_size, actual_size);

fc::variant row_var;
if( p.json ) {
auto time_left = end_time - cur_time;
try {
row_var = abis.binary_to_variant( p.table.to_string(), row_value, abi_serializer::create_yield_function( time_left ), shorten_abi_errors );
} catch (fc::exception &e) {
}
catch ( fc::exception &e )
{
row_var = fc::variant( row_value );
}
} else {
row_var = fc::variant( row_value );
}

result.rows.emplace_back( std::move(row_var) );
++count;

Expand Down
1 change: 1 addition & 0 deletions plugins/chain_plugin/include/eosio/chain_plugin/chain_plugin.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -350,6 +350,7 @@ class read_only {
get_table_rows_result get_table_rows( const get_table_rows_params& params )const;

constexpr uint32_t prefix_size() const { return 1 + 2 * sizeof(uint64_t); }
void convert_key(const string& index_type, const string& encode_type, const string& index_value, vector<char>& bin)const;
void make_prefix(eosio::name table_name, eosio::name index_name, uint8_t status, vector<char> &prefix)const;
get_table_rows_result get_kv_table_rows( const get_kv_table_rows_params& params )const;
get_table_rows_result get_kv_table_rows_context( const read_only::get_kv_table_rows_params& p, eosio::chain::kv_context &kv_context, const abi_def &abi )const;
Expand Down
7 changes: 7 additions & 0 deletions programs/cleos/eosc.pot
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -88,6 +88,13 @@ msgid ""
" eosc get code ${1}"
msgstr ""

msgid ""
"The ABI for the code on account \"${1}\" does not specify kv_table \"${2}\".\n"
"\n"
"Please check the account and kv_table name, and verify that the account has the expected code using:\n"
" eosc get code ${1}"
msgstr ""

msgid "Error locating help text: ${code} ${what}"
msgstr ""

Expand Down
5 changes: 5 additions & 0 deletions programs/cleos/help_text.cpp.in
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,10 @@ const char* unknown_abi_table_help_text = _(R"text(The ABI for the code on accou
Please check the account and table name, and verify that the account has the expected code using:
@CLI_CLIENT_EXECUTABLE_NAME@ get code ${1})text");

const char* unknown_abi_kv_table_help_text = _(R"text(The ABI for the code on account "${1}" does not specify kv_table "${2}".
Please check the account and kv_table name, and verify that the account has the expected code using:
@CLI_CLIENT_EXECUTABLE_NAME@ get code ${1})text");

const char* failed_to_find_transaction_text = _("Failed to fetch information for transaction: \033[1m${1}\033[0m from the history plugin\n\n"
"\033[32mIf you know the block number which included this transaction you providing it with the \033[2m--block-hint\033[22m option may help\033[0m");

Expand All @@ -86,6 +90,7 @@ const std::vector<std::pair<const char*, std::vector<const char *>>> error_help_
{"Wallet is locked: ([\\S]*)", {locked_wallet_help_text}},
{"Key already in wallet[^\\x00]*wallet/import_key.*postdata\":\\[\"([^\"]*)\"", {duplicate_key_import_help_text}},
{"ABI does not define table[^\\x00]*get_table_rows.*code\":\"([^\"]*)\",\"table\":\"([^\"]*)\"", {unknown_abi_table_help_text}},
{"ABI does not define kv_table[^\\x00]*get_kv_table_rows.*code\":\"([^\"]*)\",\"kv_table\":\"([^\"]*)\"", {unknown_abi_kv_table_help_text}},
{"Transaction ([^ ]{8})[^ ]* not found in history and no block hint was given", {failed_to_find_transaction_text, history_plugin_advice_text}},
{"Transaction ([^ ]{8})[^ ]* not found in history or in block number ([0-9]*)", {failed_to_find_transaction_with_block_text, history_plugin_advice_text}},
};
Expand Down
1 change: 1 addition & 0 deletions programs/cleos/httpc.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -89,6 +89,7 @@ namespace eosio { namespace client { namespace http {
const string get_block_header_state_func = chain_func_base + "/get_block_header_state";
const string get_account_func = chain_func_base + "/get_account";
const string get_table_func = chain_func_base + "/get_table_rows";
const string get_kv_table_func = chain_func_base + "/get_kv_table_rows";
const string get_table_by_scope_func = chain_func_base + "/get_table_by_scope";
const string get_code_func = chain_func_base + "/get_code";
const string get_code_hash_func = chain_func_base + "/get_code_hash";
Expand Down
Loading