diff --git a/include/leo_object_storage.hrl b/include/leo_object_storage.hrl index 0fdecb0..fe8ea01 100644 --- a/include/leo_object_storage.hrl +++ b/include/leo_object_storage.hrl @@ -277,125 +277,166 @@ }). -record(metadata, { %% - leofs-v1.0.0-pre3 - key = <<>> :: binary(), %% filename - addr_id = 0 :: integer(), %% ring-address id (MD5 > hex-to-integer) - ksize = 0 :: integer(), %% file-path size - dsize = 0 :: integer(), %% data size - msize = 0 :: integer(), %% custom-metadata size - - csize = 0 :: integer(), %% * chunked data size (for large-object) - cnumber = 0 :: integer(), %% * # of chunked objects (for large-object) - cindex = 0 :: integer(), %% * chunked object index (for large-object) - - offset = 0 :: integer(), %% object-container's offset - clock = 0 :: integer(), %% clock - timestamp = 0 :: integer(), %% timestamp - checksum = 0 :: integer(), %% checksum (MD5 > hex-to-integer) - ring_hash = 0 :: integer(), %% RING's Hash(CRC32) when write an object. - del = ?DEL_FALSE :: del_flag() %% [{0,not_deleted}, {1,deleted}] + key = <<>> :: binary(), %% filename + addr_id = 0 :: integer(), %% ring-address id (MD5 > hex-to-integer) + ksize = 0 :: integer(), %% file-path size + dsize = 0 :: integer(), %% data size + msize = 0 :: integer(), %% custom-metadata size + csize = 0 :: integer(), %% * chunked data size (for large-object) + cnumber = 0 :: integer(), %% * # of chunked objects (for large-object) + cindex = 0 :: integer(), %% * chunked object index (for large-object) + offset = 0 :: integer(), %% object-container's offset + clock = 0 :: integer(), %% clock + timestamp = 0 :: integer(), %% timestamp + checksum = 0 :: integer(), %% checksum (MD5 > hex-to-integer) + ring_hash = 0 :: integer(), %% RING's Hash(CRC32) when write an object. + del = ?DEL_FALSE :: del_flag() %% [{0,not_deleted}, {1,deleted}] }). -record(metadata_1, { %% leofs-v1.0.0 - v1.3.0 - key = <<>> :: binary(), %% filename - addr_id = 0 :: integer(), %% ring-address id (MD5 > hex-to-integer) - ksize = 0 :: integer(), %% file-path size - dsize = 0 :: integer(), %% data size - msize = 0 :: integer(), %% custom-metadata size - - csize = 0 :: integer(), %% * chunked data size (for large-object) - cnumber = 0 :: integer(), %% * # of chunked objects (for large-object) - cindex = 0 :: integer(), %% * chunked object index (for large-object) - - offset = 0 :: integer(), %% object-container's offset - clock = 0 :: integer(), %% clock - timestamp = 0 :: integer(), %% timestamp - checksum = 0 :: integer(), %% checksum (MD5 > hex-to-integer) - ring_hash = 0 :: integer(), %% RING's Hash(CRC32) when write an object. - - cluster_id :: atom(), %% cluster-id for the mdc-replication - num_of_replicas = 0 :: integer(), %% # of replicas for the mdc-replication - ver = 0 :: integer(), %% version number - - del = ?DEL_FALSE :: del_flag() %% [{0,not_deleted}, {1,deleted}] + key = <<>> :: binary(), %% filename + addr_id = 0 :: integer(), %% ring-address id (MD5 > hex-to-integer) + ksize = 0 :: integer(), %% file-path size + dsize = 0 :: integer(), %% data size + msize = 0 :: integer(), %% custom-metadata size + csize = 0 :: integer(), %% * chunked data size (for large-object) + cnumber = 0 :: integer(), %% * # of chunked objects (for large-object) + cindex = 0 :: integer(), %% * chunked object index (for large-object) + offset = 0 :: integer(), %% object-container's offset + clock = 0 :: integer(), %% clock + timestamp = 0 :: integer(), %% timestamp + checksum = 0 :: integer(), %% checksum (MD5 > hex-to-integer) + ring_hash = 0 :: integer(), %% RING's Hash(CRC32) when write an object. + cluster_id :: atom(), %% [+] cluster-id for the mdc-replication + num_of_replicas = 0 :: integer(), %% [+] # of replicas for the mdc-replication + ver = 0 :: integer(), %% [+] version number + del = ?DEL_FALSE :: del_flag() %% [{0,not_deleted}, {1,deleted}] }). --record(metadata_2, { %% leofs-v1.3.1 - the latest version - key = <<>> :: binary(), %% filename - addr_id = 0 :: integer(), %% ring-address id (MD5 > hex-to-integer) - ksize = 0 :: integer(), %% file-path size - dsize = 0 :: integer(), %% data size - - meta = <<>> :: binary(), %% custom-metadata (user defined metadata) - msize = 0 :: integer(), %% custom-metadata size - - csize = 0 :: integer(), %% * chunked data size (for large-object) - cnumber = 0 :: integer(), %% * # of chunked objects (for large-object) - cindex = 0 :: integer(), %% * chunked object index (for large-object) - - offset = 0 :: integer(), %% object-container's offset - clock = 0 :: integer(), %% clock - timestamp = 0 :: integer(), %% timestamp - checksum = 0 :: integer(), %% checksum (MD5 > hex-to-integer) - ring_hash = 0 :: integer(), %% RING's Hash(CRC32) when write an object. - - cluster_id :: atom(), %% cluster-id for the mdc-replication +-record(metadata_2, { %% leofs-v1.3.1 - v1.3.2.1 + key = <<>> :: binary(), %% filename + addr_id = 0 :: integer(), %% ring-address id (MD5 > hex-to-integer) + ksize = 0 :: integer(), %% file-path size + dsize = 0 :: integer(), %% data size + meta = <<>> :: binary(), %% [+] custom-metadata (user defined metadata) + msize = 0 :: integer(), %% custom-metadata size + csize = 0 :: integer(), %% * chunked data size (for large-object) + cnumber = 0 :: integer(), %% * # of chunked objects (for large-object) + cindex = 0 :: integer(), %% * chunked object index (for large-object) + offset = 0 :: integer(), %% object-container's offset + clock = 0 :: integer(), %% clock + timestamp = 0 :: integer(), %% timestamp + checksum = 0 :: integer(), %% checksum (MD5 > hex-to-integer) + ring_hash = 0 :: integer(), %% RING's Hash(CRC32) when write an object. + cluster_id :: atom(), %% cluster-id for the mdc-replication num_of_replicas = 0 :: integer(), %% # of replicas for the mdc-replication - ver = 0 :: integer(), %% version number + ver = 0 :: integer(), %% version number + del = ?DEL_FALSE :: del_flag() %% [{0,not_deleted}, {1,deleted}] + }). - del = ?DEL_FALSE :: del_flag() %% [{0,not_deleted}, {1,deleted}] +-record(metadata_3, { %% leofs-v1.3.1 - the latest version + key = <<>> :: binary(), %% filename + addr_id = 0 :: integer(), %% ring-address id (MD5 > hex-to-integer) + ksize = 0 :: integer(), %% file-path size + dsize = 0 :: integer(), %% data size + meta = <<>> :: binary(), %% custom-metadata (user defined metadata) + msize = 0 :: integer(), %% custom-metadata size + csize = 0 :: integer(), %% * chunked data size (for large-object) + cnumber = 0 :: integer(), %% * # of chunked objects (for large-object) + cindex = 0 :: integer(), %% * chunked object index (for large-object) + offset = 0 :: integer(), %% object-container's offset + clock = 0 :: integer(), %% clock + timestamp = 0 :: integer(), %% timestamp + checksum = 0 :: integer(), %% checksum (MD5 > hex-to-integer) + ring_hash = 0 :: integer(), %% RING's Hash(CRC32) when write an object. + cluster_id :: atom(), %% cluster-id for the mdc-replication + num_of_replicas = 0 :: non_neg_integer(), %% [mdcr/bucket] # of replicas for the mdc-replication + %% - [0: no effects, + %% 1..*: preferred value of the data-replicatino] + %% as well as preferred_r, preferred_w, preferred_d + preferred_r = 0 :: non_neg_integer(), %% [+] [mdcr/bucket] # of replicas needed for a successful READ operation + preferred_w = 0 :: non_neg_integer(), %% [+] [mdcr/bucket] # of replicas needed for a successful WRITE operation + preferred_d = 0 :: non_neg_integer(), %% [+] [mdcr/bucket] # of replicas needed for a successful DELETE operation + ver = 0 :: integer(), %% version number + del = ?DEL_FALSE :: del_flag() %% [{0,not_deleted}, {1,deleted}] }). --define(METADATA, 'metadata_2'). +-define(METADATA, 'metadata_3'). -record(object, { %% - leofs-v1.0.0-pre3 method, - key = <<>> :: binary(), %% filename - addr_id = 0 :: integer(), %% ring-address id (MD5 > hex-to-integer) - data = <<>> :: binary(), %% file - meta = <<>> :: binary(), %% custom-metadata - ksize = 0 :: integer(), %% filename size - dsize = 0 :: integer(), %% data size - msize = 0 :: integer(), %% custom-metadata size - - csize = 0 :: integer(), %% * chunked data size (for large-object) - cnumber = 0 :: integer(), %% * # of chunked objects (for large-object) - cindex = 0 :: integer(), %% * chunked object index (for large-object) - - offset = 0 :: integer(), %% object-container's offset - clock = 0 :: integer(), %% clock - timestamp = 0 :: integer(), %% timestamp - checksum = 0 :: integer(), %% checksum (MD5 > hex-to-integer) - ring_hash = 0 :: integer(), %% RING's Hash(CRC32) when write an object. - req_id = 0 :: integer(), %% request id - del = ?DEL_FALSE :: del_flag() %% delete flag + key = <<>> :: binary(), %% filename + addr_id = 0 :: integer(), %% ring-address id (MD5 > hex-to-integer) + data = <<>> :: binary(), %% file + meta = <<>> :: binary(), %% custom-metadata + ksize = 0 :: integer(), %% filename size + dsize = 0 :: integer(), %% data size + msize = 0 :: integer(), %% custom-metadata size + csize = 0 :: integer(), %% * chunked data size (for large-object) + cnumber = 0 :: integer(), %% * # of chunked objects (for large-object) + cindex = 0 :: integer(), %% * chunked object index (for large-object) + offset = 0 :: integer(), %% object-container's offset + clock = 0 :: integer(), %% clock + timestamp = 0 :: integer(), %% timestamp + checksum = 0 :: integer(), %% checksum (MD5 > hex-to-integer) + ring_hash = 0 :: integer(), %% RING's Hash(CRC32) when write an object. + req_id = 0 :: integer(), %% request id + del = ?DEL_FALSE :: del_flag() %% delete flag }). --record(object_1, { %% leofs-v1.0.0 - current ver +-record(object_1, { %% leofs-v1.0.0 - v1.3.2.1 method, - key = <<>> :: binary(), %% filename - addr_id = 0 :: integer(), %% ring-address id (MD5 > hex-to-integer) - data = <<>> :: binary(), %% file - meta = <<>> :: binary(), %% custom-metadata (user defined metadata) - ksize = 0 :: integer(), %% filename size - dsize = 0 :: integer(), %% data size - msize = 0 :: integer(), %% custom-metadata size - - csize = 0 :: integer(), %% * chunked data size (for large-object) - cnumber = 0 :: integer(), %% * # of chunked objects (for large-object) - cindex = 0 :: integer(), %% * chunked object index (for large-object) - - offset = 0 :: integer(), %% object-container's offset - clock = 0 :: integer(), %% clock - timestamp = 0 :: integer(), %% timestamp - checksum = 0 :: integer(), %% checksum (MD5 > hex-to-integer) - ring_hash = 0 :: integer(), %% RING's Hash(CRC32) when write an object. - req_id = 0 :: integer(), %% request id - - cluster_id :: atom(), %% cluster-id for the mdc-replication - num_of_replicas = 0 :: integer(), %% # of replicas for the mdc-replication - ver = 0 :: integer(), %% version number - del = ?DEL_FALSE :: del_flag() %% delete flag + key = <<>> :: binary(), %% filename + addr_id = 0 :: integer(), %% ring-address id (MD5 > hex-to-integer) + data = <<>> :: binary(), %% file + meta = <<>> :: binary(), %% custom-metadata (user defined metadata) + ksize = 0 :: integer(), %% filename size + dsize = 0 :: integer(), %% data size + msize = 0 :: integer(), %% custom-metadata size + csize = 0 :: integer(), %% * chunked data size (for large-object) + cnumber = 0 :: integer(), %% * # of chunked objects (for large-object) + cindex = 0 :: integer(), %% * chunked object index (for large-object) + offset = 0 :: integer(), %% object-container's offset + clock = 0 :: integer(), %% clock + timestamp = 0 :: integer(), %% timestamp + checksum = 0 :: integer(), %% checksum (MD5 > hex-to-integer) + ring_hash = 0 :: integer(), %% RING's Hash(CRC32) when write an object. + req_id= 0 :: integer(), %% request id + cluster_id :: atom(), %% [+] cluster-id for the mdc-replication + num_of_replicas = 0 :: integer(), %% [+] # of replicas for the mdc-replication + ver = 0 :: integer(), %% [+] version number + del = ?DEL_FALSE :: del_flag() %% delete flag + }). + +-record(object_2, { %% leofs-v1.3.3 - the latest version + method, + key = <<>> :: binary(), %% filename + addr_id = 0 :: integer(), %% ring-address id (MD5 > hex-to-integer) + data = <<>> :: binary(), %% file + meta = <<>> :: binary(), %% custom-metadata (user defined metadata) + ksize = 0 :: integer(), %% filename size + dsize = 0 :: integer(), %% data size + msize = 0 :: integer(), %% custom-metadata size + csize = 0 :: integer(), %% * chunked data size (for large-object) + cnumber = 0 :: integer(), %% * # of chunked objects (for large-object) + cindex = 0 :: integer(), %% * chunked object index (for large-object) + offset = 0 :: integer(), %% object-container's offset + clock = 0 :: integer(), %% clock + timestamp = 0 :: integer(), %% timestamp + checksum = 0 :: integer(), %% checksum (MD5 > hex-to-integer) + ring_hash = 0 :: integer(), %% RING's Hash(CRC32) when write an object. + req_id= 0 :: integer(), %% request id + cluster_id :: atom(), %% cluster-id for the mdc-replication + num_of_replicas = 0 :: non_neg_integer(), %% [mdcr/bucket] # of replicas for the mdc-replication + %% - [0: no effects, + %% 1..*: preferred value of the data-replicatino] + %% as well as preferred_r, preferred_w, preferred_d + preferred_r = 0 :: non_neg_integer(), %% [+] [mdcr/bucket] # of replicas needed for a successful READ operation + preferred_w = 0 :: non_neg_integer(), %% [+] [mdcr/bucket] # of replicas needed for a successful WRITE operation + preferred_d = 0 :: non_neg_integer(), %% [+] [mdcr/bucket] # of replicas needed for a successful DELETE operation + ver = 0 :: integer(), %% version number + del = ?DEL_FALSE :: del_flag() %% delete flag }). --define(OBJECT, 'object_1'). +-define(OBJECT, 'object_2'). -record(storage_stats, { file_path = [] :: string(), @@ -590,6 +631,9 @@ %% custom-metadata's items for MDC-replication: -define(PROP_CMETA_CLUSTER_ID, 'cluster_id'). -define(PROP_CMETA_NUM_OF_REPLICAS, 'num_of_replicas'). +-define(PROP_CMETA_PREFERRED_R, 'preferred_r'). +-define(PROP_CMETA_PREFERRED_W, 'preferred_w'). +-define(PROP_CMETA_PREFERRED_D, 'preferred_d'). -define(PROP_CMETA_VER, 'ver'). -define(PROP_CMETA_UDM, 'udm'). %% user defined metadata: [{, }] diff --git a/src/leo_object_storage_transformer.erl b/src/leo_object_storage_transformer.erl index 1ae3410..103b6df 100644 --- a/src/leo_object_storage_transformer.erl +++ b/src/leo_object_storage_transformer.erl @@ -45,105 +45,149 @@ %% @doc Transform from a metadata to an object -spec(metadata_to_object(Metadata) -> #?OBJECT{} | {error, invaid_record} when Metadata::#metadata{} | #metadata_1{} | #?METADATA{}). -metadata_to_object(#metadata{key = Key, +metadata_to_object(#metadata{key = Key, addr_id = AddrId, - ksize = KSize, - dsize = DSize, - msize = MSize, - csize = CSize, + ksize = KSize, + dsize = DSize, + msize = MSize, + csize = CSize, cnumber = CNum, - cindex = CIndex, - offset = Offset, - clock = Clock, + cindex = CIndex, + offset = Offset, + clock = Clock, timestamp = Timestamp, - checksum = Checksum, + checksum = Checksum, ring_hash = RingHash, del = Del}) -> - #?OBJECT{key = Key, + #?OBJECT{key = Key, addr_id = AddrId, - ksize = KSize, - dsize = DSize, - msize = MSize, - csize = CSize, + ksize = KSize, + dsize = DSize, + msize = MSize, + csize = CSize, cnumber = CNum, - cindex = CIndex, - offset = Offset, - clock = Clock, + cindex = CIndex, + offset = Offset, + clock = Clock, timestamp = Timestamp, - checksum = Checksum, + checksum = Checksum, ring_hash = RingHash, del = Del}; metadata_to_object(#metadata_1{} = Metadata) -> - #metadata_1{key = Key, + #metadata_1{key = Key, addr_id = AddrId, - ksize = KSize, - dsize = DSize, - msize = MSize, - csize = CSize, + ksize = KSize, + dsize = DSize, + msize = MSize, + csize = CSize, cnumber = CNum, - cindex = CIndex, - offset = Offset, - clock = Clock, + cindex = CIndex, + offset = Offset, + clock = Clock, timestamp = Timestamp, - checksum = Checksum, + checksum = Checksum, ring_hash = RingHash, cluster_id = ClusterId, num_of_replicas = NumOfReplicas, ver = Ver, del = Del} = Metadata, - #?OBJECT{key = Key, + #?OBJECT{key = Key, addr_id = AddrId, - ksize = KSize, - dsize = DSize, - msize = MSize, - csize = CSize, + ksize = KSize, + dsize = DSize, + msize = MSize, + csize = CSize, cnumber = CNum, - cindex = CIndex, - offset = Offset, - clock = Clock, + cindex = CIndex, + offset = Offset, + clock = Clock, timestamp = Timestamp, - checksum = Checksum, + checksum = Checksum, ring_hash = RingHash, cluster_id = ClusterId, num_of_replicas = NumOfReplicas, ver = Ver, del = Del}; + +metadata_to_object(#metadata_2{} = Metadata) -> + #metadata_2{key = Key, + addr_id = AddrId, + ksize = KSize, + dsize = DSize, + meta = CMeta, + msize = MSize, + csize = CSize, + cnumber = CNum, + cindex = CIndex, + offset = Offset, + clock = Clock, + timestamp = Timestamp, + checksum = Checksum, + ring_hash = RingHash, + cluster_id = ClusterId, + num_of_replicas = NumOfReplicas, + ver = Ver, + del = Del} = Metadata, + #?OBJECT{key = Key, + addr_id = AddrId, + ksize = KSize, + dsize = DSize, + meta = CMeta, + msize = MSize, + csize = CSize, + cnumber = CNum, + cindex = CIndex, + offset = Offset, + clock = Clock, + timestamp = Timestamp, + checksum = Checksum, + ring_hash = RingHash, + cluster_id = ClusterId, + num_of_replicas = NumOfReplicas, + ver = Ver, + del = Del}; + metadata_to_object(#?METADATA{} = Metadata) -> - #?METADATA{key = Key, + #?METADATA{key = Key, addr_id = AddrId, - ksize = KSize, - dsize = DSize, - meta = CMeta, - msize = MSize, - csize = CSize, + ksize = KSize, + dsize = DSize, + meta = CMeta, + msize = MSize, + csize = CSize, cnumber = CNum, - cindex = CIndex, - offset = Offset, - clock = Clock, + cindex = CIndex, + offset = Offset, + clock = Clock, timestamp = Timestamp, - checksum = Checksum, + checksum = Checksum, ring_hash = RingHash, cluster_id = ClusterId, num_of_replicas = NumOfReplicas, + preferred_r = Preferred_R, + preferred_w = Preferred_W, + preferred_d = Preferred_D, ver = Ver, del = Del} = Metadata, - - #?OBJECT{key = Key, + #?OBJECT{key = Key, addr_id = AddrId, - ksize = KSize, - dsize = DSize, - meta = CMeta, - msize = MSize, - csize = CSize, + ksize = KSize, + dsize = DSize, + meta = CMeta, + msize = MSize, + csize = CSize, cnumber = CNum, - cindex = CIndex, - offset = Offset, - clock = Clock, + cindex = CIndex, + offset = Offset, + clock = Clock, timestamp = Timestamp, - checksum = Checksum, + checksum = Checksum, ring_hash = RingHash, cluster_id = ClusterId, num_of_replicas = NumOfReplicas, + preferred_r = Preferred_R, + preferred_w = Preferred_W, + preferred_d = Preferred_D, ver = Ver, del = Del}; metadata_to_object(_M) -> @@ -172,68 +216,110 @@ metadata_to_object(Bin, Metadata) -> %% @doc Transfer object to metadata -spec(object_to_metadata(Object) -> #?METADATA{} when Object::#object{}|#object_1{}). -object_to_metadata(#object{key = Key, +object_to_metadata(#object{key = Key, addr_id = AddrId, - ksize = KSize, - dsize = DSize, - msize = MSize, - csize = CSize, + ksize = KSize, + dsize = DSize, + msize = MSize, + csize = CSize, cnumber = CNum, - cindex = CIndex, - offset = Offset, - clock = Clock, + cindex = CIndex, + offset = Offset, + clock = Clock, timestamp = Timestamp, - checksum = Checksum, + checksum = Checksum, ring_hash = RingHash, - del = Del}) -> - #?METADATA{key = Key, + del = Del}) -> + #?METADATA{key = Key, + addr_id = AddrId, + ksize = KSize, + dsize = DSize, + msize = MSize, + csize = CSize, + cnumber = CNum, + cindex = CIndex, + offset = Offset, + clock = Clock, + timestamp = Timestamp, + checksum = Checksum, + ring_hash = RingHash, + del = Del}; +object_to_metadata(#object_1{key = Key, + addr_id = AddrId, + ksize = KSize, + dsize = DSize, + meta = CMeta, + msize = MSize, + csize = CSize, + cnumber = CNum, + cindex = CIndex, + offset = Offset, + clock = Clock, + timestamp = Timestamp, + checksum = Checksum, + ring_hash = RingHash, + cluster_id = ClusterId, + num_of_replicas = NumOfReplicas, + ver = Ver, + del = Del}) -> + #?METADATA{key = Key, addr_id = AddrId, - ksize = KSize, - dsize = DSize, - msize = MSize, - csize = CSize, + ksize = KSize, + dsize = DSize, + meta = CMeta, + msize = MSize, + csize = CSize, cnumber = CNum, - cindex = CIndex, - offset = Offset, - clock = Clock, + cindex = CIndex, + offset = Offset, + clock = Clock, timestamp = Timestamp, - checksum = Checksum, + checksum = Checksum, ring_hash = RingHash, - del = Del}; -object_to_metadata(#?OBJECT{key = Key, + cluster_id = ClusterId, + num_of_replicas = NumOfReplicas, + ver = Ver, + del = Del}; +object_to_metadata(#?OBJECT{key = Key, addr_id = AddrId, - ksize = KSize, - dsize = DSize, - meta = CMeta, - msize = MSize, - csize = CSize, + ksize = KSize, + dsize = DSize, + meta = CMeta, + msize = MSize, + csize = CSize, cnumber = CNum, - cindex = CIndex, - offset = Offset, - clock = Clock, + cindex = CIndex, + offset = Offset, + clock = Clock, timestamp = Timestamp, - checksum = Checksum, + checksum = Checksum, ring_hash = RingHash, cluster_id = ClusterId, num_of_replicas = NumOfReplicas, + preferred_r = Preferred_R, + preferred_w = Preferred_W, + preferred_d = Preferred_D, ver = Ver, del = Del}) -> - #?METADATA{key = Key, + #?METADATA{key = Key, addr_id = AddrId, - ksize = KSize, - dsize = DSize, - meta = CMeta, - msize = MSize, - csize = CSize, + ksize = KSize, + dsize = DSize, + meta = CMeta, + msize = MSize, + csize = CSize, cnumber = CNum, - cindex = CIndex, - offset = Offset, - clock = Clock, + cindex = CIndex, + offset = Offset, + clock = Clock, timestamp = Timestamp, - checksum = Checksum, + checksum = Checksum, ring_hash = RingHash, cluster_id = ClusterId, num_of_replicas = NumOfReplicas, + preferred_r = Preferred_R, + preferred_w = Preferred_W, + preferred_d = Preferred_D, ver = Ver, del = Del}; object_to_metadata(_) -> @@ -244,32 +330,32 @@ object_to_metadata(_) -> -spec(transform_metadata(Metadata) -> #metadata_1{} | {error, invaid_record} when Metadata::#metadata{} | #metadata_1{}). -transform_metadata(#metadata{key = Key, +transform_metadata(#metadata{key = Key, addr_id = AddrId, - ksize = KSize, - dsize = DSize, - msize = MSize, - csize = CSize, + ksize = KSize, + dsize = DSize, + msize = MSize, + csize = CSize, cnumber = CNum, - cindex = CIndex, - offset = Offset, - clock = Clock, + cindex = CIndex, + offset = Offset, + clock = Clock, timestamp = Timestamp, - checksum = Checksum, + checksum = Checksum, ring_hash = RingHash, del = Del}) -> - #?METADATA{key = Key, + #?METADATA{key = Key, addr_id = AddrId, - ksize = KSize, - dsize = DSize, - msize = MSize, - csize = CSize, + ksize = KSize, + dsize = DSize, + msize = MSize, + csize = CSize, cnumber = CNum, - cindex = CIndex, - offset = Offset, - clock = Clock, + cindex = CIndex, + offset = Offset, + clock = Clock, timestamp = Timestamp, - checksum = Checksum, + checksum = Checksum, ring_hash = RingHash, del = Del}; @@ -290,18 +376,54 @@ transform_metadata(#metadata_1{key = Key, num_of_replicas = NumOfReplicas, ver = Ver, del = Del}) -> - #?METADATA{key = Key, + #?METADATA{key = Key, addr_id = AddrId, - ksize = KSize, - dsize = DSize, - msize = MSize, - csize = CSize, + ksize = KSize, + dsize = DSize, + msize = MSize, + csize = CSize, cnumber = CNum, - cindex = CIndex, - offset = Offset, - clock = Clock, + cindex = CIndex, + offset = Offset, + clock = Clock, timestamp = Timestamp, - checksum = Checksum, + checksum = Checksum, + ring_hash = RingHash, + cluster_id = ClusterId, + num_of_replicas = NumOfReplicas, + ver = Ver, + del = Del}; +transform_metadata(#metadata_2{key = Key, + addr_id = AddrId, + ksize = KSize, + dsize = DSize, + meta = CMeta, + msize = MSize, + csize = CSize, + cnumber = CNum, + cindex = CIndex, + offset = Offset, + clock = Clock, + timestamp = Timestamp, + checksum = Checksum, + ring_hash = RingHash, + cluster_id = ClusterId, + num_of_replicas = NumOfReplicas, + ver = Ver, + del = Del}) -> + #?METADATA{key = Key, + addr_id = AddrId, + ksize = KSize, + dsize = DSize, + meta = CMeta, + msize = MSize, + csize = CSize, + cnumber = CNum, + cindex = CIndex, + offset = Offset, + clock = Clock, + timestamp = Timestamp, + checksum = Checksum, ring_hash = RingHash, cluster_id = ClusterId, num_of_replicas = NumOfReplicas, @@ -360,17 +482,17 @@ header_bin_to_metadata(Bin) -> case (Timestamp /= 0) of true -> Meta = #?METADATA{addr_id = AddrId, - ksize = KSize, - msize = MSize, - dsize = DSize, - csize = CSize, - cnumber = CNum, - cindex = CIndex, - offset = Offset, - clock = Clock, - checksum = Checksum, - timestamp = Timestamp, - del = Del}, + ksize = KSize, + msize = MSize, + dsize = DSize, + csize = CSize, + cnumber = CNum, + cindex = CIndex, + offset = Offset, + clock = Clock, + checksum = Checksum, + timestamp = Timestamp, + del = Del}, case is_invalid_header(Meta) of true -> {error, {invalid_format, over_limit}}; @@ -392,8 +514,7 @@ header_bin_to_metadata(Bin) -> Metadata::#?METADATA{}, UDM::[{UDM_Key, UDM_Val}], UDM_Key::binary(), - UDM_Val::binary() - ). + UDM_Val::binary()). cmeta_bin_into_metadata(<<>>, Metadata) -> {ok, {Metadata, []}}; cmeta_bin_into_metadata(CustomMetadataBin, Metadata) -> @@ -401,10 +522,16 @@ cmeta_bin_into_metadata(CustomMetadataBin, Metadata) -> CustomMetadata = binary_to_term(CustomMetadataBin), ClusterId = leo_misc:get_value(?PROP_CMETA_CLUSTER_ID, CustomMetadata, []), NumOfReplicas = leo_misc:get_value(?PROP_CMETA_NUM_OF_REPLICAS, CustomMetadata, 0), + Preferred_R = leo_misc:get_value(?PROP_CMETA_PREFERRED_R, CustomMetadata, 0), + Preferred_W = leo_misc:get_value(?PROP_CMETA_PREFERRED_W, CustomMetadata, 0), + Preferred_D = leo_misc:get_value(?PROP_CMETA_PREFERRED_D, CustomMetadata, 0), Ver = leo_misc:get_value(?PROP_CMETA_VER, CustomMetadata, 0), UDM =leo_misc:get_value(?PROP_CMETA_UDM, CustomMetadata, []), {ok, {Metadata#?METADATA{cluster_id = ClusterId, num_of_replicas = NumOfReplicas, + preferred_r = Preferred_R, + preferred_w = Preferred_W, + preferred_d = Preferred_D, ver = Ver}, UDM}} catch @@ -436,11 +563,17 @@ get_udm_from_cmeta_bin(CustomMetadataBin)-> list_to_cmeta_bin(CustomMetadata) -> ClusterId = leo_misc:get_value(?PROP_CMETA_CLUSTER_ID, CustomMetadata, []), NumOfReplicas = leo_misc:get_value(?PROP_CMETA_NUM_OF_REPLICAS, CustomMetadata, 0), + Preferred_R = leo_misc:get_value(?PROP_CMETA_PREFERRED_R, CustomMetadata, 0), + Preferred_W = leo_misc:get_value(?PROP_CMETA_PREFERRED_W, CustomMetadata, 0), + Preferred_D = leo_misc:get_value(?PROP_CMETA_PREFERRED_D, CustomMetadata, 0), Ver = leo_misc:get_value(?PROP_CMETA_VER, CustomMetadata, 0), UDM = leo_misc:get_value(?PROP_CMETA_UDM, CustomMetadata, []), term_to_binary([{?PROP_CMETA_CLUSTER_ID, ClusterId}, {?PROP_CMETA_NUM_OF_REPLICAS, NumOfReplicas}, + {?PROP_CMETA_PREFERRED_R, Preferred_R}, + {?PROP_CMETA_PREFERRED_W, Preferred_W}, + {?PROP_CMETA_PREFERRED_D, Preferred_D}, {?PROP_CMETA_VER, Ver}, {?PROP_CMETA_UDM, UDM} ]). diff --git a/test/leo_object_storage_api_tests.erl b/test/leo_object_storage_api_tests.erl index 2a8246e..565bf1a 100644 --- a/test/leo_object_storage_api_tests.erl +++ b/test/leo_object_storage_api_tests.erl @@ -532,6 +532,9 @@ put_regular_bin_with_cmeta(Index, Counter) -> CMeta = [{?PROP_CMETA_CLUSTER_ID, 'leofs_1'}, {?PROP_CMETA_NUM_OF_REPLICAS, 3}, + {?PROP_CMETA_PREFERRED_R, 1}, + {?PROP_CMETA_PREFERRED_W, 2}, + {?PROP_CMETA_PREFERRED_D, 2}, {?PROP_CMETA_VER, leo_date:clock()}, {?PROP_CMETA_UDM, [{<<"name">>, <<"LeoFS">>}, {<<"category">>, <<"distributed storage">>}, @@ -1049,6 +1052,9 @@ compact_2() -> ], CMeta_1 = [{?PROP_CMETA_CLUSTER_ID, 'leofs_1'}, {?PROP_CMETA_NUM_OF_REPLICAS, 3}, + {?PROP_CMETA_PREFERRED_R, 1}, + {?PROP_CMETA_PREFERRED_W, 2}, + {?PROP_CMETA_PREFERRED_D, 2}, {?PROP_CMETA_VER, leo_date:clock()}, {?PROP_CMETA_UDM, UDM_1} ], diff --git a/test/leo_object_transformer_tests.erl b/test/leo_object_transformer_tests.erl new file mode 100644 index 0000000..4ab4256 --- /dev/null +++ b/test/leo_object_transformer_tests.erl @@ -0,0 +1,454 @@ +%%==================================================================== +%% +%% Leo Object Storage +%% +%% Copyright (c) 2012-2017 Rakuten, Inc. +%% +%% This file is provided to you under the Apache License, +%% Version 2.0 (the "License"); you may not use this file +%% except in compliance with the License. You may obtain +%% a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, +%% software distributed under the License is distributed on an +%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +%% KIND, either express or implied. See the License for the +%% specific language governing permissions and limitations +%% under the License. +%% +%%==================================================================== +-module(leo_object_transformer_tests). +-author('yosuke hara'). + +-include_lib("eunit/include/eunit.hrl"). +-include("leo_object_storage.hrl"). + + +-ifdef(EUNIT). + +leo_object_transformer_testssuit_test_() -> + {setup, + fun ( ) -> + ok + end, + fun (_) -> + ok + end, + [ + {"Tests all transformers", + {timeout, 1000, fun transform/0}} + ]}. + + +transform() -> + Key = <<"leofs/distributed/storage">>, + AddrId = 328119305811, + KSize = erlang:byte_size(Key), + DSize = 128, + MSize = 384, + CMeta = <<"user-defined-metadata">>, + CSize = 64, + CNum = 1, + CIndex = 1, + Offset = 512, + Clock = leo_date:clock(), + Timestamp = leo_date:now(), + Checksum = 1549499612212564499221795376818021, + RingHash = 1116532140187397365077604119361862, + ClusterId = 'leofs_cluster_1', + NumOfReplicas = 3, + Preferred_R = 1, + Preferred_W = 2, + Preferred_D = 2, + Ver = 1, + Del = 0, + + #?OBJECT{key = Key, + addr_id = AddrId, + ksize = KSize, + dsize = DSize, + msize = MSize, + csize = CSize, + cnumber = CNum, + cindex = CIndex, + offset = Offset, + clock = Clock, + timestamp = Timestamp, + checksum = Checksum, + ring_hash = RingHash, + del = Del} = leo_object_storage_transformer:metadata_to_object(#metadata{key = Key, + addr_id = AddrId, + ksize = KSize, + dsize = DSize, + msize = MSize, + csize = CSize, + cnumber = CNum, + cindex = CIndex, + offset = Offset, + clock = Clock, + timestamp = Timestamp, + checksum = Checksum, + ring_hash = RingHash, + del = Del}), + #?OBJECT{key = Key, + addr_id = AddrId, + ksize = KSize, + dsize = DSize, + msize = MSize, + csize = CSize, + cnumber = CNum, + cindex = CIndex, + offset = Offset, + clock = Clock, + timestamp = Timestamp, + checksum = Checksum, + ring_hash = RingHash, + cluster_id = ClusterId, + num_of_replicas = NumOfReplicas, + ver = Ver, + del = Del} = leo_object_storage_transformer:metadata_to_object(#metadata_1{key = Key, + addr_id = AddrId, + ksize = KSize, + dsize = DSize, + msize = MSize, + csize = CSize, + cnumber = CNum, + cindex = CIndex, + offset = Offset, + clock = Clock, + timestamp = Timestamp, + checksum = Checksum, + ring_hash = RingHash, + cluster_id = ClusterId, + num_of_replicas = NumOfReplicas, + ver = Ver, + del = Del}), + #?OBJECT{key = Key, + addr_id = AddrId, + ksize = KSize, + dsize = DSize, + meta = CMeta, + msize = MSize, + csize = CSize, + cnumber = CNum, + cindex = CIndex, + offset = Offset, + clock = Clock, + timestamp = Timestamp, + checksum = Checksum, + ring_hash = RingHash, + cluster_id = ClusterId, + num_of_replicas = NumOfReplicas, + ver = Ver, + del = Del} = leo_object_storage_transformer:metadata_to_object(#metadata_2{key = Key, + addr_id = AddrId, + ksize = KSize, + dsize = DSize, + meta = CMeta, + msize = MSize, + csize = CSize, + cnumber = CNum, + cindex = CIndex, + offset = Offset, + clock = Clock, + timestamp = Timestamp, + checksum = Checksum, + ring_hash = RingHash, + cluster_id = ClusterId, + num_of_replicas = NumOfReplicas, + ver = Ver, + del = Del}), + #?OBJECT{key = Key, + addr_id = AddrId, + ksize = KSize, + dsize = DSize, + meta = CMeta, + msize = MSize, + csize = CSize, + cnumber = CNum, + cindex = CIndex, + offset = Offset, + clock = Clock, + timestamp = Timestamp, + checksum = Checksum, + ring_hash = RingHash, + cluster_id = ClusterId, + num_of_replicas = NumOfReplicas, + preferred_r = Preferred_R, + preferred_w = Preferred_W, + preferred_d = Preferred_D, + ver = Ver, + del = Del} = leo_object_storage_transformer:metadata_to_object(#?METADATA{key = Key, + addr_id = AddrId, + ksize = KSize, + dsize = DSize, + meta = CMeta, + msize = MSize, + csize = CSize, + cnumber = CNum, + cindex = CIndex, + offset = Offset, + clock = Clock, + timestamp = Timestamp, + checksum = Checksum, + ring_hash = RingHash, + cluster_id = ClusterId, + num_of_replicas = NumOfReplicas, + preferred_r = Preferred_R, + preferred_w = Preferred_W, + preferred_d = Preferred_D, + ver = Ver, + del = Del}), + ?debugVal("## Passed: metadata_to_object_test"), + + + #?METADATA{key = Key, + addr_id = AddrId, + ksize = KSize, + dsize = DSize, + msize = MSize, + csize = CSize, + cnumber = CNum, + cindex = CIndex, + offset = Offset, + clock = Clock, + timestamp = Timestamp, + checksum = Checksum, + ring_hash = RingHash, + del = Del} = leo_object_storage_transformer:object_to_metadata(#object{key = Key, + addr_id = AddrId, + ksize = KSize, + dsize = DSize, + msize = MSize, + csize = CSize, + cnumber = CNum, + cindex = CIndex, + offset = Offset, + clock = Clock, + timestamp = Timestamp, + checksum = Checksum, + ring_hash = RingHash, + del = Del}), + #?METADATA{key = Key, + addr_id = AddrId, + ksize = KSize, + dsize = DSize, + meta = CMeta, + msize = MSize, + csize = CSize, + cnumber = CNum, + cindex = CIndex, + offset = Offset, + clock = Clock, + timestamp = Timestamp, + checksum = Checksum, + ring_hash = RingHash, + cluster_id = ClusterId, + num_of_replicas = NumOfReplicas, + ver = Ver, + del = Del} = leo_object_storage_transformer:object_to_metadata(#object_1{key = Key, + addr_id = AddrId, + ksize = KSize, + dsize = DSize, + meta = CMeta, + msize = MSize, + csize = CSize, + cnumber = CNum, + cindex = CIndex, + offset = Offset, + clock = Clock, + timestamp = Timestamp, + checksum = Checksum, + ring_hash = RingHash, + cluster_id = ClusterId, + num_of_replicas = NumOfReplicas, + ver = Ver, + del = Del}), + #?METADATA{key = Key, + addr_id = AddrId, + ksize = KSize, + dsize = DSize, + meta = CMeta, + msize = MSize, + csize = CSize, + cnumber = CNum, + cindex = CIndex, + offset = Offset, + clock = Clock, + timestamp = Timestamp, + checksum = Checksum, + ring_hash = RingHash, + cluster_id = ClusterId, + num_of_replicas = NumOfReplicas, + preferred_r = Preferred_R, + preferred_w = Preferred_W, + preferred_d = Preferred_D, + ver = Ver, + del = Del} = leo_object_storage_transformer:object_to_metadata(#?OBJECT{key = Key, + addr_id = AddrId, + ksize = KSize, + dsize = DSize, + meta = CMeta, + msize = MSize, + csize = CSize, + cnumber = CNum, + cindex = CIndex, + offset = Offset, + clock = Clock, + timestamp = Timestamp, + checksum = Checksum, + ring_hash = RingHash, + cluster_id = ClusterId, + num_of_replicas = NumOfReplicas, + preferred_r = Preferred_R, + preferred_w = Preferred_W, + preferred_d = Preferred_D, + ver = Ver, + del = Del}), + ?debugVal("## Passed: object_to_metadata_test"), + + + #?METADATA{key = Key, + addr_id = AddrId, + ksize = KSize, + dsize = DSize, + msize = MSize, + csize = CSize, + cnumber = CNum, + cindex = CIndex, + offset = Offset, + clock = Clock, + timestamp = Timestamp, + checksum = Checksum, + ring_hash = RingHash, + del = Del} = leo_object_storage_transformer:transform_metadata(#metadata{key = Key, + addr_id = AddrId, + ksize = KSize, + dsize = DSize, + msize = MSize, + csize = CSize, + cnumber = CNum, + cindex = CIndex, + offset = Offset, + clock = Clock, + timestamp = Timestamp, + checksum = Checksum, + ring_hash = RingHash, + del = Del}), + #?METADATA{key = Key, + addr_id = AddrId, + ksize = KSize, + dsize = DSize, + msize = MSize, + csize = CSize, + cnumber = CNum, + cindex = CIndex, + offset = Offset, + clock = Clock, + timestamp = Timestamp, + checksum = Checksum, + ring_hash = RingHash, + cluster_id = ClusterId, + num_of_replicas = NumOfReplicas, + ver = Ver, + del = Del} = leo_object_storage_transformer:transform_metadata(#metadata_1{key = Key, + addr_id = AddrId, + ksize = KSize, + dsize = DSize, + msize = MSize, + csize = CSize, + cnumber = CNum, + cindex = CIndex, + offset = Offset, + clock = Clock, + timestamp = Timestamp, + checksum = Checksum, + ring_hash = RingHash, + cluster_id = ClusterId, + num_of_replicas = NumOfReplicas, + ver = Ver, + del = Del}), + #?METADATA{key = Key, + addr_id = AddrId, + ksize = KSize, + dsize = DSize, + meta = CMeta, + msize = MSize, + csize = CSize, + cnumber = CNum, + cindex = CIndex, + offset = Offset, + clock = Clock, + timestamp = Timestamp, + checksum = Checksum, + ring_hash = RingHash, + cluster_id = ClusterId, + num_of_replicas = NumOfReplicas, + ver = Ver, + del = Del} = leo_object_storage_transformer:transform_metadata(#metadata_2{key = Key, + addr_id = AddrId, + ksize = KSize, + dsize = DSize, + meta = CMeta, + msize = MSize, + csize = CSize, + cnumber = CNum, + cindex = CIndex, + offset = Offset, + clock = Clock, + timestamp = Timestamp, + checksum = Checksum, + ring_hash = RingHash, + cluster_id = ClusterId, + num_of_replicas = NumOfReplicas, + ver = Ver, + del = Del}), + ?debugVal("## Passed: transform_metadata_test"), + + + CustomMetadataBin = leo_object_storage_transformer:list_to_cmeta_bin( + [{?PROP_CMETA_CLUSTER_ID, ClusterId}, + {?PROP_CMETA_NUM_OF_REPLICAS, NumOfReplicas}, + {?PROP_CMETA_PREFERRED_R, Preferred_R}, + {?PROP_CMETA_PREFERRED_W, Preferred_W}, + {?PROP_CMETA_PREFERRED_D, Preferred_D}, + {?PROP_CMETA_VER, Ver}, + {?PROP_CMETA_UDM, CMeta} + ]), + {ok, {Metadata_1, CMeta_1}} = leo_object_storage_transformer:cmeta_bin_into_metadata( + CustomMetadataBin, + #?METADATA{key = Key, + addr_id = AddrId, + ksize = KSize, + dsize = DSize, + meta = CMeta, + msize = MSize, + csize = CSize, + cnumber = CNum, + cindex = CIndex, + offset = Offset, + clock = Clock, + timestamp = Timestamp, + checksum = Checksum, + ring_hash = RingHash, + del = Del}), + ?assertEqual(ClusterId, Metadata_1#?METADATA.cluster_id), + ?assertEqual(NumOfReplicas, Metadata_1#?METADATA.num_of_replicas), + ?assertEqual(Preferred_R, Metadata_1#?METADATA.preferred_r), + ?assertEqual(Preferred_W, Metadata_1#?METADATA.preferred_w), + ?assertEqual(Preferred_D, Metadata_1#?METADATA.preferred_d), + ?assertEqual(Ver, Metadata_1#?METADATA.ver), + ?assertEqual(CMeta, CMeta_1), + %% ?debugVal(Metadata_1#?METADATA.cluster_id), + %% ?debugVal(Metadata_1#?METADATA.num_of_replicas), + %% ?debugVal(Metadata_1#?METADATA.preferred_r), + %% ?debugVal(Metadata_1#?METADATA.preferred_w), + %% ?debugVal(Metadata_1#?METADATA.preferred_d), + %% ?debugVal(Metadata_1#?METADATA.ver), + %% ?debugVal(CMeta_1), + ?debugVal("## Passed: cmeta_bin_into_metadata_test"), + ok. + +-endif.