update bindings and bindingtester that uses versionstamps to use new …

…protocol

issue apple#148

bindings/bindingtester/tests/__init__.py

@@ -20,6 +20,7 @@
 
 import math
 import re
+import struct
 
 import fdb
 
@@ -80,6 +81,20 @@ def get_expected_results(self):
     def validate(self, db, args):
         return []
 
+    def versionstamp_key(self, raw_bytes, version_pos):
+        if hasattr(self, 'api_version') and self.api_version < 520:
+            return raw_bytes + struct.pack('<H', version_pos)
+        else:
+            return raw_bytes + struct.pack('<L', version_pos)
+
+    def versionstamp_value(self, raw_bytes, version_pos=0):
+        if hasattr(self, 'api_version') and self.api_version < 520:
+            if version_pos != 0:
+                raise ValueError("unable to set non-zero version position before 520 in values")
+            return raw_bytes
+        else:
+            return raw_bytes + struct.pack('<L', version_pos)
+
     @classmethod
     def create_test(cls, name, subspace):
         target = 'bindingtester.tests.%s' % name

bindings/bindingtester/tests/api.py

@@ -57,6 +57,7 @@ def setup(self, args):
         self.generated_keys = []
         self.outstanding_ops = []
         self.random = test_util.RandomGenerator(args.max_int_bits, args.api_version, args.types)
+        self.api_version = args.api_version
 
     def add_stack_items(self, num):
         self.stack_size += num
@@ -151,7 +152,7 @@ def generate(self, args, thread_number):
         snapshot_versions = ['GET_READ_VERSION_SNAPSHOT']
         tuples = ['TUPLE_PACK', 'TUPLE_UNPACK', 'TUPLE_RANGE', 'TUPLE_SORT', 'SUB', 'ENCODE_FLOAT', 'ENCODE_DOUBLE', 'DECODE_DOUBLE', 'DECODE_FLOAT']
         if 'versionstamp' in args.types:
-            tuples += ['TUPLE_PACK_VERSIONSTAMPED_KEY', 'TUPLE_PACK_VERSIONSTAMPED_VALUE']
+            tuples.append('TUPLE_PACK_WITH_VERSIONSTAMP')
         resets = ['ON_ERROR', 'RESET', 'CANCEL']
         read_conflicts = ['READ_CONFLICT_RANGE', 'READ_CONFLICT_KEY']
         write_conflicts = ['WRITE_CONFLICT_RANGE', 'WRITE_CONFLICT_KEY', 'DISABLE_WRITE_CONFLICT']
@@ -362,13 +363,16 @@ def generate(self, args, thread_number):
                 rand_str2 = prefix + fdb.tuple.Versionstamp._UNSET_TR_VERSION + suffix
                 key3 = self.versionstamped_keys.pack() + rand_str2
                 index = len(self.versionstamped_keys.pack()) + len(prefix)
-                key3 += chr(index % 256) + chr(index / 256)
+                key3 = self.versionstamp_key(key3, index)
 
-                instructions.push_args(u'SET_VERSIONSTAMPED_VALUE', key1, fdb.tuple.Versionstamp._UNSET_TR_VERSION + rand_str2)
+                instructions.push_args(u'SET_VERSIONSTAMPED_VALUE',
+                                       key1,
+                                       self.versionstamp_value(fdb.tuple.Versionstamp._UNSET_TR_VERSION + rand_str2))
                 instructions.append('ATOMIC_OP')
 
-                instructions.push_args(u'SET_VERSIONSTAMPED_VALUE_POS', key2, rand_str2 + struct.pack('<L', len(prefix)))
-                instructions.append('ATOMIC_OP')
+                if args.api_version >= 520:
+                    instructions.push_args(u'SET_VERSIONSTAMPED_VALUE', key2, self.versionstamp_value(rand_str2, len(prefix)))
+                    instructions.append('ATOMIC_OP')
 
                 instructions.push_args(u'SET_VERSIONSTAMPED_KEY', key3, rand_str1)
                 instructions.append('ATOMIC_OP')
@@ -439,9 +443,9 @@ def generate(self, args, thread_number):
                 else:
                     self.add_strings(2)
 
-            elif op == 'TUPLE_PACK_VERSIONSTAMPED_KEY' or op == 'TUPLE_PACK_VERSIONSTAMPED_VALUE':
+            elif op == 'TUPLE_PACK_WITH_VERSIONSTAMP':
                 tup = (self.random.random_string(20),) + self.random.random_tuple(10, incomplete_versionstamps=True)
-                prefix = self.versionstamped_keys.pack() if op == 'TUPLE_PACK_VERSIONSTAMPED_KEY' else ''
+                prefix = self.versionstamped_keys.pack()
                 instructions.push_args(prefix, len(tup), *tup)
                 instructions.append(op)
                 self.add_strings(1)
@@ -455,30 +459,20 @@ def generate(self, args, thread_number):
                 if first_incomplete >= 0 and second_incomplete < 0:
                     rand_str = self.random.random_string(100)
 
-                    if op == 'TUPLE_PACK_VERSIONSTAMPED_KEY':
-                        instructions.push_args(rand_str)
-                        test_util.to_front(instructions, 1)
-                        instructions.push_args(u'SET_VERSIONSTAMPED_KEY')
-                        instructions.append('ATOMIC_OP')
+                    instructions.push_args(rand_str)
+                    test_util.to_front(instructions, 1)
+                    instructions.push_args(u'SET_VERSIONSTAMPED_KEY')
+                    instructions.append('ATOMIC_OP')
 
+                    if self.api_version >= 520:
                         version_value_key_2 = self.versionstamped_values_2.pack((rand_str,))
-                        versionstamped_value = fdb.tuple.pack(tup) + struct.pack('<L', first_incomplete - len(prefix))
-                        instructions.push_args(u'SET_VERSIONSTAMPED_VALUE_POS', version_value_key_2, versionstamped_value)
-                        instructions.append('ATOMIC_OP')
-
-                    else:
-                        instructions.push_args(self.versionstamped_values_2.pack((rand_str,)))
-                        instructions.push_args(u'SET_VERSIONSTAMPED_VALUE_POS')
-                        instructions.append('ATOMIC_OP')
-
-                        versionstamped_key = self.versionstamped_keys.pack() + versionstamp_param \
-                            + struct.pack('<H', first_incomplete + len(self.versionstamped_keys.pack()))
-                        instructions.push_args(u'SET_VERSIONSTAMPED_KEY', versionstamped_key, rand_str)
+                        versionstamped_value = self.versionstamp_value(fdb.tuple.pack(tup), first_incomplete - len(prefix))
+                        instructions.push_args(u'SET_VERSIONSTAMPED_VALUE', version_value_key_2, versionstamped_value)
                         instructions.append('ATOMIC_OP')
 
                     version_value_key = self.versionstamped_values.pack((rand_str,))
                     instructions.push_args(u'SET_VERSIONSTAMPED_VALUE', version_value_key,
-                                           fdb.tuple.Versionstamp._UNSET_TR_VERSION + fdb.tuple.pack(tup))
+                                           self.versionstamp_value(fdb.tuple.Versionstamp._UNSET_TR_VERSION + fdb.tuple.pack(tup)))
                     instructions.append('ATOMIC_OP')
                     self.can_use_key_selectors = False
 
@@ -538,7 +532,7 @@ def generate(self, args, thread_number):
                 self.add_strings(1)
 
             else:
-                assert False
+                assert False, 'Unknown operation: ' + op
 
             if read_performed and op not in database_reads:
                 self.outstanding_ops.append((self.stack_size, len(instructions) - 1))
@@ -576,11 +570,12 @@ def check_versionstamps(self, tr, begin_key, limit):
                 util.get_logger().error('    %s != %s', repr(tr[versioned_key]), repr(random_id))
                 incorrect_versionstamps += 1
 
-            k2 = self.versionstamped_values_2.pack((random_id,))
-            if tr[k2] != versioned_value:
-                util.get_logger().error('  INCORRECT VERSIONSTAMP:')
-                util.get_logger().error('    %s != %s', repr(tr[k2]), repr(versioned_value))
-                incorrect_versionstamps += 1
+            if self.api_version >= 520:
+                k2 = self.versionstamped_values_2.pack((random_id,))
+                if tr[k2] != versioned_value:
+                    util.get_logger().error('  INCORRECT VERSIONSTAMP:')
+                    util.get_logger().error('    %s != %s', repr(tr[k2]), repr(versioned_value))
+                    incorrect_versionstamps += 1
 
         return (next_begin, incorrect_versionstamps)
 

bindings/bindingtester/tests/scripted.py

@@ -54,6 +54,8 @@ def setup(self, args):
         if args.bisect:
             raise Exception('Scripted tests cannot be bisected')
 
+        self.api_version = args.api_version
+
     def generate(self, args, thread_number):
         self.results = []
 
@@ -272,12 +274,17 @@ def generate(self, args, thread_number):
 
         stampKey = 'stampedXXXXXXXXXXsuffix'
         stampKeyIndex = stampKey.find('XXXXXXXXXX')
-        stampKeyStr = chr(stampKeyIndex % 256) + chr(stampKeyIndex / 256)
-        main_thread.push_args(u'SET_VERSIONSTAMPED_KEY', stampKey + stampKeyStr, 'stampedBar')
+        main_thread.push_args(u'SET_VERSIONSTAMPED_KEY', self.versionstamp_key(stampKey, stampKeyIndex), 'stampedBar')
         main_thread.append('ATOMIC_OP')
-        main_thread.push_args(u'SET_VERSIONSTAMPED_VALUE', 'stampedValue', 'XXXXXXXXXX')
+        main_thread.push_args(u'SET_VERSIONSTAMPED_VALUE', 'stampedValue', self.versionstamp_value('XXXXXXXXXX'))
         main_thread.append('ATOMIC_OP')
 
+        if self.api_version >= 520:
+            stampValue = 'stampedXXXXXXXXXXsuffix'
+            stampValueIndex = stampValue.find('XXXXXXXXXX')
+            main_thread.push_args(u'SET_VERSIONSTAMPED_VALUE', 'stampedValue2', self.versionstamp_value(stampValue, stampValueIndex))
+            main_thread.append('ATOMIC_OP')
+
         main_thread.push_args('suffix')
         main_thread.append('GET_VERSIONSTAMP')
         test_util.blocking_commit(main_thread)
@@ -296,6 +303,12 @@ def generate(self, args, thread_number):
         main_thread.append('GET')
         self.add_result(main_thread, args, 'stampedBar')
 
+        if self.api_version >= 520:
+            main_thread.push_args('stampedValue2')
+            main_thread.append('GET')
+            main_thread.append('GET')
+            self.add_result(main_thread, args, 'stampedBar')
+
         main_thread.append('GET_VERSIONSTAMP')
         test_util.blocking_commit(main_thread)
         self.add_result(main_thread, args, 'RESULT_NOT_PRESENT')

bindings/flow/tester/Tester.actor.cpp

@@ -1676,7 +1676,6 @@ void populateAtomicOpMap() {
 	optionInfo["MIN"] = FDBMutationType::FDB_MUTATION_TYPE_MIN;
 	optionInfo["SET_VERSIONSTAMPED_KEY"] = FDBMutationType::FDB_MUTATION_TYPE_SET_VERSIONSTAMPED_KEY;
 	optionInfo["SET_VERSIONSTAMPED_VALUE"] = FDBMutationType::FDB_MUTATION_TYPE_SET_VERSIONSTAMPED_VALUE;
-	optionInfo["SET_VERSIONSTAMPED_VALUE_POS"] = FDBMutationType::FDB_MUTATION_TYPE_SET_VERSIONSTAMPED_VALUE_POS;
 	optionInfo["BYTE_MIN"] = FDBMutationType::FDB_MUTATION_TYPE_BYTE_MIN;
 	optionInfo["BYTE_MAX"] = FDBMutationType::FDB_MUTATION_TYPE_BYTE_MAX;
 }

bindings/go/src/fdb/generated.go

@@ -445,12 +445,12 @@ func (t Transaction) Min(key KeyConvertible, param []byte) {
 	t.atomicOp(key.FDBKey(), param, 13)
 }
 
-// SetVersionstampedKey transforms ``key`` using a versionstamp for the transaction. Sets the transformed key in the database to ``param``. A versionstamp is a 10 byte, unique, monotonically (but not sequentially) increasing value for each committed transaction. The first 8 bytes are the committed version of the database. The last 2 bytes are monotonic in the serialization order for transactions. WARNING: At this time versionstamps are compatible with the Tuple layer only in the Java and Python bindings. Note that this implies versionstamped keys may not be used with the Subspace and Directory layers except in those languages.
+// SetVersionstampedKey transforms ``key`` using a versionstamp for the transaction. Sets the transformed key in the database to ``param``. The key is transformed by removing the final four bytes from the key and reading those as a little-Endian 32-bit integer to get a position ``pos``. The 10 bytes of the key from ``pos`` to ``pos + 10`` are replaced with the versionstamp of the transaction used. The first byte of the key is position 0. A versionstamp is a 10 byte, unique, monotonically (but not sequentially) increasing value for each committed transaction. The first 8 bytes are the committed version of the database (serialized in big-Endian order). The last 2 bytes are monotonic in the serialization order for transactions. WARNING: At this time, versionstamps are compatible with the Tuple layer only in the Java and Python bindings. Also, note that prior to API version 520, the offset was computed from only the final two bytes rather than the final four bytes.
 func (t Transaction) SetVersionstampedKey(key KeyConvertible, param []byte) {
 	t.atomicOp(key.FDBKey(), param, 14)
 }
 
-// SetVersionstampedValue transforms ``param`` using a versionstamp for the transaction. Sets ``key`` in the database to the transformed parameter. A versionstamp is a 10 byte, unique, monotonically (but not sequentially) increasing value for each committed transaction. The first 8 bytes are the committed version of the database. The last 2 bytes are monotonic in the serialization order for transactions. WARNING: At this time versionstamped values are not compatible with the Tuple layer.
+// SetVersionstampedValue transforms ``param`` using a versionstamp for the transaction. Sets the ``key`` given to the transformed ``param``. The parameter is transformed by removing the final four bytes from ``param`` and reading those as a little-Endian 32-bit integer to get a position ``pos``. The 10 bytes of the parameter from ``pos`` to ``pos + 10`` are replaced with the versionstamp of the transaction used. The first byte of the parameter is position 0. A versionstamp is a 10 byte, unique, monotonically (but not sequentially) increasing value for each committed transaction. The first 8 bytes are the committed version of the database (serialized in big-Endian order). The last 2 bytes are monotonic in the serialization order for transactions. WARNING: At this time, versionstamps are compatible with the Tuple layer only in the Java and Python bindings. Also, note that prior to API version 520, the versionstamp was always placed at the beginning of the parameter rather than computing an offset.
 func (t Transaction) SetVersionstampedValue(key KeyConvertible, param []byte) {
 	t.atomicOp(key.FDBKey(), param, 15)
 }

bindings/java/src/main/com/apple/foundationdb/subspace/Subspace.java

@@ -199,23 +199,6 @@ public byte[] pack(Tuple tuple) {
 		return tuple.pack(rawPrefix);
 	}
 
-	/**
-	 * Deprecated. Use {@link #packVersionstampedKey(Tuple) packVersionstampedKey()} instead. Because
-	 * the serialization formats are different for versionstamped keys and values, this method was
-	 * deprecated to encourage developers to choose appropriately between
-	 * {@link #packVersionstampedKey(Tuple) packVersionstampedKey()} and
-	 * {@link #packVersionstampedValue(Tuple)} packVersionstampedValue()}.
-	 *
-	 * @param tuple the {@code Tuple} to be packed
-	 * @return the key encoding the specified tuple in this {@code Subspace}
-	 * @throws IllegalArgumentException if {@code tuple} does not contain exactly one incomplete {@link Versionstamp}
-	 * @deprecated Since 5.2.0, replaced with {@link #packVersionstampedKey(Tuple) packVersionstampedKey()}
-	 */
-	@Deprecated
-	public byte[] packWithVersionstamp(Tuple tuple) {
-		return packVersionstampedKey(tuple);
-	}
-
 	/**
 	 * Gets the key encoding the specified tuple in this {@code Subspace} for use with
 	 * {@link com.apple.foundationdb.MutationType#SET_VERSIONSTAMPED_KEY MutationType.SET_VERSIONSTAMPED_KEY}.
@@ -230,22 +213,8 @@ public byte[] packWithVersionstamp(Tuple tuple) {
 	 * @return the key encoding the specified tuple in this {@code Subspace}
 	 * @throws IllegalArgumentException if {@code tuple} does not contain exactly one incomplete {@link Versionstamp}
 	 */
-	public byte[] packVersionstampedKey(Tuple tuple) {
-		return tuple.packVersionstampedKey(rawPrefix);
-	}
-
-	/**
-	 * Gets the key encoding the specified tuple in this {@code Subspace} for use with
-	 * {@link com.apple.foundationdb.MutationType#SET_VERSIONSTAMPED_VALUE MutationType.SET_VERSIONSTAMPED_VALUE}.
-	 * This works just like {@link #packVersionstampedKey(Tuple) packVersionstampedKey()} except that
-	 * the final serialization format is different.
-	 *
-	 * @param tuple the {@code Tuple} to be packed
-	 * @return the key encoding the specified tuple in this {@code Subspace}
-	 * @throws IllegalArgumentException if {@code tuple} does not contain exactly one incomplete {@link Versionstamp}
-	 */
-	public byte[] packVersionstampedValue(Tuple tuple) {
-		return tuple.packVersionstampedValue(rawPrefix);
+	public byte[] packWithVersionstamp(Tuple tuple) {
+		return tuple.packWithVersionstamp(rawPrefix);
 	}
 
 	/**