Unify SET_VERSIONSTAMPED_KEY and SET_VERSIONSTAMPED_VALUE API

bindings/bindingtester/tests/__init__.py

@@ -20,6 +20,7 @@
 
 import math
 import re
+import struct
 
 import fdb
 
@@ -80,6 +81,20 @@ def get_expected_results(self):
     def validate(self, db, args):
         return []
 
+    def versionstamp_key(self, raw_bytes, version_pos):
+        if hasattr(self, 'api_version') and self.api_version < 520:
+            return raw_bytes + struct.pack('<H', version_pos)
+        else:
+            return raw_bytes + struct.pack('<L', version_pos)
+
+    def versionstamp_value(self, raw_bytes, version_pos=0):
+        if hasattr(self, 'api_version') and self.api_version < 520:
+            if version_pos != 0:
+                raise ValueError("unable to set non-zero version position before 520 in values")
+            return raw_bytes
+        else:
+            return raw_bytes + struct.pack('<L', version_pos)
+
     @classmethod
     def create_test(cls, name, subspace):
         target = 'bindingtester.tests.%s' % name

bindings/bindingtester/tests/api.py

@@ -40,6 +40,7 @@ def __init__(self, subspace):
         self.stack_subspace = self.subspace['stack']
 
         self.versionstamped_values = self.scratch['versionstamped_values']
+        self.versionstamped_values_2 = self.scratch['versionstamped_values_2']
         self.versionstamped_keys = self.scratch['versionstamped_keys']
 
     def setup(self, args):
@@ -56,6 +57,7 @@ def setup(self, args):
         self.generated_keys = []
         self.outstanding_ops = []
         self.random = test_util.RandomGenerator(args.max_int_bits, args.api_version, args.types)
+        self.api_version = args.api_version
 
     def add_stack_items(self, num):
         self.stack_size += num
@@ -349,17 +351,30 @@ def generate(self, args, thread_number):
             elif op == 'VERSIONSTAMP':
                 rand_str1 = self.random.random_string(100)
                 key1 = self.versionstamped_values.pack((rand_str1,))
+                key2 = self.versionstamped_values_2.pack((rand_str1,))
 
                 split = random.randint(0, 70)
-                rand_str2 = self.random.random_string(20 + split) + fdb.tuple.Versionstamp._UNSET_TR_VERSION + self.random.random_string(70 - split)
-                key2 = self.versionstamped_keys.pack() + rand_str2
-                index = key2.find(fdb.tuple.Versionstamp._UNSET_TR_VERSION)
-                key2 += chr(index % 256) + chr(index / 256)
-
-                instructions.push_args(u'SET_VERSIONSTAMPED_VALUE', key1, fdb.tuple.Versionstamp._UNSET_TR_VERSION + rand_str2)
+                prefix = self.random.random_string(20 + split)
+                if prefix.endswith('\xff'):
+                    # Necessary to make sure that the SET_VERSIONSTAMPED_VALUE check
+                    # correctly finds where the version is supposed to fit in.
+                    prefix += '\x00'
+                suffix = self.random.random_string(70 - split)
+                rand_str2 = prefix + fdb.tuple.Versionstamp._UNSET_TR_VERSION + suffix
+                key3 = self.versionstamped_keys.pack() + rand_str2
+                index = len(self.versionstamped_keys.pack()) + len(prefix)
+                key3 = self.versionstamp_key(key3, index)
+
+                instructions.push_args(u'SET_VERSIONSTAMPED_VALUE',
+                                       key1,
+                                       self.versionstamp_value(fdb.tuple.Versionstamp._UNSET_TR_VERSION + rand_str2))
                 instructions.append('ATOMIC_OP')
 
-                instructions.push_args(u'SET_VERSIONSTAMPED_KEY', key2, rand_str1)
+                if args.api_version >= 520:
+                    instructions.push_args(u'SET_VERSIONSTAMPED_VALUE', key2, self.versionstamp_value(rand_str2, len(prefix)))
+                    instructions.append('ATOMIC_OP')
+
+                instructions.push_args(u'SET_VERSIONSTAMPED_KEY', key3, rand_str1)
                 instructions.append('ATOMIC_OP')
                 self.can_use_key_selectors = False
 
@@ -430,16 +445,17 @@ def generate(self, args, thread_number):
 
             elif op == 'TUPLE_PACK_WITH_VERSIONSTAMP':
                 tup = (self.random.random_string(20),) + self.random.random_tuple(10, incomplete_versionstamps=True)
-                instructions.push_args(self.versionstamped_keys.pack(), len(tup), *tup)
+                prefix = self.versionstamped_keys.pack()
+                instructions.push_args(prefix, len(tup), *tup)
                 instructions.append(op)
                 self.add_strings(1)
 
-                version_key = self.versionstamped_keys.pack(tup)
-                first_incomplete = version_key.find(fdb.tuple.Versionstamp._UNSET_TR_VERSION)
+                versionstamp_param = prefix + fdb.tuple.pack(tup)
+                first_incomplete = versionstamp_param.find(fdb.tuple.Versionstamp._UNSET_TR_VERSION)
                 second_incomplete = -1 if first_incomplete < 0 else \
-                    version_key.find(fdb.tuple.Versionstamp._UNSET_TR_VERSION, first_incomplete + len(fdb.tuple.Versionstamp._UNSET_TR_VERSION) + 1)
+                    versionstamp_param.find(fdb.tuple.Versionstamp._UNSET_TR_VERSION, first_incomplete + len(fdb.tuple.Versionstamp._UNSET_TR_VERSION) + 1)
 
-                # If there is exactly one incomplete versionstamp, perform the versionstamped key operation.
+                # If there is exactly one incomplete versionstamp, perform the versionstamp operation.
                 if first_incomplete >= 0 and second_incomplete < 0:
                     rand_str = self.random.random_string(100)
 
@@ -448,9 +464,15 @@ def generate(self, args, thread_number):
                     instructions.push_args(u'SET_VERSIONSTAMPED_KEY')
                     instructions.append('ATOMIC_OP')
 
+                    if self.api_version >= 520:
+                        version_value_key_2 = self.versionstamped_values_2.pack((rand_str,))
+                        versionstamped_value = self.versionstamp_value(fdb.tuple.pack(tup), first_incomplete - len(prefix))
+                        instructions.push_args(u'SET_VERSIONSTAMPED_VALUE', version_value_key_2, versionstamped_value)
+                        instructions.append('ATOMIC_OP')
+
                     version_value_key = self.versionstamped_values.pack((rand_str,))
                     instructions.push_args(u'SET_VERSIONSTAMPED_VALUE', version_value_key,
-                                           fdb.tuple.Versionstamp._UNSET_TR_VERSION + fdb.tuple.pack(tup))
+                                           self.versionstamp_value(fdb.tuple.Versionstamp._UNSET_TR_VERSION + fdb.tuple.pack(tup)))
                     instructions.append('ATOMIC_OP')
                     self.can_use_key_selectors = False
 
@@ -510,7 +532,7 @@ def generate(self, args, thread_number):
                 self.add_strings(1)
 
             else:
-                assert False
+                assert False, 'Unknown operation: ' + op
 
             if read_performed and op not in database_reads:
                 self.outstanding_ops.append((self.stack_size, len(instructions) - 1))
@@ -539,13 +561,22 @@ def check_versionstamps(self, tr, begin_key, limit):
         incorrect_versionstamps = 0
         for k, v in tr.get_range(begin_key, self.versionstamped_values.range().stop, limit=limit):
             next_begin = k + '\x00'
-            tup = fdb.tuple.unpack(k)
-            key = self.versionstamped_keys.pack() + v[10:].replace(fdb.tuple.Versionstamp._UNSET_TR_VERSION, v[:10], 1)
-            if tr[key] != tup[-1]:
+            random_id = self.versionstamped_values.unpack(k)[0]
+            versioned_value = v[10:].replace(fdb.tuple.Versionstamp._UNSET_TR_VERSION, v[:10], 1)
+
+            versioned_key = self.versionstamped_keys.pack() + versioned_value
+            if tr[versioned_key] != random_id:
                 util.get_logger().error('  INCORRECT VERSIONSTAMP:')
-                util.get_logger().error('    %s != %s', repr(tr[key]), repr(tup[-1]))
+                util.get_logger().error('    %s != %s', repr(tr[versioned_key]), repr(random_id))
                 incorrect_versionstamps += 1
 
+            if self.api_version >= 520:
+                k2 = self.versionstamped_values_2.pack((random_id,))
+                if tr[k2] != versioned_value:
+                    util.get_logger().error('  INCORRECT VERSIONSTAMP:')
+                    util.get_logger().error('    %s != %s', repr(tr[k2]), repr(versioned_value))
+                    incorrect_versionstamps += 1
+
         return (next_begin, incorrect_versionstamps)
 
     def validate(self, db, args):

bindings/bindingtester/tests/scripted.py

@@ -54,6 +54,8 @@ def setup(self, args):
         if args.bisect:
             raise Exception('Scripted tests cannot be bisected')
 
+        self.api_version = args.api_version
+
     def generate(self, args, thread_number):
         self.results = []
 
@@ -272,12 +274,17 @@ def generate(self, args, thread_number):
 
         stampKey = 'stampedXXXXXXXXXXsuffix'
         stampKeyIndex = stampKey.find('XXXXXXXXXX')
-        stampKeyStr = chr(stampKeyIndex % 256) + chr(stampKeyIndex / 256)
-        main_thread.push_args(u'SET_VERSIONSTAMPED_KEY', stampKey + stampKeyStr, 'stampedBar')
+        main_thread.push_args(u'SET_VERSIONSTAMPED_KEY', self.versionstamp_key(stampKey, stampKeyIndex), 'stampedBar')
         main_thread.append('ATOMIC_OP')
-        main_thread.push_args(u'SET_VERSIONSTAMPED_VALUE', 'stampedValue', 'XXXXXXXXXX')
+        main_thread.push_args(u'SET_VERSIONSTAMPED_VALUE', 'stampedValue', self.versionstamp_value('XXXXXXXXXX'))
         main_thread.append('ATOMIC_OP')
 
+        if self.api_version >= 520:
+            stampValue = 'stampedXXXXXXXXXXsuffix'
+            stampValueIndex = stampValue.find('XXXXXXXXXX')
+            main_thread.push_args(u'SET_VERSIONSTAMPED_VALUE', 'stampedValue2', self.versionstamp_value(stampValue, stampValueIndex))
+            main_thread.append('ATOMIC_OP')
+
         main_thread.push_args('suffix')
         main_thread.append('GET_VERSIONSTAMP')
         test_util.blocking_commit(main_thread)
@@ -296,6 +303,12 @@ def generate(self, args, thread_number):
         main_thread.append('GET')
         self.add_result(main_thread, args, 'stampedBar')
 
+        if self.api_version >= 520:
+            main_thread.push_args('stampedValue2')
+            main_thread.append('GET')
+            main_thread.append('GET')
+            self.add_result(main_thread, args, 'stampedBar')
+
         main_thread.append('GET_VERSIONSTAMP')
         test_util.blocking_commit(main_thread)
         self.add_result(main_thread, args, 'RESULT_NOT_PRESENT')

bindings/go/src/fdb/generated.go

@@ -445,12 +445,12 @@ func (t Transaction) Min(key KeyConvertible, param []byte) {
 	t.atomicOp(key.FDBKey(), param, 13)
 }
 
-// SetVersionstampedKey transforms ``key`` using a versionstamp for the transaction. Sets the transformed key in the database to ``param``. A versionstamp is a 10 byte, unique, monotonically (but not sequentially) increasing value for each committed transaction. The first 8 bytes are the committed version of the database. The last 2 bytes are monotonic in the serialization order for transactions. WARNING: At this time versionstamps are compatible with the Tuple layer only in the Java and Python bindings. Note that this implies versionstamped keys may not be used with the Subspace and Directory layers except in those languages.
+// SetVersionstampedKey transforms ``key`` using a versionstamp for the transaction. Sets the transformed key in the database to ``param``. The key is transformed by removing the final four bytes from the key and reading those as a little-Endian 32-bit integer to get a position ``pos``. The 10 bytes of the key from ``pos`` to ``pos + 10`` are replaced with the versionstamp of the transaction used. The first byte of the key is position 0. A versionstamp is a 10 byte, unique, monotonically (but not sequentially) increasing value for each committed transaction. The first 8 bytes are the committed version of the database (serialized in big-Endian order). The last 2 bytes are monotonic in the serialization order for transactions. WARNING: At this time, versionstamps are compatible with the Tuple layer only in the Java and Python bindings. Also, note that prior to API version 520, the offset was computed from only the final two bytes rather than the final four bytes.
 func (t Transaction) SetVersionstampedKey(key KeyConvertible, param []byte) {
 	t.atomicOp(key.FDBKey(), param, 14)
 }
 
-// SetVersionstampedValue transforms ``param`` using a versionstamp for the transaction. Sets ``key`` in the database to the transformed parameter. A versionstamp is a 10 byte, unique, monotonically (but not sequentially) increasing value for each committed transaction. The first 8 bytes are the committed version of the database. The last 2 bytes are monotonic in the serialization order for transactions. WARNING: At this time versionstamped values are not compatible with the Tuple layer.
+// SetVersionstampedValue transforms ``param`` using a versionstamp for the transaction. Sets the ``key`` given to the transformed ``param``. The parameter is transformed by removing the final four bytes from ``param`` and reading those as a little-Endian 32-bit integer to get a position ``pos``. The 10 bytes of the parameter from ``pos`` to ``pos + 10`` are replaced with the versionstamp of the transaction used. The first byte of the parameter is position 0. A versionstamp is a 10 byte, unique, monotonically (but not sequentially) increasing value for each committed transaction. The first 8 bytes are the committed version of the database (serialized in big-Endian order). The last 2 bytes are monotonic in the serialization order for transactions. WARNING: At this time, versionstamps are compatible with the Tuple layer only in the Java and Python bindings. Also, note that prior to API version 520, the versionstamp was always placed at the beginning of the parameter rather than computing an offset.
 func (t Transaction) SetVersionstampedValue(key KeyConvertible, param []byte) {
 	t.atomicOp(key.FDBKey(), param, 15)
 }

bindings/java/src/main/com/apple/foundationdb/tuple/TupleUtil.java

@@ -33,6 +33,8 @@
 import java.util.UUID;
 import java.util.stream.Stream;
 
+import com.apple.foundationdb.FDB;
+
 class TupleUtil {
 	private static final byte nil = 0x00;
 	private static final BigInteger[] size_limits;
@@ -569,7 +571,7 @@ static EncodeResult encodeAll(List<Object> items, byte[] prefix, List<byte[]> en
 	}
 
 	static byte[] pack(List<Object> items, byte[] prefix) {
-		List<byte[]> encoded = new ArrayList<byte[]>(2 * items.size() + (prefix == null ? 0 : 1));
+		List<byte[]> encoded = new ArrayList<>(2 * items.size() + (prefix == null ? 0 : 1));
 		EncodeResult result = encodeAll(items, prefix, encoded);
 		if(result.versionPos > 0) {
 			throw new IllegalArgumentException("Incomplete Versionstamp included in vanilla tuple pack");
@@ -579,15 +581,19 @@ static byte[] pack(List<Object> items, byte[] prefix) {
 	}
 
 	static byte[] packWithVersionstamp(List<Object> items, byte[] prefix) {
-		List<byte[]> encoded = new ArrayList<byte[]>(2 * items.size() + (prefix == null ? 1 : 2));
+		List<byte[]> encoded = new ArrayList<>(2 * items.size() + (prefix == null ? 1 : 2));
 		EncodeResult result = encodeAll(items, prefix, encoded);
 		if(result.versionPos < 0) {
 			throw new IllegalArgumentException("No incomplete Versionstamp included in tuple pack with versionstamp");
 		} else {
 			if(result.versionPos > 0xffff) {
 				throw new IllegalArgumentException("Tuple has incomplete version at position " + result.versionPos + " which is greater than the maximum " + 0xffff);
 			}
-			encoded.add(ByteBuffer.allocate(2).order(ByteOrder.LITTLE_ENDIAN).putShort((short)result.versionPos).array());
+			if (FDB.instance().getAPIVersion() < 520) {
+				encoded.add(ByteBuffer.allocate(2).order(ByteOrder.LITTLE_ENDIAN).putShort((short)result.versionPos).array());
+			} else {
+				encoded.add(ByteBuffer.allocate(4).order(ByteOrder.LITTLE_ENDIAN).putInt(result.versionPos).array());
+			}
 			return ByteArrayUtil.join(null, encoded);
 		}
 	}

bindings/python/fdb/tuple.py

@@ -384,7 +384,10 @@ def _pack_maybe_with_versionstamp(t, prefix=None):
     if version_pos >= 0:
         version_pos += len(prefix) if prefix is not None else 0
         bytes_list.extend(child_bytes)
-        bytes_list.append(struct.pack('<H', version_pos))
+        if fdb.is_api_version_selected() and fdb.get_api_version() < 520:
+            bytes_list.append(struct.pack('<H', version_pos))
+        else:
+            bytes_list.append(struct.pack('<L', version_pos))
     else:
         bytes_list.extend(child_bytes)
 

bindings/python/tests/tester.py

@@ -32,14 +32,7 @@
 
 sys.path[:0] = [os.path.join(os.path.dirname(__file__), '..')]
 import fdb
-
-assert not fdb.is_api_version_selected()
-try:
-    fdb.get_api_version()
-except RuntimeError as e:
-    assert str(e) == 'API version is not set'
 fdb.api_version(int(sys.argv[2]))
-assert int(sys.argv[2]) == fdb.get_api_version()
 
 from fdb import six
 from fdb.impl import strinc
@@ -528,19 +521,6 @@ def run(self):
                     Tester.wait_empty(self.db, prefix)
                     inst.push(b"WAITED_FOR_EMPTY")
                 elif inst.op == six.u("UNIT_TESTS"):
-                    assert fdb.is_api_version_selected()
-                    api_version = fdb.get_api_version()
-                    try:
-                        fdb.api_version(api_version + 1)
-                        raise RuntimeError('Was not stopped from selecting two API versions')
-                    except RuntimeError as e:
-                        assert str(e) == 'FDB API already loaded at version {}'.format(api_version)
-                    try:
-                        fdb.api_version(api_version - 1)
-                        raise RuntimeError('Was not stopped from selecting two API versions')
-                    except RuntimeError as e:
-                        assert str(e) == 'FDB API already loaded at version {}'.format(api_version)
-                    fdb.api_version(api_version)
                     try:
                         db.options.set_location_cache_size(100001)