ddl: add exclude_null support for indexes

Add exclude_null support for indexes. The option was introduced in
Tarantool 2.8.1 [1] for nullable TREE indexes.

Since ddl approach is "require explicit fields rather than fill with
sane defaults" (for example, one must explicitly set is_nullable for
each field), we do not add "if exclude_null=true and is_nullable is not
provided, set is_nullable=true" rule from the core Tarantool [1]:
we require to explicitly specify both fields, if one wants to use
exclude_null features.

1. tarantool/tarantool@17c9c03

Closes #110

CHANGELOG.md

@@ -4,6 +4,12 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/)
 and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html).
 
+## Unreleased
+
+### Added
+
+- Added exclude_null support for indexes (#110).
+
 ## [1.6.3] - 2023-06-08
 
 ### Added

ddl/check.lua

@@ -386,9 +386,47 @@ local function check_index_part(i, index, space)
         end
     end
 
+    do -- check exclude_null has correct format
+        if part.exclude_null ~= nil and type(part.exclude_null) ~= 'boolean' then
+            return nil, string.format(
+                "spaces[%q].indexes[%q].parts[%d].exclude_null: bad value" ..
+                " (boolean expected, got %s)",
+                space.name, index.name, i, type(part.exclude_null)
+            )
+        end
+    end
+
+    do -- check exclude_null support
+        if not db.exclude_null_allowed() and part.exclude_null ~= nil then
+            return nil, string.format(
+                "spaces[%q].indexes[%q].parts[%d]: exclude_null" ..
+                " isn't allowed in your Tarantool version (%s)",
+                space.name, index.name, i, _TARANTOOL
+            )
+        end
+
+        if part.exclude_null == true and index.type ~= 'TREE' then
+            return nil, string.format(
+                "spaces[%q].indexes[%q].parts[%d]: exclude_null" ..
+                " isn't allowed for a %s index",
+                space.name, index.name, i, index.type
+            )
+        end
+    end
+
+    do -- check that exclude_null and field nullability is compatible
+        if part.exclude_null == true and part.is_nullable == false then
+            return nil, string.format(
+                "spaces[%q].indexes[%q].parts[%d]:" ..
+                " exclude_null=true and is_nullable=false are incompatible",
+                space.name, index.name, i
+            )
+        end
+    end
+
     do -- check redundant keys
         local k = utils.redundant_key(part,
-            {'path', 'type', 'collation', 'is_nullable'}
+            {'path', 'type', 'collation', 'is_nullable', 'exclude_null'}
         )
         if k ~= nil then
             return nil, string.format(

ddl/db.lua

@@ -1,41 +1,122 @@
-local function check_version(expected_major, expected_minor)
-    local db_major, db_minor = string.match(_TARANTOOL, '^(%d+)%.(%d+)')
-    local major, minor = tonumber(db_major), tonumber(db_minor)
-
-    if major < expected_major then
-        return false
-    elseif major > expected_major then
-        return true
+-- Utilities borrowed from tarantool/crud
+-- https://github.com/tarantool/crud/blob/2d3d47937fd02d938424659bc659fdc24a32dc8a/crud/common/utils.lua#L434-L555
+
+local function get_version_suffix(suffix_candidate)
+    if type(suffix_candidate) ~= 'string' then
+        return nil
+    end
+
+    if suffix_candidate:find('^entrypoint$')
+    or suffix_candidate:find('^alpha%d$')
+    or suffix_candidate:find('^beta%d$')
+    or suffix_candidate:find('^rc%d$') then
+        return suffix_candidate
+    end
+
+    return nil
+end
+
+local suffix_with_digit_weight = {
+    alpha = -3000,
+    beta  = -2000,
+    rc    = -1000,
+}
+
+local function get_version_suffix_weight(suffix)
+    if suffix == nil then
+        return 0
     end
 
-    if minor < expected_minor then
-        return false
+    if suffix:find('^entrypoint$') then
+        return -math.huge
     end
+
+    for header, weight in pairs(suffix_with_digit_weight) do
+        local pos, _, digits = suffix:find('^' .. header .. '(%d)$')
+        if pos ~= nil then
+            return weight + tonumber(digits)
+        end
+    end
+
+    error(('Unexpected suffix %q, parse with "utils.get_version_suffix" first'):format(suffix))
+end
+
+local function is_version_ge(major, minor,
+                             patch, suffix,
+                             major_to_compare, minor_to_compare,
+                             patch_to_compare, suffix_to_compare)
+    major = major or 0
+    minor = minor or 0
+    patch = patch or 0
+    local suffix_weight = get_version_suffix_weight(suffix)
+
+    major_to_compare = major_to_compare or 0
+    minor_to_compare = minor_to_compare or 0
+    patch_to_compare = patch_to_compare or 0
+    local suffix_weight_to_compare = get_version_suffix_weight(suffix_to_compare)
+
+    if major > major_to_compare then return true end
+    if major < major_to_compare then return false end
+
+    if minor > minor_to_compare then return true end
+    if minor < minor_to_compare then return false end
+
+    if patch > patch_to_compare then return true end
+    if patch < patch_to_compare then return false end
+
+    if suffix_weight > suffix_weight_to_compare then return true end
+    if suffix_weight < suffix_weight_to_compare then return false end
+
     return true
 end
 
+local function get_tarantool_version()
+    local version_parts = rawget(_G, '_TARANTOOL'):split('-', 1)
+
+    local major_minor_patch_parts = version_parts[1]:split('.', 2)
+    local major = tonumber(major_minor_patch_parts[1])
+    local minor = tonumber(major_minor_patch_parts[2])
+    local patch = tonumber(major_minor_patch_parts[3])
+
+    local suffix = get_version_suffix(version_parts[2])
+
+    return major, minor, patch, suffix
+end
+
+local function tarantool_version_at_least(wanted_major, wanted_minor, wanted_patch)
+    local major, minor, patch, suffix = get_tarantool_version()
+
+    return is_version_ge(major, minor, patch, suffix,
+                         wanted_major, wanted_minor, wanted_patch, nil)
+end
+
+
 local function json_path_allowed()
-    return check_version(2, 1)
+    return tarantool_version_at_least(2, 1)
 end
 
 local function multikey_path_allowed()
-    return check_version(2, 2)
+    return tarantool_version_at_least(2, 2)
 end
 
 local function varbinary_allowed()
-    return check_version(2, 2)
+    return tarantool_version_at_least(2, 2)
 end
 
 local function datetime_allowed()
-    return check_version(2, 10)
+    return tarantool_version_at_least(2, 10)
 end
 
-
 -- https://github.com/tarantool/tarantool/issues/4083
 local function transactional_ddl_allowed()
-    return check_version(2, 2)
+    return tarantool_version_at_least(2, 2)
+end
+
+local function exclude_null_allowed()
+    return tarantool_version_at_least(2, 8, 1)
 end
 
+
 local function atomic_tail(status, ...)
     if not status then
         box.rollback()
@@ -74,6 +155,7 @@ return {
     multikey_path_allowed = multikey_path_allowed,
     transactional_ddl_allowed = transactional_ddl_allowed,
     datetime_allowed = datetime_allowed,
+    exclude_null_allowed = exclude_null_allowed,
 
     call_atomic = call_atomic,
     call_dry_run = call_dry_run,

ddl/set.lua

@@ -9,6 +9,7 @@ local function create_index(box_space, ddl_index)
             ddl_index_part.path, ddl_index_part.type,
             is_nullable = ddl_index_part.is_nullable,
             collation = ddl_index_part.collation,
+            exclude_null = ddl_index_part.exclude_null,
         }
         table.insert(index_parts, index_part)
     end

test/check_schema_test.lua

@@ -1,6 +1,7 @@
 #!/usr/bin/env tarantool
 
 local t = require('luatest')
+local utils = require('luatest.utils')
 local db = require('test.db')
 local ddl_check = require('ddl.check')
 local ddl = require('ddl')
@@ -1454,3 +1455,135 @@ g.test_gh_108_fieldno_index_outside_space_format = function()
     }}})
     t.assert_equals(err, nil)
 end
+
+local function get_nullable_field_schema(second_index)
+    return {spaces = {my_space = {
+        engine = 'memtx',
+        is_local = false,
+        temporary = false,
+        format = {
+            {name = 'id', type = 'unsigned', is_nullable = false},
+            {name = 'field', type = 'string', is_nullable = true},
+        },
+        indexes = {
+            {
+                name = 'pk',
+                type = 'TREE',
+                parts = {
+                    {path = 'id', type = 'unsigned', is_nullable = false},
+                },
+                unique = true,
+            },
+            second_index,
+        },
+    }}}
+end
+
+g.test_exclude_null = function()
+    local version = utils.get_tarantool_version()
+    t.skip_if(utils.version_ge(utils.version(2, 8, 1), version),
+        'Tarantool does not support exclude_null')
+
+    local schema = get_nullable_field_schema({
+        name = 'nullable_index',
+        type = 'TREE',
+        parts = {
+            {path = 'field', type = 'string', is_nullable = true, exclude_null = true},
+        },
+        unique = true,
+    })
+    local _, err = ddl.set_schema(schema)
+    t.assert_equals(err, nil)
+end
+
+g.test_exclude_null_invalid_param = function()
+    local version = utils.get_tarantool_version()
+    t.skip_if(utils.version_ge(utils.version(2, 8, 1), version),
+        'Tarantool does not support exclude_null')
+
+    local schema = get_nullable_field_schema({
+        name = 'nullable_index',
+        type = 'TREE',
+        parts = {
+            {path = 'field', type = 'string', is_nullable = true, exclude_null = 1},
+        },
+        unique = true,
+    })
+    local _, err = ddl.set_schema(schema)
+    t.assert_str_contains(err, 'spaces["my_space"].indexes["nullable_index"].parts[1].exclude_null: ' ..
+                               'bad value (boolean expected, got number)')
+end
+
+g.test_exclude_null_invalid_index_type = function()
+    local version = utils.get_tarantool_version()
+    t.skip_if(utils.version_ge(utils.version(2, 8, 1), version),
+        'Tarantool does not support exclude_null')
+
+    local schema = get_nullable_field_schema({
+        name = 'nullable_index',
+        type = 'HASH',
+        parts = {
+            {path = 'field', type = 'string', is_nullable = true, exclude_null = true},
+        },
+        unique = true,
+    })
+    local _, err = ddl.set_schema(schema)
+    t.assert_str_contains(err, 'spaces["my_space"].indexes["nullable_index"].parts[1]: ' ..
+                               'exclude_null isn\'t allowed for a HASH index')
+end
+
+g.test_exclude_null_invalid_nullability = function()
+    local version = utils.get_tarantool_version()
+    t.skip_if(utils.version_ge(utils.version(2, 8, 1), version),
+        'Tarantool does not support exclude_null')
+
+    local schema = {spaces = {my_space = {
+        engine = 'memtx',
+        is_local = false,
+        temporary = false,
+        format = {
+            {name = 'id', type = 'unsigned', is_nullable = false},
+            {name = 'field', type = 'string', is_nullable = false},
+        },
+        indexes = {
+            {
+                name = 'pk',
+                type = 'TREE',
+                parts = {
+                    {path = 'id', type = 'unsigned', is_nullable = false},
+                },
+                unique = true,
+            },
+            {
+                name = 'nullable_index',
+                type = 'TREE',
+                parts = {
+                    {path = 'field', type = 'string', is_nullable = false, exclude_null = true},
+                },
+                unique = true,
+            },
+        },
+    }}}
+
+    local _, err = ddl.set_schema(schema)
+    t.assert_str_contains(err, 'spaces["my_space"].indexes["nullable_index"].parts[1]: ' ..
+                               'exclude_null=true and is_nullable=false are incompatible')
+end
+
+g.test_exclude_null_unsupported = function()
+    local version = utils.get_tarantool_version()
+    t.skip_if(utils.version_ge(version, utils.version(2, 8, 1)),
+        'Tarantool supports exclude_null')
+
+    local schema = get_nullable_field_schema({
+        name = 'nullable_index',
+        type = 'TREE',
+        parts = {
+            {path = 'field', type = 'string', is_nullable = true, exclude_null = true},
+        },
+        unique = true,
+    })
+    local _, err = ddl.set_schema(schema)
+    t.assert_str_contains(err, 'spaces["my_space"].indexes["nullable_index"].parts[1]: ' ..
+                               'exclude_null isn\'t allowed in your Tarantool version')
+end