diff --git a/CHANGELOG.md b/CHANGELOG.md index db17e2fb..f4dcbb7e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,10 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. * Ignoring `opts.first` on `crud.pairs` call * External `keydef` compatibility with built-in `merger` +### Added + +* Added jsonpath indexes support for queries. + ## [0.7.0] - 2021-05-27 ### Fixed diff --git a/crud/common/compat.lua b/crud/common/compat.lua index 3b3c52c8..2eaf6969 100644 --- a/crud/common/compat.lua +++ b/crud/common/compat.lua @@ -30,4 +30,21 @@ function compat.require(module_name, builtin_module_name) return module end +function compat.exists(module_name, builtin_module_name) + local module_cached = rawget(_G, string.format('__crud_%s_cached', module_name)) + if module_cached ~= nil then + return true + end + + if package.search(module_name) then + return true + end + + if package.loaded[builtin_module_name] ~= nil then + return true + end + + return false +end + return compat diff --git a/crud/common/utils.lua b/crud/common/utils.lua index b7ec5de0..b8243037 100644 --- a/crud/common/utils.lua +++ b/crud/common/utils.lua @@ -205,6 +205,10 @@ local function determine_enabled_features() -- since Tarantool 2.4 enabled_tarantool_features.uuids = major >= 2 and (minor > 4 or minor == 4 and patch >= 1) + + -- since Tarantool 2.6.3 / 2.7.2 / 2.8.1 + enabled_tarantool_features.jsonpath_indexes = major >= 3 or (major >= 2 and ((minor >= 6 and patch >= 3) + or (minor >= 7 and patch >= 2) or (minor >= 8 and patch >= 1) or minor >= 9)) end function utils.tarantool_supports_fieldpaths() @@ -223,6 +227,14 @@ function utils.tarantool_supports_uuids() return enabled_tarantool_features.uuids end +function utils.tarantool_supports_jsonpath_indexes() + if enabled_tarantool_features.jsonpath_indexes == nil then + determine_enabled_features() + end + + return enabled_tarantool_features.jsonpath_indexes +end + local function add_nullable_fields_recursive(operations, operations_map, space_format, tuple, id) if id < 2 or tuple[id - 1] ~= box.NULL then return operations diff --git a/crud/compare/comparators.lua b/crud/compare/comparators.lua index eb781b5b..e59bf053 100644 --- a/crud/compare/comparators.lua +++ b/crud/compare/comparators.lua @@ -2,10 +2,18 @@ local errors = require('errors') local compare_conditions = require('crud.compare.conditions') local type_comparators = require('crud.compare.type_comparators') -local operators = compare_conditions.operators - local utils = require('crud.common.utils') +local compat = require('crud.common.compat') +local has_keydef = compat.exists('tuple.keydef', 'key_def') + +local keydef_lib +if has_keydef then + keydef_lib = compat.require('tuple.keydef', 'key_def') +end + +local operators = compare_conditions.operators + local ComparatorsError = errors.new_class('ComparatorsError') local comparators = {} @@ -102,7 +110,8 @@ function comparators.update_key_parts_by_field_names(space_format, field_names, local field_name = space_format[part.fieldno].name local updated_part = {type = part.type, fieldno = fields_positions[field_name], - is_nullable = part.is_nullable} + is_nullable = part.is_nullable, + path = part.path} table.insert(updated_key_parts, updated_part) end @@ -150,13 +159,22 @@ function comparators.gen_tuples_comparator(cmp_operator, key_parts, field_names, local updated_key_parts = comparators.update_key_parts_by_field_names( space_format, field_names, key_parts ) - local keys_comparator = comparators.gen_func(cmp_operator, updated_key_parts) - return function(lhs, rhs) - local lhs_key = utils.extract_key(lhs, updated_key_parts) - local rhs_key = utils.extract_key(rhs, updated_key_parts) + local keys_comparator = comparators.gen_func(cmp_operator, updated_key_parts) - return keys_comparator(lhs_key, rhs_key) + if has_keydef then + local key_def = keydef_lib.new(updated_key_parts) + return function(lhs, rhs) + local lhs_key = key_def:extract_key(lhs) + local rhs_key = key_def:extract_key(rhs) + return keys_comparator(lhs_key, rhs_key) + end + else + return function(lhs, rhs) + local lhs_key = utils.extract_key(lhs, updated_key_parts) + local rhs_key = utils.extract_key(rhs, updated_key_parts) + return keys_comparator(lhs_key, rhs_key) + end end end diff --git a/crud/select/executor.lua b/crud/select/executor.lua index a75d1b50..04615dbd 100644 --- a/crud/select/executor.lua +++ b/crud/select/executor.lua @@ -2,6 +2,13 @@ local errors = require('errors') local dev_checks = require('crud.common.dev_checks') local select_comparators = require('crud.compare.comparators') +local compat = require('crud.common.compat') +local has_keydef = compat.exists('tuple.keydef', 'key_def') + +local keydef_lib +if has_keydef then + keydef_lib = compat.require('tuple.keydef', 'key_def') +end local utils = require('crud.common.utils') @@ -31,6 +38,26 @@ local function scroll_to_after_tuple(gen, space, scan_index, tarantool_iter, aft end end +local generate_value + +if has_keydef then + generate_value = function(after_tuple, scan_value, index_parts) + local key_def = keydef_lib.new(index_parts) + if key_def:compare_with_key(after_tuple, scan_value) < 0 then + return key_def:extract_key(after_tuple) + end + end +else + generate_value = function(after_tuple, scan_value, index_parts, tarantool_iter) + local cmp_operator = select_comparators.get_cmp_operator(tarantool_iter) + local scan_comparator = select_comparators.gen_tuples_comparator(cmp_operator, index_parts) + local after_tuple_key = utils.extract_key(after_tuple, index_parts) + if scan_comparator(after_tuple_key, scan_value) then + return after_tuple_key + end + end +end + function executor.execute(space, index, filter_func, opts) dev_checks('table', 'table', 'function', { scan_value = 'table', @@ -53,11 +80,9 @@ function executor.execute(space, index, filter_func, opts) if value == nil then value = opts.after_tuple else - local cmp_operator = select_comparators.get_cmp_operator(opts.tarantool_iter) - local scan_comparator = select_comparators.gen_tuples_comparator(cmp_operator, index.parts) - local after_tuple_key = utils.extract_key(opts.after_tuple, index.parts) - if scan_comparator(after_tuple_key, opts.scan_value) then - value = after_tuple_key + local new_value = generate_value(opts.after_tuple, opts.scan_value, index.parts, opts.tarantool_iter) + if new_value ~= nil then + value = new_value end end end diff --git a/crud/select/filters.lua b/crud/select/filters.lua index 6cf502b6..145e0190 100644 --- a/crud/select/filters.lua +++ b/crud/select/filters.lua @@ -44,7 +44,11 @@ local function get_index_fieldnos(index) local index_fieldnos = {} for _, part in ipairs(index.parts) do - table.insert(index_fieldnos, part.fieldno) + if part.path ~= nil then + table.insert(index_fieldnos, string.format("[%d]%s", part.fieldno, part.path)) + else + table.insert(index_fieldnos, part.fieldno) + end end return index_fieldnos @@ -91,7 +95,6 @@ local function parse(space, conditions, opts) end local filter_conditions = {} - for i, condition in ipairs(conditions) do if i ~= opts.scan_condition_num then -- Index check (including one and multicolumn) diff --git a/crud/select/merger.lua b/crud/select/merger.lua index 0ea63285..66b6e19e 100644 --- a/crud/select/merger.lua +++ b/crud/select/merger.lua @@ -136,7 +136,6 @@ local reverse_tarantool_iters = { local function new(replicasets, space, index_id, func_name, func_args, opts) opts = opts or {} local call_opts = opts.call_opts - local mode = call_opts.mode or 'read' local vshard_call_name = call.get_vshard_call_name(mode, call_opts.prefer_replica, call_opts.balance) diff --git a/crud/select/plan.lua b/crud/select/plan.lua index f6dbf5c6..1662eb2c 100644 --- a/crud/select/plan.lua +++ b/crud/select/plan.lua @@ -4,6 +4,14 @@ local compare_conditions = require('crud.compare.conditions') local utils = require('crud.common.utils') local dev_checks = require('crud.common.dev_checks') +local compat = require('crud.common.compat') +local has_keydef = compat.exists('tuple.keydef', 'key_def') + +local keydef_lib +if has_keydef then + keydef_lib = compat.require('tuple.keydef', 'key_def') +end + local select_plan = {} local IndexTypeError = errors.new_class('IndexTypeError', {capture_stack = false}) @@ -184,7 +192,12 @@ function select_plan.new(space, conditions, opts) scan_condition_num = nil if scan_after_tuple ~= nil then - scan_value = utils.extract_key(scan_after_tuple, scan_index.parts) + if has_keydef then + local key_def = keydef_lib.new(scan_index.parts) + scan_value = key_def:extract_key(scan_after_tuple) + else + scan_value = utils.extract_key(scan_after_tuple, scan_index.parts) + end else scan_value = nil end diff --git a/test/entrypoint/srv_select.lua b/test/entrypoint/srv_select.lua index 98081b5d..56d0d3d3 100755 --- a/test/entrypoint/srv_select.lua +++ b/test/entrypoint/srv_select.lua @@ -151,6 +151,43 @@ package.preload['customers-storage'] = function() unique = false, if_not_exists = true, }) + + if crud_utils.tarantool_supports_jsonpath_indexes() then + local cars_space = box.schema.space.create('cars', { + format = { + {name = 'id', type = 'map'}, + {name = 'bucket_id', type = 'unsigned'}, + {name = 'age', type = 'number'}, + {name = 'manufacturer', type = 'string'}, + {name = 'data', type = 'map'} + }, + if_not_exists = true, + engine = engine, + }) + + -- primary index + cars_space:create_index('id_ind', { + parts = { + {1, 'unsigned', path = 'car_id.signed'}, + }, + if_not_exists = true, + }) + + cars_space:create_index('bucket_id', { + parts = { 'bucket_id' }, + unique = false, + if_not_exists = true, + }) + + cars_space:create_index('data_index', { + parts = { + {5, 'str', path = 'car.color'}, + {5, 'str', path = 'car.model'}, + }, + unique = false, + if_not_exists = true, + }) + end end, } end diff --git a/test/integration/select_test.lua b/test/integration/select_test.lua index 1d243216..f1f6fba0 100644 --- a/test/integration/select_test.lua +++ b/test/integration/select_test.lua @@ -32,6 +32,7 @@ pgroup:set_after_all(function(g) helpers.stop_cluster(g.cluster) end) pgroup:set_before_each(function(g) helpers.truncate_space_on_cluster(g.cluster, 'customers') helpers.truncate_space_on_cluster(g.cluster, 'developers') + helpers.truncate_space_on_cluster(g.cluster, 'cars') end) @@ -1254,3 +1255,221 @@ pgroup:add('test_jsonpath', function(g) } t.assert_equals(objects, expected_objects) end) + +pgroup:add('test_jsonpath_index_field', function(g) + t.skip_if( + not crud_utils.tarantool_supports_jsonpath_indexes(), + "Jsonpath indexes supported since 2.6.3/2.7.2/2.8.1" + ) + + helpers.insert_objects(g, 'cars', { + { + id = {car_id = {signed = 1}}, + age = 2, + manufacturer = 'VAG', + data = {car = { model = 'BMW', color = 'Black' }}, + }, + { + id = {car_id = {signed = 2}}, + age = 5, + manufacturer = 'FIAT', + data = {car = { model = 'Cadillac', color = 'White' }}, + }, + { + id = {car_id = {signed = 3}}, + age = 17, + manufacturer = 'Ford', + data = {car = { model = 'BMW', color = 'Yellow' }}, + }, + { + id = {car_id = {signed = 4}}, + age = 3, + manufacturer = 'General Motors', + data = {car = { model = 'Mercedes', color = 'Yellow' }}, + }, + }) + + -- PK jsonpath index + local result, err = g.cluster.main_server.net_box:call('crud.select', + {'cars', {{'<=', 'id_ind', 3}, {'<=', 'age', 5}}, {fields = {'id', 'age'}}}) + t.assert_equals(err, nil) + + local objects = crud.unflatten_rows(result.rows, result.metadata) + local expected_objects = { + { + id = {car_id = {signed = 2}}, + age = 5, + }, + { + id = {car_id = {signed = 1}}, + age = 2, + }} + + t.assert_equals(objects, expected_objects) + + -- Secondary jsonpath index (partial) + local result, err = g.cluster.main_server.net_box:call('crud.select', + {'cars', {{'==', 'data_index', 'Yellow'}}, {fields = {'id', 'age'}}}) + t.assert_equals(err, nil) + + local objects = crud.unflatten_rows(result.rows, result.metadata) + local expected_objects = { + { + id = {car_id = {signed = 3}}, + age = 17, + data = {car = { model = 'BMW', color = 'Yellow' }}, + }, + { + id = {car_id = {signed = 4}}, + age = 3, + data = {car = { model = 'Mercedes', color = 'Yellow' }} + }} + + t.assert_equals(objects, expected_objects) + + -- Seconday jsonpath index (full) + local result, err = g.cluster.main_server.net_box:call('crud.select', + {'cars', {{'==', 'data_index', {'Yellow', 'Mercedes'}}}, {fields = {'id', 'age'}}}) + t.assert_equals(err, nil) + + local objects = crud.unflatten_rows(result.rows, result.metadata) + local expected_objects = { + { + id = {car_id = {signed = 4}}, + age = 3, + data = {car = { model = 'Mercedes', color = 'Yellow' }} + }} + + t.assert_equals(objects, expected_objects) +end) + +pgroup:add('test_jsonpath_index_field_pagination', function(g) + t.skip_if( + not crud_utils.tarantool_supports_jsonpath_indexes(), + "Jsonpath indexes supported since 2.6.3/2.7.2/2.8.1" + ) + + local cars = helpers.insert_objects(g, 'cars', { + { + id = {car_id = {signed = 1}}, + age = 5, + manufacturer = 'VAG', + data = {car = { model = 'A', color = 'Yellow' }}, + }, + { + id = {car_id = {signed = 2}}, + age = 17, + manufacturer = 'FIAT', + data = {car = { model = 'B', color = 'Yellow' }}, + }, + { + id = {car_id = {signed = 3}}, + age = 5, + manufacturer = 'Ford', + data = {car = { model = 'C', color = 'Yellow' }}, + }, + { + id = {car_id = {signed = 4}}, + age = 3, + manufacturer = 'General Motors', + data = {car = { model = 'D', color = 'Yellow' }}, + }, + { + id = {car_id = {signed = 5}}, + age = 3, + manufacturer = 'General Motors', + data = {car = { model = 'E', color = 'Yellow' }}, + }, + { + id = {car_id = {signed = 6}}, + age = 3, + manufacturer = 'General Motors', + data = {car = { model = 'F', color = 'Yellow' }}, + }, + { + id = {car_id = {signed = 7}}, + age = 3, + manufacturer = 'General Motors', + data = {car = { model = 'G', color = 'Yellow' }}, + }, + { + id = {car_id = {signed = 8}}, + age = 3, + manufacturer = 'General Motors', + data = {car = { model = 'H', color = 'Yellow' }}, + }, + }) + + + -- Pagination (primary index) + local result, err = g.cluster.main_server.net_box:call('crud.select', + {'cars', nil, {first = 2}}) + t.assert_equals(err, nil) + + local objects = crud.unflatten_rows(result.rows, result.metadata) + t.assert_equals(objects, helpers.get_objects_by_idxs(cars, {1, 2})) + + local result, err = g.cluster.main_server.net_box:call('crud.select', + {'cars', nil, {first = 2, after = result.rows[2]}}) + t.assert_equals(err, nil) + + local objects = crud.unflatten_rows(result.rows, result.metadata) + t.assert_equals(objects, helpers.get_objects_by_idxs(cars, {3, 4})) + + -- Reverse pagination (primary index) + local result, err = g.cluster.main_server.net_box:call('crud.select', + {'cars', nil, {first = -2, after = result.rows[1]}}) + t.assert_equals(err, nil) + + local objects = crud.unflatten_rows(result.rows, result.metadata) + t.assert_equals(objects, helpers.get_objects_by_idxs(cars, {1, 2})) + +--[==[ + -- Uncomment after https://github.com/tarantool/crud/issues/170 + -- Pagination (secondary index - 1 field) + local result, err = g.cluster.main_server.net_box:call('crud.select', + {'cars', {{'==', 'data_index', 'Yellow'}}, {first = 2}}) + t.assert_equals(err, nil) + + local objects = crud.unflatten_rows(result.rows, result.metadata) + t.assert_equals(objects, helpers.get_objects_by_idxs(cars, {1, 2})) + + local result, err = g.cluster.main_server.net_box:call('crud.select', + {'cars', {{'==', 'data_index', 'Yellow'}}, {first = 2, after = result.rows[2]}}) + t.assert_equals(err, nil) + + local objects = crud.unflatten_rows(result.rows, result.metadata) + t.assert_equals(objects, helpers.get_objects_by_idxs(cars, {3, 4})) + + -- Reverse pagination (secondary index - 1 field) + local result, err = g.cluster.main_server.net_box:call('crud.select', + {'cars', {{'==', 'data_index', 'Yellow'}}, {first = -2, after = result.rows[1]}}) + t.assert_equals(err, nil) +]==] + + local objects = crud.unflatten_rows(result.rows, result.metadata) + t.assert_equals(objects, helpers.get_objects_by_idxs(cars, {1, 2})) + + -- Pagination (secondary index - 2 fields) + local result, err = g.cluster.main_server.net_box:call('crud.select', + {'cars', {{'>=', 'data_index', {'Yellow', 'E'}}}, {first = 2}}) + t.assert_equals(err, nil) + + local objects = crud.unflatten_rows(result.rows, result.metadata) + t.assert_equals(objects, helpers.get_objects_by_idxs(cars, {5, 6})) + + local result, err = g.cluster.main_server.net_box:call('crud.select', + {'cars', {{'>=', 'data_index', {'Yellow', 'E'}}}, {first = 2, after = result.rows[2]}}) + t.assert_equals(err, nil) + + local objects = crud.unflatten_rows(result.rows, result.metadata) + t.assert_equals(objects, helpers.get_objects_by_idxs(cars, {7, 8})) + + local result, err = g.cluster.main_server.net_box:call('crud.select', + {'cars', {{'>=', 'data_index', {'Yellow', 'B'}}, {'<=', 'id_ind', 3}}, + {first = -3, after = result.rows[1]}}) + t.assert_equals(err, nil) + + local objects = crud.unflatten_rows(result.rows, result.metadata) + t.assert_equals(objects, helpers.get_objects_by_idxs(cars, {2, 3})) +end) diff --git a/test/unit/select_filters_test.lua b/test/unit/select_filters_test.lua index 6b4cc246..38ad3859 100644 --- a/test/unit/select_filters_test.lua +++ b/test/unit/select_filters_test.lua @@ -9,6 +9,8 @@ local collations = require('crud.common.collations') local t = require('luatest') local g = t.group('select_filters') +local crud_utils = require('crud.common.utils') + local helpers = require('test.helper') g.before_all = function() @@ -43,10 +45,47 @@ g.before_all = function() unique = false, if_not_exists = true, }) + + if crud_utils.tarantool_supports_jsonpath_indexes() then + local cars_space = box.schema.space.create('cars', { + format = { + {name = 'id', type = 'map'}, + {name = 'bucket_id', type = 'unsigned'}, + {name = 'age', type = 'number'}, + {name = 'manufacturer', type = 'string'}, + {name = 'data', type = 'map'} + }, + if_not_exists = true, + }) + + -- primary index + cars_space:create_index('id_ind', { + parts = { + {1, 'unsigned', path = 'car_id.signed'}, + }, + if_not_exists = true, + }) + + cars_space:create_index('bucket_id', { + parts = { 'bucket_id' }, + unique = false, + if_not_exists = true, + }) + + cars_space:create_index('data_index', { + parts = { + {5, 'str', path = 'car.color'}, + {5, 'str', path = 'car.model'}, + }, + unique = false, + if_not_exists = true, + }) + end end g.after_all(function() box.space.customers:drop() + box.space.cars:drop() end) g.test_empty_conditions = function() @@ -815,5 +854,34 @@ return M]] t.assert_equals({ filter_func(box.tuple.new({{field_1 = 5, f2 = 3}, {fld_1 = "jsonpath_test"}, 23})) }, {false, false}) end +g.test_jsonpath_indexes = function() + t.skip_if( + not crud_utils.tarantool_supports_jsonpath_indexes(), + "Jsonpath indexes supported since 2.6.3/2.7.2/2.8.1" + ) + + local conditions = { + cond_funcs.gt('id', 20), + cond_funcs.eq('data_index', {'Yellow', 'BMW'}) + } + + local plan, err = select_plan.new(box.space.cars, conditions) + t.assert_equals(err, nil) + + local filter_conditions, err = select_filters.internal.parse(box.space.cars, conditions, { + scan_condition_num = plan.scan_condition_num, + tarantool_iter = plan.tarantool_iter, + }) + + t.assert_equals(err, nil) + + local data_condition = filter_conditions[1] + t.assert_type(data_condition, 'table') + t.assert_equals(data_condition.fields, {"[5]car.color", "[5]car.model"}) + t.assert_equals(data_condition.operator, compare_conditions.operators.EQ) + t.assert_equals(data_condition.values, {'Yellow', 'BMW'}) + t.assert_equals(data_condition.types, {'string', 'string'}) + t.assert_equals(data_condition.early_exit_is_possible, false) +end -- luacheck: pop