From bcc2b2adbb92ef888f697e9e6a5884aa96c9e532 Mon Sep 17 00:00:00 2001 From: Alexey Romanov Date: Sat, 15 May 2021 22:21:54 +0300 Subject: [PATCH 01/28] Jsonpath indexes --- crud/common/utils.lua | 68 ++++++++++++++++++++++++++++++++++++ crud/compare/comparators.lua | 7 ++-- crud/compare/keydef.lua | 1 + crud/select.lua | 1 + crud/select/executor.lua | 7 +++- crud/select/filters.lua | 9 +++-- crud/select/merger.lua | 1 - crud/select/plan.lua | 6 +++- 8 files changed, 91 insertions(+), 9 deletions(-) diff --git a/crud/common/utils.lua b/crud/common/utils.lua index b7ec5de0..e7de9d62 100644 --- a/crud/common/utils.lua +++ b/crud/common/utils.lua @@ -12,6 +12,9 @@ local ShardingError = errors.new_class('ShardingError', {capture_stack = false} local GetSpaceFormatError = errors.new_class('GetSpaceFormatError', {capture_stack = false}) local FilterFieldsError = errors.new_class('FilterFieldsError', {capture_stack = false}) +local compat = require('crud.common.compat') +local keydef_lib = compat.require('tuple.keydef', 'key_def') + local utils = {} local space_format_cache = setmetatable({}, {__mode = 'k'}) @@ -148,6 +151,49 @@ function utils.extract_key(tuple, key_parts) return key end +function utils.extract_jsonpath_keys(tuple, index_parts, condition_map) + if tuple == nil then + return nil + end + + local key_def = nil + local extracted_values + local curr_jsonpath_ind = nil + local key = {} + + for i, part in ipairs(index_parts) do + if part.path ~= nil and type(tuple[part.fieldno]) == 'table' then + if curr_jsonpath_ind == nil then + curr_jsonpath_ind = i + end + + if key_def == nil then + key_def = keydef_lib.new(index_parts) + extracted_values = key_def:extract_key(tuple) + end + + -- We need this check (and also condition map) for composite + -- jsonpath indexes. For example, we have to filtering values + -- by partial jsonpath index, but index is composite and fields + -- are jsonpaths to the same space field. In this case, we + -- have to extract only needed value, specified in condtion. + if condition_map ~= nil then + if condition_map[extracted_values[curr_jsonpath_ind]] then + key[i] = extracted_values[curr_jsonpath_ind] + end + else + key[i] = extracted_values[curr_jsonpath_ind] + end + + curr_jsonpath_ind = curr_jsonpath_ind + 1 + else + key[i] = tuple[part.fieldno] + end + end + + return key +end + function utils.merge_primary_key_parts(key_parts, pk_parts) local merged_parts = {} local key_fieldnos = {} @@ -512,4 +558,26 @@ function utils.flatten_obj_reload(space_name, obj) return schema.wrap_func_reload(flatten_obj, space_name, obj) end +function utils.get_index_paths(index_parts) + local parts = {} + for _, v in ipairs(index_parts) do + table.insert(parts, v.path) + end + + return parts +end + +function utils.get_condition_values_map(conditions) + if conditions == nil or conditions.values == nil then + return {} + end + + local map = {} + for _, v in pairs(conditions.values) do + map[v] = true + end + + return map +end + return utils diff --git a/crud/compare/comparators.lua b/crud/compare/comparators.lua index eb781b5b..34c54d0b 100644 --- a/crud/compare/comparators.lua +++ b/crud/compare/comparators.lua @@ -102,7 +102,8 @@ function comparators.update_key_parts_by_field_names(space_format, field_names, local field_name = space_format[part.fieldno].name local updated_part = {type = part.type, fieldno = fields_positions[field_name], - is_nullable = part.is_nullable} + is_nullable = part.is_nullable, + path = part.path} table.insert(updated_key_parts, updated_part) end @@ -153,8 +154,8 @@ function comparators.gen_tuples_comparator(cmp_operator, key_parts, field_names, local keys_comparator = comparators.gen_func(cmp_operator, updated_key_parts) return function(lhs, rhs) - local lhs_key = utils.extract_key(lhs, updated_key_parts) - local rhs_key = utils.extract_key(rhs, updated_key_parts) + local lhs_key = utils.extract_jsonpath_keys(lhs, updated_key_parts) + local rhs_key = utils.extract_jsonpath_keys(rhs, updated_key_parts) return keys_comparator(lhs_key, rhs_key) end diff --git a/crud/compare/keydef.lua b/crud/compare/keydef.lua index 6a2eced8..76a9a9e0 100644 --- a/crud/compare/keydef.lua +++ b/crud/compare/keydef.lua @@ -1,5 +1,6 @@ local comparators = require('crud.compare.comparators') local collations = require('crud.common.collations') +local utils = require('crud.common.utils') local compat = require('crud.common.compat') local keydef_lib = compat.require('tuple.keydef', 'key_def') diff --git a/crud/select.lua b/crud/select.lua index 41964e1f..d9ed390b 100644 --- a/crud/select.lua +++ b/crud/select.lua @@ -69,6 +69,7 @@ local function select_on_storage(space_name, index_id, conditions, opts) after_tuple = opts.after_tuple, tarantool_iter = opts.tarantool_iter, limit = opts.limit, + conditions = conditions, }) if err ~= nil then return nil, SelectError:new("Failed to execute select: %s", err) diff --git a/crud/select/executor.lua b/crud/select/executor.lua index a75d1b50..f8f30aad 100644 --- a/crud/select/executor.lua +++ b/crud/select/executor.lua @@ -37,6 +37,7 @@ function executor.execute(space, index, filter_func, opts) after_tuple = '?table', tarantool_iter = 'number', limit = '?number', + conditions = '?table' }) opts = opts or {} @@ -55,7 +56,11 @@ function executor.execute(space, index, filter_func, opts) else local cmp_operator = select_comparators.get_cmp_operator(opts.tarantool_iter) local scan_comparator = select_comparators.gen_tuples_comparator(cmp_operator, index.parts) - local after_tuple_key = utils.extract_key(opts.after_tuple, index.parts) + local after_tuple_key = utils.extract_jsonpath_keys( + opts.after_tuple, + index.parts, + utils.get_condition_values_map(opts.conditions) + ) if scan_comparator(after_tuple_key, opts.scan_value) then value = after_tuple_key end diff --git a/crud/select/filters.lua b/crud/select/filters.lua index 6cf502b6..40151608 100644 --- a/crud/select/filters.lua +++ b/crud/select/filters.lua @@ -40,11 +40,15 @@ local function is_early_exit_possible(index, tarantool_iter, condition) return false end -local function get_index_fieldnos(index) +local function get_index_fieldnos(index, space_format) local index_fieldnos = {} for _, part in ipairs(index.parts) do - table.insert(index_fieldnos, part.fieldno) + if part.path ~= nil then + table.insert(index_fieldnos, string.format("%s.%s", space_format[part.fieldno].name, part.path)) + else + table.insert(index_fieldnos, part.fieldno) + end end return index_fieldnos @@ -91,7 +95,6 @@ local function parse(space, conditions, opts) end local filter_conditions = {} - for i, condition in ipairs(conditions) do if i ~= opts.scan_condition_num then -- Index check (including one and multicolumn) diff --git a/crud/select/merger.lua b/crud/select/merger.lua index 0ea63285..66b6e19e 100644 --- a/crud/select/merger.lua +++ b/crud/select/merger.lua @@ -136,7 +136,6 @@ local reverse_tarantool_iters = { local function new(replicasets, space, index_id, func_name, func_args, opts) opts = opts or {} local call_opts = opts.call_opts - local mode = call_opts.mode or 'read' local vshard_call_name = call.get_vshard_call_name(mode, call_opts.prefer_replica, call_opts.balance) diff --git a/crud/select/plan.lua b/crud/select/plan.lua index f6dbf5c6..d2d095f3 100644 --- a/crud/select/plan.lua +++ b/crud/select/plan.lua @@ -184,7 +184,11 @@ function select_plan.new(space, conditions, opts) scan_condition_num = nil if scan_after_tuple ~= nil then - scan_value = utils.extract_key(scan_after_tuple, scan_index.parts) + scan_value = utils.extract_jsonpath_keys( + scan_after_tuple, + scan_index.parts, + utils.get_condition_values_map(conditions) + ) else scan_value = nil end From d4a36364509d375e2d7248e9f833671a664252d8 Mon Sep 17 00:00:00 2001 From: Alexey Romanov Date: Wed, 26 May 2021 17:59:32 +0300 Subject: [PATCH 02/28] Tests --- crud/common/utils.lua | 12 +++++++++++ test/entrypoint/srv_select.lua | 37 ++++++++++++++++++++++++++++++++ test/integration/select_test.lua | 4 ++++ 3 files changed, 53 insertions(+) diff --git a/crud/common/utils.lua b/crud/common/utils.lua index e7de9d62..9b31db99 100644 --- a/crud/common/utils.lua +++ b/crud/common/utils.lua @@ -251,6 +251,10 @@ local function determine_enabled_features() -- since Tarantool 2.4 enabled_tarantool_features.uuids = major >= 2 and (minor > 4 or minor == 4 and patch >= 1) + + -- since Tarantool 2.6.3 / 2.7.2 / 2.8.1 + enabled_tarantool_features.jsonpath_indexes = major >= 3 or (major >= 2 and ((minor >= 6 and patch >= 3) + or (minor >= 7 and patch >= 2) or (minor >= 8 and patch >= 1) or minor >= 9)) end function utils.tarantool_supports_fieldpaths() @@ -269,6 +273,14 @@ function utils.tarantool_supports_uuids() return enabled_tarantool_features.uuids end +function utils.tarantool_supports_jsonpath_indexes() + if enabled_tarantool_features.jsonpath_indexes == nil then + determine_enabled_features() + end + + return enabled_tarantool_features.jsonpath_indexes +end + local function add_nullable_fields_recursive(operations, operations_map, space_format, tuple, id) if id < 2 or tuple[id - 1] ~= box.NULL then return operations diff --git a/test/entrypoint/srv_select.lua b/test/entrypoint/srv_select.lua index 98081b5d..56d0d3d3 100755 --- a/test/entrypoint/srv_select.lua +++ b/test/entrypoint/srv_select.lua @@ -151,6 +151,43 @@ package.preload['customers-storage'] = function() unique = false, if_not_exists = true, }) + + if crud_utils.tarantool_supports_jsonpath_indexes() then + local cars_space = box.schema.space.create('cars', { + format = { + {name = 'id', type = 'map'}, + {name = 'bucket_id', type = 'unsigned'}, + {name = 'age', type = 'number'}, + {name = 'manufacturer', type = 'string'}, + {name = 'data', type = 'map'} + }, + if_not_exists = true, + engine = engine, + }) + + -- primary index + cars_space:create_index('id_ind', { + parts = { + {1, 'unsigned', path = 'car_id.signed'}, + }, + if_not_exists = true, + }) + + cars_space:create_index('bucket_id', { + parts = { 'bucket_id' }, + unique = false, + if_not_exists = true, + }) + + cars_space:create_index('data_index', { + parts = { + {5, 'str', path = 'car.color'}, + {5, 'str', path = 'car.model'}, + }, + unique = false, + if_not_exists = true, + }) + end end, } end diff --git a/test/integration/select_test.lua b/test/integration/select_test.lua index 1d243216..72014b73 100644 --- a/test/integration/select_test.lua +++ b/test/integration/select_test.lua @@ -31,7 +31,11 @@ pgroup:set_after_all(function(g) helpers.stop_cluster(g.cluster) end) pgroup:set_before_each(function(g) helpers.truncate_space_on_cluster(g.cluster, 'customers') +<<<<<<< HEAD helpers.truncate_space_on_cluster(g.cluster, 'developers') +======= + helpers.truncate_space_on_cluster(g.cluster, 'cars') +>>>>>>> 7b67809 (Tests) end) From 2315fd1a0080f780f123a4f69142826e24941b7b Mon Sep 17 00:00:00 2001 From: Alexey Romanov Date: Thu, 27 May 2021 15:34:30 +0300 Subject: [PATCH 03/28] Fix tests --- crud/select/filters.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crud/select/filters.lua b/crud/select/filters.lua index 40151608..c1a23730 100644 --- a/crud/select/filters.lua +++ b/crud/select/filters.lua @@ -105,7 +105,7 @@ local function parse(space, conditions, opts) local index = space_indexes[condition.operand] if index ~= nil then - fields = get_index_fieldnos(index) + fields = get_index_fieldnos(index, space_format) fields_types = get_index_fields_types(index) values_opts = get_values_opts(index) else From 5b4c5c9d8840a79f7274fc1d6093dc12208d54f1 Mon Sep 17 00:00:00 2001 From: Alexey Romanov Date: Thu, 27 May 2021 15:36:58 +0300 Subject: [PATCH 04/28] Changelog --- CHANGELOG.md | 4 + test/integration/select_test.lua | 217 ++++++++++++++++++++++++++++++- 2 files changed, 218 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index db17e2fb..5972dcd0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,10 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. * Ignoring `opts.first` on `crud.pairs` call * External `keydef` compatibility with built-in `merger` +### Added + +* Added support for jsonpath indexes for ``crud.select`` + ## [0.7.0] - 2021-05-27 ### Fixed diff --git a/test/integration/select_test.lua b/test/integration/select_test.lua index 72014b73..0c1f2712 100644 --- a/test/integration/select_test.lua +++ b/test/integration/select_test.lua @@ -31,11 +31,8 @@ pgroup:set_after_all(function(g) helpers.stop_cluster(g.cluster) end) pgroup:set_before_each(function(g) helpers.truncate_space_on_cluster(g.cluster, 'customers') -<<<<<<< HEAD helpers.truncate_space_on_cluster(g.cluster, 'developers') -======= helpers.truncate_space_on_cluster(g.cluster, 'cars') ->>>>>>> 7b67809 (Tests) end) @@ -1258,3 +1255,217 @@ pgroup:add('test_jsonpath', function(g) } t.assert_equals(objects, expected_objects) end) + +pgroup:add('test_jsonpath_index_field', function(g) + t.skip_if( + not crud_utils.tarantool_supports_jsonpath_indexes(), + "Jsonpath indexes supported since 2.6.3/2.7.2/2.8.1" + ) + + helpers.insert_objects(g, 'cars', { + { + id = {car_id = {signed = 1}}, + age = 2, + manufacturer = 'VAG', + data = {car = { model = 'BMW', color = 'Black' }}, + }, + { + id = {car_id = {signed = 2}}, + age = 5, + manufacturer = 'FIAT', + data = {car = { model = 'Cadillac', color = 'White' }}, + }, + { + id = {car_id = {signed = 3}}, + age = 17, + manufacturer = 'Ford', + data = {car = { model = 'BMW', color = 'Yellow' }}, + }, + { + id = {car_id = {signed = 4}}, + age = 3, + manufacturer = 'General Motors', + data = {car = { model = 'Mercedes', color = 'Yellow' }}, + }, + }) + + -- PK jsonpath index + local result, err = g.cluster.main_server.net_box:call('crud.select', + {'cars', {{'<=', 'id_ind', 3}, {'<=', 'age', 5}}, {fields = {'id', 'age'}}}) + t.assert_equals(err, nil) + + local objects = crud.unflatten_rows(result.rows, result.metadata) + local expected_objects = { + { + id = {car_id = {signed = 2}}, + age = 5, + }, + { + id = {car_id = {signed = 1}}, + age = 2, + }} + + t.assert_equals(objects, expected_objects) + + -- Secondary jsonpath index (partial) + local result, err = g.cluster.main_server.net_box:call('crud.select', + {'cars', {{'==', 'data_index', 'Yellow'}}, {fields = {'id', 'age'}}}) + t.assert_equals(err, nil) + + local objects = crud.unflatten_rows(result.rows, result.metadata) + local expected_objects = { + { + id = {car_id = {signed = 3}}, + age = 17, + data = {car = { model = 'BMW', color = 'Yellow' }}, + }, + { + id = {car_id = {signed = 4}}, + age = 3, + data = {car = { model = 'Mercedes', color = 'Yellow' }} + }} + + t.assert_equals(objects, expected_objects) + + -- Seconday jsonpath index (full) + local result, err = g.cluster.main_server.net_box:call('crud.select', + {'cars', {{'==', 'data_index', {'Yellow', 'Mercedes'}}}, {fields = {'id', 'age'}}}) + t.assert_equals(err, nil) + + local objects = crud.unflatten_rows(result.rows, result.metadata) + local expected_objects = { + { + id = {car_id = {signed = 4}}, + age = 3, + data = {car = { model = 'Mercedes', color = 'Yellow' }} + }} + + t.assert_equals(objects, expected_objects) +end) + +pgroup:add('test_jsonpath_index_field_pagination', function(g) + t.skip_if( + not crud_utils.tarantool_supports_jsonpath_indexes(), + "Jsonpath indexes supported since 2.6.3/2.7.2/2.8.1" + ) + + local cars = helpers.insert_objects(g, 'cars', { + { + id = {car_id = {signed = 1}}, + age = 5, + manufacturer = 'VAG', + data = {car = { model = 'A', color = 'Yellow' }}, + }, + { + id = {car_id = {signed = 2}}, + age = 17, + manufacturer = 'FIAT', + data = {car = { model = 'B', color = 'Yellow' }}, + }, + { + id = {car_id = {signed = 3}}, + age = 5, + manufacturer = 'Ford', + data = {car = { model = 'C', color = 'Yellow' }}, + }, + { + id = {car_id = {signed = 4}}, + age = 3, + manufacturer = 'General Motors', + data = {car = { model = 'D', color = 'Yellow' }}, + }, + { + id = {car_id = {signed = 5}}, + age = 3, + manufacturer = 'General Motors', + data = {car = { model = 'E', color = 'Yellow' }}, + }, + { + id = {car_id = {signed = 6}}, + age = 3, + manufacturer = 'General Motors', + data = {car = { model = 'F', color = 'Yellow' }}, + }, + { + id = {car_id = {signed = 7}}, + age = 3, + manufacturer = 'General Motors', + data = {car = { model = 'G', color = 'Yellow' }}, + }, + { + id = {car_id = {signed = 8}}, + age = 3, + manufacturer = 'General Motors', + data = {car = { model = 'H', color = 'Yellow' }}, + }, + }) + + -- Pagination (primary index) + local result, err = g.cluster.main_server.net_box:call('crud.select', + {'cars', nil, {first = 2}}) + t.assert_equals(err, nil) + + local objects = crud.unflatten_rows(result.rows, result.metadata) + t.assert_equals(objects, helpers.get_objects_by_idxs(cars, {1, 2})) + + local result, err = g.cluster.main_server.net_box:call('crud.select', + {'cars', nil, {first = 2, after = result.rows[2]}}) + t.assert_equals(err, nil) + + local objects = crud.unflatten_rows(result.rows, result.metadata) + t.assert_equals(objects, helpers.get_objects_by_idxs(cars, {3, 4})) + + -- Reverse pagination (primary index) + local result, err = g.cluster.main_server.net_box:call('crud.select', + {'cars', nil, {first = -2, after = result.rows[1]}}) + t.assert_equals(err, nil) + + local objects = crud.unflatten_rows(result.rows, result.metadata) + t.assert_equals(objects, helpers.get_objects_by_idxs(cars, {1, 2})) + + -- Pagination (secondary index - 1 field) + local result, err = g.cluster.main_server.net_box:call('crud.select', + {'cars', {{'==', 'data_index', 'Yellow'}}, {first = 2}}) + t.assert_equals(err, nil) + + local objects = crud.unflatten_rows(result.rows, result.metadata) + t.assert_equals(objects, helpers.get_objects_by_idxs(cars, {1, 2})) + + local result, err = g.cluster.main_server.net_box:call('crud.select', + {'cars', {{'==', 'data_index', 'Yellow'}}, {first = 2, after = result.rows[2]}}) + t.assert_equals(err, nil) + + local objects = crud.unflatten_rows(result.rows, result.metadata) + t.assert_equals(objects, helpers.get_objects_by_idxs(cars, {3, 4})) + + -- Reverse pagination (secondary index - 1 field) + local result, err = g.cluster.main_server.net_box:call('crud.select', + {'cars', {{'==', 'data_index', 'Yellow'}}, {first = -2, after = result.rows[1]}}) + t.assert_equals(err, nil) + + local objects = crud.unflatten_rows(result.rows, result.metadata) + t.assert_equals(objects, helpers.get_objects_by_idxs(cars, {1, 2})) + + -- Pagination (secondary index - 2 fields) + local result, err = g.cluster.main_server.net_box:call('crud.select', + {'cars', {{'>=', 'data_index', {'Yellow', 'E'}}}, {first = 2}}) + t.assert_equals(err, nil) + + local objects = crud.unflatten_rows(result.rows, result.metadata) + t.assert_equals(objects, helpers.get_objects_by_idxs(cars, {5, 6})) + + local result, err = g.cluster.main_server.net_box:call('crud.select', + {'cars', {{'>=', 'data_index', {'Yellow', 'E'}}}, {first = 2, after = result.rows[2]}}) + t.assert_equals(err, nil) + + local objects = crud.unflatten_rows(result.rows, result.metadata) + t.assert_equals(objects, helpers.get_objects_by_idxs(cars, {7, 8})) + + local result, err = g.cluster.main_server.net_box:call('crud.select', + {'cars', {{'>=', 'data_index', {'Yellow', 'B'}}, {'<=', 'id_ind', 3}}, + {first = -3, after = result.rows[1]}}) + t.assert_equals(err, nil) + + local objects = crud.unflatten_rows(result.rows, result.metadata) + t.assert_equals(objects, helpers.get_objects_by_idxs(cars, {2, 3})) +end) From ba8a4d4c6bf238ee8e0c1254642149b29de42204 Mon Sep 17 00:00:00 2001 From: Alexey Romanov Date: Thu, 27 May 2021 15:39:39 +0300 Subject: [PATCH 05/28] Fix style --- crud/compare/keydef.lua | 1 - 1 file changed, 1 deletion(-) diff --git a/crud/compare/keydef.lua b/crud/compare/keydef.lua index 76a9a9e0..6a2eced8 100644 --- a/crud/compare/keydef.lua +++ b/crud/compare/keydef.lua @@ -1,6 +1,5 @@ local comparators = require('crud.compare.comparators') local collations = require('crud.common.collations') -local utils = require('crud.common.utils') local compat = require('crud.common.compat') local keydef_lib = compat.require('tuple.keydef', 'key_def') From 21f62d06214ae08906b4cb76fe74618d23fc217f Mon Sep 17 00:00:00 2001 From: Alexey Romanov Date: Tue, 8 Jun 2021 08:08:01 +0300 Subject: [PATCH 06/28] Fix PR issues --- crud/common/utils.lua | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/crud/common/utils.lua b/crud/common/utils.lua index 9b31db99..d6150abf 100644 --- a/crud/common/utils.lua +++ b/crud/common/utils.lua @@ -152,10 +152,6 @@ function utils.extract_key(tuple, key_parts) end function utils.extract_jsonpath_keys(tuple, index_parts, condition_map) - if tuple == nil then - return nil - end - local key_def = nil local extracted_values local curr_jsonpath_ind = nil @@ -570,15 +566,6 @@ function utils.flatten_obj_reload(space_name, obj) return schema.wrap_func_reload(flatten_obj, space_name, obj) end -function utils.get_index_paths(index_parts) - local parts = {} - for _, v in ipairs(index_parts) do - table.insert(parts, v.path) - end - - return parts -end - function utils.get_condition_values_map(conditions) if conditions == nil or conditions.values == nil then return {} From 2711ba5d6f16d81c58b48f58d0b6b62f3e1fe32d Mon Sep 17 00:00:00 2001 From: Alexey Romanov Date: Thu, 17 Jun 2021 13:28:39 +0300 Subject: [PATCH 07/28] Fix some logic --- crud/common/utils.lua | 53 ++------------------------------ crud/select/executor.lua | 17 ---------- crud/select/plan.lua | 6 +--- test/integration/select_test.lua | 4 +++ 4 files changed, 8 insertions(+), 72 deletions(-) diff --git a/crud/common/utils.lua b/crud/common/utils.lua index d6150abf..64b17eeb 100644 --- a/crud/common/utils.lua +++ b/crud/common/utils.lua @@ -151,43 +151,9 @@ function utils.extract_key(tuple, key_parts) return key end -function utils.extract_jsonpath_keys(tuple, index_parts, condition_map) - local key_def = nil - local extracted_values - local curr_jsonpath_ind = nil - local key = {} - - for i, part in ipairs(index_parts) do - if part.path ~= nil and type(tuple[part.fieldno]) == 'table' then - if curr_jsonpath_ind == nil then - curr_jsonpath_ind = i - end - - if key_def == nil then - key_def = keydef_lib.new(index_parts) - extracted_values = key_def:extract_key(tuple) - end - - -- We need this check (and also condition map) for composite - -- jsonpath indexes. For example, we have to filtering values - -- by partial jsonpath index, but index is composite and fields - -- are jsonpaths to the same space field. In this case, we - -- have to extract only needed value, specified in condtion. - if condition_map ~= nil then - if condition_map[extracted_values[curr_jsonpath_ind]] then - key[i] = extracted_values[curr_jsonpath_ind] - end - else - key[i] = extracted_values[curr_jsonpath_ind] - end - - curr_jsonpath_ind = curr_jsonpath_ind + 1 - else - key[i] = tuple[part.fieldno] - end - end - - return key +function utils.extract_jsonpath_keys(tuple, index_parts) + local key_def = keydef_lib.new(index_parts) + return key_def:extract_key(tuple) end function utils.merge_primary_key_parts(key_parts, pk_parts) @@ -566,17 +532,4 @@ function utils.flatten_obj_reload(space_name, obj) return schema.wrap_func_reload(flatten_obj, space_name, obj) end -function utils.get_condition_values_map(conditions) - if conditions == nil or conditions.values == nil then - return {} - end - - local map = {} - for _, v in pairs(conditions.values) do - map[v] = true - end - - return map -end - return utils diff --git a/crud/select/executor.lua b/crud/select/executor.lua index f8f30aad..fad42deb 100644 --- a/crud/select/executor.lua +++ b/crud/select/executor.lua @@ -50,23 +50,6 @@ function executor.execute(space, index, filter_func, opts) local tuples_count = 0 local value = opts.scan_value - if opts.after_tuple ~= nil then - if value == nil then - value = opts.after_tuple - else - local cmp_operator = select_comparators.get_cmp_operator(opts.tarantool_iter) - local scan_comparator = select_comparators.gen_tuples_comparator(cmp_operator, index.parts) - local after_tuple_key = utils.extract_jsonpath_keys( - opts.after_tuple, - index.parts, - utils.get_condition_values_map(opts.conditions) - ) - if scan_comparator(after_tuple_key, opts.scan_value) then - value = after_tuple_key - end - end - end - local tuple local gen = index:pairs(value, {iterator = opts.tarantool_iter}) diff --git a/crud/select/plan.lua b/crud/select/plan.lua index d2d095f3..fb4ff1cf 100644 --- a/crud/select/plan.lua +++ b/crud/select/plan.lua @@ -184,11 +184,7 @@ function select_plan.new(space, conditions, opts) scan_condition_num = nil if scan_after_tuple ~= nil then - scan_value = utils.extract_jsonpath_keys( - scan_after_tuple, - scan_index.parts, - utils.get_condition_values_map(conditions) - ) + scan_value = utils.extract_jsonpath_keys(scan_after_tuple, scan_index.parts) else scan_value = nil end diff --git a/test/integration/select_test.lua b/test/integration/select_test.lua index 0c1f2712..f1f6fba0 100644 --- a/test/integration/select_test.lua +++ b/test/integration/select_test.lua @@ -1400,6 +1400,7 @@ pgroup:add('test_jsonpath_index_field_pagination', function(g) }, }) + -- Pagination (primary index) local result, err = g.cluster.main_server.net_box:call('crud.select', {'cars', nil, {first = 2}}) @@ -1423,6 +1424,8 @@ pgroup:add('test_jsonpath_index_field_pagination', function(g) local objects = crud.unflatten_rows(result.rows, result.metadata) t.assert_equals(objects, helpers.get_objects_by_idxs(cars, {1, 2})) +--[==[ + -- Uncomment after https://github.com/tarantool/crud/issues/170 -- Pagination (secondary index - 1 field) local result, err = g.cluster.main_server.net_box:call('crud.select', {'cars', {{'==', 'data_index', 'Yellow'}}, {first = 2}}) @@ -1442,6 +1445,7 @@ pgroup:add('test_jsonpath_index_field_pagination', function(g) local result, err = g.cluster.main_server.net_box:call('crud.select', {'cars', {{'==', 'data_index', 'Yellow'}}, {first = -2, after = result.rows[1]}}) t.assert_equals(err, nil) +]==] local objects = crud.unflatten_rows(result.rows, result.metadata) t.assert_equals(objects, helpers.get_objects_by_idxs(cars, {1, 2})) From 2fb7eefa7749ce1269eeb6901a71ee88d9b33416 Mon Sep 17 00:00:00 2001 From: Alexey Romanov Date: Thu, 17 Jun 2021 21:26:33 +0300 Subject: [PATCH 08/28] Fix PR issues --- crud/common/utils.lua | 18 +++++++++++++++--- crud/compare/comparators.lua | 9 +++++++-- crud/select/executor.lua | 13 +++++++++++++ 3 files changed, 35 insertions(+), 5 deletions(-) diff --git a/crud/common/utils.lua b/crud/common/utils.lua index 64b17eeb..56961768 100644 --- a/crud/common/utils.lua +++ b/crud/common/utils.lua @@ -151,9 +151,21 @@ function utils.extract_key(tuple, key_parts) return key end -function utils.extract_jsonpath_keys(tuple, index_parts) - local key_def = keydef_lib.new(index_parts) - return key_def:extract_key(tuple) +function utils.extract_jsonpath_keys(tuple, index_parts, key_def) + key_def = key_def or keydef_lib.new(index_parts) + + -- We have case, when we get already extracted values in + -- tuple variable, and we will get error like this: + -- 'Tuple field [5] required by space format is missing'. + local ok, extracted_values = pcall(function() + return key_def:extract_key(tuple) + end) + + if ok == false then + return tuple + end + + return extracted_values end function utils.merge_primary_key_parts(key_parts, pk_parts) diff --git a/crud/compare/comparators.lua b/crud/compare/comparators.lua index 34c54d0b..aa3a0b7e 100644 --- a/crud/compare/comparators.lua +++ b/crud/compare/comparators.lua @@ -4,6 +4,9 @@ local compare_conditions = require('crud.compare.conditions') local type_comparators = require('crud.compare.type_comparators') local operators = compare_conditions.operators +local compat = require('crud.common.compat') +local keydef_lib = compat.require('tuple.keydef', 'key_def') + local utils = require('crud.common.utils') local ComparatorsError = errors.new_class('ComparatorsError') @@ -151,11 +154,13 @@ function comparators.gen_tuples_comparator(cmp_operator, key_parts, field_names, local updated_key_parts = comparators.update_key_parts_by_field_names( space_format, field_names, key_parts ) + local keys_comparator = comparators.gen_func(cmp_operator, updated_key_parts) + local key_def = keydef_lib.new(updated_key_parts) return function(lhs, rhs) - local lhs_key = utils.extract_jsonpath_keys(lhs, updated_key_parts) - local rhs_key = utils.extract_jsonpath_keys(rhs, updated_key_parts) + local lhs_key = utils.extract_jsonpath_keys(lhs, updated_key_parts, key_def) + local rhs_key = utils.extract_jsonpath_keys(rhs, updated_key_parts, key_def) return keys_comparator(lhs_key, rhs_key) end diff --git a/crud/select/executor.lua b/crud/select/executor.lua index fad42deb..bae1a292 100644 --- a/crud/select/executor.lua +++ b/crud/select/executor.lua @@ -50,6 +50,19 @@ function executor.execute(space, index, filter_func, opts) local tuples_count = 0 local value = opts.scan_value + if opts.after_tuple ~= nil then + if value == nil then + value = opts.after_tuple + else + local cmp_operator = select_comparators.get_cmp_operator(opts.tarantool_iter) + local scan_comparator = select_comparators.gen_tuples_comparator(cmp_operator, index.parts) + local after_tuple_key = utils.extract_jsonpath_keys(opts.after_tuple, index.parts) + if scan_comparator(after_tuple_key, opts.scan_value) then + value = after_tuple_key + end + end + end + local tuple local gen = index:pairs(value, {iterator = opts.tarantool_iter}) From 77b55861c7525a0b52795aee8cb687b8f41f1bac Mon Sep 17 00:00:00 2001 From: Alexey Romanov Date: Fri, 18 Jun 2021 15:46:11 +0300 Subject: [PATCH 09/28] Update extract logic --- crud/common/utils.lua | 23 +++++++++++++---------- crud/select/executor.lua | 14 +++++++++++--- crud/select/plan.lua | 9 ++++++++- 3 files changed, 32 insertions(+), 14 deletions(-) diff --git a/crud/common/utils.lua b/crud/common/utils.lua index 56961768..234ecd09 100644 --- a/crud/common/utils.lua +++ b/crud/common/utils.lua @@ -12,9 +12,6 @@ local ShardingError = errors.new_class('ShardingError', {capture_stack = false} local GetSpaceFormatError = errors.new_class('GetSpaceFormatError', {capture_stack = false}) local FilterFieldsError = errors.new_class('FilterFieldsError', {capture_stack = false}) -local compat = require('crud.common.compat') -local keydef_lib = compat.require('tuple.keydef', 'key_def') - local utils = {} local space_format_cache = setmetatable({}, {__mode = 'k'}) @@ -152,20 +149,26 @@ function utils.extract_key(tuple, key_parts) end function utils.extract_jsonpath_keys(tuple, index_parts, key_def) - key_def = key_def or keydef_lib.new(index_parts) + local keys = {} -- We have case, when we get already extracted values in -- tuple variable, and we will get error like this: -- 'Tuple field [5] required by space format is missing'. - local ok, extracted_values = pcall(function() - return key_def:extract_key(tuple) - end) + -- For example, we can get tuple {'Yellow'} (already extrated jsonpath key) + -- and index parts like this: {{fieldno = 3, path = data.color}}. + -- When we try to key_def:extract_key, we wil get error. + for _, part in ipairs(index_parts) do + if part.path ~= nil and type(tuple[part.fieldno]) == 'table' then + keys = key_def:extract_key(tuple) + break + end + end - if ok == false then - return tuple + if #keys == 0 then + return utils.extract_key(tuple, index_parts) end - return extracted_values + return keys end function utils.merge_primary_key_parts(key_parts, pk_parts) diff --git a/crud/select/executor.lua b/crud/select/executor.lua index bae1a292..7c8a647d 100644 --- a/crud/select/executor.lua +++ b/crud/select/executor.lua @@ -3,6 +3,9 @@ local errors = require('errors') local dev_checks = require('crud.common.dev_checks') local select_comparators = require('crud.compare.comparators') +local compat = require('crud.common.compat') +local keydef_lib = compat.require('tuple.keydef', 'key_def') + local utils = require('crud.common.utils') local ExecuteSelectError = errors.new_class('ExecuteSelectError') @@ -25,7 +28,7 @@ local function scroll_to_after_tuple(gen, space, scan_index, tarantool_iter, aft return nil end - if scroll_comparator(tuple, after_tuple) then + if scroll_comparator(tuple, after_tuple, false, true) then return tuple end end @@ -56,8 +59,13 @@ function executor.execute(space, index, filter_func, opts) else local cmp_operator = select_comparators.get_cmp_operator(opts.tarantool_iter) local scan_comparator = select_comparators.gen_tuples_comparator(cmp_operator, index.parts) - local after_tuple_key = utils.extract_jsonpath_keys(opts.after_tuple, index.parts) - if scan_comparator(after_tuple_key, opts.scan_value) then + local after_tuple_key = utils.extract_jsonpath_keys( + opts.after_tuple, + index.parts, + keydef_lib.new(index.parts) + ) + + if scan_comparator(after_tuple_key, opts.scan_value, true, true) then value = after_tuple_key end end diff --git a/crud/select/plan.lua b/crud/select/plan.lua index fb4ff1cf..384fb7c1 100644 --- a/crud/select/plan.lua +++ b/crud/select/plan.lua @@ -4,6 +4,9 @@ local compare_conditions = require('crud.compare.conditions') local utils = require('crud.common.utils') local dev_checks = require('crud.common.dev_checks') +local compat = require('crud.common.compat') +local keydef_lib = compat.require('tuple.keydef', 'key_def') + local select_plan = {} local IndexTypeError = errors.new_class('IndexTypeError', {capture_stack = false}) @@ -184,7 +187,11 @@ function select_plan.new(space, conditions, opts) scan_condition_num = nil if scan_after_tuple ~= nil then - scan_value = utils.extract_jsonpath_keys(scan_after_tuple, scan_index.parts) + scan_value = utils.extract_jsonpath_keys( + scan_after_tuple, + scan_index.parts, + keydef_lib.new(scan_index.parts) + ) else scan_value = nil end From 5376645bc0aa205d5141a85264ea88309ca5cd8d Mon Sep 17 00:00:00 2001 From: Alexey Romanov Date: Fri, 18 Jun 2021 15:57:03 +0300 Subject: [PATCH 10/28] Remove redundant code --- crud/select/executor.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crud/select/executor.lua b/crud/select/executor.lua index 7c8a647d..21935bd1 100644 --- a/crud/select/executor.lua +++ b/crud/select/executor.lua @@ -65,7 +65,7 @@ function executor.execute(space, index, filter_func, opts) keydef_lib.new(index.parts) ) - if scan_comparator(after_tuple_key, opts.scan_value, true, true) then + if scan_comparator(after_tuple_key, opts.scan_value) then value = after_tuple_key end end From 3cdee5102cda2817955c2035dba538ee80556a8a Mon Sep 17 00:00:00 2001 From: Alexey Romanov Date: Fri, 18 Jun 2021 16:33:44 +0300 Subject: [PATCH 11/28] Fix extract logic --- crud/common/utils.lua | 21 +-------------------- crud/select/executor.lua | 13 ++++--------- 2 files changed, 5 insertions(+), 29 deletions(-) diff --git a/crud/common/utils.lua b/crud/common/utils.lua index 234ecd09..c8bb82a1 100644 --- a/crud/common/utils.lua +++ b/crud/common/utils.lua @@ -149,26 +149,7 @@ function utils.extract_key(tuple, key_parts) end function utils.extract_jsonpath_keys(tuple, index_parts, key_def) - local keys = {} - - -- We have case, when we get already extracted values in - -- tuple variable, and we will get error like this: - -- 'Tuple field [5] required by space format is missing'. - -- For example, we can get tuple {'Yellow'} (already extrated jsonpath key) - -- and index parts like this: {{fieldno = 3, path = data.color}}. - -- When we try to key_def:extract_key, we wil get error. - for _, part in ipairs(index_parts) do - if part.path ~= nil and type(tuple[part.fieldno]) == 'table' then - keys = key_def:extract_key(tuple) - break - end - end - - if #keys == 0 then - return utils.extract_key(tuple, index_parts) - end - - return keys + return key_def:extract_key(tuple) end function utils.merge_primary_key_parts(key_parts, pk_parts) diff --git a/crud/select/executor.lua b/crud/select/executor.lua index 21935bd1..44caef3d 100644 --- a/crud/select/executor.lua +++ b/crud/select/executor.lua @@ -57,15 +57,10 @@ function executor.execute(space, index, filter_func, opts) if value == nil then value = opts.after_tuple else - local cmp_operator = select_comparators.get_cmp_operator(opts.tarantool_iter) - local scan_comparator = select_comparators.gen_tuples_comparator(cmp_operator, index.parts) - local after_tuple_key = utils.extract_jsonpath_keys( - opts.after_tuple, - index.parts, - keydef_lib.new(index.parts) - ) - - if scan_comparator(after_tuple_key, opts.scan_value) then + local key_def = keydef_lib.new(index.parts) + local after_tuple_key = utils.extract_jsonpath_keys(opts.after_tuple, index.parts, key_def) + + if key_def:compare_with_key(opts.after_tuple, opts.scan_value) == 0 then value = after_tuple_key end end From 70383f20d8f65bb321b37ecd9b3fe5755e6738c8 Mon Sep 17 00:00:00 2001 From: Alexey Romanov Date: Fri, 18 Jun 2021 16:38:46 +0300 Subject: [PATCH 12/28] Fix style --- crud/common/utils.lua | 4 ---- crud/compare/comparators.lua | 6 ++---- crud/select/executor.lua | 5 +++-- crud/select/plan.lua | 7 ++----- 4 files changed, 7 insertions(+), 15 deletions(-) diff --git a/crud/common/utils.lua b/crud/common/utils.lua index c8bb82a1..b8243037 100644 --- a/crud/common/utils.lua +++ b/crud/common/utils.lua @@ -148,10 +148,6 @@ function utils.extract_key(tuple, key_parts) return key end -function utils.extract_jsonpath_keys(tuple, index_parts, key_def) - return key_def:extract_key(tuple) -end - function utils.merge_primary_key_parts(key_parts, pk_parts) local merged_parts = {} local key_fieldnos = {} diff --git a/crud/compare/comparators.lua b/crud/compare/comparators.lua index aa3a0b7e..6b280a47 100644 --- a/crud/compare/comparators.lua +++ b/crud/compare/comparators.lua @@ -7,8 +7,6 @@ local operators = compare_conditions.operators local compat = require('crud.common.compat') local keydef_lib = compat.require('tuple.keydef', 'key_def') -local utils = require('crud.common.utils') - local ComparatorsError = errors.new_class('ComparatorsError') local comparators = {} @@ -159,8 +157,8 @@ function comparators.gen_tuples_comparator(cmp_operator, key_parts, field_names, local key_def = keydef_lib.new(updated_key_parts) return function(lhs, rhs) - local lhs_key = utils.extract_jsonpath_keys(lhs, updated_key_parts, key_def) - local rhs_key = utils.extract_jsonpath_keys(rhs, updated_key_parts, key_def) + local lhs_key = key_def:extract_key(lhs) + local rhs_key = key_def:extract_key(rhs) return keys_comparator(lhs_key, rhs_key) end diff --git a/crud/select/executor.lua b/crud/select/executor.lua index 44caef3d..5667cdff 100644 --- a/crud/select/executor.lua +++ b/crud/select/executor.lua @@ -28,7 +28,7 @@ local function scroll_to_after_tuple(gen, space, scan_index, tarantool_iter, aft return nil end - if scroll_comparator(tuple, after_tuple, false, true) then + if scroll_comparator(tuple, after_tuple) then return tuple end end @@ -58,7 +58,8 @@ function executor.execute(space, index, filter_func, opts) value = opts.after_tuple else local key_def = keydef_lib.new(index.parts) - local after_tuple_key = utils.extract_jsonpath_keys(opts.after_tuple, index.parts, key_def) + local after_tuple_key = key_def:extract_key(opts.after_tuple) + --local after_tuple_key = utils.extract_jsonpath_keys(opts.after_tuple, index.parts, key_def) if key_def:compare_with_key(opts.after_tuple, opts.scan_value) == 0 then value = after_tuple_key diff --git a/crud/select/plan.lua b/crud/select/plan.lua index 384fb7c1..281f1c8c 100644 --- a/crud/select/plan.lua +++ b/crud/select/plan.lua @@ -187,11 +187,8 @@ function select_plan.new(space, conditions, opts) scan_condition_num = nil if scan_after_tuple ~= nil then - scan_value = utils.extract_jsonpath_keys( - scan_after_tuple, - scan_index.parts, - keydef_lib.new(scan_index.parts) - ) + local key_def = keydef_lib.new(scan_index.parts) + scan_value = key_def:extract_key(scan_after_tuple) else scan_value = nil end From f6b349a2062a991890b104e328a7c961d8d390d7 Mon Sep 17 00:00:00 2001 From: Alexey Romanov Date: Fri, 18 Jun 2021 16:39:56 +0300 Subject: [PATCH 13/28] Remove redundant code --- crud/select/executor.lua | 1 - 1 file changed, 1 deletion(-) diff --git a/crud/select/executor.lua b/crud/select/executor.lua index 5667cdff..a9502f3b 100644 --- a/crud/select/executor.lua +++ b/crud/select/executor.lua @@ -59,7 +59,6 @@ function executor.execute(space, index, filter_func, opts) else local key_def = keydef_lib.new(index.parts) local after_tuple_key = key_def:extract_key(opts.after_tuple) - --local after_tuple_key = utils.extract_jsonpath_keys(opts.after_tuple, index.parts, key_def) if key_def:compare_with_key(opts.after_tuple, opts.scan_value) == 0 then value = after_tuple_key From 43307f698a517f9c43c5fbaffa990506ff3f0045 Mon Sep 17 00:00:00 2001 From: Alexey Romanov Date: Fri, 18 Jun 2021 20:02:15 +0300 Subject: [PATCH 14/28] Compare fix --- crud/select/executor.lua | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/crud/select/executor.lua b/crud/select/executor.lua index a9502f3b..5cde223c 100644 --- a/crud/select/executor.lua +++ b/crud/select/executor.lua @@ -58,10 +58,8 @@ function executor.execute(space, index, filter_func, opts) value = opts.after_tuple else local key_def = keydef_lib.new(index.parts) - local after_tuple_key = key_def:extract_key(opts.after_tuple) - - if key_def:compare_with_key(opts.after_tuple, opts.scan_value) == 0 then - value = after_tuple_key + if key_def:compare_with_key(opts.after_tuple, opts.scan_value) < 0 then + value = key_def:extract_key(opts.after_tuple) end end end From fed05e20ce782488c465c3eb9fe1d378478313bf Mon Sep 17 00:00:00 2001 From: Alexey Romanov Date: Fri, 18 Jun 2021 20:40:44 +0300 Subject: [PATCH 15/28] Changelog --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5972dcd0..f4dcbb7e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,7 +17,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ### Added -* Added support for jsonpath indexes for ``crud.select`` +* Added jsonpath indexes support for queries. ## [0.7.0] - 2021-05-27 From 83707ed33c8339741cf9bf2585b456c7a2934aa3 Mon Sep 17 00:00:00 2001 From: Alexey Romanov Date: Sun, 20 Jun 2021 19:03:54 +0300 Subject: [PATCH 16/28] Jsonpath indexes now use fieldno --- crud/select/filters.lua | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/crud/select/filters.lua b/crud/select/filters.lua index c1a23730..145e0190 100644 --- a/crud/select/filters.lua +++ b/crud/select/filters.lua @@ -40,12 +40,12 @@ local function is_early_exit_possible(index, tarantool_iter, condition) return false end -local function get_index_fieldnos(index, space_format) +local function get_index_fieldnos(index) local index_fieldnos = {} for _, part in ipairs(index.parts) do if part.path ~= nil then - table.insert(index_fieldnos, string.format("%s.%s", space_format[part.fieldno].name, part.path)) + table.insert(index_fieldnos, string.format("[%d]%s", part.fieldno, part.path)) else table.insert(index_fieldnos, part.fieldno) end @@ -105,7 +105,7 @@ local function parse(space, conditions, opts) local index = space_indexes[condition.operand] if index ~= nil then - fields = get_index_fieldnos(index, space_format) + fields = get_index_fieldnos(index) fields_types = get_index_fields_types(index) values_opts = get_values_opts(index) else From bb0f72750d4adc3c49f1d9f7ee0fba5f0697b457 Mon Sep 17 00:00:00 2001 From: Alexey Romanov Date: Mon, 21 Jun 2021 18:19:38 +0300 Subject: [PATCH 17/28] Unit tests --- test/unit/select_filters_test.lua | 68 +++++++++++++++++++++++++++++++ 1 file changed, 68 insertions(+) diff --git a/test/unit/select_filters_test.lua b/test/unit/select_filters_test.lua index 6b4cc246..38ad3859 100644 --- a/test/unit/select_filters_test.lua +++ b/test/unit/select_filters_test.lua @@ -9,6 +9,8 @@ local collations = require('crud.common.collations') local t = require('luatest') local g = t.group('select_filters') +local crud_utils = require('crud.common.utils') + local helpers = require('test.helper') g.before_all = function() @@ -43,10 +45,47 @@ g.before_all = function() unique = false, if_not_exists = true, }) + + if crud_utils.tarantool_supports_jsonpath_indexes() then + local cars_space = box.schema.space.create('cars', { + format = { + {name = 'id', type = 'map'}, + {name = 'bucket_id', type = 'unsigned'}, + {name = 'age', type = 'number'}, + {name = 'manufacturer', type = 'string'}, + {name = 'data', type = 'map'} + }, + if_not_exists = true, + }) + + -- primary index + cars_space:create_index('id_ind', { + parts = { + {1, 'unsigned', path = 'car_id.signed'}, + }, + if_not_exists = true, + }) + + cars_space:create_index('bucket_id', { + parts = { 'bucket_id' }, + unique = false, + if_not_exists = true, + }) + + cars_space:create_index('data_index', { + parts = { + {5, 'str', path = 'car.color'}, + {5, 'str', path = 'car.model'}, + }, + unique = false, + if_not_exists = true, + }) + end end g.after_all(function() box.space.customers:drop() + box.space.cars:drop() end) g.test_empty_conditions = function() @@ -815,5 +854,34 @@ return M]] t.assert_equals({ filter_func(box.tuple.new({{field_1 = 5, f2 = 3}, {fld_1 = "jsonpath_test"}, 23})) }, {false, false}) end +g.test_jsonpath_indexes = function() + t.skip_if( + not crud_utils.tarantool_supports_jsonpath_indexes(), + "Jsonpath indexes supported since 2.6.3/2.7.2/2.8.1" + ) + + local conditions = { + cond_funcs.gt('id', 20), + cond_funcs.eq('data_index', {'Yellow', 'BMW'}) + } + + local plan, err = select_plan.new(box.space.cars, conditions) + t.assert_equals(err, nil) + + local filter_conditions, err = select_filters.internal.parse(box.space.cars, conditions, { + scan_condition_num = plan.scan_condition_num, + tarantool_iter = plan.tarantool_iter, + }) + + t.assert_equals(err, nil) + + local data_condition = filter_conditions[1] + t.assert_type(data_condition, 'table') + t.assert_equals(data_condition.fields, {"[5]car.color", "[5]car.model"}) + t.assert_equals(data_condition.operator, compare_conditions.operators.EQ) + t.assert_equals(data_condition.values, {'Yellow', 'BMW'}) + t.assert_equals(data_condition.types, {'string', 'string'}) + t.assert_equals(data_condition.early_exit_is_possible, false) +end -- luacheck: pop From 6dd1fac2ecd4bc38d8335e95e66c403cbc88e2ea Mon Sep 17 00:00:00 2001 From: Alexey Romanov Date: Mon, 21 Jun 2021 19:02:03 +0300 Subject: [PATCH 18/28] Fix compatibility with 1.10.6 --- crud/select/executor.lua | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/crud/select/executor.lua b/crud/select/executor.lua index 5cde223c..b931e6cf 100644 --- a/crud/select/executor.lua +++ b/crud/select/executor.lua @@ -2,9 +2,8 @@ local errors = require('errors') local dev_checks = require('crud.common.dev_checks') local select_comparators = require('crud.compare.comparators') - local compat = require('crud.common.compat') -local keydef_lib = compat.require('tuple.keydef', 'key_def') +local has_keydef, keydef_lib = pcall(compat.require, 'tuple.keydef', 'key_def') local utils = require('crud.common.utils') @@ -57,9 +56,18 @@ function executor.execute(space, index, filter_func, opts) if value == nil then value = opts.after_tuple else - local key_def = keydef_lib.new(index.parts) - if key_def:compare_with_key(opts.after_tuple, opts.scan_value) < 0 then - value = key_def:extract_key(opts.after_tuple) + if has_keydef then + local key_def = keydef_lib.new(index.parts) + if key_def:compare_with_key(opts.after_tuple, opts.scan_value) < 0 then + value = key_def:extract_key(opts.after_tuple) + end + else + local cmp_operator = select_comparators.get_cmp_operator(opts.tarantool_iter) + local scan_comparator = select_comparators.gen_tuples_comparator(cmp_operator, index.parts) + local after_tuple_key = utils.extract_key(opts.after_tuple, index.parts) + if scan_comparator(after_tuple_key, opts.scan_value) then + value = after_tuple_key + end end end end From aac61b32b267ad6446a60c91075114cb5cef2c1e Mon Sep 17 00:00:00 2001 From: Alexey Romanov Date: Mon, 21 Jun 2021 19:22:50 +0300 Subject: [PATCH 19/28] Fix compatibility with v1.10.6 --- crud/common/utils.lua | 3 +++ crud/compare/comparators.lua | 21 ++++++++++++++++----- crud/select/plan.lua | 10 +++++++--- 3 files changed, 26 insertions(+), 8 deletions(-) diff --git a/crud/common/utils.lua b/crud/common/utils.lua index b8243037..bab64c89 100644 --- a/crud/common/utils.lua +++ b/crud/common/utils.lua @@ -5,6 +5,9 @@ local vshard = require('vshard') local schema = require('crud.common.schema') local dev_checks = require('crud.common.dev_checks') +local compat = require('crud.common.compat') +local has_keydef, keydef_lib = pcall(compat.require, 'tuple.keydef', 'key_def') + local FlattenError = errors.new_class("FlattenError", {capture_stack = false}) local UnflattenError = errors.new_class("UnflattenError", {capture_stack = false}) local ParseOperationsError = errors.new_class('ParseOperationsError', {capture_stack = false}) diff --git a/crud/compare/comparators.lua b/crud/compare/comparators.lua index 6b280a47..127620dd 100644 --- a/crud/compare/comparators.lua +++ b/crud/compare/comparators.lua @@ -2,10 +2,12 @@ local errors = require('errors') local compare_conditions = require('crud.compare.conditions') local type_comparators = require('crud.compare.type_comparators') -local operators = compare_conditions.operators +local utils = require('crud.common.utils') local compat = require('crud.common.compat') -local keydef_lib = compat.require('tuple.keydef', 'key_def') +local has_keydef, keydef_lib = pcall(compat.require, 'tuple.keydef', 'key_def') + +local operators = compare_conditions.operators local ComparatorsError = errors.new_class('ComparatorsError') @@ -154,12 +156,21 @@ function comparators.gen_tuples_comparator(cmp_operator, key_parts, field_names, ) local keys_comparator = comparators.gen_func(cmp_operator, updated_key_parts) - local key_def = keydef_lib.new(updated_key_parts) + + local key_def + if has_keydef then + key_def = keydef_lib.new(updated_key_parts) + end return function(lhs, rhs) - local lhs_key = key_def:extract_key(lhs) - local rhs_key = key_def:extract_key(rhs) + if has_keydef then + local lhs_key = key_def:extract_key(lhs) + local rhs_key = key_def:extract_key(rhs) + return keys_comparator(lhs_key, rhs_key) + end + local lhs_key = utils.extract_key(lhs, updated_key_parts) + local rhs_key = utils.extract_key(rhs, updated_key_parts) return keys_comparator(lhs_key, rhs_key) end end diff --git a/crud/select/plan.lua b/crud/select/plan.lua index 281f1c8c..b73eff6a 100644 --- a/crud/select/plan.lua +++ b/crud/select/plan.lua @@ -5,7 +5,7 @@ local utils = require('crud.common.utils') local dev_checks = require('crud.common.dev_checks') local compat = require('crud.common.compat') -local keydef_lib = compat.require('tuple.keydef', 'key_def') +local has_keydef, keydef_lib = pcall(compat.require, 'tuple.keydef', 'key_def') local select_plan = {} @@ -187,8 +187,12 @@ function select_plan.new(space, conditions, opts) scan_condition_num = nil if scan_after_tuple ~= nil then - local key_def = keydef_lib.new(scan_index.parts) - scan_value = key_def:extract_key(scan_after_tuple) + if has_keydef then + local key_def = keydef_lib.new(scan_index.parts) + scan_value = key_def:extract_key(scan_after_tuple) + else + scan_value = utils.extract_key(scan_after_tuple) + end else scan_value = nil end From fcdc489897c526b1d391ffe47a44e51d6e65bc9b Mon Sep 17 00:00:00 2001 From: Alexey Romanov Date: Mon, 21 Jun 2021 19:26:03 +0300 Subject: [PATCH 20/28] Lint fix --- crud/common/utils.lua | 3 --- 1 file changed, 3 deletions(-) diff --git a/crud/common/utils.lua b/crud/common/utils.lua index bab64c89..b8243037 100644 --- a/crud/common/utils.lua +++ b/crud/common/utils.lua @@ -5,9 +5,6 @@ local vshard = require('vshard') local schema = require('crud.common.schema') local dev_checks = require('crud.common.dev_checks') -local compat = require('crud.common.compat') -local has_keydef, keydef_lib = pcall(compat.require, 'tuple.keydef', 'key_def') - local FlattenError = errors.new_class("FlattenError", {capture_stack = false}) local UnflattenError = errors.new_class("UnflattenError", {capture_stack = false}) local ParseOperationsError = errors.new_class('ParseOperationsError', {capture_stack = false}) From 205166b5ea687d856eebbb8a607a37bc86046ee9 Mon Sep 17 00:00:00 2001 From: Alexey Romanov Date: Mon, 21 Jun 2021 19:32:40 +0300 Subject: [PATCH 21/28] Typo fix --- crud/select/plan.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crud/select/plan.lua b/crud/select/plan.lua index b73eff6a..000e365f 100644 --- a/crud/select/plan.lua +++ b/crud/select/plan.lua @@ -191,7 +191,7 @@ function select_plan.new(space, conditions, opts) local key_def = keydef_lib.new(scan_index.parts) scan_value = key_def:extract_key(scan_after_tuple) else - scan_value = utils.extract_key(scan_after_tuple) + scan_value = utils.extract_key(scan_after_tuple, index.parts) end else scan_value = nil From 92490a237a513b5986930d877378ea9bd8c9ca2a Mon Sep 17 00:00:00 2001 From: Alexey Romanov Date: Mon, 21 Jun 2021 19:33:01 +0300 Subject: [PATCH 22/28] Typo fix v2 --- crud/select/plan.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crud/select/plan.lua b/crud/select/plan.lua index 000e365f..af92e809 100644 --- a/crud/select/plan.lua +++ b/crud/select/plan.lua @@ -191,7 +191,7 @@ function select_plan.new(space, conditions, opts) local key_def = keydef_lib.new(scan_index.parts) scan_value = key_def:extract_key(scan_after_tuple) else - scan_value = utils.extract_key(scan_after_tuple, index.parts) + scan_value = utils.extract_key(scan_after_tuple, scan_index.parts) end else scan_value = nil From d01812dd1bfefbc68f975bddf88a1df9eaaa03d5 Mon Sep 17 00:00:00 2001 From: Alexey Romanov Date: Mon, 21 Jun 2021 19:47:22 +0300 Subject: [PATCH 23/28] Fix PR issues --- crud/compare/comparators.lua | 18 ++++++++---------- crud/select.lua | 1 - 2 files changed, 8 insertions(+), 11 deletions(-) diff --git a/crud/compare/comparators.lua b/crud/compare/comparators.lua index 127620dd..4c1239b8 100644 --- a/crud/compare/comparators.lua +++ b/crud/compare/comparators.lua @@ -157,21 +157,19 @@ function comparators.gen_tuples_comparator(cmp_operator, key_parts, field_names, local keys_comparator = comparators.gen_func(cmp_operator, updated_key_parts) - local key_def if has_keydef then - key_def = keydef_lib.new(updated_key_parts) - end - - return function(lhs, rhs) - if has_keydef then + local key_def = keydef_lib.new(updated_key_parts) + return function(lhs, rhs) local lhs_key = key_def:extract_key(lhs) local rhs_key = key_def:extract_key(rhs) return keys_comparator(lhs_key, rhs_key) end - - local lhs_key = utils.extract_key(lhs, updated_key_parts) - local rhs_key = utils.extract_key(rhs, updated_key_parts) - return keys_comparator(lhs_key, rhs_key) + else + return function(lhs, rhs) + local lhs_key = utils.extract_key(lhs, updated_key_parts) + local rhs_key = utils.extract_key(rhs, updated_key_parts) + return keys_comparator(lhs_key, rhs_key) + end end end diff --git a/crud/select.lua b/crud/select.lua index d9ed390b..41964e1f 100644 --- a/crud/select.lua +++ b/crud/select.lua @@ -69,7 +69,6 @@ local function select_on_storage(space_name, index_id, conditions, opts) after_tuple = opts.after_tuple, tarantool_iter = opts.tarantool_iter, limit = opts.limit, - conditions = conditions, }) if err ~= nil then return nil, SelectError:new("Failed to execute select: %s", err) From 0818132ce94abc5d086c8a17c8094334c32af7ad Mon Sep 17 00:00:00 2001 From: Alexey Romanov Date: Mon, 21 Jun 2021 19:55:15 +0300 Subject: [PATCH 24/28] Remove redundant code --- crud/select/executor.lua | 1 - 1 file changed, 1 deletion(-) diff --git a/crud/select/executor.lua b/crud/select/executor.lua index b931e6cf..1bc7a971 100644 --- a/crud/select/executor.lua +++ b/crud/select/executor.lua @@ -39,7 +39,6 @@ function executor.execute(space, index, filter_func, opts) after_tuple = '?table', tarantool_iter = 'number', limit = '?number', - conditions = '?table' }) opts = opts or {} From 89a342a672d5ebc73a68caf1abed64c3979f8f9d Mon Sep 17 00:00:00 2001 From: Alexey Romanov Date: Mon, 21 Jun 2021 20:27:15 +0300 Subject: [PATCH 25/28] Encapsulated module existence check --- crud/common/compat.lua | 17 +++++++++++++++++ crud/compare/comparators.lua | 7 ++++++- crud/compare/keydef.lua | 7 ++++++- crud/select/executor.lua | 7 ++++++- crud/select/plan.lua | 7 ++++++- 5 files changed, 41 insertions(+), 4 deletions(-) diff --git a/crud/common/compat.lua b/crud/common/compat.lua index 3b3c52c8..2eaf6969 100644 --- a/crud/common/compat.lua +++ b/crud/common/compat.lua @@ -30,4 +30,21 @@ function compat.require(module_name, builtin_module_name) return module end +function compat.exists(module_name, builtin_module_name) + local module_cached = rawget(_G, string.format('__crud_%s_cached', module_name)) + if module_cached ~= nil then + return true + end + + if package.search(module_name) then + return true + end + + if package.loaded[builtin_module_name] ~= nil then + return true + end + + return false +end + return compat diff --git a/crud/compare/comparators.lua b/crud/compare/comparators.lua index 4c1239b8..e59bf053 100644 --- a/crud/compare/comparators.lua +++ b/crud/compare/comparators.lua @@ -5,7 +5,12 @@ local type_comparators = require('crud.compare.type_comparators') local utils = require('crud.common.utils') local compat = require('crud.common.compat') -local has_keydef, keydef_lib = pcall(compat.require, 'tuple.keydef', 'key_def') +local has_keydef = compat.exists('tuple.keydef', 'key_def') + +local keydef_lib +if has_keydef then + keydef_lib = compat.require('tuple.keydef', 'key_def') +end local operators = compare_conditions.operators diff --git a/crud/compare/keydef.lua b/crud/compare/keydef.lua index 6a2eced8..8aa2932b 100644 --- a/crud/compare/keydef.lua +++ b/crud/compare/keydef.lua @@ -2,7 +2,12 @@ local comparators = require('crud.compare.comparators') local collations = require('crud.common.collations') local compat = require('crud.common.compat') -local keydef_lib = compat.require('tuple.keydef', 'key_def') +local has_keydef = compat.exists('tuple.keydef', 'key_def') + +local keydef_lib +if has_keydef then + keydef_lib = compat.require('tuple.keydef', 'key_def') +end -- As "tuple.key_def" doesn't support collation_id -- we manually change it to collation diff --git a/crud/select/executor.lua b/crud/select/executor.lua index 1bc7a971..ec355d3c 100644 --- a/crud/select/executor.lua +++ b/crud/select/executor.lua @@ -3,7 +3,12 @@ local errors = require('errors') local dev_checks = require('crud.common.dev_checks') local select_comparators = require('crud.compare.comparators') local compat = require('crud.common.compat') -local has_keydef, keydef_lib = pcall(compat.require, 'tuple.keydef', 'key_def') +local has_keydef = compat.exists('tuple.keydef', 'key_def') + +local keydef_lib +if has_keydef then + keydef_lib = compat.require('tuple.keydef', 'key_def') +end local utils = require('crud.common.utils') diff --git a/crud/select/plan.lua b/crud/select/plan.lua index af92e809..1662eb2c 100644 --- a/crud/select/plan.lua +++ b/crud/select/plan.lua @@ -5,7 +5,12 @@ local utils = require('crud.common.utils') local dev_checks = require('crud.common.dev_checks') local compat = require('crud.common.compat') -local has_keydef, keydef_lib = pcall(compat.require, 'tuple.keydef', 'key_def') +local has_keydef = compat.exists('tuple.keydef', 'key_def') + +local keydef_lib +if has_keydef then + keydef_lib = compat.require('tuple.keydef', 'key_def') +end local select_plan = {} From 9a344031e08386c67edd6d58bb5758059361588b Mon Sep 17 00:00:00 2001 From: Alexey Romanov Date: Mon, 21 Jun 2021 20:34:44 +0300 Subject: [PATCH 26/28] Decompose update value in executor function --- crud/select/executor.lua | 31 +++++++++++++++++++------------ 1 file changed, 19 insertions(+), 12 deletions(-) diff --git a/crud/select/executor.lua b/crud/select/executor.lua index ec355d3c..bffeaec0 100644 --- a/crud/select/executor.lua +++ b/crud/select/executor.lua @@ -38,6 +38,22 @@ local function scroll_to_after_tuple(gen, space, scan_index, tarantool_iter, aft end end +local function generate_value(after_tuple, scan_value, index_parts, tarantool_iter) + if has_keydef then + local key_def = keydef_lib.new(index_parts) + if key_def:compare_with_key(after_tuple, scan_value) < 0 then + return key_def:extract_key(after_tuple) + end + else + local cmp_operator = select_comparators.get_cmp_operator(tarantool_iter) + local scan_comparator = select_comparators.gen_tuples_comparator(cmp_operator, index_parts) + local after_tuple_key = utils.extract_key(after_tuple, index_parts) + if scan_comparator(after_tuple_key, scan_value) then + return after_tuple_key + end + end +end + function executor.execute(space, index, filter_func, opts) dev_checks('table', 'table', 'function', { scan_value = 'table', @@ -60,18 +76,9 @@ function executor.execute(space, index, filter_func, opts) if value == nil then value = opts.after_tuple else - if has_keydef then - local key_def = keydef_lib.new(index.parts) - if key_def:compare_with_key(opts.after_tuple, opts.scan_value) < 0 then - value = key_def:extract_key(opts.after_tuple) - end - else - local cmp_operator = select_comparators.get_cmp_operator(opts.tarantool_iter) - local scan_comparator = select_comparators.gen_tuples_comparator(cmp_operator, index.parts) - local after_tuple_key = utils.extract_key(opts.after_tuple, index.parts) - if scan_comparator(after_tuple_key, opts.scan_value) then - value = after_tuple_key - end + local new_value = generate_value(opts.after_tuple, opts.scan_value, index.parts, opts.tarantool_iter) + if new_value ~= nil then + value = new_value end end end From 56d4f7ea3513740653d6ccd31a1d12b6eb5900e3 Mon Sep 17 00:00:00 2001 From: Alexey Romanov Date: Mon, 21 Jun 2021 21:30:23 +0300 Subject: [PATCH 27/28] Fix 1.10.6 compatibility --- crud/compare/keydef.lua | 7 +------ crud/select/executor.lua | 10 +++++++--- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/crud/compare/keydef.lua b/crud/compare/keydef.lua index 8aa2932b..6a2eced8 100644 --- a/crud/compare/keydef.lua +++ b/crud/compare/keydef.lua @@ -2,12 +2,7 @@ local comparators = require('crud.compare.comparators') local collations = require('crud.common.collations') local compat = require('crud.common.compat') -local has_keydef = compat.exists('tuple.keydef', 'key_def') - -local keydef_lib -if has_keydef then - keydef_lib = compat.require('tuple.keydef', 'key_def') -end +local keydef_lib = compat.require('tuple.keydef', 'key_def') -- As "tuple.key_def" doesn't support collation_id -- we manually change it to collation diff --git a/crud/select/executor.lua b/crud/select/executor.lua index bffeaec0..826a8cf4 100644 --- a/crud/select/executor.lua +++ b/crud/select/executor.lua @@ -38,13 +38,17 @@ local function scroll_to_after_tuple(gen, space, scan_index, tarantool_iter, aft end end -local function generate_value(after_tuple, scan_value, index_parts, tarantool_iter) - if has_keydef then +local generate_value + +if has_keydef then + generate_value = function(after_tuple, scan_value, index_parts, tarantool_iter) local key_def = keydef_lib.new(index_parts) if key_def:compare_with_key(after_tuple, scan_value) < 0 then return key_def:extract_key(after_tuple) end - else + end +else + generate_value = function(after_tuple, scan_value, index_parts, tarantool_iter) local cmp_operator = select_comparators.get_cmp_operator(tarantool_iter) local scan_comparator = select_comparators.gen_tuples_comparator(cmp_operator, index_parts) local after_tuple_key = utils.extract_key(after_tuple, index_parts) From 4f6c1daed697550f65355b21a3fe9991c65b06a1 Mon Sep 17 00:00:00 2001 From: Alexey Romanov Date: Mon, 21 Jun 2021 21:32:56 +0300 Subject: [PATCH 28/28] Lint fix --- crud/select/executor.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crud/select/executor.lua b/crud/select/executor.lua index 826a8cf4..04615dbd 100644 --- a/crud/select/executor.lua +++ b/crud/select/executor.lua @@ -41,7 +41,7 @@ end local generate_value if has_keydef then - generate_value = function(after_tuple, scan_value, index_parts, tarantool_iter) + generate_value = function(after_tuple, scan_value, index_parts) local key_def = keydef_lib.new(index_parts) if key_def:compare_with_key(after_tuple, scan_value) < 0 then return key_def:extract_key(after_tuple)