Refactor arguments generation / converting

* Generate object arguments in avro-schema format first.
* Generate object arguments for subrecords.
* Use full path for a GraphQL type name where possible.

Also:

* Added apidoc-lint target and added to CI.

* Part of #166.
* Fixes #73 (2nd bullet).
* Fixes #163.
* Fixes #46.

Author: Totktonada

.gitignore

@@ -19,3 +19,6 @@ doc/apidoc
 
 # benchmarks results
 bench.*.result.txt
+
+# temporary directory for make apidoc-lint
+doc/apidoc-lint-tmp

.travis.yml

@@ -58,6 +58,8 @@ install:
   - cd ../graphql
   - sudo luarocks install luacheck
   - sudo pip install virtualenv
+  - sudo luarocks install ldoc
 
 script:
+  - make apidoc-lint
   - make test

Makefile

@@ -45,6 +45,12 @@ pure-bench:
 clean:
 	rm -rf test/var
 
+.PHONY: apidoc-lint
+apidoc-lint:
+	! ldoc -d doc/apidoc-lint-tmp graphql --all -f markdown 2>&1 >/dev/null | \
+		grep -v ': no module() call found; no initial doc comment$$\|: contains no items$$'
+	rm -rf doc/apidoc-lint-tmp
+
 .PHONY: apidoc
 apidoc:
 	ldoc -d doc/apidoc graphql --all -f markdown

graphql/accessor_general.lua

@@ -291,7 +291,7 @@ end
 --- (directly or indirectly using the `accessor_space.new` or the
 --- `accessor_shard.new` function); this function uses the
 --- `self.index_cache` prebuild table representing available indexes
-
+---
 --- @tparam string collection_name name of a collection of whose indexes the
 --- function will search through
 ---
@@ -332,7 +332,7 @@ end
 --- `nil`, or contains `value_list` field to pass to a GT (great-then) index,
 --- or contains `filter` field to use in `process_tuple` for find the pivot in
 --- a select result
-local get_index_name = function(self, collection_name, from, filter, args)
+local function get_index_name(self, collection_name, from, filter, args)
     assert(type(self) == 'table',
         'self must be a table, got ' .. type(self))
     assert(type(collection_name) == 'string',
@@ -772,36 +772,42 @@ end
 local function match_using_re(obj, pcre)
     if pcre == nil then return true end
 
+    assert(rex ~= nil, 'we should not pass over :compile() ' ..
+        'with a query contains PCRE matching when there are '..
+        'no lrexlib-pcre (rex_pcre) module present')
+
     for field_name, re in pairs(pcre) do
         -- skip an object with null in a string* field
         if obj[field_name] == nil then
             return false
         end
-        assert(rex ~= nil, 'we should not pass over :compile() ' ..
-            'with a query contains PCRE matching when there are '..
-            'no lrexlib-pcre (rex_pcre) module present')
-        local flags = rex.flags()
-        -- emulate behaviour of (?i) on libpcre (libpcre2 supports it)
-        local cfg = 0
-        if not is_pcre2 then
-            local cnt
-            re, cnt = re:gsub('^%(%?i%)', '')
-            if cnt > 0 then
-                cfg = bit.bor(cfg, flags.CASELESS)
-            end
-        end
-        -- enable UTF-8
-        if is_pcre2 then
-            cfg = bit.bor(cfg, flags.UTF)
-            cfg = bit.bor(cfg, flags.UCP)
+        if type(re) == 'table' then
+            local match = match_using_re(obj[field_name], re)
+            if not match then return false end
         else
-            cfg = bit.bor(cfg, flags.UTF8)
-            cfg = bit.bor(cfg, flags.UCP)
-        end
-        -- XXX: compile re once
-        local re = rex.new(re, cfg)
-        if not re:match(obj[field_name]) then
-            return false
+            local flags = rex.flags()
+            -- emulate behaviour of (?i) on libpcre (libpcre2 supports it)
+            local cfg = 0
+            if not is_pcre2 then
+                local cnt
+                re, cnt = re:gsub('^%(%?i%)', '')
+                if cnt > 0 then
+                    cfg = bit.bor(cfg, flags.CASELESS)
+                end
+            end
+            -- enable UTF-8
+            if is_pcre2 then
+                cfg = bit.bor(cfg, flags.UTF)
+                cfg = bit.bor(cfg, flags.UCP)
+            else
+                cfg = bit.bor(cfg, flags.UTF8)
+                cfg = bit.bor(cfg, flags.UCP)
+            end
+            -- XXX: compile re once
+            local re = rex.new(re, cfg)
+            if not re:match(obj[field_name]) then
+                return false
+            end
         end
     end
 
@@ -908,23 +914,6 @@ local function process_tuple(state, tuple, opts)
     return true
 end
 
---- Get schema name by a collection name.
----
---- @tparam table self data accessor instance
----
---- @tparam string collection_name
----
---- @treturn string `schema_name`
-local function get_schema_name(self, collection_name)
-    local collection = self.collections[collection_name]
-    assert(collection ~= nil,
-        ('cannot find the collection "%s"'):format(collection_name))
-    local schema_name = collection.schema_name
-    assert(type(schema_name) == 'string',
-        'schema_name must be a string, got ' .. type(schema_name))
-    return schema_name
-end
-
 --- Call one of accessor function: `update_tuple` or `delete_tuple` for each
 --- selected object.
 ---
@@ -1141,7 +1130,7 @@ local function insert_internal(self, collection_name, from, filter, args, extra)
     check(from.collection_name, 'from.collection_name', 'nil')
 
     -- convert object -> tuple (set default values from a schema)
-    local schema_name = get_schema_name(self, collection_name)
+    local schema_name = db_schema_helpers.get_schema_name(self, collection_name)
     local default_flatten_object = self.default_flatten_object[schema_name]
     assert(default_flatten_object ~= nil,
         ('cannot find default_flatten_object ' ..
@@ -1183,7 +1172,7 @@ local function update_internal(self, collection_name, extra, selected)
     assert(next(extra.extra_args, next(extra.extra_args)) == nil, err_msg)
 
     -- convert xobject -> update statements
-    local schema_name = get_schema_name(self, collection_name)
+    local schema_name = db_schema_helpers.get_schema_name(self, collection_name)
     local default_xflatten = self.default_xflatten[schema_name]
     assert(default_xflatten ~= nil,
         ('cannot find default_xflatten ' ..
@@ -1199,10 +1188,16 @@ end
 
 --- Delete an object.
 ---
---- Parameters are the same as for @{select_internal}.
+--- Corresponding parameters are the same as for @{select_internal}.
+---
+--- @tparam table self
+---
+--- @tparam string collection_name
 ---
 --- @tparam table extra `extra.extra_args.delete` is used
 ---
+--- @tparam table selected objects to delete
+---
 --- @treturn table `new_objects` list of deleted objects (in the order of the
 --- `selected` parameter)
 local function delete_internal(self, collection_name, extra, selected)
@@ -1212,7 +1207,7 @@ local function delete_internal(self, collection_name, extra, selected)
         'arguments'
     assert(next(extra.extra_args, next(extra.extra_args)) == nil, err_msg)
 
-    local schema_name = get_schema_name(self, collection_name)
+    local schema_name = db_schema_helpers.get_schema_name(self, collection_name)
 
     return perform_primary_key_operation(self, collection_name, schema_name,
         selected, 'delete_tuple')

graphql/avro_helpers.lua

@@ -28,7 +28,7 @@ local avro_helpers = {}
 ---
 --- * `raise_on_nullable` (boolean) raise an error on nullable type
 ---
---- @result `result` (string or table) nullable avro type
+--- @return `result` (string or table) nullable avro type
 function avro_helpers.make_avro_type_nullable(avro, opts)
     assert(avro ~= nil, "avro must not be nil")
     local opts = opts or {}
@@ -69,7 +69,8 @@ function avro_helpers.make_avro_type_nullable(avro, opts)
         end
         return avro
     elseif value_type == 'table' and #avro == 0 then
-        return avro_helpers.make_avro_type_nullable(avro.type, opts)
+        avro.type = avro_helpers.make_avro_type_nullable(avro.type, opts)
+        return avro
     end
 
     error("avro should be a string or a table, got " .. value_type)
@@ -108,24 +109,6 @@ function avro_helpers.is_scalar_type(avro_schema_type)
     return scalar_types[avro_schema_type] or false
 end
 
-function avro_helpers.is_comparable_scalar_type(avro_schema_type)
-    check(avro_schema_type, 'avro_schema_type', 'string')
-
-    local scalar_types = {
-        ['int'] = true,
-        ['int*'] = true,
-        ['long'] = true,
-        ['long*'] = true,
-        ['boolean'] = true,
-        ['boolean*'] = true,
-        ['string'] = true,
-        ['string*'] = true,
-        ['null'] = true,
-    }
-
-    return scalar_types[avro_schema_type] or false
-end
-
 function avro_helpers.is_compound_type(avro_schema_type)
     check(avro_schema_type, 'avro_schema_type', 'string')
 
@@ -136,30 +119,123 @@ function avro_helpers.is_compound_type(avro_schema_type)
         ['array*'] = true,
         ['map'] = true,
         ['map*'] = true,
+        ['union'] = true,
     }
 
     return compound_types[avro_schema_type] or false
 end
 
+--- Get type of an avro-schema.
+---
+--- @param avro_schema (table or string) input avro-schema
+---
+--- @tparam[opt] table opts the following options:
+---
+--- * allow_references (boolean)
+---
+--- @treturn string `avro_t` type of the avro-schema
+---
+--- @treturn boolean `is_ref` whether the avro-schema is reference to another
+--- avro-schema type
 function avro_helpers.avro_type(avro_schema, opts)
     local opts = opts or {}
     local allow_references = opts.allow_references or false
 
     if type(avro_schema) == 'table' then
         if utils.is_array(avro_schema) then
-            return 'union'
+            return 'union', false
         elseif avro_helpers.is_compound_type(avro_schema.type) then
-            return avro_schema.type
+            return avro_schema.type, false
         elseif allow_references then
-            return avro_schema
+            return avro_schema, true
         end
     elseif type(avro_schema) == 'string' then
         if avro_helpers.is_scalar_type(avro_schema) then
-            return avro_schema
+            return avro_schema, false
         elseif allow_references then
-            return avro_schema
+            return avro_schema, true
+        end
+    end
+
+    error('unrecognized avro-schema type: ' .. json.encode(avro_schema))
+end
+
+--- Expand avro-schema references.
+---
+--- @param avro_schema (table or string) input avro-schema
+---
+--- @tparam[opt] table opts the following options:
+---
+--- * definitions (table) processed avro-schemas to expand further references
+---
+--- @return generated expanded avro-schema
+function avro_helpers.expand_references(avro_schema, opts)
+    local opts = opts or {}
+    local definitions = opts.definitions or {}
+
+    local avro_t, is_ref = avro_helpers.avro_type(avro_schema,
+        {allow_references = true})
+
+    if is_ref then
+        assert(definitions[avro_t] ~= nil,
+            ('undefined reference: %s'):format(avro_t))
+        return definitions[avro_t]
+    elseif avro_t == 'union' then
+        local res = {}
+        for _, child in ipairs(avro_schema) do
+            table.insert(res, avro_helpers.expand_references(child,
+                {definitions = definitions}))
         end
+        return res
+    elseif avro_t == 'record' or avro_t == 'record*' then
+        local res = table.copy(avro_schema)
+        res.fields = {}
+
+        local res_nonnull
+        local res_nullable
+        if avro_t == 'record' then
+            res_nonnull = res
+            res_nullable = table.copy(res)
+            res_nullable.type = 'record*'
+            res_nullable.fields = res.fields
+        else
+            res_nonnull = table.copy(res)
+            res_nonnull.type = 'record'
+            res_nonnull.fields = res.fields
+            res_nullable = res
+        end
+
+        -- Saving type before traverse deeper allows to use reference to it
+        -- inside (it is allowed by our avro-schema implementation for nullable
+        -- fields, union, array and map).
+        local name = avro_schema.name
+        assert(definitions[name] == nil and definitions[name .. '*'] == nil,
+            ('multiple definitions of %s'):format(name))
+        definitions[name] = res_nonnull
+        definitions[name .. '*'] = res_nullable
+
+        for _, field in ipairs(avro_schema.fields) do
+            local field = table.copy(field)
+            field.type = avro_helpers.expand_references(field.type,
+                {definitions = definitions})
+            table.insert(res.fields, field)
+        end
+
+        return res
+    elseif avro_t == 'array' or avro_t == 'array*' then
+        local res = table.copy(avro_schema)
+        res.items = avro_helpers.expand_references(avro_schema.items,
+            {definitons = definitions})
+        return res
+    elseif avro_t == 'map' or avro_t == 'map*' then
+        local res = table.copy(avro_schema)
+        res.values = avro_helpers.expand_references(avro_schema.values,
+            {definitons = definitions})
+        return res
+    elseif avro_helpers.is_scalar_type(avro_t) then
+        return avro_schema
     end
+
     error('unrecognized avro-schema type: ' .. json.encode(avro_schema))
 end
 

graphql/config_complement.lua

@@ -203,7 +203,7 @@ end
 --- @tparam table connections cfg.connections - user-defined collections
 --- @tparam table indexes cfg.indexes - {[collection_name] = collection_indexes, ...}
 --- @treturn table `collections` is complemented collections
-local function complement_connections(collections, connections, indexes, schemas)
+local function complement_connections(collections, connections, indexes)
     if connections == nil then
         return collections
     end