box/lua/console: Add support for lua output format

@TarantoolBot document
Title: document \set output lua

Historically we use YAML format to print results of operation to
a console. Moreover our test engine is aiming YAML as a primary format
to compare results of test runs. Still we need an ability to print
results in a different fasion, in particular one may need to use
the console in a REPL way so that the results would be copied and
pased back to further processing.

For this sake we introduce that named "output" command which allows
to specify which exactly output format to use. Currently only yaml
and lua formats are supported.

To specify lua output format type

 | tarantool> \set output lua

in the console. lua mode supports line oriented output (default) or
block mode.

For example

 | tarantool> a={1,2,3}
 | tarantool> a
 | ---
 | - - 1
 |   - 2
 |   - 3
 | ...
 | tarantool> \set output lua
 | true
 | tarantool> a
 | {1, 2, 3}
 | tarantool> \set output lua,block
 | true
 | tarantool> a
 | {
 |   1,
 |   2,
 |   3
 | }

By default YAML output format is kept for now, simply to not
break the test engine. The output is bound to a session, thus every
new session should setup own conversion if needed.

Since serializing lua data is not a trivial task we use "serpent"
third party module to convert data.

Part-of #3834

Author: cyrillos

src/box/CMakeLists.txt

@@ -11,6 +11,7 @@ lua_source(lua_sources lua/feedback_daemon.lua)
 lua_source(lua_sources lua/net_box.lua)
 lua_source(lua_sources lua/upgrade.lua)
 lua_source(lua_sources lua/console.lua)
+lua_source(lua_sources ../../third_party/serpent/src/serpent.lua)
 lua_source(lua_sources lua/xlog.lua)
 lua_source(lua_sources lua/key_def.lua)
 lua_source(lua_sources lua/merger.lua)

src/box/lua/console.c

@@ -46,6 +46,8 @@
 #include <stdlib.h>
 #include <ctype.h>
 
+extern char serpent_lua[];
+
 static struct luaL_serializer *luaL_yaml_default = NULL;
 
 /*
@@ -431,6 +433,24 @@ console_session_push(struct session *session, uint64_t sync, struct port *port)
 				     TIMEOUT_INFINITY);
 }
 
+static void
+lua_serpent_init(struct lua_State *L)
+{
+	static const char modname[] = "serpent";
+	const char *modfile;
+
+	lua_getfield(L, LUA_REGISTRYINDEX, "_LOADED");
+	modfile = lua_pushfstring(L, "@builtin/%s.lua", modname);
+	if (luaL_loadbuffer(L, serpent_lua, strlen(serpent_lua), modfile)) {
+		panic("Error loading Lua module %s...: %s",
+		      modname, lua_tostring(L, -1));
+	}
+
+	lua_call(L, 0, 1);
+	lua_setfield(L, -3, modname);  /* _LOADED[modname] = new table */
+	lua_pop(L, 2);
+}
+
 void
 tarantool_lua_console_init(struct lua_State *L)
 {
@@ -471,6 +491,12 @@ tarantool_lua_console_init(struct lua_State *L)
 	};
 	session_vtab_registry[SESSION_TYPE_CONSOLE] = console_session_vtab;
 	session_vtab_registry[SESSION_TYPE_REPL] = console_session_vtab;
+
+	/*
+	 * Register serpent serializer as we
+	 * need it inside console REPL mode.
+	 */
+	lua_serpent_init(L);
 }
 
 /*

src/box/lua/console.lua

@@ -1,5 +1,6 @@
 -- console.lua -- internal file
 
+local serpent = require('serpent')
 local internal = require('console')
 local session_internal = require('box.internal.session')
 local fiber = require('fiber')
@@ -13,7 +14,9 @@ local net_box = require('net.box')
 local YAML_TERM = '\n...\n'
 local PUSH_TAG_HANDLE = '!push!'
 
-local function format(status, ...)
+local output_handlers = { }
+
+output_handlers["yaml"] = function(status, opts, ...)
     local err
     if status then
         -- serializer can raise an exception
@@ -33,6 +36,92 @@ local function format(status, ...)
     return internal.format({ error = err })
 end
 
+output_handlers["lua"] = function(status, opts, ...)
+    local data = ...
+    --
+    -- Don't print nil if there is no data
+    if data == nil then
+        return ""
+    end
+    --
+    -- Map internal symbols which serpent doesn't know
+    -- about to a known representation.
+    local map_symbols = function(tag, head, body, tail, level)
+        local symbols = {
+            ['"cdata<void %*>: NULL"']  = 'box.NULL'
+        }
+        for k,v in pairs(symbols) do
+            body = body:gsub(k, v)
+        end
+        return tag..head..body..tail
+    end
+    local serpent_opts = {
+        custom  = map_symbols,
+        comment = false,
+    }
+    if opts == "block" then
+        return serpent.block(..., serpent_opts)
+    end
+    return serpent.line(..., serpent_opts)
+end
+
+local function output_verify_opts(fmt, opts)
+    if opts == nil then
+        return true, nil
+    end
+    if fmt == "lua" then
+        if opts ~= "line" and opts ~= "block" then
+            local msg = 'Wrong option "%s", expecting: line or block.'
+            return false, msg:format(opts)
+        end
+    end
+    return true, nil
+end
+
+local function output_parse(value)
+    local fmt, opts
+    if value:match("([^,]+),([^,]+)") ~= nil then
+        fmt, opts = value:match("([^,]+),([^,]+)")
+    else
+        fmt = value
+    end
+    for k, _ in pairs(output_handlers) do
+        if k == fmt then
+            local status, err = output_verify_opts(fmt, opts)
+            if status ~= true then
+                return false, err
+            end
+            return true, nil, fmt, opts
+        end
+    end
+    local msg = 'Invalid format "%s", supported languages: lua and yaml.'
+    return false, msg:format(value)
+end
+
+local function output_save(fmt, opts)
+    --
+    -- Output format descriptors are saved per
+    -- session thus each console may specify
+    -- own mode.
+    box.session.storage.console_output = {
+        ["fmt"] = fmt, ["opts"] = opts
+    }
+end
+
+local function format(status, ...)
+    local d = box.session.storage.console_output
+    --
+    -- If there was no assignment yet provide
+    -- a default value; for now it is YAML
+    -- for backward compatibility sake (don't
+    -- forget about test results which are in
+    -- YAML format).
+    if d == nil then
+        d = { ["fmt"] = "yaml", ["opts"] = nil }
+    end
+    return output_handlers[d["fmt"]](status, d["opts"], ...)
+end
+
 --
 -- Set delimiter
 --
@@ -71,11 +160,22 @@ local function set_language(storage, value)
     return true
 end
 
+local function set_output(storage, value)
+    local status, err, fmt, opts = output_parse(value)
+    if status ~= true then
+        return error(err)
+    end
+    output_save(fmt, opts)
+    return true
+end
+
 local function set_param(storage, func, param, value)
     local params = {
         language = set_language,
         lang = set_language,
         l = set_language,
+        output = set_output,
+        o = set_output,
         delimiter = set_delimiter,
         delim = set_delimiter,
         d = set_delimiter

src/lua/help_en_US.lua

@@ -8,6 +8,7 @@ To start the interactive Tarantool tutorial, type 'tutorial()' here.
 Available backslash commands:
 
   \set language <language>   -- set language (lua or sql)
+  \set output <format>       -- set output format (lua[,line|block] or yaml)
   \set delimiter <delimiter> -- set expression delimiter
   \help                      -- show this screen
   \quit                      -- quit interactive console

test/box/admin.result

@@ -20,6 +20,7 @@ help()
     Available backslash commands:
 
       \set language <language>   -- set language (lua or sql)
+      \set output <format>       -- set output format (lua[,line|block] or yaml)
       \set delimiter <delimiter> -- set expression delimiter
       \help                      -- show this screen
       \quit                      -- quit interactive console