Support for std.parseYaml.

Author:    Greg Roodt <[email protected]>
Co-authored-by:    Stanisław Barzowski <[email protected]>

Author: groodt

.travis.yml

@@ -1,4 +1,5 @@
 language: generic
+dist: bionic
 
 matrix:
   include:
@@ -9,14 +10,16 @@ matrix:
           packages:
             - g++-5
           sources: &sources
-            - llvm-toolchain-precise-3.8
+            - llvm-toolchain-bionic-11
+            - sourceline: 'deb https://apt.llvm.org/bionic/ llvm-toolchain-bionic-11 main'
+              key_url: 'https://apt.llvm.org/llvm-snapshot.gpg.key'
             - ubuntu-toolchain-r-test    
     - os: linux
-      env: COMPILER_NAME=clang CXX=clang++-3.8 CC=clang-3.8 COMMON_OS_NAME=${TRAVIS_OS_NAME}
+      env: COMPILER_NAME=clang CXX=clang++-11 CC=clang-11 COMMON_OS_NAME=${TRAVIS_OS_NAME}
       addons:
         apt:
           packages:
-            - clang-3.8
+            - clang-11
           sources: *sources
     - os: osx
       osx_image: xcode11.3

CMakeLists.txt

@@ -121,6 +121,7 @@ endif()
 add_subdirectory(include)
 add_subdirectory(stdlib)
 add_subdirectory(third_party/md5)
+add_subdirectory(third_party/rapidyaml/rapidyaml ryml)
 add_subdirectory(core)
 add_subdirectory(cpp)
 add_subdirectory(cmd)

Makefile

@@ -32,10 +32,10 @@ OPT ?= -O3
 PREFIX ?= /usr/local
 
 CXXFLAGS ?= -g $(OPT) -Wall -Wextra -Woverloaded-virtual -pedantic -std=c++0x -fPIC
-CXXFLAGS += -Iinclude -Ithird_party/md5 -Ithird_party/json
+CXXFLAGS += -Iinclude -Ithird_party/md5 -Ithird_party/json -Ithird_party/rapidyaml/rapidyaml/src/ -Ithird_party/rapidyaml/rapidyaml/ext/c4core/src/
 CFLAGS ?= -g $(OPT) -Wall -Wextra -pedantic -std=c99 -fPIC
 CFLAGS += -Iinclude
-MAKEDEPENDFLAGS += -Iinclude -Ithird_party/md5 -Ithird_party/json
+MAKEDEPENDFLAGS += -Iinclude -Ithird_party/md5 -Ithird_party/json -Ithird_party/rapidyaml/rapidyaml/src/ -Ithird_party/rapidyaml/rapidyaml/ext/c4core/src/
 EMCXXFLAGS = $(CXXFLAGS) --memory-init-file 0 -s DISABLE_EXCEPTION_CATCHING=0 -s INLINING_LIMIT=50 -s RESERVED_FUNCTION_POINTERS=20 -s ASSERTIONS=1 -s ALLOW_MEMORY_GROWTH=1
 EMCFLAGS = $(CFLAGS) --memory-init-file 0 -s DISABLE_EXCEPTION_CATCHING=0 -s ASSERTIONS=1 -s ALLOW_MEMORY_GROWTH=1
 LDFLAGS ?=
@@ -50,6 +50,10 @@ SOVERSION = 0
 # End of user-servicable parts
 ################################################################################
 
+RAPIDYAML_SRC = \
+	$(wildcard third_party/rapidyaml/rapidyaml/src/c4/yml/*.cpp) \
+	$(wildcard third_party/rapidyaml/rapidyaml/ext/c4core/src/c4/*.cpp)
+
 LIB_SRC = \
 	core/desugarer.cpp \
 	core/formatter.cpp \
@@ -60,7 +64,8 @@ LIB_SRC = \
 	core/static_analysis.cpp \
 	core/string_utils.cpp \
 	core/vm.cpp \
-	third_party/md5/md5.cpp
+	third_party/md5/md5.cpp \
+	$(RAPIDYAML_SRC)
 
 LIB_OBJ = $(LIB_SRC:.cpp=.o)
 
@@ -110,6 +115,7 @@ ALL_HEADERS = \
 	core/std.jsonnet.h \
 	third_party/md5/md5.h \
 	third_party/json/json.hpp \
+	third_party/rapidyaml/rapidyaml/src/ryml_std.hpp \
 	$(INCS)
 
 

core/BUILD

@@ -36,6 +36,7 @@ cc_library(
         "//stdlib:std",
         "//third_party/json",
         "//third_party/md5:libmd5",
+        "//third_party/rapidyaml:ryml",
     ],
 )
 

core/CMakeLists.txt

@@ -30,7 +30,7 @@ set(LIBJSONNET_SOURCE
 
 add_library(libjsonnet SHARED ${LIBJSONNET_HEADERS} ${LIBJSONNET_SOURCE})
 add_dependencies(libjsonnet md5 stdlib)
-target_link_libraries(libjsonnet md5 nlohmann_json::nlohmann_json)
+target_link_libraries(libjsonnet md5 nlohmann_json::nlohmann_json ryml)
 
 file(STRINGS ${CMAKE_CURRENT_SOURCE_DIR}/../include/libjsonnet.h JSONNET_VERSION_DEF
      REGEX "[#]define[ \t]+LIB_JSONNET_VERSION[ \t]+")
@@ -55,7 +55,7 @@ if (BUILD_STATIC_LIBS)
     # Static library for jsonnet command-line tool.
     add_library(libjsonnet_static STATIC ${LIBJSONNET_SOURCE})
     add_dependencies(libjsonnet_static md5 stdlib)
-    target_link_libraries(libjsonnet_static md5 nlohmann_json::nlohmann_json)
+    target_link_libraries(libjsonnet_static md5 nlohmann_json::nlohmann_json ryml)
     set_target_properties(libjsonnet_static PROPERTIES OUTPUT_NAME jsonnet)
     install(TARGETS libjsonnet_static DESTINATION "${CMAKE_INSTALL_LIBDIR}")
     target_include_directories(libjsonnet_static INTERFACE

core/desugarer.cpp

@@ -34,7 +34,7 @@ struct BuiltinDecl {
     std::vector<UString> params;
 };
 
-static unsigned long max_builtin = 37;
+static unsigned long max_builtin = 38;
 BuiltinDecl jsonnet_builtin_decl(unsigned long builtin)
 {
     switch (builtin) {
@@ -74,8 +74,9 @@ BuiltinDecl jsonnet_builtin_decl(unsigned long builtin)
         case 33: return {U"asciiUpper", {U"str"}};
         case 34: return {U"join", {U"sep", U"arr"}};
         case 35: return {U"parseJson", {U"str"}};
-        case 36: return {U"encodeUTF8", {U"str"}};
-        case 37: return {U"decodeUTF8", {U"arr"}};
+        case 36: return {U"parseYaml", {U"str"}};
+        case 37: return {U"encodeUTF8", {U"str"}};
+        case 38: return {U"decodeUTF8", {U"arr"}};
         default:
             std::cerr << "INTERNAL ERROR: Unrecognized builtin function: " << builtin << std::endl;
             std::abort();

core/vm.cpp

@@ -26,6 +26,8 @@ limitations under the License.
 #include "json.hpp"
 #include "md5.h"
 #include "parser.h"
+#include "ryml_std.hpp" // include this before any other ryml header
+#include "ryml.hpp"
 #include "state.h"
 #include "static_analysis.h"
 #include "string_utils.h"
@@ -930,6 +932,7 @@ class Interpreter {
         builtins["asciiUpper"] = &Interpreter::builtinAsciiUpper;
         builtins["join"] = &Interpreter::builtinJoin;
         builtins["parseJson"] = &Interpreter::builtinParseJson;
+        builtins["parseYaml"] = &Interpreter::builtinParseYaml;
         builtins["encodeUTF8"] = &Interpreter::builtinEncodeUTF8;
         builtins["decodeUTF8"] = &Interpreter::builtinDecodeUTF8;
 
@@ -1591,6 +1594,65 @@ class Interpreter {
         return nullptr;
     }
 
+    const AST *builtinParseYaml(const LocationRange &loc, const std::vector<Value> &args)
+    {
+        validateBuiltinArgs(loc, "parseYaml", args, {Value::STRING});
+
+        std::string value = encode_utf8(static_cast<HeapString *>(args[0].v.h)->value);
+
+        ryml::Tree tree = treeFromString(value);
+
+        json j;
+        if (tree.is_stream(tree.root_id())) {
+            // Split into individual yaml documents
+            std::stringstream ss;
+            ss << tree;
+            std::vector<std::string> v = split(ss.str(), "---\n");
+
+            // Convert yaml to json and push onto json array
+            ryml::Tree doc;
+            for (int i = 0; i < v.size(); ++i) {
+                if (!v[i].empty()) {
+                    doc = treeFromString(v[i]);
+                    j.push_back(yamlTreeToJson(doc));
+                }
+            }
+        } else {
+            j = yamlTreeToJson(tree);
+        }
+
+        bool filled;
+
+        otherJsonToHeap(j, filled, scratch);
+
+        return nullptr;
+    }
+
+    const ryml::Tree treeFromString(const std::string& s) {
+        return ryml::parse(c4::to_csubstr(s));
+    }
+
+    const std::vector<std::string> split(const std::string& s, const std::string& delimiter) {
+        size_t pos_start = 0, pos_end, delim_len = delimiter.length();
+        std::string token;
+        std::vector<std::string> res;
+
+        while ((pos_end = s.find(delimiter, pos_start)) != std::string::npos) {
+            token = s.substr(pos_start, pos_end - pos_start);
+            pos_start = pos_end + delim_len;
+            res.push_back(token);
+        }
+
+        res.push_back(s.substr(pos_start));
+        return res;
+    }
+
+    const json yamlTreeToJson(const ryml::Tree& tree) {
+        std::ostringstream jsonStream;
+        jsonStream << ryml::as_json(tree);
+        return json::parse(jsonStream.str());
+    }
+
     void otherJsonToHeap(const json &v, bool &filled, Value &attach) {
         // In order to not anger the garbage collector, assign to attach immediately after
         // making the heap object.

doc/_stdlib_gen/stdlib-content.jsonnet

@@ -554,6 +554,24 @@ local html = import 'html.libsonnet';
             },
           ],
         },
+        {
+          name: 'parseYaml',
+          availableSince: 'x.y.z',
+          params: ['str'],
+          description: |||
+            Parses a YAML string. This is provided as a "best-effort" mechanism and should not be relied on to provide
+            a fully standards compliant YAML parser. YAML is a superset of JSON, consequently "downcasting" or
+            manifestation of YAML into JSON or Jsonnet values will only succeed when using the subset of YAML that is
+            compatible with JSON. The parser does not support YAML documents with scalar values at the root. The
+            root node of a YAML document must start with either a YAML sequence or map to be successfully parsed.
+          |||,
+          examples: [
+            {
+              input: 'std.parseYaml(\'foo: bar\')',
+              output: std.parseYaml('foo: bar'),
+            },
+          ],
+        },
         {
           name: 'encodeUTF8',
           params: ['str'],

setup.py

@@ -29,7 +29,19 @@
     'core/static_analysis.o',
     'core/string_utils.o',
     'core/vm.o',
-    'third_party/md5/md5.o'
+    'third_party/md5/md5.o',
+    'third_party/rapidyaml/rapidyaml/ext/c4core/src/c4/char_traits.o',
+    'third_party/rapidyaml/rapidyaml/ext/c4core/src/c4/base64.o',
+    'third_party/rapidyaml/rapidyaml/ext/c4core/src/c4/language.o',
+    'third_party/rapidyaml/rapidyaml/ext/c4core/src/c4/memory_util.o',
+    'third_party/rapidyaml/rapidyaml/ext/c4core/src/c4/format.o',
+    'third_party/rapidyaml/rapidyaml/ext/c4core/src/c4/time.o',
+    'third_party/rapidyaml/rapidyaml/ext/c4core/src/c4/memory_resource.o',
+    'third_party/rapidyaml/rapidyaml/ext/c4core/src/c4/error.o',
+    'third_party/rapidyaml/rapidyaml/src/c4/yml/parse.o',
+    'third_party/rapidyaml/rapidyaml/src/c4/yml/preprocess.o',
+    'third_party/rapidyaml/rapidyaml/src/c4/yml/common.o',
+    'third_party/rapidyaml/rapidyaml/src/c4/yml/tree.o',
 ]
 
 MODULE_SOURCES = ['python/_jsonnet.c']

test_suite/stdlib.jsonnet

@@ -1035,6 +1035,47 @@ std.assertEqual(std.parseJson('12'), 12) &&
 std.assertEqual(std.parseJson('12.123'), 12.123) &&
 std.assertEqual(std.parseJson('{"a": {"b": ["c", 42]}}'), { a: { b: ['c', 42] } }) &&
 
+std.assertEqual(std.parseYaml('{}'), {}) &&
+std.assertEqual(std.parseYaml('[]'), []) &&
+std.assertEqual(
+  std.parseYaml(
+    |||
+      foo:
+        bar:
+        - true
+        - 42
+        - 1.0
+    |||
+  ), { foo: { bar: [true, 42, 1] } }
+) &&
+std.assertEqual(
+  std.parseYaml(
+    |||
+      ---
+      foo:
+        bar:
+        - true
+        - 42
+        - 1.0
+      ---
+      wibble:
+        wobble:
+        - true
+        - 42
+        - 1.0
+    |||
+  ), [{ foo: { bar: [true, 42, 1] } }, { wibble: { wobble: [true, 42, 1] } }]
+) &&
+std.assertEqual(
+  std.parseYaml(
+    |||
+      - 1
+      - 2
+      - 3
+    |||
+  ), [1, 2, 3]
+) &&
+
 std.assertEqual(std.asciiUpper('!@#$%&*()asdfghFGHJKL09876 '), '!@#$%&*()ASDFGHFGHJKL09876 ') &&
 std.assertEqual(std.asciiLower('!@#$%&*()asdfghFGHJKL09876 '), '!@#$%&*()asdfghfghjkl09876 ') &&
 

test_suite/unicode_bmp.jsonnet

@@ -1,6 +1,6 @@
 {
-  "unicode_bmp1.json (import)": import 'unicode_bmp1.jsonnet.in',
-  "unicode_bmp1.json (importstr+parseJson)": std.parseJson(importstr 'unicode_bmp1.jsonnet.in'),
-  "unicode_bmp2.json (import)": import 'unicode_bmp2.jsonnet.in',
-  "unicode_bmp2.json (importstr+parseJson)": std.parseJson(importstr 'unicode_bmp2.jsonnet.in'),
+  'unicode_bmp1.json (import)': import 'unicode_bmp1.jsonnet.in',
+  'unicode_bmp1.json (importstr+parseJson)': std.parseJson(importstr 'unicode_bmp1.jsonnet.in'),
+  'unicode_bmp2.json (import)': import 'unicode_bmp2.jsonnet.in',
+  'unicode_bmp2.json (importstr+parseJson)': std.parseJson(importstr 'unicode_bmp2.jsonnet.in'),
 }

third_party/rapidyaml/BUILD

@@ -0,0 +1,43 @@
+licenses(["permissive"])
+package(default_visibility = ["//visibility:private"])
+
+cc_library(
+    name = "debugbreak",
+    hdrs = glob(["rapidyaml/ext/c4core/src/c4/ext/debugbreak/debugbreak.h"]),
+    srcs = [],
+)
+
+cc_library(
+    name = "fastfloat",
+    includes = ["rapidyaml/ext/c4core/src/c4/ext"],
+    hdrs = glob(["rapidyaml/ext/c4core/src/c4/ext/fast_float.hpp", "rapidyaml/ext/c4core/src/c4/ext/fast_float/**/*.h"]),
+    srcs = [],
+)
+
+cc_library(
+    name = "c4core",
+    deps = [
+      ":debugbreak",
+      ":fastfloat",
+    ],
+    includes = ["rapidyaml/ext/c4core/src"],
+    hdrs = glob(
+        ["rapidyaml/ext/c4core/src/**/*.hpp"],
+        exclude = ["rapidyaml/ext/c4core/src/c4/ext/**/*"],
+    ),
+    srcs = glob(
+        ["rapidyaml/ext/c4core/src/**/*.cpp"],
+        exclude = ["rapidyaml/ext/c4core/src/c4/ext/**/*"],
+    ),
+)
+
+cc_library(
+    name = "ryml",
+    visibility = ["//visibility:public"],
+    deps = [
+      ":c4core",
+    ],
+    includes = ["rapidyaml/src"],
+    hdrs = glob(["rapidyaml/src/**/*.hpp"]),
+    srcs = glob(["rapidyaml/src/**/*.cpp"]),
+)

third_party/rapidyaml/README.md

@@ -0,0 +1,12 @@
+# Vendoring
+
+```
+git clone \
+    --depth=1 \
+    --branch=master \
+    --recursive \
+    [email protected]:biojppm/rapidyaml \
+    third_party/rapidyaml/rapidyaml \
+&& \
+rm -rf third_party/rapidyaml/rapidyaml/.git
+```