Adds std.parseYaml (#339)

Add support for std.parseYaml.

Co-authored-by: Stanisław Barzowski <[email protected]>

Author: groodt

.travis.yml

@@ -3,8 +3,6 @@ sudo: false
 matrix:
   include:
     - go: 1.x
-    - go: 1.11.x
-    - go: 1.12.x
     - go: 1.13.x
     - go: 1.x
       arch: amd64

BUILD.bazel

@@ -22,6 +22,7 @@ go_library(
         "util.go",
         "value.go",
         "vm.go",
+        "yaml.go",
     ],
     importpath = "github.com/google/go-jsonnet",
     visibility = ["//visibility:public"],
@@ -31,6 +32,7 @@ go_library(
         "//internal/errors:go_default_library",
         "//internal/parser:go_default_library",
         "//internal/program:go_default_library",
+        "@io_k8s_sigs_yaml//:go_default_library",
     ],
 )
 

bazel/deps.bzl

@@ -80,7 +80,6 @@ def jsonnet_go_dependencies():
         sum = "h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk=",
         version = "v1.4.0",
     )
-
     go_repository(
         name = "in_gopkg_check_v1",
         importpath = "gopkg.in/check.v1",
@@ -90,8 +89,14 @@ def jsonnet_go_dependencies():
     go_repository(
         name = "in_gopkg_yaml_v2",
         importpath = "gopkg.in/yaml.v2",
-        sum = "h1:/eiJrUcujPVeJ3xlSWaiNi3uSVmDGBK1pDHUHAnao1I=",
-        version = "v2.2.4",
+        sum = "h1:VUgggvou5XRW9mHwD/yXxIYSMtY0zoKQf/v226p2nyo=",
+        version = "v2.2.7",
+    )
+    go_repository(
+        name = "io_k8s_sigs_yaml",
+        importpath = "sigs.k8s.io/yaml",
+        sum = "h1:4A07+ZFc2wgJwo8YNlQpr1rVlgUDlxXHhPJciaPY5gs=",
+        version = "v1.1.0",
     )
     go_repository(
         name = "org_golang_x_sys",

builtins.go

@@ -23,6 +23,7 @@ import (
 	"encoding/hex"
 	"encoding/json"
 	"fmt"
+	"io"
 	"math"
 	"reflect"
 	"sort"
@@ -1194,6 +1195,34 @@ func builtinParseJSON(i *interpreter, str value) (value, error) {
 	return jsonToValue(i, parsedJSON)
 }
 
+func builtinParseYAML(i *interpreter, str value) (value, error) {
+	sval, err := i.getString(str)
+	if err != nil {
+		return nil, err
+	}
+	s := sval.getGoString()
+
+	isYamlStream := strings.Contains(s, "---")
+
+	elems := []interface{}{}
+	d := NewYAMLToJSONDecoder(strings.NewReader(s))
+	for {
+		var elem interface{}
+		if err := d.Decode(&elem); err != nil {
+			if err == io.EOF {
+				break
+			}
+			return nil, i.Error(fmt.Sprintf("failed to parse YAML: %v", err.Error()))
+		}
+		elems = append(elems, elem)
+	}
+
+	if isYamlStream {
+		return jsonToValue(i, elems)
+	}
+	return jsonToValue(i, elems[0])
+}
+
 func jsonEncode(v interface{}) (string, error) {
 	buf := new(bytes.Buffer)
 	enc := json.NewEncoder(buf)
@@ -1620,6 +1649,7 @@ var funcBuiltins = buildBuiltinMap([]builtin{
 	&unaryBuiltin{name: "base64Decode", function: builtinBase64Decode, params: ast.Identifiers{"str"}},
 	&unaryBuiltin{name: "base64DecodeBytes", function: builtinBase64DecodeBytes, params: ast.Identifiers{"str"}},
 	&unaryBuiltin{name: "parseJson", function: builtinParseJSON, params: ast.Identifiers{"str"}},
+	&unaryBuiltin{name: "parseYaml", function: builtinParseYAML, params: ast.Identifiers{"str"}},
 	&generalBuiltin{name: "manifestJsonEx", function: builtinManifestJSONEx, params: []generalBuiltinParameter{{name: "value"}, {name: "indent"},
 		{name: "newline", defaultValue: &valueFlatString{value: []rune("\n")}},
 		{name: "key_val_sep", defaultValue: &valueFlatString{value: []rune(": ")}}}},

cpp-jsonnet

@@ -5,4 +5,6 @@ go 1.13
 require (
 	github.com/fatih/color v1.10.0
 	github.com/sergi/go-diff v1.1.0
+	gopkg.in/yaml.v2 v2.2.7 // indirect
+	sigs.k8s.io/yaml v1.1.0
 )

go.mod

@@ -28,3 +28,7 @@ gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.4 h1:/eiJrUcujPVeJ3xlSWaiNi3uSVmDGBK1pDHUHAnao1I=
 gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.7 h1:VUgggvou5XRW9mHwD/yXxIYSMtY0zoKQf/v226p2nyo=
+gopkg.in/yaml.v2 v2.2.7/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+sigs.k8s.io/yaml v1.1.0 h1:4A07+ZFc2wgJwo8YNlQpr1rVlgUDlxXHhPJciaPY5gs=
+sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o=

go.sum

@@ -100,6 +100,7 @@ func prepareStdlib(g *typeGraph) {
 		"parseOctal": g.newSimpleFuncType(numberType, "str"),
 		"parseHex":   g.newSimpleFuncType(numberType, "str"),
 		"parseJson":  g.newSimpleFuncType(jsonType, "str"),
+		"parseYaml":  g.newSimpleFuncType(jsonType, "str"),
 		"encodeUTF8": g.newSimpleFuncType(arrayOfNumber, "str"),
 		"decodeUTF8": g.newSimpleFuncType(stringType, "arr"),
 

linter/internal/types/stdlib.go

@@ -0,0 +1,10 @@
+{
+   "aaa": { },
+   "foo": "bar",
+   "xxx": [
+      42,
+      "asdf",
+      { }
+   ],
+   "ąę": "ćż"
+}

testdata/parseYaml.golden

@@ -0,0 +1,11 @@
+std.parseYaml(
+  |||
+   foo: bar
+   aaa: {}
+   ąę: ćż
+   xxx:
+   - 42
+   - asdf
+   - {}
+  |||
+)

testdata/parseYaml.jsonnet

@@ -0,0 +1,139 @@
+/*
+Copyright 2019 Google Inc. All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+	http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package jsonnet
+
+import (
+	"bufio"
+	"bytes"
+	"io"
+	"strings"
+	"unicode"
+
+	"sigs.k8s.io/yaml"
+)
+
+const separator = "---"
+
+// YAMLToJSONDecoder decodes YAML documents from an io.Reader by
+// separating individual documents. It first converts the YAML
+// body to JSON, then unmarshals the JSON.
+type YAMLToJSONDecoder struct {
+	reader Reader
+}
+
+// NewYAMLToJSONDecoder decodes YAML documents from the provided
+// stream in chunks by converting each document (as defined by
+// the YAML spec) into its own chunk, converting it to JSON via
+// yaml.YAMLToJSON, and then passing it to json.Decoder.
+func NewYAMLToJSONDecoder(r io.Reader) *YAMLToJSONDecoder {
+	reader := bufio.NewReader(r)
+	return &YAMLToJSONDecoder{
+		reader: NewYAMLReader(reader),
+	}
+}
+
+// Decode reads a YAML document as JSON from the stream or returns
+// an error. The decoding rules match json.Unmarshal, not
+// yaml.Unmarshal.
+func (d *YAMLToJSONDecoder) Decode(into interface{}) error {
+	bytes, err := d.reader.Read()
+	if err != nil && err != io.EOF {
+		return err
+	}
+
+	if len(bytes) != 0 {
+		err := yaml.Unmarshal(bytes, into)
+		if err != nil {
+			return err
+		}
+	}
+	return err
+}
+
+// Reader reads bytes
+type Reader interface {
+	Read() ([]byte, error)
+}
+
+// YAMLReader reads YAML
+type YAMLReader struct {
+	reader Reader
+}
+
+// NewYAMLReader creates a new YAMLReader
+func NewYAMLReader(r *bufio.Reader) *YAMLReader {
+	return &YAMLReader{
+		reader: &LineReader{reader: r},
+	}
+}
+
+// Read returns a full YAML document.
+func (r *YAMLReader) Read() ([]byte, error) {
+	var buffer bytes.Buffer
+	for {
+		line, err := r.reader.Read()
+		if err != nil && err != io.EOF {
+			return nil, err
+		}
+
+		sep := len([]byte(separator))
+		if i := bytes.Index(line, []byte(separator)); i == 0 {
+			// We have a potential document terminator
+			i += sep
+			after := line[i:]
+			if len(strings.TrimRightFunc(string(after), unicode.IsSpace)) == 0 {
+				if buffer.Len() != 0 {
+					return buffer.Bytes(), nil
+				}
+				if err == io.EOF {
+					return nil, err
+				}
+			}
+		}
+		if err == io.EOF {
+			if buffer.Len() != 0 {
+				// If we're at EOF, we have a final, non-terminated line. Return it.
+				return buffer.Bytes(), nil
+			}
+			return nil, err
+		}
+		buffer.Write(line)
+	}
+}
+
+// LineReader reads single lines.
+type LineReader struct {
+	reader *bufio.Reader
+}
+
+// Read returns a single line (with '\n' ended) from the underlying reader.
+// An error is returned iff there is an error with the underlying reader.
+func (r *LineReader) Read() ([]byte, error) {
+	var (
+		isPrefix bool  = true
+		err      error = nil
+		line     []byte
+		buffer   bytes.Buffer
+	)
+
+	for isPrefix && err == nil {
+		line, isPrefix, err = r.reader.ReadLine()
+		buffer.Write(line)
+	}
+	buffer.WriteByte('\n')
+	return buffer.Bytes(), err
+}