From d49537a304741b54af7c0d348810bda6acbc006c Mon Sep 17 00:00:00 2001 From: Brian Hendriks Date: Wed, 9 Jun 2021 16:28:53 -0700 Subject: [PATCH 1/6] Bh/quoted col fix (#1) --- go.mod | 5 ++ go.sum | 11 +++++ jsonpath.go | 23 +++++++++ jsonpath_test.go | 122 ++++++++++++++++------------------------------- 4 files changed, 80 insertions(+), 81 deletions(-) create mode 100644 go.mod create mode 100644 go.sum diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..f31d50c --- /dev/null +++ b/go.mod @@ -0,0 +1,5 @@ +module github.com/oliveagle/jsonpath + +go 1.15 + +require github.com/stretchr/testify v1.7.0 diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..acb88a4 --- /dev/null +++ b/go.sum @@ -0,0 +1,11 @@ +github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/jsonpath.go b/jsonpath.go index 00dc6fd..7fd542d 100644 --- a/jsonpath.go +++ b/jsonpath.go @@ -144,9 +144,27 @@ func tokenize(query string) ([]string, error) { // token_start := false // token_end := false token := "" + quoteChar := rune(0) // fmt.Println("-------------------------------------------------- start") for idx, x := range query { + if quoteChar != 0 { + if x == quoteChar { + quoteChar = 0 + } else { + token += string(x) + } + + continue + } else if x == '"' { + if token == "." { + token = "" + } + + quoteChar = x + continue + } + token += string(x) // //fmt.Printf("idx: %d, x: %s, token: %s, tokens: %v\n", idx, string(x), token, tokens) if idx == 0 { @@ -193,6 +211,11 @@ func tokenize(query string) ([]string, error) { } } } + + if quoteChar != 0 { + token = string(quoteChar) + token + } + if len(token) > 0 { if token[0] == '.' { token = token[1:] diff --git a/jsonpath_test.go b/jsonpath_test.go index 90f05b7..0540a20 100644 --- a/jsonpath_test.go +++ b/jsonpath_test.go @@ -8,6 +8,8 @@ import ( "reflect" "regexp" "testing" + + "github.com/stretchr/testify/assert" ) var json_data interface{} @@ -68,6 +70,12 @@ func Test_jsonpath_JsonPathLookup_1(t *testing.T) { t.Errorf("$.store.book[0].price should be 8.95") } + // quoted - single index + res, _ = JsonPathLookup(json_data, `$."store"."book"[0]."price"`) + if res_v, ok := res.(float64); ok != true || res_v != 8.95 { + t.Errorf(`$."store"."book"[0]."price" should be 8.95`) + } + // nagtive single index res, _ = JsonPathLookup(json_data, "$.store.book[-1].isbn") if res_v, ok := res.(string); ok != true || res_v != "0-395-19395-8" { @@ -153,90 +161,42 @@ func Test_jsonpath_authors_of_all_books(t *testing.T) { t.Log(res, expected) } -var token_cases = []map[string]interface{}{ - map[string]interface{}{ - "query": "$..author", - "tokens": []string{"$", "*", "author"}, - }, - map[string]interface{}{ - "query": "$.store.*", - "tokens": []string{"$", "store", "*"}, - }, - map[string]interface{}{ - "query": "$.store..price", - "tokens": []string{"$", "store", "*", "price"}, - }, - map[string]interface{}{ - "query": "$.store.book[*].author", - "tokens": []string{"$", "store", "book[*]", "author"}, - }, - map[string]interface{}{ - "query": "$..book[2]", - "tokens": []string{"$", "*", "book[2]"}, - }, - map[string]interface{}{ - "query": "$..book[(@.length-1)]", - "tokens": []string{"$", "*", "book[(@.length-1)]"}, - }, - map[string]interface{}{ - "query": "$..book[0,1]", - "tokens": []string{"$", "*", "book[0,1]"}, - }, - map[string]interface{}{ - "query": "$..book[:2]", - "tokens": []string{"$", "*", "book[:2]"}, - }, - map[string]interface{}{ - "query": "$..book[?(@.isbn)]", - "tokens": []string{"$", "*", "book[?(@.isbn)]"}, - }, - map[string]interface{}{ - "query": "$.store.book[?(@.price < 10)]", - "tokens": []string{"$", "store", "book[?(@.price < 10)]"}, - }, - map[string]interface{}{ - "query": "$..book[?(@.price <= $.expensive)]", - "tokens": []string{"$", "*", "book[?(@.price <= $.expensive)]"}, - }, - map[string]interface{}{ - "query": "$..book[?(@.author =~ /.*REES/i)]", - "tokens": []string{"$", "*", "book[?(@.author =~ /.*REES/i)]"}, - }, - map[string]interface{}{ - "query": "$..book[?(@.author =~ /.*REES\\]/i)]", - "tokens": []string{"$", "*", "book[?(@.author =~ /.*REES\\]/i)]"}, - }, - map[string]interface{}{ - "query": "$..*", - "tokens": []string{"$", "*"}, - }, - map[string]interface{}{ - "query": "$....author", - "tokens": []string{"$", "*", "author"}, - }, +var token_cases = []struct { + query string + expected []string +}{ + {"$..author", []string{"$", "*", "author"}}, + {"$.store.*", []string{"$", "store", "*"}}, + {"$.store..price", []string{"$", "store", "*", "price"}}, + {"$.store.book[*].author", []string{"$", "store", "book[*]", "author"}}, + {"$..book[2]", []string{"$", "*", "book[2]"}}, + {"$..book[(@.length-1)]", []string{"$", "*", "book[(@.length-1)]"}}, + {"$..book[0,1]", []string{"$", "*", "book[0,1]"}}, + {"$..book[:2]", []string{"$", "*", "book[:2]"}}, + {"$..book[?(@.isbn)]", []string{"$", "*", "book[?(@.isbn)]"}}, + {"$.store.book[?(@.price < 10)]", []string{"$", "store", "book[?(@.price < 10)]"}}, + {"$..book[?(@.price <= $.expensive)]", []string{"$", "*", "book[?(@.price <= $.expensive)]"}}, + {"$..book[?(@.author =~ /.*REES/i)]", []string{"$", "*", "book[?(@.author =~ /.*REES/i)]"}}, + {"$..book[?(@.author =~ /.*REES\\]/i)]", []string{"$", "*", "book[?(@.author =~ /.*REES\\]/i)]"}}, + {"$..*", []string{"$", "*"}}, + {"$....author", []string{"$", "*", "author"}}, + {`$."col"`, []string{"$", "col"}}, + {`$."col.with.dots"."sub.with.dots"`, []string{"$", "col.with.dots", "sub.with.dots"}}, + {`$."unterminated`, []string{"$", `"unterminated`}}, + {`$."col with spaces"."sub with spaces"`, []string{"$", "col with spaces", "sub with spaces"}}, } func Test_jsonpath_tokenize(t *testing.T) { - for idx, tcase := range token_cases { - t.Logf("idx[%d], tcase: %v", idx, tcase) - query := tcase["query"].(string) - expected_tokens := tcase["tokens"].([]string) - tokens, err := tokenize(query) - t.Log(err, tokens, expected_tokens) - if len(tokens) != len(expected_tokens) { - t.Errorf("different length: (got)%v, (expected)%v", len(tokens), len(expected_tokens)) - continue - } - for i := 0; i < len(expected_tokens); i++ { - if tokens[i] != expected_tokens[i] { - t.Errorf("not expected: [%d], (got)%v != (expected)%v", i, tokens[i], expected_tokens[i]) - } - } + for _, tcase := range token_cases { + t.Run(tcase.query, func(t *testing.T) { + tokens, err := tokenize(tcase.query) + assert.NoError(t, err) + assert.Equal(t, tcase.expected, tokens) + }) } } var parse_token_cases = []map[string]interface{}{ - map[string]interface{}{ "token": "$", "op": "root", @@ -1179,13 +1139,13 @@ func Test_jsonpath_rootnode_is_array_range(t *testing.T) { t.Logf("idx: %v, v: %v", idx, v) } if len(ares) != 2 { - t.Fatal("len is not 2. got: %v", len(ares)) + t.Fatalf("len is not 2. got: %v", len(ares)) } if ares[0].(float64) != 12.34 { - t.Fatal("idx: 0, should be 12.34. got: %v", ares[0]) + t.Fatalf("idx: 0, should be 12.34. got: %v", ares[0]) } if ares[1].(float64) != 13.34 { - t.Fatal("idx: 0, should be 12.34. got: %v", ares[1]) + t.Fatalf("idx: 0, should be 12.34. got: %v", ares[1]) } } @@ -1232,7 +1192,7 @@ func Test_jsonpath_rootnode_is_nested_array_range(t *testing.T) { t.Logf("idx: %v, v: %v", idx, v) } if len(ares) != 2 { - t.Fatal("len is not 2. got: %v", len(ares)) + t.Fatalf("len is not 2. got: %v", len(ares)) } //FIXME: `$[:1].[0].test` got wrong result From 21915318c934d5628bc41a1486bd21751b42acaf Mon Sep 17 00:00:00 2001 From: Aaron Son Date: Wed, 19 Apr 2023 14:17:21 -0700 Subject: [PATCH 2/6] go.mod: Rename go module to github.com/dolthub/jsonpath. --- LICENSE | 2 +- go.mod | 2 +- jsonpath.go | 6 ++++++ jsonpath_test.go | 6 ++++++ readme.md | 4 ++-- 5 files changed, 16 insertions(+), 4 deletions(-) diff --git a/LICENSE b/LICENSE index 530afca..50ad671 100644 --- a/LICENSE +++ b/LICENSE @@ -1,6 +1,6 @@ The MIT License (MIT) -Copyright (c) 2015 oliver +Copyright (c) 2021, 2015; DoltHub Authors, oliver Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/go.mod b/go.mod index f31d50c..99304ec 100644 --- a/go.mod +++ b/go.mod @@ -1,4 +1,4 @@ -module github.com/oliveagle/jsonpath +module github.com/dolthub/jsonpath go 1.15 diff --git a/jsonpath.go b/jsonpath.go index 7fd542d..6e69912 100644 --- a/jsonpath.go +++ b/jsonpath.go @@ -1,3 +1,9 @@ +// Copyright 2015, 2021; oliver, DoltHub Authors +// +// Use of this source code is governed by an MIT-style +// license that can be found in the LICENSE file or at +// https://opensource.org/licenses/MIT. + package jsonpath import ( diff --git a/jsonpath_test.go b/jsonpath_test.go index 0540a20..9e343e5 100644 --- a/jsonpath_test.go +++ b/jsonpath_test.go @@ -1,3 +1,9 @@ +// Copyright 2015, 2021; oliver, DoltHub Authors +// +// Use of this source code is governed by an MIT-style +// license that can be found in the LICENSE file or at +// https://opensource.org/licenses/MIT. + package jsonpath import ( diff --git a/readme.md b/readme.md index a8ee2db..bfdcbdc 100644 --- a/readme.md +++ b/readme.md @@ -9,7 +9,7 @@ but also with some minor differences. this library is till bleeding edge, so use it at your own risk. :D -**Golang Version Required**: 1.5+ +**Golang Version Required**: 1.15+ Get Started ------------ @@ -111,4 +111,4 @@ example json path syntax. | $.store.book[:].price | [8.9.5, 12.99, 8.9.9, 22.99] | | $.store.book[?(@.author =~ /(?i).*REES/)].author | "Nigel Rees" | -> Note: golang support regular expression flags in form of `(?imsU)pattern` \ No newline at end of file +> Note: golang support regular expression flags in form of `(?imsU)pattern` From 77b8157e4af516102c3f118da95e6763948b6e6c Mon Sep 17 00:00:00 2001 From: James Cor Date: Tue, 23 May 2023 16:11:45 -0700 Subject: [PATCH 3/6] add support for scan op (#3) additionally, converts `fmt.Print` to `t.Log` and improves an error message --- jsonpath.go | 46 +++++++++++++++-- jsonpath_test.go | 128 +++++++++++++++++++++++++++++++++++++++-------- 2 files changed, 150 insertions(+), 24 deletions(-) diff --git a/jsonpath.go b/jsonpath.go index 6e69912..bbcb181 100644 --- a/jsonpath.go +++ b/jsonpath.go @@ -76,7 +76,7 @@ func (c *Compiled) String() string { func (c *Compiled) Lookup(obj interface{}) (interface{}, error) { var err error for _, s := range c.steps { - // "key", "idx" + // "key", "idx", "range", "filter", "scan" switch s.op { case "key": obj, err = get_key(obj, s.key) @@ -138,8 +138,13 @@ func (c *Compiled) Lookup(obj interface{}) (interface{}, error) { if err != nil { return nil, err } + case "scan": + obj, err = get_scan(obj) + if err != nil { + return nil, err + } default: - return nil, fmt.Errorf("expression don't support in filter") + return nil, fmt.Errorf("unsupported jsonpath operation: %s", s.op) } } return obj, nil @@ -354,7 +359,7 @@ func filter_get_from_explicit_path(obj interface{}, path string) (interface{}, e return nil, err } default: - return nil, fmt.Errorf("expression don't support in filter") + return nil, fmt.Errorf("unsupported jsonpath operation %s in filter", op) } } return xobj, nil @@ -550,6 +555,41 @@ func get_filtered(obj, root interface{}, filter string) ([]interface{}, error) { return res, nil } +func get_scan(obj interface{}) (interface{}, error) { + if reflect.TypeOf(obj) == nil { + return nil, ErrGetFromNullObj + } + switch reflect.TypeOf(obj).Kind() { + case reflect.Map: + var res []interface{} + if jsonMap, ok := obj.(map[string]interface{}); ok { + for _, v := range jsonMap { + res = append(res, v) + } + return res, nil + } + iter := reflect.ValueOf(obj).MapRange() + for iter.Next() { + res = append(res, iter.Value().Interface()) + } + return res, nil + case reflect.Slice: + // slice we should get from all objects in it. + var res []interface{} + for i := 0; i < reflect.ValueOf(obj).Len(); i++ { + tmp := reflect.ValueOf(obj).Index(i).Interface() + newObj, err := get_scan(tmp) + if err != nil { + return nil, err + } + res = append(res, newObj.([]interface{})...) + } + return res, nil + default: + return nil, fmt.Errorf("object is not scanable: %v", reflect.TypeOf(obj).Kind()) + } +} + // @.isbn => @.isbn, exists, nil // @.price < 10 => @.price, <, 10 // @.price <= $.expensive => @.price, <=, $.expensive diff --git a/jsonpath_test.go b/jsonpath_test.go index 9e343e5..90865d6 100644 --- a/jsonpath_test.go +++ b/jsonpath_test.go @@ -337,7 +337,7 @@ func Test_jsonpath_parse_token(t *testing.T) { if op == "range" { if args_v, ok := args.([2]interface{}); ok == true { - fmt.Println(args_v) + t.Logf("%v", args_v) exp_from := exp_args.([2]interface{})[0] exp_to := exp_args.([2]interface{})[1] if args_v[0] != exp_from { @@ -356,7 +356,7 @@ func Test_jsonpath_parse_token(t *testing.T) { if op == "filter" { if args_v, ok := args.(string); ok == true { - fmt.Println(args_v) + t.Logf(args_v) if exp_args.(string) != args_v { t.Errorf("len(args) not expected: (got)%v != (exp)%v", len(args_v), len(exp_args.([]string))) return @@ -374,7 +374,7 @@ func Test_jsonpath_get_key(t *testing.T) { "key": 1, } res, err := get_key(obj, "key") - fmt.Println(err, res) + t.Logf("err: %v, res: %v", err, res) if err != nil { t.Errorf("failed to get key: %v", err) return @@ -385,7 +385,7 @@ func Test_jsonpath_get_key(t *testing.T) { } res, err = get_key(obj, "hah") - fmt.Println(err, res) + t.Logf("err: %v, res: %v", err, res) if err == nil { t.Errorf("key error not raised") return @@ -397,7 +397,7 @@ func Test_jsonpath_get_key(t *testing.T) { obj2 := 1 res, err = get_key(obj2, "key") - fmt.Println(err, res) + t.Logf("err: %v, res: %v", err, res) if err == nil { t.Errorf("object is not map error not raised") @@ -406,7 +406,7 @@ func Test_jsonpath_get_key(t *testing.T) { obj3 := map[string]string{"key": "hah"} res, err = get_key(obj3, "key") if res_v, ok := res.(string); ok != true || res_v != "hah" { - fmt.Println(err, res) + t.Logf("err: %v, res: %v", err, res) t.Errorf("map[string]string support failed") } @@ -419,13 +419,13 @@ func Test_jsonpath_get_key(t *testing.T) { }, } res, err = get_key(obj4, "a") - fmt.Println(err, res) + t.Logf("err: %v, res: %v", err, res) } func Test_jsonpath_get_idx(t *testing.T) { obj := []interface{}{1, 2, 3, 4} res, err := get_idx(obj, 0) - fmt.Println(err, res) + t.Logf("err: %v, res: %v", err, res) if err != nil { t.Errorf("failed to get_idx(obj,0): %v", err) return @@ -435,19 +435,19 @@ func Test_jsonpath_get_idx(t *testing.T) { } res, err = get_idx(obj, 2) - fmt.Println(err, res) + t.Logf("err: %v, res: %v", err, res) if v, ok := res.(int); ok != true || v != 3 { t.Errorf("failed to get int 3") } res, err = get_idx(obj, 4) - fmt.Println(err, res) + t.Logf("err: %v, res: %v", err, res) if err == nil { t.Errorf("index out of range error not raised") return } res, err = get_idx(obj, -1) - fmt.Println(err, res) + t.Logf("err: %v, res: %v", err, res) if err != nil { t.Errorf("failed to get_idx(obj, -1): %v", err) return @@ -457,13 +457,13 @@ func Test_jsonpath_get_idx(t *testing.T) { } res, err = get_idx(obj, -4) - fmt.Println(err, res) + t.Logf("err: %v, res: %v", err, res) if v, ok := res.(int); ok != true || v != 1 { t.Errorf("failed to get int 1") } res, err = get_idx(obj, -5) - fmt.Println(err, res) + t.Logf("err: %v, res: %v", err, res) if err == nil { t.Errorf("index out of range error not raised") return @@ -478,7 +478,7 @@ func Test_jsonpath_get_idx(t *testing.T) { obj2 := []int{1, 2, 3, 4} res, err = get_idx(obj2, 0) - fmt.Println(err, res) + t.Logf("err: %v, res: %v", err, res) if err != nil { t.Errorf("failed to get_idx(obj2,0): %v", err) return @@ -492,7 +492,7 @@ func Test_jsonpath_get_range(t *testing.T) { obj := []int{1, 2, 3, 4, 5} res, err := get_range(obj, 0, 2) - fmt.Println(err, res) + t.Logf("err: %v, res: %v", err, res) if err != nil { t.Errorf("failed to get_range: %v", err) } @@ -502,11 +502,11 @@ func Test_jsonpath_get_range(t *testing.T) { obj1 := []interface{}{1, 2, 3, 4, 5} res, err = get_range(obj1, 3, -1) - fmt.Println(err, res) + t.Logf("err: %v, res: %v", err, res) if err != nil { t.Errorf("failed to get_range: %v", err) } - fmt.Println(res.([]interface{})) + t.Logf("%v", res.([]interface{})) if res.([]interface{})[0] != 4 || res.([]interface{})[1] != 5 { t.Errorf("failed get_range: %v, expect: [4,5]", res) } @@ -531,16 +531,102 @@ func Test_jsonpath_get_range(t *testing.T) { obj2 := 2 res, err = get_range(obj2, 0, 1) - fmt.Println(err, res) + t.Logf("err: %v, res: %v", err, res) if err == nil { t.Errorf("object is Slice error not raised") } } +func Test_jsonpath_get_scan(t *testing.T) { + obj := map[string]interface{}{ + "key": 1, + } + res, err := get_scan(obj) + t.Logf("err: %v, res: %v", err, res) + if err != nil { + t.Errorf("failed to scan: %v", err) + return + } + if res.([]interface{})[0] != 1 { + t.Errorf("scanned value is not 1: %v", res) + return + } + + obj2 := 1 + res, err = get_scan(obj2) + t.Logf("err: %v, res: %v", err, res) + if err == nil { + t.Errorf("object is not scanable error not raised") + return + } + + obj3 := map[string]string{"key1": "hah1", "key2": "hah2", "key3": "hah3"} + res, err = get_scan(obj3) + if err != nil { + t.Errorf("failed to scan: %v", err) + return + } + res_v, ok := res.([]interface{}) + if !ok { + t.Errorf("scanned result is not a slice") + } + if len(res_v) != 3 { + t.Errorf("scanned result is of wrong length") + } + // order of items in maps can't be guaranteed + for _, v := range res_v { + val, _ := v.(string) + if val != "hah1" && val != "hah2" && val != "hah3" { + t.Errorf("scanned result contains unexpected value: %v", val) + } + } + + obj4 := map[string]interface{}{ + "key1" : "abc", + "key2" : 123, + "key3" : map[string]interface{}{ + "a": 1, + "b": 2, + "c": 3, + }, + "key4" : []interface{}{1,2,3}, + } + res, err = get_scan(obj4) + res_v, ok = res.([]interface{}) + if !ok { + t.Errorf("scanned result is not a slice") + } + if len(res_v) != 4 { + t.Errorf("scanned result is of wrong length") + } + // order of items in maps can't be guaranteed + for _, v := range res_v { + switch v.(type) { + case string: + if v_str, ok := v.(string); ok && v_str == "abc" { + continue + } + case int: + if v_int, ok := v.(int); ok && v_int == 123 { + continue + } + case map[string]interface{}: + if v_map, ok := v.(map[string]interface{}); ok && v_map["a"].(int) == 1 && v_map["b"].(int) == 2 && v_map["c"].(int) == 3 { + continue + } + case []interface{}: + if v_slice, ok := v.([]interface{}); ok && v_slice[0].(int) == 1 && v_slice[1].(int) == 2 && v_slice[2].(int) == 3 { + continue + } + } + t.Errorf("scanned result contains unexpected value: %v", v) + } +} + func Test_jsonpath_types_eval(t *testing.T) { fset := token.NewFileSet() res, err := types.Eval(fset, nil, 0, "1 < 2") - fmt.Println(err, res, res.Type, res.Value, res.IsValue()) + t.Logf("err: %v, res: %v, res.Type: %v, res.Value: %v, res.IsValue: %v", err, res, res.Type, res.Value, res.IsValue()) } var tcase_parse_filter = []map[string]interface{}{ @@ -747,7 +833,7 @@ var tcase_eval_filter = []map[string]interface{}{ func Test_jsonpath_eval_filter(t *testing.T) { for idx, tcase := range tcase_eval_filter[1:] { - fmt.Println("------------------------------") + t.Logf("------------------------------") obj := tcase["obj"].(map[string]interface{}) root := tcase["root"].(map[string]interface{}) lp := tcase["lp"].(string) @@ -1074,7 +1160,7 @@ var tcases_reg_op = []struct { func TestRegOp(t *testing.T) { for idx, tcase := range tcases_reg_op { - fmt.Println("idx: ", idx, "tcase: ", tcase) + t.Logf("idx: %v, tcase: %v", idx, tcase) res, err := regFilterCompile(tcase.Line) if tcase.Err == true { if err == nil { From 8dc13778fd7281a30107d6adbc275b3848c9b66a Mon Sep 17 00:00:00 2001 From: James Cor Date: Thu, 25 May 2023 11:06:05 -0700 Subject: [PATCH 4/6] fix `scan` over `nil` and enforce ordering (#4) --- jsonpath.go | 40 ++++++++++++++++++++++++++++------ jsonpath_test.go | 56 +++++++++++++++++++++--------------------------- 2 files changed, 58 insertions(+), 38 deletions(-) diff --git a/jsonpath.go b/jsonpath.go index bbcb181..4f55bc8 100644 --- a/jsonpath.go +++ b/jsonpath.go @@ -13,6 +13,7 @@ import ( "go/types" "reflect" "regexp" + "sort" "strconv" "strings" ) @@ -143,6 +144,13 @@ func (c *Compiled) Lookup(obj interface{}) (interface{}, error) { if err != nil { return nil, err } + if obj == nil { + continue + } + // empty scan is NULL + if len(obj.([]interface{})) == 0 { + obj = nil + } default: return nil, fmt.Errorf("unsupported jsonpath operation: %s", s.op) } @@ -557,20 +565,38 @@ func get_filtered(obj, root interface{}, filter string) ([]interface{}, error) { func get_scan(obj interface{}) (interface{}, error) { if reflect.TypeOf(obj) == nil { - return nil, ErrGetFromNullObj + return nil, nil } switch reflect.TypeOf(obj).Kind() { case reflect.Map: + // iterate over keys in sorted by length, then alphabetically var res []interface{} if jsonMap, ok := obj.(map[string]interface{}); ok { - for _, v := range jsonMap { - res = append(res, v) + var sortedKeys []string + for k := range jsonMap { + sortedKeys = append(sortedKeys, k) + } + sort.Slice(sortedKeys, func(i, j int) bool { + if len(sortedKeys[i]) != len(sortedKeys[j]) { + return len(sortedKeys[i]) < len(sortedKeys[j]) + } + return sortedKeys[i] < sortedKeys[j] + }) + for _, k := range sortedKeys { + res = append(res, jsonMap[k]) } return res, nil } - iter := reflect.ValueOf(obj).MapRange() - for iter.Next() { - res = append(res, iter.Value().Interface()) + keys := reflect.ValueOf(obj).MapKeys() + sort.Slice(keys, func(i, j int) bool { + ki, kj := keys[i].String(), keys[j].String() + if len(ki) != len(kj) { + return len(ki) < len(kj) + } + return ki < kj + }) + for _, k := range keys { + res = append(res, reflect.ValueOf(obj).MapIndex(k).Interface()) } return res, nil case reflect.Slice: @@ -586,7 +612,7 @@ func get_scan(obj interface{}) (interface{}, error) { } return res, nil default: - return nil, fmt.Errorf("object is not scanable: %v", reflect.TypeOf(obj).Kind()) + return nil, fmt.Errorf("object is not scannable: %v", reflect.TypeOf(obj).Kind()) } } diff --git a/jsonpath_test.go b/jsonpath_test.go index 90865d6..b98dd53 100644 --- a/jsonpath_test.go +++ b/jsonpath_test.go @@ -542,7 +542,6 @@ func Test_jsonpath_get_scan(t *testing.T) { "key": 1, } res, err := get_scan(obj) - t.Logf("err: %v, res: %v", err, res) if err != nil { t.Errorf("failed to scan: %v", err) return @@ -554,9 +553,8 @@ func Test_jsonpath_get_scan(t *testing.T) { obj2 := 1 res, err = get_scan(obj2) - t.Logf("err: %v, res: %v", err, res) - if err == nil { - t.Errorf("object is not scanable error not raised") + if err == nil || err.Error() != "object is not scannable: int" { + t.Errorf("object is not scannable error not raised") return } @@ -573,12 +571,14 @@ func Test_jsonpath_get_scan(t *testing.T) { if len(res_v) != 3 { t.Errorf("scanned result is of wrong length") } - // order of items in maps can't be guaranteed - for _, v := range res_v { - val, _ := v.(string) - if val != "hah1" && val != "hah2" && val != "hah3" { - t.Errorf("scanned result contains unexpected value: %v", val) - } + if v, ok := res_v[0].(string); !ok || v != "hah1" { + t.Errorf("scanned result contains unexpected value: %v", v) + } + if v, ok := res_v[1].(string); !ok || v != "hah2" { + t.Errorf("scanned result contains unexpected value: %v", v) + } + if v, ok := res_v[2].(string); !ok || v != "hah3" { + t.Errorf("scanned result contains unexpected value: %v", v) } obj4 := map[string]interface{}{ @@ -590,37 +590,31 @@ func Test_jsonpath_get_scan(t *testing.T) { "c": 3, }, "key4" : []interface{}{1,2,3}, + "key5" : nil, } res, err = get_scan(obj4) res_v, ok = res.([]interface{}) if !ok { t.Errorf("scanned result is not a slice") } - if len(res_v) != 4 { + if len(res_v) != 5 { t.Errorf("scanned result is of wrong length") } - // order of items in maps can't be guaranteed - for _, v := range res_v { - switch v.(type) { - case string: - if v_str, ok := v.(string); ok && v_str == "abc" { - continue - } - case int: - if v_int, ok := v.(int); ok && v_int == 123 { - continue - } - case map[string]interface{}: - if v_map, ok := v.(map[string]interface{}); ok && v_map["a"].(int) == 1 && v_map["b"].(int) == 2 && v_map["c"].(int) == 3 { - continue - } - case []interface{}: - if v_slice, ok := v.([]interface{}); ok && v_slice[0].(int) == 1 && v_slice[1].(int) == 2 && v_slice[2].(int) == 3 { - continue - } - } + if v, ok := res_v[0].(string); !ok || v != "abc" { + t.Errorf("scanned result contains unexpected value: %v", v) + } + if v, ok := res_v[1].(int); !ok || v != 123 { t.Errorf("scanned result contains unexpected value: %v", v) } + if v, ok := res_v[2].(map[string]interface{}); !ok || v["a"].(int) != 1 || v["b"].(int) != 2 || v["c"].(int) != 3 { + t.Errorf("scanned result contains unexpected value: %v", v) + } + if v, ok := res_v[3].([]interface{}); !ok || v[0].(int) != 1 || v[1].(int) != 2 || v[2].(int) != 3 { + t.Errorf("scanned result contains unexpected value: %v", v) + } + if res_v[4] != nil { + t.Errorf("scanned result contains unexpected value: %v", res_v[4]) + } } func Test_jsonpath_types_eval(t *testing.T) { From 392940944c154219e1893d6fa1984e9d7de9f586 Mon Sep 17 00:00:00 2001 From: James Cor Date: Wed, 31 Jan 2024 16:30:50 -0800 Subject: [PATCH 5/6] Make KeyError public --- go.mod | 6 +++++- go.sum | 4 ++++ jsonpath.go | 8 ++++++-- 3 files changed, 15 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 99304ec..488830e 100644 --- a/go.mod +++ b/go.mod @@ -2,4 +2,8 @@ module github.com/dolthub/jsonpath go 1.15 -require github.com/stretchr/testify v1.7.0 +require ( + github.com/pkg/errors v0.9.1 // indirect + github.com/stretchr/testify v1.7.0 + gopkg.in/src-d/go-errors.v1 v1.0.0 +) diff --git a/go.sum b/go.sum index acb88a4..1ea467e 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,7 @@ github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= @@ -7,5 +9,7 @@ github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5Cc github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/src-d/go-errors.v1 v1.0.0 h1:cooGdZnCjYbeS1zb1s6pVAAimTdKceRrpn7aKOnNIfc= +gopkg.in/src-d/go-errors.v1 v1.0.0/go.mod h1:q1cBlomlw2FnDBDNGlnh6X0jPihy+QxZfMMNxPCbdYg= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/jsonpath.go b/jsonpath.go index 4f55bc8..d1d04a0 100644 --- a/jsonpath.go +++ b/jsonpath.go @@ -16,9 +16,13 @@ import ( "sort" "strconv" "strings" + + errKind "gopkg.in/src-d/go-errors.v1" ) var ErrGetFromNullObj = errors.New("get attribute from null object") +var ErrKeyError = errKind.NewKind("key error: %s not found in object") + func JsonPathLookup(obj interface{}, jpath string) (interface{}, error) { c, err := Compile(jpath) @@ -385,7 +389,7 @@ func get_key(obj interface{}, key string) (interface{}, error) { if jsonMap, ok := obj.(map[string]interface{}); ok { val, exists := jsonMap[key] if !exists { - return nil, fmt.Errorf("key error: %s not found in object", key) + return nil, ErrKeyError.New(key) } return val, nil } @@ -395,7 +399,7 @@ func get_key(obj interface{}, key string) (interface{}, error) { return reflect.ValueOf(obj).MapIndex(kv).Interface(), nil } } - return nil, fmt.Errorf("key error: %s not found in object", key) + return nil, ErrKeyError.New(key) case reflect.Slice: // slice we should get from all objects in it. res := []interface{}{} From 19675ab05c71df43bda05c9f24e73942a5bb9483 Mon Sep 17 00:00:00 2001 From: James Cor Date: Tue, 27 Feb 2024 12:06:19 -0800 Subject: [PATCH 6/6] fix panic for empty jsonpaths (#6) --- jsonpath.go | 3 +++ jsonpath_test.go | 10 ++++++++-- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/jsonpath.go b/jsonpath.go index d1d04a0..dc2b279 100644 --- a/jsonpath.go +++ b/jsonpath.go @@ -56,6 +56,9 @@ func Compile(jpath string) (*Compiled, error) { if err != nil { return nil, err } + if len(tokens) == 0 { + return nil, fmt.Errorf("empty path") + } if tokens[0] != "@" && tokens[0] != "$" { return nil, fmt.Errorf("$ or @ should in front of path") } diff --git a/jsonpath_test.go b/jsonpath_test.go index b98dd53..f7c4a9a 100644 --- a/jsonpath_test.go +++ b/jsonpath_test.go @@ -64,8 +64,14 @@ func init() { } func Test_jsonpath_JsonPathLookup_1(t *testing.T) { + // empty string + res, err := JsonPathLookup(json_data, "") + if err == nil { + t.Errorf("expected error from empty jsonpath") + } + // key from root - res, _ := JsonPathLookup(json_data, "$.expensive") + res, _ = JsonPathLookup(json_data, "$.expensive") if res_v, ok := res.(float64); ok != true || res_v != 10.0 { t.Errorf("expensive should be 10") } @@ -89,7 +95,7 @@ func Test_jsonpath_JsonPathLookup_1(t *testing.T) { } // multiple index - res, err := JsonPathLookup(json_data, "$.store.book[0,1].price") + res, err = JsonPathLookup(json_data, "$.store.book[0,1].price") t.Log(err, res) if res_v, ok := res.([]interface{}); ok != true || res_v[0].(float64) != 8.95 || res_v[1].(float64) != 12.99 { t.Errorf("exp: [8.95, 12.99], got: %v", res)