-
-
get json values quickly
GJSON is a Go package that provides a [fast](#performance) and [simple](#get-a-value) way to get values from a json document. @@ -126,11 +123,12 @@ nil, for JSON null To directly access the value: ```go -result.Type // can be String, Number, True, False, Null, or JSON -result.Str // holds the string -result.Num // holds the float64 number -result.Raw // holds the raw json -result.Index // index of raw value in original json, zero means index unknown +result.Type // can be String, Number, True, False, Null, or JSON +result.Str // holds the string +result.Num // holds the float64 number +result.Raw // holds the raw json +result.Index // index of raw value in original json, zero means index unknown +result.Indexes // indexes of all the elements that match on a path containing the '#' query character. ``` There are a variety of handy functions that work on a result: @@ -153,10 +151,6 @@ result.Less(token Result, caseSensitive bool) bool The `result.Value()` function returns an `interface{}` which requires type assertion and is one of the following Go types: -The `result.Array()` function returns back an array of values. -If the result represents a non-existent value, then an empty array will be returned. -If the result is not a JSON array, the return value will be an array containing one result. - ```go boolean >> bool number >> float64 @@ -166,6 +160,10 @@ array >> []interface{} object >> map[string]interface{} ``` +The `result.Array()` function returns back an array of values. +If the result represents a non-existent value, then an empty array will be returned. +If the result is not a JSON array, the return value will be an array containing one result. + ### 64-bit integers The `result.Int()` and `result.Uint()` calls are capable of reading all 64 bits, allowing for large JSON integers. @@ -193,11 +191,15 @@ we'll get `children` array and reverse the order: "children|@reverse|0" >> "Jack" ``` -There are currently three built-in modifiers: +There are currently the following built-in modifiers: - `@reverse`: Reverse an array or the members of an object. - `@ugly`: Remove all whitespace from a json document. - `@pretty`: Make the json document more human readable. +- `@this`: Returns the current element. It can be used to retrieve the root element. +- `@valid`: Ensure the json document is valid. +- `@flatten`: Flattens an array. +- `@join`: Joins multiple objects into a single object. ### Modifier arguments @@ -472,7 +474,7 @@ JSON document used: } ``` -Each operation was rotated though one of the following search paths: +Each operation was rotated through one of the following search paths: ``` widget.window.name @@ -480,12 +482,4 @@ widget.image.hOffset widget.text.onMouseUp ``` -*These benchmarks were run on a MacBook Pro 15" 2.8 GHz Intel Core i7 using Go 1.8 and can be be found [here](https://github.com/tidwall/gjson-benchmarks).* - - -## Contact -Josh Baker [@tidwall](http://twitter.com/tidwall) - -## License - -GJSON source code is available under the MIT [License](/LICENSE). +*These benchmarks were run on a MacBook Pro 15" 2.8 GHz Intel Core i7 using Go 1.8 and can be found [here](https://github.com/tidwall/gjson-benchmarks).* diff --git a/vendor/github.com/tidwall/gjson/SYNTAX.md b/vendor/github.com/tidwall/gjson/SYNTAX.md index ed584de..34fdccf 100644 --- a/vendor/github.com/tidwall/gjson/SYNTAX.md +++ b/vendor/github.com/tidwall/gjson/SYNTAX.md @@ -77,6 +77,21 @@ Special purpose characters, such as `.`, `*`, and `?` can be escaped with `\`. fav\.movie "Deer Hunter" ``` +You'll also need to make sure that the `\` character is correctly escaped when hardcoding a path in you source code. + +```go +// Go +val := gjson.Get(json, "fav\\.movie") // must escape the slash +val := gjson.Get(json, `fav\.movie`) // no need to escape the slash +``` + +```rust +// Rust +let val = gjson::get(json, "fav\\.movie") // must escape the slash +let val = gjson::get(json, r#"fav\.movie"#) // no need to escape the slash +``` + + ### Arrays The `#` character allows for digging into JSON Arrays. @@ -120,6 +135,37 @@ changed in v1.3.0 as to avoid confusion with the new [multipath](#multipaths) syntax. For backwards compatibility, `#[...]` will continue to work until the next major release.* +The `~` (tilde) operator will convert a value to a boolean before comparison. + +For example, using the following JSON: + +```json +{ + "vals": [ + { "a": 1, "b": true }, + { "a": 2, "b": true }, + { "a": 3, "b": false }, + { "a": 4, "b": "0" }, + { "a": 5, "b": 0 }, + { "a": 6, "b": "1" }, + { "a": 7, "b": 1 }, + { "a": 8, "b": "true" }, + { "a": 9, "b": false }, + { "a": 10, "b": null }, + { "a": 11 } + ] +} +``` + +You can now query for all true(ish) or false(ish) values: + +``` +vals.#(b==~true)#.a >> [1,2,6,7,8] +vals.#(b==~false)#.a >> [3,4,5,9,10,11] +``` + +The last value which was non-existent is treated as `false` + ### Dot vs Pipe The `.` is standard separator, but it's also possible to use a `|`. @@ -181,12 +227,15 @@ children.@reverse ["Jack","Alex","Sara"] children.@reverse.0 "Jack" ``` -There are currently three built-in modifiers: +There are currently the following built-in modifiers: - `@reverse`: Reverse an array or the members of an object. - `@ugly`: Remove all whitespace from JSON. - `@pretty`: Make the JSON more human readable. -- `@this`: Returns the current element. Can be used to retrieve the root element. +- `@this`: Returns the current element. It can be used to retrieve the root element. +- `@valid`: Ensure the json document is valid. +- `@flatten`: Flattens an array. +- `@join`: Joins multiple objects into a single object. #### Modifier arguments @@ -237,11 +286,13 @@ gjson.AddModifier("case", func(json, arg string) string { "children.@case:lower.@reverse" ["jack","alex","sara"] ``` +*Note: Custom modifiers are not yet available in the Rust version* + ### Multipaths Starting with v1.3.0, GJSON added the ability to join multiple paths together -to form new documents. Wrapping comma-separated paths between `{...}` or -`[...]` will result in a new array or object, respectively. +to form new documents. Wrapping comma-separated paths between `[...]` or +`{...}` will result in a new array or object, respectively. For example, using the given multipath diff --git a/vendor/github.com/tidwall/gjson/gjson.go b/vendor/github.com/tidwall/gjson/gjson.go index bac29a9..95d210b 100644 --- a/vendor/github.com/tidwall/gjson/gjson.go +++ b/vendor/github.com/tidwall/gjson/gjson.go @@ -8,6 +8,7 @@ import ( "time" "unicode/utf16" "unicode/utf8" + "unsafe" "github.com/tidwall/match" "github.com/tidwall/pretty" @@ -63,6 +64,9 @@ type Result struct { Num float64 // Index of raw value in original json, zero means index unknown Index int + // Indexes of all the elements that match on a path containing the '#' + // query character. + Indexes []int } // String returns a string representation of the value. @@ -104,7 +108,8 @@ func (t Result) Bool() bool { case True: return true case String: - return t.Str != "" && t.Str != "0" && t.Str != "false" + b, _ := strconv.ParseBool(strings.ToLower(t.Str)) + return b case Number: return t.Num != 0 } @@ -122,16 +127,17 @@ func (t Result) Int() int64 { return n case Number: // try to directly convert the float64 to int64 - n, ok := floatToInt(t.Num) - if !ok { - // now try to parse the raw string - n, ok = parseInt(t.Raw) - if !ok { - // fallback to a standard conversion - return int64(t.Num) - } + i, ok := safeInt(t.Num) + if ok { + return i } - return n + // now try to parse the raw string + i, ok = parseInt(t.Raw) + if ok { + return i + } + // fallback to a standard conversion + return int64(t.Num) } } @@ -147,16 +153,17 @@ func (t Result) Uint() uint64 { return n case Number: // try to directly convert the float64 to uint64 - n, ok := floatToUint(t.Num) - if !ok { - // now try to parse the raw string - n, ok = parseUint(t.Raw) - if !ok { - // fallback to a standard conversion - return uint64(t.Num) - } + i, ok := safeInt(t.Num) + if ok && i >= 0 { + return uint64(i) } - return n + // now try to parse the raw string + u, ok := parseUint(t.Raw) + if ok { + return u + } + // fallback to a standard conversion + return uint64(t.Num) } } @@ -277,7 +284,8 @@ func (t Result) ForEach(iterator func(key, value Result) bool) { } } -// Map returns back an map of values. The result should be a JSON array. +// Map returns back a map of values. The result should be a JSON object. +// If the result is not a JSON object, the return value will be an empty map. func (t Result) Map() map[string]Result { if t.Type != JSON { return map[string]Result{} @@ -493,6 +501,9 @@ func squash(json string) string { } } if depth == 0 { + if i >= len(json) { + return json + } return json[:i+1] } case '{', '[', '(': @@ -577,7 +588,7 @@ func tostr(json string) (raw string, str string) { continue } } - break + return json[:i+1], unescape(json[1:i]) } } var ret string @@ -707,10 +718,10 @@ type arrayPathResult struct { alogkey string query struct { on bool + all bool path string op string value string - all bool } } @@ -724,8 +735,13 @@ func parseArrayPath(path string) (r arrayPathResult) { } if path[i] == '.' { r.part = path[:i] - r.path = path[i+1:] - r.more = true + if !r.arrch && i < len(path)-1 && isDotPiperChar(path[i+1]) { + r.pipe = path[i+1:] + r.piped = true + } else { + r.path = path[i+1:] + r.more = true + } return } if path[i] == '#' { @@ -738,120 +754,27 @@ func parseArrayPath(path string) (r arrayPathResult) { } else if path[1] == '[' || path[1] == '(' { // query r.query.on = true - if true { - qpath, op, value, _, fi, ok := parseQuery(path[i:]) - if !ok { - // bad query, end now - break - } - r.query.path = qpath - r.query.op = op - r.query.value = value - i = fi - 1 - if i+1 < len(path) && path[i+1] == '#' { - r.query.all = true - } - } else { - var end byte - if path[1] == '[' { - end = ']' - } else { - end = ')' - } - i += 2 - // whitespace - for ; i < len(path); i++ { - if path[i] > ' ' { - break - } - } - s := i - for ; i < len(path); i++ { - if path[i] <= ' ' || - path[i] == '!' || - path[i] == '=' || - path[i] == '<' || - path[i] == '>' || - path[i] == '%' || - path[i] == end { - break - } - } - r.query.path = path[s:i] - // whitespace - for ; i < len(path); i++ { - if path[i] > ' ' { - break - } - } - if i < len(path) { - s = i - if path[i] == '!' { - if i < len(path)-1 && (path[i+1] == '=' || - path[i+1] == '%') { - i++ - } - } else if path[i] == '<' || path[i] == '>' { - if i < len(path)-1 && path[i+1] == '=' { - i++ - } - } else if path[i] == '=' { - if i < len(path)-1 && path[i+1] == '=' { - s++ - i++ - } - } - i++ - r.query.op = path[s:i] - // whitespace - for ; i < len(path); i++ { - if path[i] > ' ' { - break - } - } - s = i - for ; i < len(path); i++ { - if path[i] == '"' { - i++ - s2 := i - for ; i < len(path); i++ { - if path[i] > '\\' { - continue - } - if path[i] == '"' { - // look for an escaped slash - if path[i-1] == '\\' { - n := 0 - for j := i - 2; j > s2-1; j-- { - if path[j] != '\\' { - break - } - n++ - } - if n%2 == 0 { - continue - } - } - break - } - } - } else if path[i] == end { - if i+1 < len(path) && path[i+1] == '#' { - r.query.all = true - } - break - } - } - if i > len(path) { - i = len(path) - } - v := path[s:i] - for len(v) > 0 && v[len(v)-1] <= ' ' { - v = v[:len(v)-1] - } - r.query.value = v + qpath, op, value, _, fi, vesc, ok := + parseQuery(path[i:]) + if !ok { + // bad query, end now + break + } + if len(value) > 2 && value[0] == '"' && + value[len(value)-1] == '"' { + value = value[1 : len(value)-1] + if vesc { + value = unescape(value) } } + r.query.path = qpath + r.query.op = op + r.query.value = value + + i = fi - 1 + if i+1 < len(path) && path[i+1] == '#' { + r.query.all = true + } } } continue @@ -877,11 +800,11 @@ func parseArrayPath(path string) (r arrayPathResult) { // # middle // .cap # right func parseQuery(query string) ( - path, op, value, remain string, i int, ok bool, + path, op, value, remain string, i int, vesc, ok bool, ) { if len(query) < 2 || query[0] != '#' || (query[1] != '(' && query[1] != '[') { - return "", "", "", "", i, false + return "", "", "", "", i, false, false } i = 2 j := 0 // start of value part @@ -909,6 +832,7 @@ func parseQuery(query string) ( i++ for ; i < len(query); i++ { if query[i] == '\\' { + vesc = true i++ } else if query[i] == '"' { break @@ -917,7 +841,7 @@ func parseQuery(query string) ( } } if depth > 0 { - return "", "", "", "", i, false + return "", "", "", "", i, false, false } if j > 0 { path = trim(query[2:j]) @@ -954,7 +878,7 @@ func parseQuery(query string) ( path = trim(query[2:i]) remain = query[i+1:] } - return path, op, value, remain, i + 1, true + return path, op, value, remain, i + 1, vesc, true } func trim(s string) string { @@ -971,6 +895,11 @@ right: return s } +// peek at the next byte and see if it's a '@', '[', or '{'. +func isDotPiperChar(c byte) bool { + return !DisableModifiers && (c == '@' || c == '[' || c == '{') +} + type objectPathResult struct { part string path string @@ -989,12 +918,8 @@ func parseObjectPath(path string) (r objectPathResult) { return } if path[i] == '.' { - // peek at the next byte and see if it's a '@', '[', or '{'. r.part = path[:i] - if !DisableModifiers && - i < len(path)-1 && - (path[i+1] == '@' || - path[i+1] == '[' || path[i+1] == '{') { + if i < len(path)-1 && isDotPiperChar(path[i+1]) { r.pipe = path[i+1:] r.piped = true } else { @@ -1024,14 +949,11 @@ func parseObjectPath(path string) (r objectPathResult) { continue } else if path[i] == '.' { r.part = string(epart) - // peek at the next byte and see if it's a '@' modifier - if !DisableModifiers && - i < len(path)-1 && path[i+1] == '@' { + if i < len(path)-1 && isDotPiperChar(path[i+1]) { r.pipe = path[i+1:] r.piped = true } else { r.path = path[i+1:] - r.more = true } r.more = true return @@ -1167,9 +1089,9 @@ func parseObject(c *parseContext, i int, path string) (int, bool) { } if rp.wild { if kesc { - pmatch = match.Match(unescape(key), rp.part) + pmatch = matchLimit(unescape(key), rp.part) } else { - pmatch = match.Match(key, rp.part) + pmatch = matchLimit(key, rp.part) } } else { if kesc { @@ -1254,10 +1176,25 @@ func parseObject(c *parseContext, i int, path string) (int, bool) { } return i, false } + +// matchLimit will limit the complexity of the match operation to avoid ReDos +// attacks from arbritary inputs. +// See the github.com/tidwall/match.MatchLimit function for more information. +func matchLimit(str, pattern string) bool { + matched, _ := match.MatchLimit(str, pattern, 10000) + return matched +} + func queryMatches(rp *arrayPathResult, value Result) bool { rpv := rp.query.value - if len(rpv) > 2 && rpv[0] == '"' && rpv[len(rpv)-1] == '"' { - rpv = rpv[1 : len(rpv)-1] + if len(rpv) > 0 && rpv[0] == '~' { + // convert to bool + rpv = rpv[1:] + if value.Bool() { + value = Result{Type: True} + } else { + value = Result{Type: False} + } } if !value.Exists() { return false @@ -1285,9 +1222,9 @@ func queryMatches(rp *arrayPathResult, value Result) bool { case ">=": return value.Str >= rpv case "%": - return match.Match(value.Str, rpv) + return matchLimit(value.Str, rpv) case "!%": - return !match.Match(value.Str, rpv) + return !matchLimit(value.Str, rpv) } case Number: rpvn, _ := strconv.ParseFloat(rpv, 64) @@ -1337,6 +1274,7 @@ func parseArray(c *parseContext, i int, path string) (int, bool) { var alog []int var partidx int var multires []byte + var queryIndexes []int rp := parseArrayPath(path) if !rp.arrch { n, ok := parseUint(rp.part) @@ -1357,6 +1295,10 @@ func parseArray(c *parseContext, i int, path string) (int, bool) { multires = append(multires, '[') } } + var tmp parseContext + tmp.value = qval + fillIndex(c.json, &tmp) + parentIndex := tmp.value.Index var res Result if qval.Type == JSON { res = qval.Get(rp.query.path) @@ -1388,6 +1330,7 @@ func parseArray(c *parseContext, i int, path string) (int, bool) { multires = append(multires, ',') } multires = append(multires, raw...) + queryIndexes = append(queryIndexes, res.Index+parentIndex) } } else { c.value = res @@ -1396,7 +1339,6 @@ func parseArray(c *parseContext, i int, path string) (int, bool) { } return false } - for i < len(c.json)+1 { if !rp.arrch { pmatch = partidx == h @@ -1553,6 +1495,7 @@ func parseArray(c *parseContext, i int, path string) (int, bool) { c.pipe = right c.piped = true } + var indexes = make([]int, 0, 64) var jsons = make([]byte, 0, 64) jsons = append(jsons, '[') for j, k := 0, 0; j < len(alog); j++ { @@ -1567,6 +1510,7 @@ func parseArray(c *parseContext, i int, path string) (int, bool) { } if idx < len(c.json) && c.json[idx] != ']' { _, res, ok := parseAny(c.json, idx, true) + parentIndex := res.Index if ok { res := res.Get(rp.alogkey) if res.Exists() { @@ -1578,6 +1522,8 @@ func parseArray(c *parseContext, i int, path string) (int, bool) { raw = res.String() } jsons = append(jsons, []byte(raw)...) + indexes = append(indexes, + res.Index+parentIndex) k++ } } @@ -1586,6 +1532,7 @@ func parseArray(c *parseContext, i int, path string) (int, bool) { jsons = append(jsons, ']') c.value.Type = JSON c.value.Raw = string(jsons) + c.value.Indexes = indexes return i + 1, true } if rp.alogok { @@ -1598,10 +1545,18 @@ func parseArray(c *parseContext, i int, path string) (int, bool) { c.calcd = true return i + 1, true } - if len(multires) > 0 && !c.value.Exists() { - c.value = Result{ - Raw: string(append(multires, ']')), - Type: JSON, + if !c.value.Exists() { + if len(multires) > 0 { + c.value = Result{ + Raw: string(append(multires, ']')), + Type: JSON, + Indexes: queryIndexes, + } + } else if rp.query.all { + c.value = Result{ + Raw: "[]", + Type: JSON, + } } } return i + 1, false @@ -1833,7 +1788,7 @@ type parseContext struct { // A path is in dot syntax, such as "name.last" or "age". // When the value is found it's returned immediately. // -// A path is a series of keys searated by a dot. +// A path is a series of keys separated by a dot. // A key may contain special wildcard characters '*' and '?'. // To access an array value use the index as the key. // To get the number of elements in an array or to access a child path, use @@ -1876,6 +1831,7 @@ func Get(json, path string) Result { if len(path) > 0 && (path[0] == '|' || path[0] == '.') { res := Get(rjson, path[1:]) res.Index = 0 + res.Indexes = nil return res } return Parse(rjson) @@ -1942,7 +1898,6 @@ func Get(json, path string) Result { } } } - var i int var c = &parseContext{json: json} if len(path) >= 2 && path[0] == '.' && path[1] == '.' { @@ -2117,7 +2072,10 @@ func parseAny(json string, i int, hit bool) (int, Result, bool) { res.Raw = val res.Type = JSON } - return i, res, true + var tmp parseContext + tmp.value = res + fillIndex(json, &tmp) + return i, tmp.value, true } if json[i] <= ' ' { continue @@ -2167,11 +2125,6 @@ func parseAny(json string, i int, hit bool) (int, Result, bool) { return i, res, false } -var ( // used for testing - testWatchForFallback bool - testLastWasFallback bool -) - // GetMany searches json for the multiple paths. // The return value is a Result array where the number of items // will be equal to the number of input paths. @@ -2372,6 +2325,12 @@ func validnumber(data []byte, i int) (outi int, ok bool) { // sign if data[i] == '-' { i++ + if i == len(data) { + return i, false + } + if data[i] < '0' || data[i] > '9' { + return i, false + } } // int if i == len(data) { @@ -2522,25 +2481,15 @@ func parseInt(s string) (n int64, ok bool) { return n, true } -const minUint53 = 0 -const maxUint53 = 4503599627370495 -const minInt53 = -2251799813685248 -const maxInt53 = 2251799813685247 - -func floatToUint(f float64) (n uint64, ok bool) { - n = uint64(f) - if float64(n) == f && n >= minUint53 && n <= maxUint53 { - return n, true - } - return 0, false -} - -func floatToInt(f float64) (n int64, ok bool) { - n = int64(f) - if float64(n) == f && n >= minInt53 && n <= maxInt53 { - return n, true +// safeInt validates a given JSON number +// ensures it lies within the minimum and maximum representable JSON numbers +func safeInt(f float64) (n int64, ok bool) { + // https://tc39.es/ecma262/#sec-number.min_safe_integer + // https://tc39.es/ecma262/#sec-number.max_safe_integer + if f < -9007199254740991 || f > 9007199254740991 { + return 0, false } - return 0, false + return int64(f), true } // execModifier parses the path to find a matching modifier function. @@ -2595,6 +2544,15 @@ func execModifier(json, path string) (pathOut, res string, ok bool) { return pathOut, res, false } +// unwrap removes the '[]' or '{}' characters around json +func unwrap(json string) string { + json = trim(json) + if len(json) >= 2 && (json[0] == '[' || json[0] == '{') { + json = json[1 : len(json)-1] + } + return json +} + // DisableModifiers will disable the modifier syntax var DisableModifiers = false @@ -2603,6 +2561,9 @@ var modifiers = map[string]func(json, arg string) string{ "ugly": modUgly, "reverse": modReverse, "this": modThis, + "flatten": modFlatten, + "join": modJoin, + "valid": modValid, } // AddModifier binds a custom modifier command to the GJSON syntax. @@ -2618,6 +2579,26 @@ func ModifierExists(name string, fn func(json, arg string) string) bool { return ok } +// cleanWS remove any non-whitespace from string +func cleanWS(s string) string { + for i := 0; i < len(s); i++ { + switch s[i] { + case ' ', '\t', '\n', '\r': + continue + default: + var s2 []byte + for i := 0; i < len(s); i++ { + switch s[i] { + case ' ', '\t', '\n', '\r': + s2 = append(s2, s[i]) + } + } + return string(s2) + } + } + return s +} + // @pretty modifier makes the json look nice. func modPretty(json, arg string) string { if len(arg) > 0 { @@ -2627,9 +2608,9 @@ func modPretty(json, arg string) string { case "sortKeys": opts.SortKeys = value.Bool() case "indent": - opts.Indent = value.String() + opts.Indent = cleanWS(value.String()) case "prefix": - opts.Prefix = value.String() + opts.Prefix = cleanWS(value.String()) case "width": opts.Width = int(value.Int()) } @@ -2691,3 +2672,213 @@ func modReverse(json, arg string) string { } return json } + +// @flatten an array with child arrays. +// [1,[2],[3,4],[5,[6,7]]] -> [1,2,3,4,5,[6,7]] +// The {"deep":true} arg can be provide for deep flattening. +// [1,[2],[3,4],[5,[6,7]]] -> [1,2,3,4,5,6,7] +// The original json is returned when the json is not an array. +func modFlatten(json, arg string) string { + res := Parse(json) + if !res.IsArray() { + return json + } + var deep bool + if arg != "" { + Parse(arg).ForEach(func(key, value Result) bool { + if key.String() == "deep" { + deep = value.Bool() + } + return true + }) + } + var out []byte + out = append(out, '[') + var idx int + res.ForEach(func(_, value Result) bool { + var raw string + if value.IsArray() { + if deep { + raw = unwrap(modFlatten(value.Raw, arg)) + } else { + raw = unwrap(value.Raw) + } + } else { + raw = value.Raw + } + raw = strings.TrimSpace(raw) + if len(raw) > 0 { + if idx > 0 { + out = append(out, ',') + } + out = append(out, raw...) + idx++ + } + return true + }) + out = append(out, ']') + return bytesString(out) +} + +// @join multiple objects into a single object. +// [{"first":"Tom"},{"last":"Smith"}] -> {"first","Tom","last":"Smith"} +// The arg can be "true" to specify that duplicate keys should be preserved. +// [{"first":"Tom","age":37},{"age":41}] -> {"first","Tom","age":37,"age":41} +// Without preserved keys: +// [{"first":"Tom","age":37},{"age":41}] -> {"first","Tom","age":41} +// The original json is returned when the json is not an object. +func modJoin(json, arg string) string { + res := Parse(json) + if !res.IsArray() { + return json + } + var preserve bool + if arg != "" { + Parse(arg).ForEach(func(key, value Result) bool { + if key.String() == "preserve" { + preserve = value.Bool() + } + return true + }) + } + var out []byte + out = append(out, '{') + if preserve { + // Preserve duplicate keys. + var idx int + res.ForEach(func(_, value Result) bool { + if !value.IsObject() { + return true + } + if idx > 0 { + out = append(out, ',') + } + out = append(out, unwrap(value.Raw)...) + idx++ + return true + }) + } else { + // Deduplicate keys and generate an object with stable ordering. + var keys []Result + kvals := make(map[string]Result) + res.ForEach(func(_, value Result) bool { + if !value.IsObject() { + return true + } + value.ForEach(func(key, value Result) bool { + k := key.String() + if _, ok := kvals[k]; !ok { + keys = append(keys, key) + } + kvals[k] = value + return true + }) + return true + }) + for i := 0; i < len(keys); i++ { + if i > 0 { + out = append(out, ',') + } + out = append(out, keys[i].Raw...) + out = append(out, ':') + out = append(out, kvals[keys[i].String()].Raw...) + } + } + out = append(out, '}') + return bytesString(out) +} + +// @valid ensures that the json is valid before moving on. An empty string is +// returned when the json is not valid, otherwise it returns the original json. +func modValid(json, arg string) string { + if !Valid(json) { + return "" + } + return json +} + +// stringHeader instead of reflect.StringHeader +type stringHeader struct { + data unsafe.Pointer + len int +} + +// sliceHeader instead of reflect.SliceHeader +type sliceHeader struct { + data unsafe.Pointer + len int + cap int +} + +// getBytes casts the input json bytes to a string and safely returns the +// results as uniquely allocated data. This operation is intended to minimize +// copies and allocations for the large json string->[]byte. +func getBytes(json []byte, path string) Result { + var result Result + if json != nil { + // unsafe cast to string + result = Get(*(*string)(unsafe.Pointer(&json)), path) + // safely get the string headers + rawhi := *(*stringHeader)(unsafe.Pointer(&result.Raw)) + strhi := *(*stringHeader)(unsafe.Pointer(&result.Str)) + // create byte slice headers + rawh := sliceHeader{data: rawhi.data, len: rawhi.len, cap: rawhi.len} + strh := sliceHeader{data: strhi.data, len: strhi.len, cap: rawhi.len} + if strh.data == nil { + // str is nil + if rawh.data == nil { + // raw is nil + result.Raw = "" + } else { + // raw has data, safely copy the slice header to a string + result.Raw = string(*(*[]byte)(unsafe.Pointer(&rawh))) + } + result.Str = "" + } else if rawh.data == nil { + // raw is nil + result.Raw = "" + // str has data, safely copy the slice header to a string + result.Str = string(*(*[]byte)(unsafe.Pointer(&strh))) + } else if uintptr(strh.data) >= uintptr(rawh.data) && + uintptr(strh.data)+uintptr(strh.len) <= + uintptr(rawh.data)+uintptr(rawh.len) { + // Str is a substring of Raw. + start := uintptr(strh.data) - uintptr(rawh.data) + // safely copy the raw slice header + result.Raw = string(*(*[]byte)(unsafe.Pointer(&rawh))) + // substring the raw + result.Str = result.Raw[start : start+uintptr(strh.len)] + } else { + // safely copy both the raw and str slice headers to strings + result.Raw = string(*(*[]byte)(unsafe.Pointer(&rawh))) + result.Str = string(*(*[]byte)(unsafe.Pointer(&strh))) + } + } + return result +} + +// fillIndex finds the position of Raw data and assigns it to the Index field +// of the resulting value. If the position cannot be found then Index zero is +// used instead. +func fillIndex(json string, c *parseContext) { + if len(c.value.Raw) > 0 && !c.calcd { + jhdr := *(*stringHeader)(unsafe.Pointer(&json)) + rhdr := *(*stringHeader)(unsafe.Pointer(&(c.value.Raw))) + c.value.Index = int(uintptr(rhdr.data) - uintptr(jhdr.data)) + if c.value.Index < 0 || c.value.Index >= len(json) { + c.value.Index = 0 + } + } +} + +func stringBytes(s string) []byte { + return *(*[]byte)(unsafe.Pointer(&sliceHeader{ + data: (*stringHeader)(unsafe.Pointer(&s)).data, + len: len(s), + cap: len(s), + })) +} + +func bytesString(b []byte) string { + return *(*string)(unsafe.Pointer(&b)) +} diff --git a/vendor/github.com/tidwall/gjson/gjson_gae.go b/vendor/github.com/tidwall/gjson/gjson_gae.go deleted file mode 100644 index 9586903..0000000 --- a/vendor/github.com/tidwall/gjson/gjson_gae.go +++ /dev/null @@ -1,18 +0,0 @@ -//+build appengine js - -package gjson - -func getBytes(json []byte, path string) Result { - return Get(string(json), path) -} -func fillIndex(json string, c *parseContext) { - // noop. Use zero for the Index value. -} - -func stringBytes(s string) []byte { - return []byte(s) -} - -func bytesString(b []byte) string { - return string(b) -} diff --git a/vendor/github.com/tidwall/gjson/gjson_ngae.go b/vendor/github.com/tidwall/gjson/gjson_ngae.go deleted file mode 100644 index bc608b5..0000000 --- a/vendor/github.com/tidwall/gjson/gjson_ngae.go +++ /dev/null @@ -1,81 +0,0 @@ -//+build !appengine -//+build !js - -package gjson - -import ( - "reflect" - "unsafe" -) - -// getBytes casts the input json bytes to a string and safely returns the -// results as uniquely allocated data. This operation is intended to minimize -// copies and allocations for the large json string->[]byte. -func getBytes(json []byte, path string) Result { - var result Result - if json != nil { - // unsafe cast to string - result = Get(*(*string)(unsafe.Pointer(&json)), path) - // safely get the string headers - rawhi := *(*reflect.StringHeader)(unsafe.Pointer(&result.Raw)) - strhi := *(*reflect.StringHeader)(unsafe.Pointer(&result.Str)) - // create byte slice headers - rawh := reflect.SliceHeader{Data: rawhi.Data, Len: rawhi.Len} - strh := reflect.SliceHeader{Data: strhi.Data, Len: strhi.Len} - if strh.Data == 0 { - // str is nil - if rawh.Data == 0 { - // raw is nil - result.Raw = "" - } else { - // raw has data, safely copy the slice header to a string - result.Raw = string(*(*[]byte)(unsafe.Pointer(&rawh))) - } - result.Str = "" - } else if rawh.Data == 0 { - // raw is nil - result.Raw = "" - // str has data, safely copy the slice header to a string - result.Str = string(*(*[]byte)(unsafe.Pointer(&strh))) - } else if strh.Data >= rawh.Data && - int(strh.Data)+strh.Len <= int(rawh.Data)+rawh.Len { - // Str is a substring of Raw. - start := int(strh.Data - rawh.Data) - // safely copy the raw slice header - result.Raw = string(*(*[]byte)(unsafe.Pointer(&rawh))) - // substring the raw - result.Str = result.Raw[start : start+strh.Len] - } else { - // safely copy both the raw and str slice headers to strings - result.Raw = string(*(*[]byte)(unsafe.Pointer(&rawh))) - result.Str = string(*(*[]byte)(unsafe.Pointer(&strh))) - } - } - return result -} - -// fillIndex finds the position of Raw data and assigns it to the Index field -// of the resulting value. If the position cannot be found then Index zero is -// used instead. -func fillIndex(json string, c *parseContext) { - if len(c.value.Raw) > 0 && !c.calcd { - jhdr := *(*reflect.StringHeader)(unsafe.Pointer(&json)) - rhdr := *(*reflect.StringHeader)(unsafe.Pointer(&(c.value.Raw))) - c.value.Index = int(rhdr.Data - jhdr.Data) - if c.value.Index < 0 || c.value.Index >= len(json) { - c.value.Index = 0 - } - } -} - -func stringBytes(s string) []byte { - return *(*[]byte)(unsafe.Pointer(&reflect.SliceHeader{ - Data: (*reflect.StringHeader)(unsafe.Pointer(&s)).Data, - Len: len(s), - Cap: len(s), - })) -} - -func bytesString(b []byte) string { - return *(*string)(unsafe.Pointer(&b)) -} diff --git a/vendor/github.com/tidwall/gjson/go.mod b/vendor/github.com/tidwall/gjson/go.mod index d851688..6f64083 100644 --- a/vendor/github.com/tidwall/gjson/go.mod +++ b/vendor/github.com/tidwall/gjson/go.mod @@ -3,6 +3,6 @@ module github.com/tidwall/gjson go 1.12 require ( - github.com/tidwall/match v1.0.1 - github.com/tidwall/pretty v1.0.0 + github.com/tidwall/match v1.1.1 + github.com/tidwall/pretty v1.2.0 ) diff --git a/vendor/github.com/tidwall/gjson/go.sum b/vendor/github.com/tidwall/gjson/go.sum index a4a2d87..be39c8c 100644 --- a/vendor/github.com/tidwall/gjson/go.sum +++ b/vendor/github.com/tidwall/gjson/go.sum @@ -1,4 +1,4 @@ -github.com/tidwall/match v1.0.1 h1:PnKP62LPNxHKTwvHHZZzdOAOCtsJTjo6dZLCwpKm5xc= -github.com/tidwall/match v1.0.1/go.mod h1:LujAq0jyVjBy028G1WhWfIzbpQfMO8bBZ6Tyb0+pL9E= -github.com/tidwall/pretty v1.0.0 h1:HsD+QiTn7sK6flMKIvNmpqz1qrpP3Ps6jOKIKMooyg4= -github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= +github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= +github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= +github.com/tidwall/pretty v1.2.0 h1:RWIZEg2iJ8/g6fDDYzMpobmaoGh5OLl4AXtGUGPcqCs= +github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= diff --git a/vendor/github.com/tidwall/match/.travis.yml b/vendor/github.com/tidwall/match/.travis.yml deleted file mode 100644 index 4f2ee4d..0000000 --- a/vendor/github.com/tidwall/match/.travis.yml +++ /dev/null @@ -1 +0,0 @@ -language: go diff --git a/vendor/github.com/tidwall/match/README.md b/vendor/github.com/tidwall/match/README.md index 2aa5bc3..5fdd4cf 100644 --- a/vendor/github.com/tidwall/match/README.md +++ b/vendor/github.com/tidwall/match/README.md @@ -1,20 +1,17 @@ -Match -===== -
-
+# Match
+
+[](https://godoc.org/github.com/tidwall/match)
Match is a very simple pattern matcher where '*' matches on any
number characters and '?' matches on any one character.
-Installing
-----------
+## Installing
```
go get -u github.com/tidwall/match
```
-Example
--------
+## Example
```go
match.Match("hello", "*llo")
@@ -23,10 +20,10 @@ match.Match("hello", "h*o")
```
-Contact
--------
+## Contact
+
Josh Baker [@tidwall](http://twitter.com/tidwall)
-License
--------
+## License
+
Redcon source code is available under the MIT [License](/LICENSE).
diff --git a/vendor/github.com/tidwall/match/go.mod b/vendor/github.com/tidwall/match/go.mod
new file mode 100644
index 0000000..df19b5f
--- /dev/null
+++ b/vendor/github.com/tidwall/match/go.mod
@@ -0,0 +1,3 @@
+module github.com/tidwall/match
+
+go 1.15
diff --git a/vendor/github.com/tidwall/match/match.go b/vendor/github.com/tidwall/match/match.go
index fcfe998..11da28f 100644
--- a/vendor/github.com/tidwall/match/match.go
+++ b/vendor/github.com/tidwall/match/match.go
@@ -1,12 +1,14 @@
-// Match provides a simple pattern matcher with unicode support.
+// Package match provides a simple pattern matcher with unicode support.
package match
-import "unicode/utf8"
+import (
+ "unicode/utf8"
+)
// Match returns true if str matches pattern. This is a very
// simple wildcard match where '*' matches on any number characters
// and '?' matches on any one character.
-
+//
// pattern:
// { term }
// term:
@@ -19,113 +21,167 @@ func Match(str, pattern string) bool {
if pattern == "*" {
return true
}
- return deepMatch(str, pattern)
+ return match(str, pattern, 0, nil, -1) == rMatch
}
-func deepMatch(str, pattern string) bool {
- for len(pattern) > 0 {
- if pattern[0] > 0x7f {
- return deepMatchRune(str, pattern)
- }
- switch pattern[0] {
- default:
- if len(str) == 0 {
- return false
- }
- if str[0] > 0x7f {
- return deepMatchRune(str, pattern)
- }
- if str[0] != pattern[0] {
- return false
- }
- case '?':
- if len(str) == 0 {
- return false
- }
- case '*':
- return deepMatch(str, pattern[1:]) ||
- (len(str) > 0 && deepMatch(str[1:], pattern))
- }
- str = str[1:]
- pattern = pattern[1:]
+
+// MatchLimit is the same as Match but will limit the complexity of the match
+// operation. This is to avoid long running matches, specifically to avoid ReDos
+// attacks from arbritary inputs.
+//
+// How it works:
+// The underlying match routine is recursive and may call itself when it
+// encounters a sandwiched wildcard pattern, such as: `user:*:name`.
+// Everytime it calls itself a counter is incremented.
+// The operation is stopped when counter > maxcomp*len(str).
+func MatchLimit(str, pattern string, maxcomp int) (matched, stopped bool) {
+ if pattern == "*" {
+ return true, false
+ }
+ counter := 0
+ r := match(str, pattern, len(str), &counter, maxcomp)
+ if r == rStop {
+ return false, true
}
- return len(str) == 0 && len(pattern) == 0
+ return r == rMatch, false
}
-func deepMatchRune(str, pattern string) bool {
- var sr, pr rune
- var srsz, prsz int
+type result int
- // read the first rune ahead of time
- if len(str) > 0 {
- if str[0] > 0x7f {
- sr, srsz = utf8.DecodeRuneInString(str)
- } else {
- sr, srsz = rune(str[0]), 1
+const (
+ rNoMatch result = iota
+ rMatch
+ rStop
+)
+
+func match(str, pat string, slen int, counter *int, maxcomp int) result {
+ // check complexity limit
+ if maxcomp > -1 {
+ if *counter > slen*maxcomp {
+ return rStop
}
- } else {
- sr, srsz = utf8.RuneError, 0
+ *counter++
}
- if len(pattern) > 0 {
- if pattern[0] > 0x7f {
- pr, prsz = utf8.DecodeRuneInString(pattern)
- } else {
- pr, prsz = rune(pattern[0]), 1
+
+ for len(pat) > 0 {
+ var wild bool
+ pc, ps := rune(pat[0]), 1
+ if pc > 0x7f {
+ pc, ps = utf8.DecodeRuneInString(pat)
}
- } else {
- pr, prsz = utf8.RuneError, 0
- }
- // done reading
- for pr != utf8.RuneError {
- switch pr {
- default:
- if srsz == utf8.RuneError {
- return false
- }
- if sr != pr {
- return false
+ var sc rune
+ var ss int
+ if len(str) > 0 {
+ sc, ss = rune(str[0]), 1
+ if sc > 0x7f {
+ sc, ss = utf8.DecodeRuneInString(str)
}
+ }
+ switch pc {
case '?':
- if srsz == utf8.RuneError {
- return false
+ if ss == 0 {
+ return rNoMatch
}
case '*':
- return deepMatchRune(str, pattern[prsz:]) ||
- (srsz > 0 && deepMatchRune(str[srsz:], pattern))
- }
- str = str[srsz:]
- pattern = pattern[prsz:]
- // read the next runes
- if len(str) > 0 {
- if str[0] > 0x7f {
- sr, srsz = utf8.DecodeRuneInString(str)
- } else {
- sr, srsz = rune(str[0]), 1
+ // Ignore repeating stars.
+ for len(pat) > 1 && pat[1] == '*' {
+ pat = pat[1:]
}
- } else {
- sr, srsz = utf8.RuneError, 0
- }
- if len(pattern) > 0 {
- if pattern[0] > 0x7f {
- pr, prsz = utf8.DecodeRuneInString(pattern)
- } else {
- pr, prsz = rune(pattern[0]), 1
+
+ // If this star is the last character then it must be a match.
+ if len(pat) == 1 {
+ return rMatch
}
- } else {
- pr, prsz = utf8.RuneError, 0
+
+ // Match and trim any non-wildcard suffix characters.
+ var ok bool
+ str, pat, ok = matchTrimSuffix(str, pat)
+ if !ok {
+ return rNoMatch
+ }
+
+ // Check for single star again.
+ if len(pat) == 1 {
+ return rMatch
+ }
+
+ // Perform recursive wildcard search.
+ r := match(str, pat[1:], slen, counter, maxcomp)
+ if r != rNoMatch {
+ return r
+ }
+ if len(str) == 0 {
+ return rNoMatch
+ }
+ wild = true
+ default:
+ if ss == 0 {
+ return rNoMatch
+ }
+ if pc == '\\' {
+ pat = pat[ps:]
+ pc, ps = utf8.DecodeRuneInString(pat)
+ if ps == 0 {
+ return rNoMatch
+ }
+ }
+ if sc != pc {
+ return rNoMatch
+ }
+ }
+ str = str[ss:]
+ if !wild {
+ pat = pat[ps:]
}
- // done reading
}
-
- return srsz == 0 && prsz == 0
+ if len(str) == 0 {
+ return rMatch
+ }
+ return rNoMatch
}
-var maxRuneBytes = func() []byte {
- b := make([]byte, 4)
- if utf8.EncodeRune(b, '\U0010FFFF') != 4 {
- panic("invalid rune encoding")
+// matchTrimSuffix matches and trims any non-wildcard suffix characters.
+// Returns the trimed string and pattern.
+//
+// This is called because the pattern contains extra data after the wildcard
+// star. Here we compare any suffix characters in the pattern to the suffix of
+// the target string. Basically a reverse match that stops when a wildcard
+// character is reached. This is a little trickier than a forward match because
+// we need to evaluate an escaped character in reverse.
+//
+// Any matched characters will be trimmed from both the target
+// string and the pattern.
+func matchTrimSuffix(str, pat string) (string, string, bool) {
+ // It's expected that the pattern has at least two bytes and the first byte
+ // is a wildcard star '*'
+ match := true
+ for len(str) > 0 && len(pat) > 1 {
+ pc, ps := utf8.DecodeLastRuneInString(pat)
+ var esc bool
+ for i := 0; ; i++ {
+ if pat[len(pat)-ps-i-1] != '\\' {
+ if i&1 == 1 {
+ esc = true
+ ps++
+ }
+ break
+ }
+ }
+ if pc == '*' && !esc {
+ match = true
+ break
+ }
+ sc, ss := utf8.DecodeLastRuneInString(str)
+ if !((pc == '?' && !esc) || pc == sc) {
+ match = false
+ break
+ }
+ str = str[:len(str)-ss]
+ pat = pat[:len(pat)-ps]
}
- return b
-}()
+ return str, pat, match
+}
+
+var maxRuneBytes = [...]byte{244, 143, 191, 191}
// Allowable parses the pattern and determines the minimum and maximum allowable
// values that the pattern can represent.
@@ -146,7 +202,7 @@ func Allowable(pattern string) (min, max string) {
}
if pattern[i] == '?' {
minb = append(minb, 0)
- maxb = append(maxb, maxRuneBytes...)
+ maxb = append(maxb, maxRuneBytes[:]...)
} else {
minb = append(minb, pattern[i])
maxb = append(maxb, pattern[i])
diff --git a/vendor/github.com/tidwall/pretty/.travis.yml b/vendor/github.com/tidwall/pretty/.travis.yml
deleted file mode 100644
index 4f2ee4d..0000000
--- a/vendor/github.com/tidwall/pretty/.travis.yml
+++ /dev/null
@@ -1 +0,0 @@
-language: go
diff --git a/vendor/github.com/tidwall/pretty/README.md b/vendor/github.com/tidwall/pretty/README.md
index d2b8864..d3be5e5 100644
--- a/vendor/github.com/tidwall/pretty/README.md
+++ b/vendor/github.com/tidwall/pretty/README.md
@@ -1,8 +1,6 @@
# Pretty
-[](https://travis-ci.org/tidwall/prettty)
-[](http://gocover.io/github.com/tidwall/pretty)
-[](https://godoc.org/github.com/tidwall/pretty)
+[](https://pkg.go.dev/github.com/tidwall/pretty)
Pretty is a Go package that provides [fast](#performance) methods for formatting JSON for human readability, or to compact JSON for smaller payloads.
@@ -81,7 +79,6 @@ Will format the json to:
{"name":{"first":"Tom","last":"Anderson"},"age":37,"children":["Sara","Alex","Jack"],"fav.movie":"Deer Hunter","friends":[{"first":"Janet","last":"Murphy","age":44}]}```
```
-
## Customized output
There's a `PrettyOptions(json, opts)` function which allows for customizing the output with the following options:
@@ -106,14 +103,15 @@ type Options struct {
Benchmarks of Pretty alongside the builtin `encoding/json` Indent/Compact methods.
```
-BenchmarkPretty-8 1000000 1283 ns/op 720 B/op 2 allocs/op
-BenchmarkUgly-8 3000000 426 ns/op 240 B/op 1 allocs/op
-BenchmarkUglyInPlace-8 5000000 340 ns/op 0 B/op 0 allocs/op
-BenchmarkJSONIndent-8 300000 4628 ns/op 1069 B/op 4 allocs/op
-BenchmarkJSONCompact-8 1000000 2469 ns/op 758 B/op 4 allocs/op
+BenchmarkPretty-16 1000000 1034 ns/op 720 B/op 2 allocs/op
+BenchmarkPrettySortKeys-16 586797 1983 ns/op 2848 B/op 14 allocs/op
+BenchmarkUgly-16 4652365 254 ns/op 240 B/op 1 allocs/op
+BenchmarkUglyInPlace-16 6481233 183 ns/op 0 B/op 0 allocs/op
+BenchmarkJSONIndent-16 450654 2687 ns/op 1221 B/op 0 allocs/op
+BenchmarkJSONCompact-16 685111 1699 ns/op 442 B/op 0 allocs/op
```
-*These benchmarks were run on a MacBook Pro 15" 2.8 GHz Intel Core i7 using Go 1.7.*
+*These benchmarks were run on a MacBook Pro 2.4 GHz 8-Core Intel Core i9.*
## Contact
Josh Baker [@tidwall](http://twitter.com/tidwall)
diff --git a/vendor/github.com/tidwall/pretty/go.mod b/vendor/github.com/tidwall/pretty/go.mod
new file mode 100644
index 0000000..6106735
--- /dev/null
+++ b/vendor/github.com/tidwall/pretty/go.mod
@@ -0,0 +1,3 @@
+module github.com/tidwall/pretty
+
+go 1.16
diff --git a/vendor/github.com/tidwall/pretty/pretty.go b/vendor/github.com/tidwall/pretty/pretty.go
index 0a922d0..f3f756a 100644
--- a/vendor/github.com/tidwall/pretty/pretty.go
+++ b/vendor/github.com/tidwall/pretty/pretty.go
@@ -1,7 +1,10 @@
package pretty
import (
+ "bytes"
+ "encoding/json"
"sort"
+ "strconv"
)
// Options is Pretty options
@@ -84,6 +87,14 @@ func ugly(dst, src []byte) []byte {
return dst
}
+func isNaNOrInf(src []byte) bool {
+ return src[0] == 'i' || //Inf
+ src[0] == 'I' || // inf
+ src[0] == '+' || // +Inf
+ src[0] == 'N' || // Nan
+ (src[0] == 'n' && len(src) > 1 && src[1] != 'u') // nan
+}
+
func appendPrettyAny(buf, json []byte, i int, pretty bool, width int, prefix, indent string, sortkeys bool, tabs, nl, max int) ([]byte, int, int, bool) {
for ; i < len(json); i++ {
if json[i] <= ' ' {
@@ -92,7 +103,8 @@ func appendPrettyAny(buf, json []byte, i int, pretty bool, width int, prefix, in
if json[i] == '"' {
return appendPrettyString(buf, json, i, nl)
}
- if (json[i] >= '0' && json[i] <= '9') || json[i] == '-' {
+
+ if (json[i] >= '0' && json[i] <= '9') || json[i] == '-' || isNaNOrInf(json[i:]) {
return appendPrettyNumber(buf, json, i, nl)
}
if json[i] == '{' {
@@ -118,25 +130,121 @@ type pair struct {
vstart, vend int
}
-type byKey struct {
+type byKeyVal struct {
sorted bool
json []byte
+ buf []byte
pairs []pair
}
-func (arr *byKey) Len() int {
+func (arr *byKeyVal) Len() int {
return len(arr.pairs)
}
-func (arr *byKey) Less(i, j int) bool {
- key1 := arr.json[arr.pairs[i].kstart+1 : arr.pairs[i].kend-1]
- key2 := arr.json[arr.pairs[j].kstart+1 : arr.pairs[j].kend-1]
- return string(key1) < string(key2)
+func (arr *byKeyVal) Less(i, j int) bool {
+ if arr.isLess(i, j, byKey) {
+ return true
+ }
+ if arr.isLess(j, i, byKey) {
+ return false
+ }
+ return arr.isLess(i, j, byVal)
}
-func (arr *byKey) Swap(i, j int) {
+func (arr *byKeyVal) Swap(i, j int) {
arr.pairs[i], arr.pairs[j] = arr.pairs[j], arr.pairs[i]
arr.sorted = true
}
+type byKind int
+
+const (
+ byKey byKind = 0
+ byVal byKind = 1
+)
+
+type jtype int
+
+const (
+ jnull jtype = iota
+ jfalse
+ jnumber
+ jstring
+ jtrue
+ jjson
+)
+
+func getjtype(v []byte) jtype {
+ if len(v) == 0 {
+ return jnull
+ }
+ switch v[0] {
+ case '"':
+ return jstring
+ case 'f':
+ return jfalse
+ case 't':
+ return jtrue
+ case 'n':
+ return jnull
+ case '[', '{':
+ return jjson
+ default:
+ return jnumber
+ }
+}
+
+func (arr *byKeyVal) isLess(i, j int, kind byKind) bool {
+ k1 := arr.json[arr.pairs[i].kstart:arr.pairs[i].kend]
+ k2 := arr.json[arr.pairs[j].kstart:arr.pairs[j].kend]
+ var v1, v2 []byte
+ if kind == byKey {
+ v1 = k1
+ v2 = k2
+ } else {
+ v1 = bytes.TrimSpace(arr.buf[arr.pairs[i].vstart:arr.pairs[i].vend])
+ v2 = bytes.TrimSpace(arr.buf[arr.pairs[j].vstart:arr.pairs[j].vend])
+ if len(v1) >= len(k1)+1 {
+ v1 = bytes.TrimSpace(v1[len(k1)+1:])
+ }
+ if len(v2) >= len(k2)+1 {
+ v2 = bytes.TrimSpace(v2[len(k2)+1:])
+ }
+ }
+ t1 := getjtype(v1)
+ t2 := getjtype(v2)
+ if t1 < t2 {
+ return true
+ }
+ if t1 > t2 {
+ return false
+ }
+ if t1 == jstring {
+ s1 := parsestr(v1)
+ s2 := parsestr(v2)
+ return string(s1) < string(s2)
+ }
+ if t1 == jnumber {
+ n1, _ := strconv.ParseFloat(string(v1), 64)
+ n2, _ := strconv.ParseFloat(string(v2), 64)
+ return n1 < n2
+ }
+ return string(v1) < string(v2)
+
+}
+
+func parsestr(s []byte) []byte {
+ for i := 1; i < len(s); i++ {
+ if s[i] == '\\' {
+ var str string
+ json.Unmarshal(s, &str)
+ return []byte(str)
+ }
+ if s[i] == '"' {
+ return s[1:i]
+ }
+ }
+ return nil
+}
+
func appendPrettyObject(buf, json []byte, i int, open, close byte, pretty bool, width int, prefix, indent string, sortkeys bool, tabs, nl, max int) ([]byte, int, int, bool) {
var ok bool
if width > 0 {
@@ -174,7 +282,11 @@ func appendPrettyObject(buf, json []byte, i int, open, close byte, pretty bool,
}
if n > 0 {
nl = len(buf)
- buf = append(buf, '\n')
+ if buf[nl-1] == ' ' {
+ buf[nl-1] = '\n'
+ } else {
+ buf = append(buf, '\n')
+ }
}
if buf[len(buf)-1] != open {
buf = appendTabs(buf, prefix, indent, tabs)
@@ -193,7 +305,11 @@ func appendPrettyObject(buf, json []byte, i int, open, close byte, pretty bool,
var p pair
if pretty {
nl = len(buf)
- buf = append(buf, '\n')
+ if buf[nl-1] == ' ' {
+ buf[nl-1] = '\n'
+ } else {
+ buf = append(buf, '\n')
+ }
if open == '{' && sortkeys {
p.kstart = i
p.vstart = len(buf)
@@ -235,8 +351,8 @@ func sortPairs(json, buf []byte, pairs []pair) []byte {
}
vstart := pairs[0].vstart
vend := pairs[len(pairs)-1].vend
- arr := byKey{false, json, pairs}
- sort.Sort(&arr)
+ arr := byKeyVal{false, json, buf, pairs}
+ sort.Stable(&arr)
if !arr.sorted {
return buf
}
@@ -305,6 +421,7 @@ func appendTabs(buf []byte, prefix, indent string, tabs int) []byte {
type Style struct {
Key, String, Number [2]string
True, False, Null [2]string
+ Escape [2]string
Append func(dst []byte, c byte) []byte
}
@@ -318,21 +435,26 @@ func hexp(p byte) byte {
}
// TerminalStyle is for terminals
-var TerminalStyle = &Style{
- Key: [2]string{"\x1B[94m", "\x1B[0m"},
- String: [2]string{"\x1B[92m", "\x1B[0m"},
- Number: [2]string{"\x1B[93m", "\x1B[0m"},
- True: [2]string{"\x1B[96m", "\x1B[0m"},
- False: [2]string{"\x1B[96m", "\x1B[0m"},
- Null: [2]string{"\x1B[91m", "\x1B[0m"},
- Append: func(dst []byte, c byte) []byte {
- if c < ' ' && (c != '\r' && c != '\n' && c != '\t' && c != '\v') {
- dst = append(dst, "\\u00"...)
- dst = append(dst, hexp((c>>4)&0xF))
- return append(dst, hexp((c)&0xF))
- }
- return append(dst, c)
- },
+var TerminalStyle *Style
+
+func init() {
+ TerminalStyle = &Style{
+ Key: [2]string{"\x1B[94m", "\x1B[0m"},
+ String: [2]string{"\x1B[92m", "\x1B[0m"},
+ Number: [2]string{"\x1B[93m", "\x1B[0m"},
+ True: [2]string{"\x1B[96m", "\x1B[0m"},
+ False: [2]string{"\x1B[96m", "\x1B[0m"},
+ Null: [2]string{"\x1B[91m", "\x1B[0m"},
+ Escape: [2]string{"\x1B[35m", "\x1B[0m"},
+ Append: func(dst []byte, c byte) []byte {
+ if c < ' ' && (c != '\r' && c != '\n' && c != '\t' && c != '\v') {
+ dst = append(dst, "\\u00"...)
+ dst = append(dst, hexp((c>>4)&0xF))
+ return append(dst, hexp((c)&0xF))
+ }
+ return append(dst, c)
+ },
+ }
}
// Color will colorize the json. The style parma is used for customizing
@@ -363,8 +485,39 @@ func Color(src []byte, style *Style) []byte {
dst = append(dst, style.String[0]...)
}
dst = apnd(dst, '"')
+ esc := false
+ uesc := 0
for i = i + 1; i < len(src); i++ {
- dst = apnd(dst, src[i])
+ if src[i] == '\\' {
+ if key {
+ dst = append(dst, style.Key[1]...)
+ } else {
+ dst = append(dst, style.String[1]...)
+ }
+ dst = append(dst, style.Escape[0]...)
+ dst = apnd(dst, src[i])
+ esc = true
+ if i+1 < len(src) && src[i+1] == 'u' {
+ uesc = 5
+ } else {
+ uesc = 1
+ }
+ } else if esc {
+ dst = apnd(dst, src[i])
+ if uesc == 1 {
+ esc = false
+ dst = append(dst, style.Escape[1]...)
+ if key {
+ dst = append(dst, style.Key[0]...)
+ } else {
+ dst = append(dst, style.String[0]...)
+ }
+ } else {
+ uesc--
+ }
+ } else {
+ dst = apnd(dst, src[i])
+ }
if src[i] == '"' {
j := i - 1
for ; ; j-- {
@@ -377,7 +530,9 @@ func Color(src []byte, style *Style) []byte {
}
}
}
- if key {
+ if esc {
+ dst = append(dst, style.Escape[1]...)
+ } else if key {
dst = append(dst, style.Key[1]...)
} else {
dst = append(dst, style.String[1]...)
@@ -393,7 +548,7 @@ func Color(src []byte, style *Style) []byte {
dst = apnd(dst, src[i])
} else {
var kind byte
- if (src[i] >= '0' && src[i] <= '9') || src[i] == '-' {
+ if (src[i] >= '0' && src[i] <= '9') || src[i] == '-' || isNaNOrInf(src[i:]) {
kind = '0'
dst = append(dst, style.Number[0]...)
} else if src[i] == 't' {
@@ -430,3 +585,90 @@ func Color(src []byte, style *Style) []byte {
}
return dst
}
+
+// Spec strips out comments and trailing commas and convert the input to a
+// valid JSON per the official spec: https://tools.ietf.org/html/rfc8259
+//
+// The resulting JSON will always be the same length as the input and it will
+// include all of the same line breaks at matching offsets. This is to ensure
+// the result can be later processed by a external parser and that that
+// parser will report messages or errors with the correct offsets.
+func Spec(src []byte) []byte {
+ return spec(src, nil)
+}
+
+// SpecInPlace is the same as Spec, but this method reuses the input json
+// buffer to avoid allocations. Do not use the original bytes slice upon return.
+func SpecInPlace(src []byte) []byte {
+ return spec(src, src)
+}
+
+func spec(src, dst []byte) []byte {
+ dst = dst[:0]
+ for i := 0; i < len(src); i++ {
+ if src[i] == '/' {
+ if i < len(src)-1 {
+ if src[i+1] == '/' {
+ dst = append(dst, ' ', ' ')
+ i += 2
+ for ; i < len(src); i++ {
+ if src[i] == '\n' {
+ dst = append(dst, '\n')
+ break
+ } else if src[i] == '\t' || src[i] == '\r' {
+ dst = append(dst, src[i])
+ } else {
+ dst = append(dst, ' ')
+ }
+ }
+ continue
+ }
+ if src[i+1] == '*' {
+ dst = append(dst, ' ', ' ')
+ i += 2
+ for ; i < len(src)-1; i++ {
+ if src[i] == '*' && src[i+1] == '/' {
+ dst = append(dst, ' ', ' ')
+ i++
+ break
+ } else if src[i] == '\n' || src[i] == '\t' ||
+ src[i] == '\r' {
+ dst = append(dst, src[i])
+ } else {
+ dst = append(dst, ' ')
+ }
+ }
+ continue
+ }
+ }
+ }
+ dst = append(dst, src[i])
+ if src[i] == '"' {
+ for i = i + 1; i < len(src); i++ {
+ dst = append(dst, src[i])
+ if src[i] == '"' {
+ j := i - 1
+ for ; ; j-- {
+ if src[j] != '\\' {
+ break
+ }
+ }
+ if (j-i)%2 != 0 {
+ break
+ }
+ }
+ }
+ } else if src[i] == '}' || src[i] == ']' {
+ for j := len(dst) - 2; j >= 0; j-- {
+ if dst[j] <= ' ' {
+ continue
+ }
+ if dst[j] == ',' {
+ dst[j] = ' '
+ }
+ break
+ }
+ }
+ }
+ return dst
+}
diff --git a/vendor/modules.txt b/vendor/modules.txt
index 062620e..362e97f 100644
--- a/vendor/modules.txt
+++ b/vendor/modules.txt
@@ -96,11 +96,11 @@ github.com/stretchr/objx
# github.com/stretchr/testify v1.3.0
github.com/stretchr/testify/assert
github.com/stretchr/testify/mock
-# github.com/tidwall/gjson v1.4.0
+# github.com/tidwall/gjson v1.9.3
github.com/tidwall/gjson
-# github.com/tidwall/match v1.0.1
+# github.com/tidwall/match v1.1.1
github.com/tidwall/match
-# github.com/tidwall/pretty v1.0.0
+# github.com/tidwall/pretty v1.2.0
github.com/tidwall/pretty
# github.com/tidwall/sjson v1.0.4
github.com/tidwall/sjson