update TOML from upstream library

Author: KristofferC

base/toml_parser.jl

@@ -194,6 +194,7 @@ end
     # Inline tables
     ErrExpectedCommaBetweenItemsInlineTable
     ErrTrailingCommaInlineTable
+    ErrInlineTableRedefine
 
     # Numbers
     ErrUnderscoreNotSurroundedByDigits
@@ -202,6 +203,7 @@ end
     ErrLeadingDot
     ErrNoTrailingDigitAfterDot
     ErrTrailingUnderscoreNumber
+    ErrSignInNonBase10Number
 
     # DateTime
     ErrParsingDateTime
@@ -229,6 +231,7 @@ const err_message = Dict(
     ErrUnexpectedEndString                  => "string literal ended unexpectedly",
     ErrExpectedEndOfTable                   => "expected end of table ']'",
     ErrAddKeyToInlineTable                  => "tried to add a new key to an inline table",
+    ErrInlineTableRedefine                  => "inline table overwrote key from other table",
     ErrArrayTreatedAsDictionary             => "tried to add a key to an array",
     ErrAddArrayToStaticArray                => "tried to append to a statically defined array",
     ErrGenericValueError                    => "failed to parse value",
@@ -244,7 +247,8 @@ const err_message = Dict(
     ErrOverflowError                        => "overflowed when parsing integer",
     ErrInvalidUnicodeScalar                 => "invalid unicode scalar",
     ErrInvalidEscapeCharacter               => "invalid escape character",
-    ErrUnexpectedEofExpectedValue           => "unexpected end of file, expected a value"
+    ErrUnexpectedEofExpectedValue           => "unexpected end of file, expected a value",
+    ErrSignInNonBase10Number                => "number not in base 10 is not allowed to have a sign",
 )
 
 for err in instances(ErrorType)
@@ -467,15 +471,15 @@ function parse_toplevel(l::Parser)::Err{Nothing}
         l.active_table = l.root
         @try parse_table(l)
         skip_ws_comment(l)
-        if !(peek(l) == '\n' || peek(l) == '\r' || peek(l) == EOF_CHAR)
+        if !(peek(l) == '\n' || peek(l) == '\r' || peek(l) == '#' || peek(l) == EOF_CHAR)
             eat_char(l)
             return ParserError(ErrExpectedNewLineKeyValue)
         end
     else
         @try parse_entry(l, l.active_table)
         skip_ws_comment(l)
         # SPEC: "There must be a newline (or EOF) after a key/value pair."
-        if !(peek(l) == '\n' || peek(l) == '\r' || peek(l) == EOF_CHAR)
+        if !(peek(l) == '\n' || peek(l) == '\r' || peek(l) == '#' || peek(l) == EOF_CHAR)
             c = eat_char(l)
             return ParserError(ErrExpectedNewLineKeyValue)
         end
@@ -563,6 +567,10 @@ function parse_entry(l::Parser, d)::Union{Nothing, ParserError}
 
     skip_ws(l)
     value = @try parse_value(l)
+    # Not allowed to overwrite a value with an inline dict
+    if value isa Dict && haskey(d, last_key_part)
+        return ParserError(ErrInlineTableRedefine)
+    end
     # TODO: Performance, hashing `last_key_part` again here
     d[last_key_part] = value
     return
@@ -789,9 +797,11 @@ function parse_number_or_date_start(l::Parser)
 
     set_marker!(l)
     sgn = 1
+    parsed_sign = false
     if accept(l, '+')
-        # do nothing
+        parsed_sign = true
     elseif accept(l, '-')
+        parsed_sign = true
         sgn = -1
     end
     if accept(l, 'i')
@@ -811,12 +821,15 @@ function parse_number_or_date_start(l::Parser)
         if ok_end_value(peek(l))
             return Int64(0)
         elseif accept(l, 'x')
+            parsed_sign && return ParserError(ErrSignInNonBase10Number)
             ate, contains_underscore = @try accept_batch_underscore(l, isvalid_hex)
             ate && return parse_int(l, contains_underscore)
         elseif accept(l, 'o')
+            parsed_sign && return ParserError(ErrSignInNonBase10Number)
             ate, contains_underscore = @try accept_batch_underscore(l, isvalid_oct)
             ate && return parse_int(l, contains_underscore)
         elseif accept(l, 'b')
+            parsed_sign && return ParserError(ErrSignInNonBase10Number)
             ate, contains_underscore = @try accept_batch_underscore(l, isvalid_binary)
             ate && return parse_int(l, contains_underscore)
         elseif accept(l, isdigit)

stdlib/TOML/Project.toml

@@ -1,12 +1,18 @@
 name = "TOML"
 uuid = "fa267f1f-6049-4f14-aa54-33bafae1ed76"
-version = "1.0.0"
+version = "1.0.3"
 
 [deps]
 Dates = "ade2ca70-3891-5945-98fb-dc099432e06a"
 
+[compat]
+julia = "1.6"
+
 [extras]
+Downloads = "f43a241f-c20a-4ad4-852c-f6b1247861c6"
 Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
+Tar = "a4e569a6-e804-4fa4-b0f3-eef7a1d5b13e"
+p7zip_jll = "3f19e933-33d8-53b3-aaab-bd5110c3b7a0"
 
 [targets]
-test = ["Test"]
+test = ["Downloads", "p7zip_jll", "Tar", "Test"]

stdlib/TOML/src/print.jl

@@ -33,6 +33,14 @@ function print_toml_escaped(io::IO, s::AbstractString)
     end
 end
 
+const MbyFunc = Union{Function, Nothing}
+const TOMLValue = Union{AbstractVector, AbstractDict, Dates.DateTime, Dates.Time, Dates.Date, Bool, Integer, AbstractFloat, AbstractString}
+
+
+########
+# Keys #
+########
+
 function printkey(io::IO, keys::Vector{String})
     for (i, k) in enumerate(keys)
         i != 1 && Base.print(io, ".")
@@ -50,48 +58,74 @@ function printkey(io::IO, keys::Vector{String})
     end
 end
 
-const MbyFunc = Union{Function, Nothing}
-const TOMLValue = Union{AbstractVector, AbstractDict, Dates.DateTime, Dates.Time, Dates.Date, Bool, Integer, AbstractFloat, AbstractString}
-function printvalue(f::MbyFunc, io::IO, value::AbstractVector; sorted=false, by=identity)
+function to_toml_value(f::MbyFunc, value)
+    if f === nothing
+        error("type `$(typeof(value))` is not a valid TOML type, pass a conversion function to `TOML.print`")
+    end
+    toml_value = f(value)
+    if !(toml_value isa TOMLValue)
+        error("TOML syntax function for type `$(typeof(value))` did not return a valid TOML type but a `$(typeof(toml_value))`")
+    end
+    return toml_value
+end
+
+##########
+# Values #
+##########
+
+# Fallback
+function printvalue(f::MbyFunc, io::IO, value)
+    toml_value = to_toml_value(f, value)
+    @invokelatest printvalue(f, io, toml_value)
+end
+
+function printvalue(f::MbyFunc, io::IO, value::AbstractVector)
     Base.print(io, "[")
     for (i, x) in enumerate(value)
         i != 1 && Base.print(io, ", ")
-        if isa(x, AbstractDict)
-            _print(f, io, x; sorted, by)
-        else
-            printvalue(f, io, x; sorted, by)
-        end
+        printvalue(f, io, x)
     end
     Base.print(io, "]")
 end
-printvalue(f::MbyFunc, io::IO, value::AbstractDict; sorted=false, by=identity) =
-    _print(f, io, value; sorted, by)
-printvalue(f::MbyFunc, io::IO, value::Dates.DateTime; _...) =
-    Base.print(io, Dates.format(value, Dates.dateformat"YYYY-mm-dd\THH:MM:SS.sss\Z"))
-printvalue(f::MbyFunc, io::IO, value::Dates.Time; _...) =
-    Base.print(io, Dates.format(value, Dates.dateformat"HH:MM:SS.sss"))
-printvalue(f::MbyFunc, io::IO, value::Dates.Date; _...) =
-    Base.print(io, Dates.format(value, Dates.dateformat"YYYY-mm-dd"))
-printvalue(f::MbyFunc, io::IO, value::Bool; _...) =
-    Base.print(io, value ? "true" : "false")
-printvalue(f::MbyFunc, io::IO, value::Integer; _...) =
-    Base.print(io, Int64(value))  # TOML specifies 64-bit signed long range for integer
-printvalue(f::MbyFunc, io::IO, value::AbstractFloat; _...) =
-    Base.print(io, isnan(value) ? "nan" :
-                   !(isfinite(value)::Bool) ? string(value > 0 ? "+" : "-", "inf") :
-                   Float64(value))  # TOML specifies IEEE 754 binary64 for float
-function printvalue(f::MbyFunc, io::IO, value::AbstractString; _...)
-    Base.print(io, "\"")
-    print_toml_escaped(io, value)
-    Base.print(io, "\"")
+
+function printvalue(f::MbyFunc, io::IO, value::TOMLValue)
+    value isa Dates.DateTime ? Base.print(io, Dates.format(value, Dates.dateformat"YYYY-mm-dd\THH:MM:SS.sss\Z")) :
+    value isa Dates.Time     ? Base.print(io, Dates.format(value, Dates.dateformat"HH:MM:SS.sss")) :
+    value isa Dates.Date     ? Base.print(io, Dates.format(value, Dates.dateformat"YYYY-mm-dd")) :
+    value isa Bool           ? Base.print(io, value ? "true" : "false") :
+    value isa Integer        ? Base.print(io, Int64(value)) :  # TOML specifies 64-bit signed long range for integer
+    value isa AbstractFloat  ? Base.print(io, isnan(value) ? "nan" :
+                                              isinf(value) ? string(value > 0 ? "+" : "-", "inf") :
+                                              Float64(value)) :  # TOML specifies IEEE 754 binary64 for float
+    value isa AbstractString ? (Base.print(io, "\"");
+                                print_toml_escaped(io, value);
+                                Base.print(io, "\"")) :
+    value isa AbstractDict ? print_inline_table(f, io, value) :
+    error("internal error in TOML printing, unhandled value")
+end
+
+function print_inline_table(f::MbyFunc, io::IO, value::AbstractDict)
+    Base.print(io, "{")
+    for (i, (k,v)) in enumerate(value)
+        i != 1 && Base.print(io, ", ")
+        printkey(io, [String(k)])
+        Base.print(io, " = ")
+        printvalue(f, io, v)
+    end
+    Base.print(io, "}")
 end
 
+
+##########
+# Tables #
+##########
+
 is_table(value)           = isa(value, AbstractDict)
 is_array_of_tables(value) = isa(value, AbstractArray) &&
                             length(value) > 0 && isa(value[1], AbstractDict)
 is_tabular(value)         = is_table(value) || is_array_of_tables(value)
 
-function _print(f::MbyFunc, io::IO, a::AbstractDict,
+function print_table(f::MbyFunc, io::IO, a::AbstractDict,
     ks::Vector{String} = String[];
     indent::Int = 0,
     first_block::Bool = true,
@@ -100,37 +134,30 @@ function _print(f::MbyFunc, io::IO, a::AbstractDict,
 )
     akeys = keys(a)
     if sorted
-        akeys = sort!(collect(akeys); by)
+        akeys = sort!(collect(akeys); by=by)
     end
 
     # First print non-tabular entries
     for key in akeys
         value = a[key]
-        is_tabular(value) && continue
         if !isa(value, TOMLValue)
-            if f === nothing
-                error("type `$(typeof(value))` is not a valid TOML type, pass a conversion function to `TOML.print`")
-            end
-            toml_value = f(value)
-            if !(toml_value isa TOMLValue)
-                error("TOML syntax function for type `$(typeof(value))` did not return a valid TOML type but a `$(typeof(toml_value))`")
-            end
-            value = toml_value
-        end
-        if is_tabular(value)
-            _print(f, io, Dict(key => value); indent, first_block, sorted, by)
-        else
-            Base.print(io, ' '^4max(0,indent-1))
-            printkey(io, [String(key)])
-            Base.print(io, " = ") # print separator
-            printvalue(f, io, value; sorted, by)
-            Base.print(io, "\n")  # new line?
+            value = to_toml_value(f, value)
         end
+        is_tabular(value) && continue
+
+        Base.print(io, ' '^4max(0,indent-1))
+        printkey(io, [String(key)])
+        Base.print(io, " = ") # print separator
+        printvalue(f, io, value)
+        Base.print(io, "\n")  # new line?
         first_block = false
     end
 
     for key in akeys
         value = a[key]
+        if !isa(value, TOMLValue)
+            value = to_toml_value(f, value)
+        end
         if is_table(value)
             push!(ks, String(key))
             header = isempty(value) || !all(is_tabular(v) for v in values(value))::Bool
@@ -144,7 +171,7 @@ function _print(f::MbyFunc, io::IO, a::AbstractDict,
                 Base.print(io,"]\n")
             end
             # Use runtime dispatch here since the type of value seems not to be enforced other than as AbstractDict
-            @invokelatest _print(f, io, value, ks; indent = indent + header, first_block = header, sorted, by)
+            @invokelatest print_table(f, io, value, ks; indent = indent + header, first_block = header, sorted=sorted, by=by)
             pop!(ks)
         elseif is_array_of_tables(value)
             # print array of tables
@@ -158,14 +185,19 @@ function _print(f::MbyFunc, io::IO, a::AbstractDict,
                 Base.print(io,"]]\n")
                 # TODO, nicer error here
                 !isa(v, AbstractDict) && error("array should contain only tables")
-                @invokelatest _print(f, io, v, ks; indent = indent + 1, sorted, by)
+                @invokelatest print_table(f, io, v, ks; indent = indent + 1, sorted=sorted, by=by)
             end
             pop!(ks)
         end
     end
 end
 
-print(f::MbyFunc, io::IO, a::AbstractDict; sorted::Bool=false, by=identity) = _print(f, io, a; sorted, by)
-print(f::MbyFunc, a::AbstractDict; sorted::Bool=false, by=identity) = print(f, stdout, a; sorted, by)
-print(io::IO, a::AbstractDict; sorted::Bool=false, by=identity) = _print(nothing, io, a; sorted, by)
-print(a::AbstractDict; sorted::Bool=false, by=identity) = print(nothing, stdout, a; sorted, by)
+
+#######
+# API #
+#######
+
+print(f::MbyFunc, io::IO, a::AbstractDict; sorted::Bool=false, by=identity) = print_table(f, io, a; sorted=sorted, by=by)
+print(f::MbyFunc, a::AbstractDict; sorted::Bool=false, by=identity) = print(f, stdout, a; sorted=sorted, by=by)
+print(io::IO, a::AbstractDict; sorted::Bool=false, by=identity) = print_table(nothing, io, a; sorted=sorted, by=by)
+print(a::AbstractDict; sorted::Bool=false, by=identity) = print(nothing, stdout, a; sorted=sorted, by=by)

stdlib/TOML/test/print.jl

@@ -71,3 +71,43 @@ end
     d = Dict("str" => string(Char(0xd800)))
     @test_throws ErrorException TOML.print(devnull, d)
 end
+
+str = """
+[[dataset.loader]]
+driver = "nested"
+loaders = ["gzip", { driver = "csv", args = {delim = "\t"}}]
+"""
+@test roundtrip(str)
+
+
+struct Foo
+    a::Int64
+    b::Float64
+end
+
+struct Bar
+    c::Float64
+    d::String
+end
+
+
+f = Foo(2,9.9)
+b = Bar(1.345, "hello")
+
+dd = Dict("hello"=>"world", "f"=>f,  "b"=>b)
+
+to_dict(foo::Foo) = Dict("a"=>foo.a, "b"=>foo.b)
+to_dict(bar::Bar) = Dict("c"=>bar.c, "d"=>bar.d)
+
+@test toml_str(to_dict, dd) ==
+"""
+hello = "world"
+
+[f]
+b = 9.9
+a = 2
+
+[b]
+c = 1.345
+d = "hello"
+"""

stdlib/TOML/test/readme.jl

@@ -613,7 +613,7 @@ contributors = [
   { name = \"Baz Qux\", email = \"[email protected]\", url = \"https://example.com/bazqux\" }
 ]
 """
-@test_broken roundtrip(str) # Printer doesn't handle inline tables in arrays?
+@test roundtrip(str)
 d = parse(str)
 @test d["integers"] == [1,2,3]
 @test d["colors"] == ["red", "yellow", "green"]

stdlib/TOML/test/runtests.jl

@@ -16,6 +16,7 @@ function roundtrip(data)
 end
 
 include("readme.jl")
+include("utils/utils.jl")
 include("toml_test.jl")
 include("values.jl")
 include("invalids.jl")