From ef2ba796d5b60654d46ccde318ef04f7e20a98b8 Mon Sep 17 00:00:00 2001 From: Rafael Fourquet Date: Sun, 28 Feb 2021 17:21:32 +0100 Subject: [PATCH 1/4] dryrun: don't give up too easily, and add "repeated" annotations Before, we were giving up when e.g. a description couldn't be eval'ed, without printing children. Also, we indicate when a testset is supposed to be repeated, whether when we don't know how many times (e.g. couldn't eval the length of testset-for iterator), or when the testset-for description is a String (doesn't depend on loop variables). --- src/ReTest.jl | 66 ++++++++++++++++++++++++++++++++++++------------ test/runtests.jl | 30 ++++++++++++++++++++++ 2 files changed, 80 insertions(+), 16 deletions(-) diff --git a/src/ReTest.jl b/src/ReTest.jl index 6292607..867d16f 100644 --- a/src/ReTest.jl +++ b/src/ReTest.jl @@ -941,39 +941,73 @@ end isindented(verbose, overall, many) = (verbose > 0) & (overall | !many) -function dryrun(mod::Module, ts::TestsetExpr, rx::Regex, align::Int=0, parentsubj="", ) +function dryrun(mod::Module, ts::TestsetExpr, rx::Regex, align::Int=0, parentsubj="" + ; evaldesc=true, repeated=nothing) ts.run || return desc = ts.desc - giveup() = println(' '^align, desc) - - if ts.loops === nothing || desc isa String - if !(desc isa String) + if ts.loops === nothing + if evaldesc && !(desc isa String) try desc = Core.eval(mod, desc) catch - return giveup() end end - subject = parentsubj * '/' * desc - if isfinal(ts) - occursin(rx, subject) || return + + subject = nothing + if parentsubj isa String && desc isa String + subject = parentsubj * '/' * desc + if isfinal(ts) + occursin(rx, subject) || return + end + end + print(' '^align, desc) + if repeated !== nothing + printstyled(" (repeated", + repeated == -1 ? ")" : " $repeated times)", '\n', + color=:light_black) + else + println() end - println(' '^align, desc) for tsc in ts.children dryrun(mod, tsc, rx, align + 2, subject) end else - loopvalues = ts.loopvalues - loopvalues === nothing && return giveup() - for x in loopvalues - descx = eval_desc(mod, ts, x) - descx === nothing && return giveup() + function dryrun_beginend(descx, repeated=nothing) # avoid repeating ourselves, transform this iteration into a "begin/end" testset beginend = TestsetExpr(ts.source, ts.mod, descx, ts.options, nothing, ts.parent, ts.children) beginend.run = true - dryrun(mod, beginend, rx, align, parentsubj) + dryrun(mod, beginend, rx, align, parentsubj; evaldesc=false, repeated=repeated) + end + + loopvalues = ts.loopvalues + if loopvalues === nothing + # ts.desc is probably a String (cf. resolve!); if so, don't print repeated + # identitical lines (caveat: if subjects of children would change randomly) + # but still try simply to evaluate the length of the iterator + repeated = -1 + if ts.desc isa String + local iterlen + try + iterlen = 1 + for loop in ts.loops + iterlen *= Core.eval(mod, :(length($(loop.args[2])))) + end + repeated = iterlen + catch + end + end + dryrun_beginend(ts.desc, repeated) + else + for (i, x) in enumerate(loopvalues) + descx = eval_desc(mod, ts, x) + if descx === nothing + @assert i == 1 + return dryrun_beginend(ts.desc, length(loopvalues)) + end + dryrun_beginend(descx === nothing ? ts.desc : descx) + end end end end diff --git a/test/runtests.jl b/test/runtests.jl index e3dd51b..05decdd 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -612,6 +612,36 @@ for dry=(true, false), retest(mod..., regex; shuffle=true, verbose=verbose, stats=stats, dry=dry) end +### dry-run + +module DryRun +using ReTest + +X = 'a' + +@testset "a" for i=1:2 + @testset "b" begin end +end +@testset "a$X" for i=1:2 + @testset "b" begin end +end +@testset "x$i" for i=1:2 + @testset "b$i" begin + @testset "c" begin end + end + @testset "d$i$j" for j=1:2 + end +end +@testset "y$i" for i=1:1 + @testset "b" for j=1:i + @testset "c" begin end + end +end + +end # DryRun + +retest(DryRun, dry=true) + ### InlineTest ############################################################### using Pkg From f017027e0c99da8d53949311a554ebe7a1730e93 Mon Sep 17 00:00:00 2001 From: Rafael Fourquet Date: Sun, 28 Feb 2021 17:35:41 +0100 Subject: [PATCH 2/4] dryrun: print in warn color when description isn't resolved Also, don't warn anymore from resolve! : it was insufficient (didn't show the line number), and it seems better to rely on dryrun for detecting that (and adding yet another flag to not be warned is annoying). --- src/ReTest.jl | 6 ++---- test/runtests.jl | 6 +++--- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/src/ReTest.jl b/src/ReTest.jl index 867d16f..7ecae0f 100644 --- a/src/ReTest.jl +++ b/src/ReTest.jl @@ -181,9 +181,6 @@ function resolve!(mod::Module, ts::TestsetExpr, rx::Regex; # and ts.run == true function giveup() - if !ts.run - @warn "could not evaluate testset description, default to inclusion" - end ts.run = true if shown # set ts.descwidth to a lower bound to reduce misalignment @@ -961,7 +958,8 @@ function dryrun(mod::Module, ts::TestsetExpr, rx::Regex, align::Int=0, parentsub occursin(rx, subject) || return end end - print(' '^align, desc) + printstyled(' '^align, desc, color = desc isa String ? :normal : Base.warn_color()) + if repeated !== nothing printstyled(" (repeated", repeated == -1 ? ")" : " $repeated times)", '\n', diff --git a/test/runtests.jl b/test/runtests.jl index 05decdd..dac219d 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -289,9 +289,9 @@ RUN = [] end end -@test_logs (:warn, r"could not evaluate testset description.*") retest(Loops1, r"asd") -@test Loops1.RUN == [1, 0, 2, 0] -empty!(Loops1.RUN) +# @test_logs (:warn, r"could not evaluate testset description.*") retest(Loops1, r"asd") +# @test Loops1.RUN == [1, 0, 2, 0] +# empty!(Loops1.RUN) retest(Loops1) # should not log @test Loops1.RUN == [1, 0, -1, 2, 0, -1] From c0e317a16215aeca362e5f36baae317615e733f3 Mon Sep 17 00:00:00 2001 From: Rafael Fourquet Date: Sun, 28 Feb 2021 18:03:31 +0100 Subject: [PATCH 3/4] dryrun: honor the verbose flag --- src/ReTest.jl | 11 ++++++----- test/runtests.jl | 2 ++ 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/src/ReTest.jl b/src/ReTest.jl index 7ecae0f..bfbaf0d 100644 --- a/src/ReTest.jl +++ b/src/ReTest.jl @@ -468,7 +468,7 @@ function retest(args::Union{Module,AbstractString,Regex}...; showmod = overall || implicitmodules showmod && println(mod) - foreach(ts -> dryrun(mod, ts, regex, showmod*2), tests) + foreach(ts -> dryrun(mod, ts, regex, showmod*2, verbose=verbose>0), tests) continue end @@ -939,8 +939,8 @@ end isindented(verbose, overall, many) = (verbose > 0) & (overall | !many) function dryrun(mod::Module, ts::TestsetExpr, rx::Regex, align::Int=0, parentsubj="" - ; evaldesc=true, repeated=nothing) - ts.run || return + ; evaldesc=true, repeated=nothing, verbose) + ts.run && verbose || return desc = ts.desc if ts.loops === nothing @@ -968,7 +968,7 @@ function dryrun(mod::Module, ts::TestsetExpr, rx::Regex, align::Int=0, parentsub println() end for tsc in ts.children - dryrun(mod, tsc, rx, align + 2, subject) + dryrun(mod, tsc, rx, align + 2, subject, verbose=ts.options.transient_verbose) end else function dryrun_beginend(descx, repeated=nothing) @@ -976,7 +976,8 @@ function dryrun(mod::Module, ts::TestsetExpr, rx::Regex, align::Int=0, parentsub beginend = TestsetExpr(ts.source, ts.mod, descx, ts.options, nothing, ts.parent, ts.children) beginend.run = true - dryrun(mod, beginend, rx, align, parentsubj; evaldesc=false, repeated=repeated) + dryrun(mod, beginend, rx, align, parentsubj; evaldesc=false, + repeated=repeated, verbose=verbose) end loopvalues = ts.loopvalues diff --git a/test/runtests.jl b/test/runtests.jl index dac219d..807a749 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -641,6 +641,8 @@ end end # DryRun retest(DryRun, dry=true) +retest(DryRun, dry=true, verbose=0) +retest(DryRun, dry=true, verbose=5) ### InlineTest ############################################################### From 3450d5ef7eaaa18480e99b89af30d31f60e5e0a8 Mon Sep 17 00:00:00 2001 From: Rafael Fourquet Date: Sun, 28 Feb 2021 18:10:09 +0100 Subject: [PATCH 4/4] dryrun: print modules in bold, and separate them with newline --- src/ReTest.jl | 7 +++++-- test/runtests.jl | 9 +++++++++ 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/src/ReTest.jl b/src/ReTest.jl index bfbaf0d..98b0633 100644 --- a/src/ReTest.jl +++ b/src/ReTest.jl @@ -466,8 +466,11 @@ function retest(args::Union{Module,AbstractString,Regex}...; if dry showmod = overall || implicitmodules - showmod && - println(mod) + if showmod + imod > 1 && verbose > 0 && + println() + printstyled(mod, '\n', bold=true) + end foreach(ts -> dryrun(mod, ts, regex, showmod*2, verbose=verbose>0), tests) continue end diff --git a/test/runtests.jl b/test/runtests.jl index 807a749..822c3dc 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -644,6 +644,15 @@ retest(DryRun, dry=true) retest(DryRun, dry=true, verbose=0) retest(DryRun, dry=true, verbose=5) +module DryRun2 +using ReTest + +@testset "just a dummy module" begin end +end # DryRun2 + +retest(DryRun, DryRun2, dry=true, verbose=0) +retest(DryRun, DryRun2, dry=true, verbose=1) + ### InlineTest ############################################################### using Pkg