better handling of descriptions with interpolation

src/ReTest.jl

@@ -152,22 +152,52 @@ function resolve!(mod::Module, ts::TestsetExpr, rx::Regex;
     ts.descwidth = 0
     ts.options.transient_verbose = shown & ((verbose > 1) | ts.options.verbose)
 
-    if desc isa String
+    function giveup()
+        if !ts.run
+            @warn "could not evaluate testset description, default to inclusion"
+        end
+        ts.run = true
         if shown
-            ts.descwidth = textwidth(desc) + 2*depth
+            # set ts.descwidth to a lower bound to reduce misalignment
+            ts.descwidth = 2*depth + mapreduce(+, desc.args) do part
+                                         if part isa String
+                                             textwidth(part)
+                                         else
+                                             4 # give 4 spaces for unknown string part
+                                         end
+                                     end
         end
-        for str in parentstrs
-            ts.run && break
-            new = str * '/' * desc
-            if occursin(rx, new)
-                ts.run = true
-            else
-                push!(strings, new)
+    end
+
+    loops = ts.loops
+    if loops === nothing || desc isa String
+        # TODO: maybe, for testset-for and !(desc isa String), still try this branch
+        # in case the the interpolation can be resolved thanks to a global binding
+        # (i.e. the description doesn't depend on loop variables)
+        gaveup = false
+        if !(desc isa String)
+            try
+                desc = Core.eval(mod, desc)
+            catch
+                giveup()
+                gaveup = true
+            end
+        end
+        if !gaveup
+            if shown
+                ts.descwidth = textwidth(desc) + 2*depth
+            end
+            for str in parentstrs
+                ts.run && break
+                new = str * '/' * desc
+                if occursin(rx, new)
+                    ts.run = true
+                else
+                    push!(strings, new)
+                end
             end
         end
     else
-        loops = ts.loops
-        @assert loops !== nothing
         xs = ()
         loopiters = Expr(:tuple, (arg.args[1] for arg in loops)...)
 
@@ -194,18 +224,14 @@ function resolve!(mod::Module, ts::TestsetExpr, rx::Regex;
             ts.loopiters = loopiters
         catch
             @assert xs == ()
-            if !ts.run
-                @warn "could not evaluate testset-for iterator, default to inclusion"
-            end
-            ts.run = true
-            if shown
-                # set ts.descwidth to a lower bound to reduce misalignment
-                ts.descwidth = 2*depth + mapreduce(textwidth, +,
-                                                   filter(x -> x isa String, desc.args))
-            end
+            giveup()
         end
         for x in xs # empty loop if eval above threw
             descx = eval_desc(mod, ts, x)
+            if descx === nothing
+                giveup()
+                break
+            end
             if shown
                 ts.descwidth = max(ts.descwidth, textwidth(descx) + 2*depth)
             end
@@ -249,11 +275,15 @@ eval_desc(mod, ts, x) =
     if ts.desc isa String
         ts.desc
     else
-        Core.eval(mod, quote
-                  let $(ts.loopiters) = $x
-                      $(ts.desc)
-                  end
-                  end)::String
+        try
+            Core.eval(mod, quote
+                      let $(ts.loopiters) = $x
+                          $(ts.desc)
+                      end
+                      end)::String
+        catch
+            nothing
+        end
     end
 
 # convert a TestsetExpr into an actually runnable testset
@@ -265,6 +295,7 @@ function make_ts(ts::TestsetExpr, rx::Regex, stats, chan)
     else
         body = make_ts(ts.body, rx, stats, chan)
     end
+
     if ts.loops === nothing
         quote
             @testset $(ts.mod) $(isfinal(ts)) $rx $(ts.desc) $(ts.options) $stats $chan $body
@@ -883,7 +914,16 @@ function dryrun(mod::Module, ts::TestsetExpr, rx::Regex, align::Int=0, parentsub
     ts.run || return
     desc = ts.desc
 
-    if desc isa String
+    giveup() = println(' '^align, desc)
+
+    if ts.loops === nothing || desc isa String
+        if !(desc isa String)
+            try
+                desc = Core.eval(mod, desc)
+            catch
+                return giveup()
+            end
+        end
         subject = parentsubj * '/' * desc
         if isfinal(ts)
             occursin(rx, subject) || return
@@ -894,13 +934,10 @@ function dryrun(mod::Module, ts::TestsetExpr, rx::Regex, align::Int=0, parentsub
         end
     else
         loopvalues = ts.loopvalues
-        if loopvalues === nothing
-            println(' '^align, desc)
-            @warn "could not evaluate testset-for iterator, default to inclusion"
-            return
-        end
+        loopvalues === nothing && return giveup()
         for x in loopvalues
             descx = eval_desc(mod, ts, x)
+            descx === nothing && return giveup()
             # avoid repeating ourselves, transform this iteration into a "begin/end" testset
             beginend = TestsetExpr(ts.source, ts.mod, descx, ts.options, nothing,
                                    ts.parent, ts.children)

src/testset.jl

@@ -425,7 +425,7 @@ function get_testset_string(remove_last=false)
 end
 
 # non-inline testset with regex filtering support
-macro testset(mod::String, isfinal::Bool, rx::Regex, desc::String, options,
+macro testset(mod::String, isfinal::Bool, rx::Regex, desc::Union{String,Expr}, options,
               stats::Bool, chan, body)
     Testset.testset_beginend(mod, isfinal, rx, desc, options, stats, chan,  body, __source__)
 end
@@ -439,12 +439,13 @@ end
 """
 Generate the code for a `@testset` with a `begin`/`end` argument
 """
-function testset_beginend(mod::String, isfinal::Bool, rx::Regex, desc::String, options,
+function testset_beginend(mod::String, isfinal::Bool, rx::Regex, desc, options,
                           stats::Bool, chan, tests, source)
     # Generate a block of code that initializes a new testset, adds
     # it to the task local storage, evaluates the test(s), before
     # finally removing the testset and giving it a chance to take
     # action (such as reporting the results)
+    desc = esc(desc)
     ex = quote
         local current_str
         if $isfinal
@@ -497,10 +498,11 @@ Generate the code for a `@testset` with a `for` loop argument
 function testset_forloop(mod::String, isfinal::Bool, rx::Regex, desc::Union{String,Expr},
                          options, stats, chan, loops, tests, source)
 
+    desc = esc(desc)
     blk = quote
         local current_str
         if $isfinal
-            current_str = string(get_testset_string(!first_iteration), '/', $(esc(desc)))
+            current_str = string(get_testset_string(!first_iteration), '/', $desc)
         end
         if !$isfinal || occursin($rx, current_str)
             # Trick to handle `break` and `continue` in the test code before
@@ -511,7 +513,7 @@ function testset_forloop(mod::String, isfinal::Bool, rx::Regex, desc::Union{Stri
                 # it's 1000 times faster to copy from tmprng rather than calling Random.seed!
                 copy!(RNG, tmprng)
             end
-            ts = ReTestSet($mod, $(esc(desc)); verbose=$(options.transient_verbose))
+            ts = ReTestSet($mod, $desc; verbose=$(options.transient_verbose))
             if nworkers() == 1 && get_testset_depth() == 0 && $(chan.preview) !== nothing
                 put!($(chan.preview), ts.description)
             end

test/runtests.jl

@@ -289,7 +289,7 @@ RUN = []
 end
 end
 
-@test_logs (:warn, r"could not evaluate testset-for iterator.*") retest(Loops1, r"asd")
+@test_logs (:warn, r"could not evaluate testset description.*") retest(Loops1, r"asd")
 @test Loops1.RUN == [1, 0, 2, 0]
 empty!(Loops1.RUN)
 retest(Loops1) # should not log
@@ -415,6 +415,91 @@ retest(LoopCollision, dry=true)
 retest(LoopCollision, dry=false)
 @test LoopCollision.RUN == [sin, cos, (sin, 1), (cos, 1)]
 
+### interpolated description #################################################
+
+module Interpolate
+using ReTest
+
+RUN = []
+X = 0
+
+@testset "a $X" verbose=true begin
+    @testset "b $X" begin
+        @test true
+        push!(RUN, 1)
+    end
+    @testset "c $X $i" for i=2:3
+        @test true
+        push!(RUN, i)
+    end
+end
+
+@testset "d $X $i" verbose=true for i=4:4
+    @test true
+    push!(RUN, i)
+    @testset "e $X" begin
+        @test true
+        push!(RUN, 5)
+    end
+end
+end # Interpolate
+
+retest(Interpolate, dry=true)
+retest(Interpolate)
+@test Interpolate.RUN == 1:5
+empty!(Interpolate.RUN)
+
+retest(Interpolate, "0")
+@test Interpolate.RUN == 1:5
+empty!(Interpolate.RUN)
+
+retest(Interpolate, "4")
+@test Interpolate.RUN == 4:5
+
+module InterpolateImpossible
+using ReTest
+
+RUN = []
+X = 0
+
+@testset "a $X" verbose=true begin
+    j = 9
+    @testset "b $X $j" begin
+        @test true
+        push!(RUN, 1)
+    end
+    @testset "c $X $j $i" for i=2:3
+        @test true
+        push!(RUN, i)
+    end
+end
+
+@testset "d $X $i" verbose=true for i=4:4
+    @test true
+    push!(RUN, i)
+    @testset "e $X $i" begin
+        @test true
+        push!(RUN, 5)
+    end
+    @testset "$i" begin # must work even "$i" is made only of Expr (no String parts)
+        @test true
+        push!(RUN, 6)
+    end
+end
+end # InterpolateImpossible
+
+retest(InterpolateImpossible, dry=true)
+retest(InterpolateImpossible)
+@test InterpolateImpossible.RUN == 1:6
+empty!(InterpolateImpossible.RUN)
+
+retest(InterpolateImpossible, "0")
+@test InterpolateImpossible.RUN == 1:6
+empty!(InterpolateImpossible.RUN)
+
+retest(InterpolateImpossible, "4") # should have a warning or two
+@test InterpolateImpossible.RUN == 4:6
+
 
 ### Failing ##################################################################