add integer ID filtering

src/ReTest.jl

@@ -58,18 +58,46 @@ struct Or <: Pattern
 end
 
 alwaysmatches(pat::And) = all(alwaysmatches, pat.xs)
-alwaysmatches(pat::Or) = any(alwaysmatches, pat.xs)
+
+alwaysmatches(pat::Or) =
+    pat.xs isa AbstractArray{<:Integer} ?
+        false : # special case for huge unit ranges; locally, this optimization seems
+                # unnecessary, i.e. alwaysmatches(Or(1:10...0)) is constant time anyway,
+                # but on CI, the any(...) below takes tooooo long
+        any(alwaysmatches, pat.xs)
+
 alwaysmatches(rx::Regex) = isempty(rx.pattern)
+alwaysmatches(id::Integer) = false
+
+matches(pat::And, x, id) = all(p -> matches(p, x, id), pat.xs)
+
+matches(pat::Or, x, id) =
+    pat.xs isa AbstractArray{<:Integer} ?
+        id ∈ pat.xs : # optimized for unit ranges
+        any(p -> matches(p, x, id), pat.xs)
 
-matches(pat::And, x) = all(p -> matches(p, x), pat.xs)
-matches(pat::Or, x) = any(p -> matches(p, x), pat.xs)
-matches(rx::Regex, x) = occursin(rx, x)
+matches(rx::Regex, x, _) = occursin(rx, x)
+matches(pat::Integer, _, id) = pat == id
 
 make_pattern(rx::Regex) = rx
+make_pattern(id::Integer) = id
 make_pattern(str::AbstractString) = VERSION >= v"1.3" ? r""i * str :
                                                         Regex(str, "i")
 
 make_pattern(pat::AbstractArray) = Or(make_pattern.(pat))
+# special case for optimizing unit-ranges:
+make_pattern(pat::AbstractArray{<:Integer}) = Or(pat)
+
+hasregex(::Regex) = true
+hasregex(::Integer) = false
+hasregex(pat::Union{And,Or}) =
+    pat.xs isa AbstractArray{<:Integer} ?
+        false :
+        any(hasregex, pat.xs)
+
+hasinteger(::Regex) = false
+hasinteger(::Integer) = true
+hasinteger(pat::Union{And,Or}) = any(hasinteger, pat.xs)
 
 
 # * TestsetExpr
@@ -80,6 +108,7 @@ Base.@kwdef mutable struct Options
 end
 
 mutable struct TestsetExpr
+    id::Int64 # unique ID per module (64 bits to be on the safe side)
     source::LineNumberNode
     mod::String # enclosing module
     desc::Union{String,Expr}
@@ -103,7 +132,7 @@ mutable struct TestsetExpr
     body::Expr
 
     TestsetExpr(source, mod, desc, options, loops, parent, children=TestsetExpr[]) =
-        new(source, mod, desc, options, loops, parent, children, String[])
+        new(0, source, mod, desc, options, loops, parent, children, String[])
 end
 
 isfor(ts::TestsetExpr) = ts.loops !== nothing
@@ -190,6 +219,7 @@ end
 # - compute ts.descwidth, to know the overall alignment of the first vertical bar
 #   (only needed when verbose is large enough)
 # - the most important: sorting out which testsets must be run
+#   (and in the process, precompute descriptions when possible, and IDs)
 #
 # Concerning the last point, we make the following compromise: as it's probably
 # rare that a Regex matches a given testset but not its children
@@ -202,14 +232,17 @@ end
 # TODO: implement the alternative to make a real comparison
 
 function resolve!(mod::Module, ts::TestsetExpr, pat::Pattern;
-                  force::Bool=false, shown::Bool=true, depth::Int=0,
-                  verbose::Int)
+                  verbose::Int, id::Int64, # external calls
+                  force::Bool=false, shown::Bool=true, depth::Int=0, # only recursive calls
+                  hasrx=hasregex(pat))
 
     strings = empty!(ts.strings)
     desc = ts.desc
     ts.run = force || alwaysmatches(pat)
     ts.loopvalues = nothing # unnecessary ?
     ts.loopiters = nothing
+    ts.id = id
+    id += 1
 
     parentstrs = ts.parent === nothing ? [""] : ts.parent.strings
     ts.descwidth = 0
@@ -219,7 +252,9 @@ function resolve!(mod::Module, ts::TestsetExpr, pat::Pattern;
     # and ts.run == true
 
     function giveup()
-        ts.run = true
+        ts.run = hasrx ? true : # conservative, we don't really know at this point
+                         matches(pat, "", ts.id)
+
         if shown
             # set ts.descwidth to a lower bound to reduce misalignment
             ts.descwidth = 2*depth + mapreduce(+, desc.args) do part
@@ -239,6 +274,7 @@ function resolve!(mod::Module, ts::TestsetExpr, pat::Pattern;
         # (i.e. the description doesn't depend on loop variables)
         gaveup = false
         if !(desc isa String)
+            # TODO: compute desc only when !ts.run (i.e. it wasn't forced) ?
             try
                 desc = Core.eval(mod, desc)
             catch
@@ -253,7 +289,7 @@ function resolve!(mod::Module, ts::TestsetExpr, pat::Pattern;
             for str in parentstrs
                 ts.run && break
                 new = str * '/' * desc
-                if matches(pat, new)
+                if matches(pat, new, ts.id)
                     ts.run = true
                 else
                     push!(strings, new)
@@ -307,7 +343,7 @@ function resolve!(mod::Module, ts::TestsetExpr, pat::Pattern;
             end
             for str in parentstrs
                 new = str * '/' * descx
-                if matches(pat, new)
+                if matches(pat, new, ts.id)
                     ts.run = true
                     break
                 else
@@ -319,10 +355,12 @@ function resolve!(mod::Module, ts::TestsetExpr, pat::Pattern;
 
     run = ts.run
     ts.hasbrokenrec = ts.hasbroken
+
     for tsc in ts.children
-        run |= resolve!(mod, tsc, pat, force=ts.run,
-                        shown=shown & ts.options.transient_verbose,
-                        depth=depth+1, verbose=verbose-1)
+        runc, id = resolve!(mod, tsc, pat, force=ts.run,
+                            shown=shown & ts.options.transient_verbose,
+                            depth=depth+1, verbose=verbose-1, id=id)
+        run |= runc
         ts.descwidth = max(ts.descwidth, tsc.descwidth)
         if tsc.run
             ts.hasbrokenrec |= tsc.hasbrokenrec
@@ -332,6 +370,7 @@ function resolve!(mod::Module, ts::TestsetExpr, pat::Pattern;
         ts.descwidth = 0
     end
     ts.run = run
+    run, id
 end
 
 eval_desc(mod, ts, x) =
@@ -361,7 +400,7 @@ function make_ts(ts::TestsetExpr, pat::Pattern, stats, chan)
 
     if ts.loops === nothing
         quote
-            @testset $(ts.mod) $(isfinal(ts)) $pat $(ts.desc) $(ts.options) $stats $chan $body
+            @testset $(ts.mod) $(isfinal(ts)) $pat $(ts.id) $(ts.desc) $(ts.options) $stats $chan $body
         end
     else
         c = count(x -> x === nothing, (ts.loopvalues, ts.loopiters))
@@ -372,7 +411,7 @@ function make_ts(ts::TestsetExpr, pat::Pattern, stats, chan)
             loops = ts.loops
         end
         quote
-            @testset $(ts.mod) $(isfinal(ts)) $pat $(ts.desc) $(ts.options) $stats $chan $loops $body
+            @testset $(ts.mod) $(isfinal(ts)) $pat $(ts.id) $(ts.desc) $(ts.options) $stats $chan $loops $body
         end
     end
 end
@@ -408,9 +447,9 @@ end
 revise_pkgid() = Base.PkgId(Base.UUID("295af30f-e4ad-537b-8983-00126c2a3abe"), "Revise")
 
 """
-    retest(m::Module..., pattern...; dry::Bool=false, stats::Bool=false,
-                                     shuffle::Bool=false, verbose::Real=true,
-                                     recursive::Bool=true)
+    retest(m::Module..., pattern...; dry::Bool=false,
+                                     stats::Bool=false, verbose::Real=true, [id::Bool],
+                                     shuffle::Bool=false, recursive::Bool=true)
 
 Run all the tests declared in `@testset` blocks, within modules `m` if specified,
 or within all currently loaded modules otherwise.
@@ -420,12 +459,14 @@ or within all currently loaded modules otherwise.
 * If `dry` is `true`, don't actually run the tests, just print the descriptions
   of the testsets which would (presumably) run.
 * If `stats` is `true`, print some time/memory statistics for each testset.
-* If `shuffle` is `true`, shuffle the order in which top-level testsets within
-  a given module are run, as well as the list of passed modules.
 * If specified, `verbose` must be an integer or `Inf` indicating the nesting level
   of testsets whose results must be printed (this is equivalent to adding the
   `verbose=true` annotation to corresponding testsets); the default behavior
   (`true` or `1`) corresponds to printing the result of top-level testsets.
+* If `id` is `true`, a unique (per module) integer ID is printed next to each testset,
+  which can be used for filtering. The default value of `id` depends on other options.
+* If `shuffle` is `true`, shuffle the order in which top-level testsets within
+  a given module are run, as well as the list of passed modules.
 * If `recursive` is `true`, the tests for all the recursive submodules of
   the passed modules `m` are also run.
 
@@ -437,8 +478,9 @@ Moreover if a testset is run, its enclosing testset, if any, also has to run
 (although not necessarily exhaustively, i.e. other nested testsets
 might be filtered out).
 
-A `pattern` can be a string, a `Regex`, or an array.
+A `pattern` can be a string, a `Regex`, an integer or an array.
 For a testset to "match" an array, it must match at least one of its elements (disjunction).
+To match an integer, its ID must be equal to this integer (cf. the `id` keyword).
 
 ### `Regex` filtering
 
@@ -470,20 +512,22 @@ the regex is simply created as `Regex(pattern, "i")`).
     this function executes each (top-level) `@testset` block using `eval` *within* the
     module in which it was written (e.g. `m`, when specified).
 """
-function retest(args::Union{Module,AbstractString,Regex,AbstractArray}...;
+function retest(args::Union{Module,AbstractString,Regex,Integer,AbstractArray}...;
                 dry::Bool=false,
                 stats::Bool=false,
                 shuffle::Bool=false,
                 group::Bool=true,
                 verbose::Real=true, # should be @nospecialize, but not supported on old Julia
                 recursive::Bool=true,
+                id=nothing,
                 )
 
     implicitmodules, modules, pat, verbose = process_args(args, verbose, shuffle, recursive)
     overall = length(modules) > 1
-    root = Testset.ReTestSet("", "Overall", true)
+    root = Testset.ReTestSet("", "Overall", overall=true)
 
-    tests_descs_hasbrokens = fetchtests.(modules, Ref(pat), verbose, overall)
+    maxidw = Ref{Int}(0) # visual width for showing IDs (Ref for mutability in hack below)
+    tests_descs_hasbrokens = fetchtests.(modules, Ref(pat), verbose, overall, Ref(maxidw))
     isempty(tests_descs_hasbrokens) &&
         throw(ArgumentError("no modules using ReTest could be found and none were passed"))
 
@@ -503,6 +547,9 @@ function retest(args::Union{Module,AbstractString,Regex,AbstractArray}...;
         return
     end
 
+    id = something(id, dry | hasinteger(pat))
+    maxidw[] = id ? maxidw[] : 0
+
     for imod in eachindex(modules)
         mod = modules[imod]
         tests = alltests[imod]
@@ -518,7 +565,8 @@ function retest(args::Union{Module,AbstractString,Regex,AbstractArray}...;
                     println()
                 printstyled(mod, '\n', bold=true)
             end
-            foreach(ts -> dryrun(mod, ts, pat, showmod*2, verbose=verbose>0), tests)
+            foreach(ts -> dryrun(mod, ts, pat, id ? 0 : showmod*2,
+                                 verbose=verbose>0, maxidw = id ? maxidw[] : 0), tests)
             continue
         end
 
@@ -556,7 +604,7 @@ function retest(args::Union{Module,AbstractString,Regex,AbstractArray}...;
         exception = Ref{Exception}()
         interrupted = Threads.Atomic{Bool}(false)
 
-        module_ts = Testset.ReTestSet("", string(mod) * ':', true)
+        module_ts = Testset.ReTestSet("", string(mod) * ':', overall=true)
         push!(root.results, module_ts)
 
         many = length(tests) > 1 || isfor(tests[1]) # FIXME: isfor when only one iteration
@@ -675,7 +723,8 @@ function retest(args::Union{Module,AbstractString,Regex,AbstractArray}...;
                         module_ts.description = chop(module_ts.description, tail=1)
                         clear_line()
                         Testset.print_test_results(module_ts, format,
-                                                   bold=true, hasbroken=hasbroken)
+                                                   bold=true, hasbroken=hasbroken,
+                                                   maxidw=maxidw[])
                     else
                         nothing
                     end
@@ -714,7 +763,8 @@ function retest(args::Union{Module,AbstractString,Regex,AbstractArray}...;
                                 rts, format;
                                 depth = Int(!rts.overall & isindented(verbose, overall, many)),
                                 bold = rts.overall | !many,
-                                hasbroken=hasbroken
+                                hasbroken=hasbroken,
+                                maxidw=maxidw[]
                             )
                         end
                         if rts.anynonpass
@@ -850,7 +900,8 @@ function retest(args::Union{Module,AbstractString,Regex,AbstractArray}...;
             println()
     end
     nmodules > 1 && !dry &&
-        Testset.print_test_results(root, format, bold=true, hasbroken=hasbroken)
+        Testset.print_test_results(root, format, bold=true,
+                                   hasbroken=hasbroken, maxidw=maxidw[])
     nothing
 end
 
@@ -948,17 +999,19 @@ function computemodules!(modules::Vector{Module}, shuffle, recursive)
     modules
 end
 
-function fetchtests(mod, pat, verbose, overall)
+function fetchtests(mod, pat, verbose, overall, maxidw)
     tests = updatetests!(mod)
     descwidth = 0
     hasbroken = false
 
+    id = 1
     for ts in tests
-        run = resolve!(mod, ts, pat, verbose=verbose)
+        run, id = resolve!(mod, ts, pat, verbose=verbose, id=id)
         run || continue
         descwidth = max(descwidth, ts.descwidth)
         hasbroken |= ts.hasbrokenrec
     end
+    maxidw[] = max(maxidw[], ndigits(id-1))
 
     tests = filter(ts -> ts.run, tests)
     many = length(tests) > 1
@@ -974,7 +1027,7 @@ end
 isindented(verbose, overall, many) = (verbose > 0) & (overall | !many)
 
 function dryrun(mod::Module, ts::TestsetExpr, pat::Pattern, align::Int=0, parentsubj=""
-                ; evaldesc=true, repeated=nothing, verbose)
+                ; evaldesc=true, repeated=nothing, verbose, maxidw::Int)
     ts.run && verbose || return
     desc = ts.desc
 
@@ -990,9 +1043,13 @@ function dryrun(mod::Module, ts::TestsetExpr, pat::Pattern, align::Int=0, parent
         if parentsubj isa String && desc isa String
             subject = parentsubj * '/' * desc
             if isfinal(ts)
-                matches(pat, subject) || return
+                matches(pat, subject, ts.id) || return
             end
         end
+
+        if maxidw > 0 # width (ndigits) of max id; <= 0 means ids not printed
+            printstyled(lpad(ts.id, maxidw), "| ", color = :light_black, bold=true)
+        end
         printstyled(' '^align, desc, color = desc isa String ? :normal : Base.warn_color())
 
         if repeated !== nothing
@@ -1003,16 +1060,18 @@ function dryrun(mod::Module, ts::TestsetExpr, pat::Pattern, align::Int=0, parent
             println()
         end
         for tsc in ts.children
-            dryrun(mod, tsc, pat, align + 2, subject, verbose=ts.options.transient_verbose)
+            dryrun(mod, tsc, pat, align + 2, subject, verbose=ts.options.transient_verbose,
+                   maxidw=maxidw)
         end
     else
         function dryrun_beginend(descx, repeated=nothing)
             # avoid repeating ourselves, transform this iteration into a "begin/end" testset
             beginend = TestsetExpr(ts.source, ts.mod, descx, ts.options, nothing,
                                    ts.parent, ts.children)
             beginend.run = true
+            beginend.id = ts.id
             dryrun(mod, beginend, pat, align, parentsubj; evaldesc=false,
-                   repeated=repeated, verbose=verbose)
+                   repeated=repeated, verbose=verbose, maxidw=maxidw)
         end
 
         loopvalues = ts.loopvalues