hijack: add revise option

Project.toml

@@ -13,10 +13,12 @@ Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
 
 [compat]
 InlineTest = "=0.1.1"
+Revise = "3.1"
 julia = "1"
 
 [extras]
 Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f"
+Revise = "295af30f-e4ad-537b-8983-00126c2a3abe"
 
 [targets]
-test = ["Pkg"]
+test = ["Pkg", "Revise"]

docs/src/index.md

@@ -243,7 +243,8 @@ uses `Test`, without modifications:
 
 - if you have a package `Package`, you can try `ReTest.hijack(Package)`, which
   will define a `PackageTests` module when successful, on which you can call
-  `retest`.
+  `retest`. To have `Revise` track changes to test files, use
+  `ReTest.hijack(Package, revise=true)`.
 - if you have a test file `"testfile.jl"`, try `ReTest.hijack("testfile.jl")`
   (this will define a fresh module like above).
 - `Base` and standard library modules can also be passed to `ReTest.hijack`

src/ReTest.jl

@@ -355,8 +355,7 @@ function updatetests!(mod::Module)
         if idx == length(tests) + 1
             push!(tests, ts)
         else
-            revise = Base.PkgId(Base.UUID("295af30f-e4ad-537b-8983-00126c2a3abe"), "Revise")
-            if !(revise in keys(Base.loaded_modules))
+            if !(revise_pkgid() in keys(Base.loaded_modules))
                 desc = ts.desc isa String ? string('"', ts.desc, '"') : ts.desc
                 source = string(ts.source.file, ':', ts.source.line)
                 @warn "duplicate description for @testset, overwriting: $desc at $source"
@@ -368,6 +367,8 @@ function updatetests!(mod::Module)
     tests
 end
 
+revise_pkgid() = Base.PkgId(Base.UUID("295af30f-e4ad-537b-8983-00126c2a3abe"), "Revise")
+
 """
     retest([m::Module...], pattern = r""; dry::Bool=false, stats::Bool=false,
                                           shuffle::Bool=false, verbose::Real=true,

src/hijack.jl

@@ -1,5 +1,6 @@
 """
-    ReTest.hijack(source, [modname]; parentmodule::Module=Main, lazy=false)
+    ReTest.hijack(source, [modname];
+                  parentmodule::Module=Main, lazy=false, revise::Bool=false)
 
 Given test files defined in `source` using the `Test` package, try to load
 them by replacing `Test` with `ReTest`, wrapping them in a module `modname`
@@ -30,11 +31,21 @@ The `lazy` keyword specifies whether some toplevel expressions should be skipped
   within these toplevel (but possible nested) blocks: `begin`, `let`, `for`, `if`;
 * `:brutal` means toplevel `@test*` macros are removed, as well as toplevel
   `begin`, `let`, `for` or `if` blocks.
+
+The `revise` keyword specifies whether `Revise` should be used to track
+the test files (in particular the testsets). If `true`, `Revise` must
+be loaded beforehand in your Julia session.
+
+!!! compat "Julia 1.5"
+    This function requires at least Julia 1.5.
 """
 function hijack end
 
 function hijack(path::AbstractString, modname=nothing; parentmodule::Module=Main,
-                                                       lazy=false)
+                                                       lazy=false, revise::Bool=false)
+    # do first, to error early if necessary
+    Revise = get_revise(revise)
+
     if modname === nothing
         modname = :TestMod
         i = 1
@@ -45,16 +56,33 @@ function hijack(path::AbstractString, modname=nothing; parentmodule::Module=Main
     end
     modname = Symbol(modname)
 
-    substitue!(x) = substitue_retest!(x, lazy)
-    newmod = @eval parentmodule module $modname
-                                    using ReTest # for files which don't have `using Test`
-                                    include($substitue!, $path)
-                                end
+    newmod = @eval parentmodule module $modname end
+    populate_mod!(newmod, path; lazy=lazy, Revise=Revise)
     newmod
 end
 
+function populate_mod!(mod, path; lazy, Revise)
+    files = Any[path]
+    substitue!(x) = substitue_retest!(x, lazy, files, dirname(path))
+
+    @eval mod begin
+        using ReTest # for files which don't have `using Test`
+        include($substitue!, $path)
+    end
+
+    if Revise !== nothing
+        filepaths = @eval mod begin
+            __revise_mode__ = :eval
+            [$(files...)]
+        end
+        for filepath in filepaths
+            Revise.track(mod, filepath)
+        end
+    end
+end
+
 function hijack(packagemod::Module, modname=nothing; parentmodule::Module=Main,
-                                                     lazy=false)
+                                                     lazy=false, revise::Bool=false)
     packagepath = pathof(packagemod)
     packagepath === nothing && packagemod !== Base &&
         throw(ArgumentError("$packagemod is not a package"))
@@ -64,17 +92,19 @@ function hijack(packagemod::Module, modname=nothing; parentmodule::Module=Main,
     end
 
     if packagemod === Base
-        hijack_base(ChooseTests.BASETESTS, base=modname, lazy=lazy)
+        hijack_base(ChooseTests.BASETESTS,
+                    base=modname, parentmodule=parentmodule, lazy=lazy, revise=revise)
     elseif startswith(packagepath, Sys.STDLIB) # packagemod is an STDLIB
-        hijack_base(string(packagemod), modname, parentmodule=parentmodule, lazy=lazy)
+        hijack_base(string(packagemod), modname,
+                    parentmodule=parentmodule, lazy=lazy, revise=revise)
     else
         path = joinpath(dirname(dirname(packagepath)), "test", "runtests.jl")
-        hijack(path, modname, parentmodule=parentmodule, lazy=lazy)
+        hijack(path, modname, parentmodule=parentmodule, lazy=lazy, revise=revise)
     end
 end
 
-function substitue_retest!(ex, lazy)
-    substitue!(x) = substitue_retest!(x, lazy)
+function substitue_retest!(ex, lazy, files=nothing, root=nothing)
+    substitue!(x) = substitue_retest!(x, lazy, files, root)
 
     if Meta.isexpr(ex, :using)
         for used in ex.args
@@ -88,6 +118,11 @@ function substitue_retest!(ex, lazy)
         if length(ex.args) > 2
             throw(ArgumentError("cannot handle include with two arguments"))
         else
+            if files !== nothing
+                newfile = :(joinpath($root, $(ex.args[2])))
+                push!(files, newfile)
+                root = :(dirname($newfile))
+            end
             insert!(ex.args, 2, substitue!)
         end
     elseif Meta.isexpr(ex, :module)
@@ -129,7 +164,8 @@ function filter_toplevel!(ex, lazy)
 end
 
 """
-    hijack_base(tests, [modname]; parentmodule::Module=Main, lazy=false)
+    hijack_base(tests, [modname];
+                parentmodule::Module=Main, lazy=false, revise::Bool=false)
 
 Similar to `ReTest.hijack`, but specifically for `Base` and stdlib tests.
 `tests` speficies which test files should be loaded, in the exact same format
@@ -151,12 +187,17 @@ module, the one under `BaseTests` or `StdLibTests` (e.g. `somedir` if any,
 or `sometest` otherwise, or `Lib_`). This unique module is then named `modname`,
 and not enclosed within `BaseTests` or `StdLibTests`.
 
-The `lazy` keyword has the same meaning as in [`ReTest.hijack`](@ref).
+The `lazy` and `revise` keywords have the same meaning as in [`ReTest.hijack`](@ref).
 Depending on the value of `lazy`, some test files are skipped when they
 are known to fail.
+
+!!! compat "Julia 1.5"
+    This function requires at least Julia 1.5.
 """
 function hijack_base(tests, modname=nothing; parentmodule::Module=Main, lazy=false,
-                     base=:BaseTests, stdlib=:StdLibTests)
+                     base=:BaseTests, stdlib=:StdLibTests, revise::Bool=false)
+
+    Revise = get_revise(revise)
     if isa(tests, AbstractString)
         tests = split(tests)
     end
@@ -174,8 +215,6 @@ function hijack_base(tests, modname=nothing; parentmodule::Module=Main, lazy=fal
         @eval parentmodule module $modname end
     end
 
-    substitue!(ex) = substitue_retest!(ex, lazy)
-
     for (test, components) in zip(tests, allcomponents)
         if test ∈ ChooseTests.BLACKLIST ||
             lazy == true && test ∈ ChooseTests.BLACKLIST_SOFT ||
@@ -216,12 +255,21 @@ function hijack_base(tests, modname=nothing; parentmodule::Module=Main, lazy=fal
         end
 
         @eval mod begin
-                      using ReTest, Random
-                      include($substitue!, $(ChooseTests.test_path(test)))
-                  end
+            using Random
+        end
+        populate_mod!(mod, ChooseTests.test_path(test), lazy=lazy, Revise=Revise)
     end
 end
 
+get_revise(revise) =
+    if revise
+        Revise = get(Base.loaded_modules, revise_pkgid(), nothing)
+        Revise === nothing &&
+            error("Revise is not loaded")
+        Revise
+    end
+
+
 
 module ChooseTests
 # choosetests.jl requires the Sockets package, which is why it's a ReTest dependency

test/Hijack/Project.toml

@@ -0,0 +1,6 @@
+name = "Hijack"
+uuid = "7dd69a16-8aa1-4e4a-bd48-85a06400df7b"
+version = "0.1.0"
+
+[deps]
+Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"

test/Hijack/src/Hijack.jl

@@ -0,0 +1,4 @@
+module Hijack
+RUN = [ ]
+
+end

test/Hijack/test/included.jl

@@ -0,0 +1,9 @@
+makepath(p) = joinpath("subdir", p)
+
+@testset "included" begin
+    @test true
+end
+
+# test that ReTest can handle non-literal arguments in include
+SUB="sub"
+include(makepath("$SUB.jl"))

test/Hijack/test/runtests.jl

@@ -0,0 +1,9 @@
+using Test, Hijack
+
+@test true
+
+@testset "runtests" begin
+    @test true
+end
+
+include("included.jl")

test/Hijack/test/subdir/sub.jl

@@ -0,0 +1,6 @@
+@test true
+
+@testset "sub" begin
+    @test true
+    push!(Hijack.RUN, 1)
+end

test/runtests.jl

@@ -660,3 +660,48 @@ Pkg.activate("./FakePackage")
 Pkg.develop(PackageSpec(path="../InlineTest"))
 Pkg.develop(PackageSpec(path="..")) # ReTest
 Pkg.test("FakePackage")
+
+### Hijack ###################################################################
+
+Pkg.activate("./Hijack")
+Pkg.develop(PackageSpec(path="..")) # ReTest
+Pkg.test("Hijack")
+
+if VERSION >= v"1.5"
+
+    using Hijack
+    ReTest.hijack(Hijack)
+    retest(HijackTests)
+    @test Hijack.RUN == [1]
+    empty!(Hijack.RUN)
+
+    @test_throws ErrorException ReTest.hijack(Hijack, :HijackTests2, revise=true)
+
+    using Revise
+    ReTest.hijack(Hijack, :HijackTests2, revise=true)
+    retest(HijackTests2)
+    @test Hijack.RUN == [1]
+    empty!(Hijack.RUN)
+
+    cp("./Hijack/test/subdir/sub.jl",
+       "./Hijack/test/subdir/sub.orig.jl", force=true)
+
+    write("./Hijack/test/subdir/sub.jl",
+          """
+@test true
+
+@testset "sub" begin
+    @test true
+    @test true
+    push!(Hijack.RUN, 2)
+end
+""")
+    Revise.revise()
+    try
+        retest(HijackTests2)
+        @test Hijack.RUN == [2]
+    finally
+        mv("./Hijack/test/subdir/sub.orig.jl",
+           "./Hijack/test/subdir/sub.jl", force=true)
+    end
+end