diff --git a/.appveyor.yml b/.appveyor.yml deleted file mode 100644 index accf97e..0000000 --- a/.appveyor.yml +++ /dev/null @@ -1,32 +0,0 @@ -# Documentation: https://github.com/JuliaCI/Appveyor.jl -environment: - matrix: - - julia_version: 1.0 - - julia_version: 1.3 - - julia_version: nightly -platform: - - x86 - - x64 -matrix: - allow_failures: - - julia_version: nightly -branches: - only: - - master - - /release-.*/ -notifications: - - provider: Email - on_build_success: false - on_build_failure: false - on_build_status_changed: true -install: - - ps: iex ((new-object net.webclient).DownloadString("https://raw.githubusercontent.com/JuliaCI/Appveyor.jl/version-1/bin/install.ps1")) -build_script: - - echo "%JL_BUILD_SCRIPT%" - - C:\julia\bin\julia -e "%JL_BUILD_SCRIPT%" -test_script: - - echo "%JL_TEST_SCRIPT%" - - C:\julia\bin\julia -e "%JL_TEST_SCRIPT%" -on_success: - - echo "%JL_CODECOV_SCRIPT%" - - C:\julia\bin\julia -e "%JL_CODECOV_SCRIPT%" diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml new file mode 100644 index 0000000..a0fa2f7 --- /dev/null +++ b/.github/workflows/CI.yml @@ -0,0 +1,47 @@ +name: CI +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true +on: + push: + paths-ignore: + - 'README.md' + branches: + - main + - master + pull_request: + paths-ignore: + - 'README.md' + branches: + - main + - master +jobs: + test: + name: Julia ${{ matrix.version }} - ${{ matrix.os }} - ${{ matrix.arch }} + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + version: + - '1' + - '1.6' + - 'nightly' + os: + - ubuntu-latest + arch: + - x64 + - x86 + steps: + - uses: actions/checkout@v3 + - uses: julia-actions/setup-julia@v1 + with: + version: ${{ matrix.version }} + arch: ${{ matrix.arch }} + - uses: julia-actions/cache@v1 + - uses: julia-actions/julia-buildpkg@v1 + - uses: julia-actions/julia-runtest@v1 + - uses: julia-actions/julia-processcoverage@v1 + - name: Upload coverage reports to Codecov + uses: codecov/codecov-action@v3 + env: + CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} diff --git a/.github/workflows/CompatHelper.yml b/.github/workflows/CompatHelper.yml new file mode 100644 index 0000000..7a9c79f --- /dev/null +++ b/.github/workflows/CompatHelper.yml @@ -0,0 +1,44 @@ +name: CompatHelper +on: + schedule: + - cron: 0 0 * * * + workflow_dispatch: +permissions: + contents: write + pull-requests: write +jobs: + CompatHelper: + runs-on: ubuntu-latest + steps: + - name: Check if Julia is already available in the PATH + id: julia_in_path + run: which julia + continue-on-error: true + - name: Install Julia, but only if it is not already available in the PATH + uses: julia-actions/setup-julia@v1 + with: + version: '1' + arch: ${{ runner.arch }} + if: steps.julia_in_path.outcome != 'success' + - name: "Add the General registry via Git" + run: | + import Pkg + ENV["JULIA_PKG_SERVER"] = "" + Pkg.Registry.add("General") + shell: julia --color=yes {0} + - name: "Install CompatHelper" + run: | + import Pkg + name = "CompatHelper" + uuid = "aa819f21-2bde-4658-8897-bab36330d9b7" + version = "3" + Pkg.add(; name, uuid, version) + shell: julia --color=yes {0} + - name: "Run CompatHelper" + run: | + import CompatHelper + CompatHelper.main() + shell: julia --color=yes {0} + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + COMPATHELPER_PRIV: ${{ secrets.DOCUMENTER_KEY }} diff --git a/.github/workflows/documenter.yml b/.github/workflows/documenter.yml new file mode 100644 index 0000000..65fe019 --- /dev/null +++ b/.github/workflows/documenter.yml @@ -0,0 +1,22 @@ +name: Documenter +on: +# push: +# branches: [main, master] +# tags: [v*] +# pull_request: +# branches: [main, master] + workflow_dispatch: +jobs: + Documenter: + name: Documentation + runs-on: ubuntu-20.04 + steps: + - uses: actions/checkout@v2 + - uses: julia-actions/setup-julia@v1 + with: + version: 1.6 + - uses: julia-actions/julia-buildpkg@latest + - uses: julia-actions/julia-docdeploy@latest + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + DOCUMENTER_KEY: ${{ secrets.DOCUMENTER_KEY }} # when run by tagbot diff --git a/.gitignore b/.gitignore index f02b970..7255c05 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,4 @@ *.jl.*.cov *.jl.mem coverage +Manifest.toml diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 73d197f..0000000 --- a/.travis.yml +++ /dev/null @@ -1,17 +0,0 @@ -# Documentation: http://docs.travis-ci.com/user/languages/julia/ -language: julia -os: - - linux - - osx -julia: - - 1.0 - - 1.5 - - nightly -matrix: - allow_failures: - - julia: nightly - fast_finish: true -notifications: - email: true -after_success: - - julia -e 'using Pkg; Pkg.add("Coverage"); using Coverage; Codecov.submit(process_folder())' diff --git a/Manifest.toml b/Manifest.toml deleted file mode 100644 index d5a7f2d..0000000 --- a/Manifest.toml +++ /dev/null @@ -1,100 +0,0 @@ -# This file is machine-generated - editing it directly is not advised - -[[Base64]] -uuid = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f" - -[[Compat]] -deps = ["Base64", "Dates", "DelimitedFiles", "Distributed", "InteractiveUtils", "LibGit2", "Libdl", "LinearAlgebra", "Markdown", "Mmap", "Pkg", "Printf", "REPL", "Random", "Serialization", "SharedArrays", "Sockets", "SparseArrays", "Statistics", "Test", "UUIDs", "Unicode"] -git-tree-sha1 = "ed2c4abadf84c53d9e58510b5fc48912c2336fbb" -uuid = "34da2185-b29b-5c13-b0c7-acf172513d20" -version = "2.2.0" - -[[Dates]] -deps = ["Printf"] -uuid = "ade2ca70-3891-5945-98fb-dc099432e06a" - -[[DeepDiffs]] -deps = ["Compat"] -git-tree-sha1 = "cd4648e98d5625c75b9ebbec516533ec1250ce57" -uuid = "ab62b9b5-e342-54a8-a765-a90f495de1a6" -version = "1.1.0" - -[[DelimitedFiles]] -deps = ["Mmap"] -uuid = "8bb1440f-4735-579b-a4ab-409b98df4dab" - -[[Distributed]] -deps = ["Random", "Serialization", "Sockets"] -uuid = "8ba89e20-285c-5b6f-9357-94700520ee1b" - -[[InteractiveUtils]] -deps = ["Markdown"] -uuid = "b77e0a4c-d291-57a0-90e8-8db25a27a240" - -[[LibGit2]] -uuid = "76f85450-5226-5b5a-8eaa-529ad045b433" - -[[Libdl]] -uuid = "8f399da3-3557-5675-b5ff-fb832c97cbdb" - -[[LinearAlgebra]] -deps = ["Libdl"] -uuid = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" - -[[Logging]] -uuid = "56ddb016-857b-54e1-b83d-db4d58db5568" - -[[Markdown]] -deps = ["Base64"] -uuid = "d6f4376e-aef5-505a-96c1-9c027394607a" - -[[Mmap]] -uuid = "a63ad114-7e13-5084-954f-fe012c677804" - -[[Pkg]] -deps = ["Dates", "LibGit2", "Markdown", "Printf", "REPL", "Random", "SHA", "UUIDs"] -uuid = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" - -[[Printf]] -deps = ["Unicode"] -uuid = "de0858da-6303-5e67-8744-51eddeeeb8d7" - -[[REPL]] -deps = ["InteractiveUtils", "Markdown", "Sockets"] -uuid = "3fa0cd96-eef1-5676-8a61-b3b8758bbffb" - -[[Random]] -deps = ["Serialization"] -uuid = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" - -[[SHA]] -uuid = "ea8e919c-243c-51af-8825-aaa63cd721ce" - -[[Serialization]] -uuid = "9e88b42a-f829-5b0c-bbe9-9e923198166b" - -[[SharedArrays]] -deps = ["Distributed", "Mmap", "Random", "Serialization"] -uuid = "1a1011a3-84de-559e-8e89-a11a2f7dc383" - -[[Sockets]] -uuid = "6462fe0b-24de-5631-8697-dd941f90decc" - -[[SparseArrays]] -deps = ["LinearAlgebra", "Random"] -uuid = "2f01184e-e22b-5df5-ae63-d93ebab69eaf" - -[[Statistics]] -deps = ["LinearAlgebra", "SparseArrays"] -uuid = "10745b16-79ce-11e8-11f9-7d13ad32a3b2" - -[[Test]] -deps = ["Distributed", "InteractiveUtils", "Logging", "Random"] -uuid = "8dfed614-e22c-5e08-85e1-65c5234f0b40" - -[[UUIDs]] -deps = ["Random", "SHA"] -uuid = "cf7118a7-6976-5b1a-9a39-7adc72f591a4" - -[[Unicode]] -uuid = "4ec0a83e-493e-50e2-b9ac-8f72acf5a8f5" diff --git a/Project.toml b/Project.toml index 27740f3..42b461f 100644 --- a/Project.toml +++ b/Project.toml @@ -1,7 +1,7 @@ name = "TestSetExtensions" uuid = "98d24dd4-01ad-11ea-1b02-c9a08f80db04" -authors = [] -version = "2.0.0" +authors = ["Spencer Russell", "Phillip Alday"] +version = "3.0.0" [deps] DeepDiffs = "ab62b9b5-e342-54a8-a765-a90f495de1a6" @@ -9,12 +9,20 @@ Distributed = "8ba89e20-285c-5b6f-9357-94700520ee1b" Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" [compat] -julia = "1" +Aqua = "0.8" DeepDiffs = "1" +Distributed = "<0.0.1, 1" +Logging = "<0.0.1, 1" +MetaTesting = "=0.1.0" +Suppressor = "0.2" +Test = "<0.0.1, 1" +julia = "1.6" [extras] +Aqua = "4c88cf16-eb10-579e-8560-4a9242c79595" Logging = "56ddb016-857b-54e1-b83d-db4d58db5568" +MetaTesting = "9e32d19f-1e4f-477a-8631-b16c78aa0f56" Suppressor = "fd094767-a336-5f1f-9728-57cf17d0bbfb" [targets] -test = ["Suppressor", "Logging"] +test = ["Suppressor", "Logging", "MetaTesting", "Aqua"] diff --git a/README.md b/README.md index 4bafbf6..4c1cbc8 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,7 @@ # TestSetExtensions -[![Build Status](https://travis-ci.org/ssfrr/TestSetExtensions.jl.svg?branch=master)](https://travis-ci.org/ssfrr/TestSetExtensions.jl) -[![Build status](https://ci.appveyor.com/api/projects/status/79m2ru7o3upt86ds/branch/master?svg=true)](https://ci.appveyor.com/project/ssfrr/testsetextensions-jl/branch/master) -[![codecov.io](http://codecov.io/github/ssfrr/TestSetExtensions.jl/coverage.svg?branch=master)](http://codecov.io/github/ssfrr/TestSetExtensions.jl?branch=master) +[![CI](https://github.com/ssfrr/TestSetExtensions.jl/actions/workflows/CI.yml/badge.svg)](https://github.com/ssfrr/TestSetExtensions.jl/actions/workflows/CI.yml) +[![codecov](https://codecov.io/gh/ssfrr/TestSetExtensions.jl/graph/badge.svg?token=OFJ613ESY8)](https://codecov.io/gh/ssfrr/TestSetExtensions.jl) ![TestSetExtensions example gif](http://ssfrr.github.io/TestSetExtensions.jl/ExtendedTestSet.gif) @@ -31,27 +30,3 @@ end ### Diff output example ![After diff output](http://ssfrr.github.io/TestSetExtensions.jl/diff_after.png) - -## `@includetests` -TestSetExtensions also provides a `@includetests` macro that makes it easy to selectively run your tests, for cases when your full test suite is large and you only need to run a subset of your tests to test a feature you're working on. The macro takes a list of test files, so you can pass it `ARGS` to allow the user to specify which tests to run from the command line. - -```julia -using Compat.Test -using TestSetExtensions - -@testset "All the tests" begin - @includetests ARGS -end -``` - -If the user doesn't provide any command-line arguments, this will look for any `*.jl` files in the same directory as the running file (usually `runtests.jl`) and `include` them. The user can also specify a list of test files: - -``` -$ julia test/runtests.jl footests bartests -``` - -Which will run `footests.jl` and `bartests.jl`. - -`@includetests` will print out each test module name as it goes (here in combination with `ExtendedTestSet`): - -![includetests output](http://ssfrr.github.io/TestSetExtensions.jl/includetests.png) diff --git a/src/TestSetExtensions.jl b/src/TestSetExtensions.jl index 61f6e65..ed4769e 100644 --- a/src/TestSetExtensions.jl +++ b/src/TestSetExtensions.jl @@ -3,45 +3,7 @@ module TestSetExtensions using Distributed, Test, DeepDiffs using Test: AbstractTestSet, DefaultTestSet using Test: Result, Fail, Error, Pass -export ExtendedTestSet, @includetests - -""" -Includes the given test files, given as a list without their ".jl" extensions. -If none are given it will scan the directory of the calling file and include all -the julia files. -""" -macro includetests(testarg...) - if length(testarg) == 0 - tests = [] - elseif length(testarg) == 1 - tests = testarg[1] - else - error("@includetests takes zero or one argument") - end - - rootfile = "$(__source__.file)" - mod = __module__ - - quote - tests = $tests - rootfile = $rootfile - - if length(tests) == 0 - tests = readdir(dirname(rootfile)) - tests = filter(f->endswith(f, ".jl") && f!= basename(rootfile), tests) - else - tests = map(f->string(f, ".jl"), tests) - end - - println(); - - for test in tests - print(splitext(test)[1], ": ") - Base.include($mod, test) - println() - end - end -end +export ExtendedTestSet struct ExtendedTestSet{T<:AbstractTestSet} <: AbstractTestSet wrapped::T @@ -78,15 +40,20 @@ function Test.record(ts::ExtendedTestSet{T}, res::Fail) where {T} elseif test_expr.args[2].head === :call && test_expr.args[3].head === :call && test_expr.args[2].args[1].head === :curly && test_expr.args[3].args[1].head === :curly deepdiff(Base.eval(test_expr.args[2].args), Base.eval(test_expr.args[3].args)) + elseif test_expr.args[2].head === :vcat && test_expr.args[3].head === :vcat + # matrices + deepdiff(test_expr.args[2].args, test_expr.args[3].args) end - - if ! isa(dd, DeepDiffs.SimpleDiff) + + # note there is an implicit `else nothing` branch to the if-block above + if !isa(dd, DeepDiffs.SimpleDiff) && dd !== nothing + # SimpleDiff has no pretty printing # The test was an comparison between things we can diff, # so display the diff printstyled("Test Failed\n"; bold = true, color = Base.error_color()) println(" Expression: ", res.orig_expr) printstyled("\nDiff:\n"; color = Base.info_color()) - display(dd) + show(dd) println() else # fallback to the default printing if we don't have a pretty diff @@ -101,7 +68,11 @@ function Test.record(ts::ExtendedTestSet{T}, res::Fail) where {T} print(res) end - Base.show_backtrace(stdout, Test.scrub_backtrace(backtrace())) + @static if VERSION < v"1.10" + Base.show_backtrace(stdout, Test.scrub_backtrace(backtrace())) + else + Base.show_backtrace(stdout, Test.scrub_backtrace(backtrace(), ts.wrapped.file, Test.extract_file(res.source))) + end # show_backtrace doesn't print a trailing newline println("\n=====================================================") end diff --git a/test/diff.jl b/test/diff.jl new file mode 100644 index 0000000..bda70ef --- /dev/null +++ b/test/diff.jl @@ -0,0 +1,60 @@ +@testset EncasedTestSet "wrapper" begin + global output + output = @capture_out begin + try + @testset ExtendedTestSet "vectors" begin + @test [3, 5, 6, 1, 6, 8] == [3, 5, 6, 1, 9, 8] + end + catch + end + end +end +@test contains(output, "Diff:\n[3, 5, 6, 1, (-)6, (+)9, 8]") + +@testset EncasedTestSet "wrapper" begin + global output + output = @capture_out begin + try + @testset ExtendedTestSet "strings" begin + @test """Lorem ipsum dolor sit amet, + consectetur adipiscing elit, sed do + eiusmod tempor incididunt ut + labore et dolore magna aliqua. + Ut enim ad minim veniam, quis nostrud + exercitation ullamco aboris.""" == + """Lorem ipsum dolor sit amet, + consectetur adipiscing elit, sed do + eiusmod temper incididunt ut + labore et dolore magna aliqua. + Ut enim ad minim veniam, quis nostrud + exercitation ullamco aboris.""" + end + catch + end + end +end +@test contains(output, """Diff: +\"\"\" + Lorem ipsum dolor sit amet, + consectetur adipiscing elit, sed do +- eiusmod tempor incididunt ut ++ eiusmod temper incididunt ut + labore et dolore magna aliqua. + Ut enim ad minim veniam, quis nostrud + exercitation ullamco aboris.\"\"\"""") + +@testset EncasedTestSet "wrapper" begin + global output + output = @capture_out begin + try + @testset ExtendedTestSet "dicts" begin + @test Dict(:foo => "bar", :baz => [1, 4, 5], :biz => nothing) == + Dict(:baz => [1, 7, 5], :biz => 42) + end + catch + end + end +end + +@test contains(output, + """Diff:\n[Dict{Symbol, Any}, (-):biz => nothing, (-):baz => [1, 4, 5], (-):foo => "bar", (+):biz => 42, (+):baz => [1, 7, 5]]""") diff --git a/test/no_diff.jl b/test/no_diff.jl new file mode 100644 index 0000000..30327e8 --- /dev/null +++ b/test/no_diff.jl @@ -0,0 +1,53 @@ +@testset EncasedTestSet "wrapper" begin + global output + output = @capture_out begin + try + @testset ExtendedTestSet "higher dimensional arrays" begin + @test [1 2; 3 4; 5 6] == [1 4; 3 4; 5 6] + end + catch + end + end +end +# XXX this should be updated when DeepDiffs.jl uses semmicolon instead of comma for matrix diff +@test contains(output, "Diff:\n[(-)1 2, (+)1 4, 3 4, 5 6]") + +@testset EncasedTestSet "wrapper" begin + global output + output = @capture_out begin + try + @testset ExtendedTestSet "inequality" begin + @test 1 > 2 + end + catch + end + end +end +@test !contains(output, "Diff:") + + +@testset EncasedTestSet "wrapper" begin + global output + output = @capture_out begin + try + @testset ExtendedTestSet "noncomparisons" begin + @test iseven(7) + end + catch + end + end +end +@test !contains(output, "Diff:") + +@testset EncasedTestSet "wrapper" begin + global output + output = @capture_out begin + try + @testset ExtendedTestSet "errors" begin + throw(ErrorException("This test is supposed to throw an error")) + end + catch + end + end +end +@test !contains(output, "Diff:") diff --git a/test/progress.jl b/test/progress.jl new file mode 100644 index 0000000..54396c5 --- /dev/null +++ b/test/progress.jl @@ -0,0 +1,19 @@ +@testset EncasedTestSet "wrapper" begin + global output + output = @capture_out begin + @testset ExtendedTestSet "top-level tests" begin + @testset "2nd-level tests 1" begin + @test true + @test 1 == 1 + end + @testset "2nd-level tests 2" begin + @test true + @test 1 == 1 + end + end + end +end + +@testset "check output dots" begin + @test split(output, '\n')[1] == "...." +end diff --git a/test/runtests.jl b/test/runtests.jl index ab872b5..1a1475d 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -1,74 +1,23 @@ -using Test, TestSetExtensions +using Test +using TestSetExtensions +using Aqua +using MetaTesting using Logging using Suppressor -output = @capture_out begin - @testset ExtendedTestSet "top-level tests" begin - @testset "2nd-level tests 1" begin - @test true - @test 1 == 1 - end - @testset "2nd-level tests 2" begin - @test true - @test 1 == 1 - end - end -end - -try - @info "You should see 3 failing tests with pretty diffs..." - include(joinpath("..", "diffdemo.jl")) -catch -end -try - @info "These 4 failing tests don't have pretty diffs to display" - @testset ExtendedTestSet "not-pretty" begin - @testset "No pretty diff for matrices" begin - @test [1 2; 3 4] == [1 4; 3 4] - end - @testset "don't diff non-equality" begin - @test 1 > 2 - end - @testset "don't diff non-comparisons" begin - @test iseven(7) - end - @testset "errors don't have diffs either" begin - throw(ErrorException("This test is supposed to throw an error")) - end - end -catch -end +using MetaTesting: EncasedTestSet @testset ExtendedTestSet "TextSetExtensions Tests" begin - @testset "check output dots" begin - @test split(output, '\n')[1] == "...." + @testset "Aqua" begin + Aqua.test_all(TestSetExtensions) end - - @testset "Auto-run test files" begin - global file1_run = false - global file2_run = false - global file3_run = false - - @includetests - - @test file1_run - @test file2_run - @test file3_run + @testset "progress" begin + include("progress.jl") end - - @testset "run selected test files" begin - global file1_run = false - global file2_run = false - global file3_run = false - - @includetests ["file1", "file3"] - - @test file1_run - @test !file2_run - @test file3_run + @testset "diff" begin + include("diff.jl") end - - @testset "more than one arg to @includetests is an error" begin - @test_throws LoadError macroexpand(@__MODULE__, :(@includetests one two)) + @testset "no diff" begin + include("no_diff.jl") end end