Surface test error if running testitem directly (#178)

* Surface test error if running testitem directly

So scripts doing this would return non-zero exit code

* Add test

* Update test/testfiles/_direct_testitem.jl

Author: nickrobinson251

src/ReTestItems.jl

@@ -919,7 +919,7 @@ end
 # when `runtestitem` called directly or `@testitem` called outside of `runtests`.
 function runtestitem(
     ti::TestItem, ctx::TestContext;
-    test_end_expr::Union{Nothing,Expr}=nothing, logs::Symbol=:eager, verbose_results::Bool=true, finish_test::Bool=true,
+    test_end_expr::Union{Nothing,Expr}=nothing, logs::Symbol=:eager, verbose_results::Bool=true, finish_test::Bool=true, catch_test_error::Bool=true,
 )
     if should_skip(ti)::Bool
         return skiptestitem(ti, ctx; verbose_results)
@@ -1017,10 +1017,16 @@ function runtestitem(
         ts1 = Test.pop_testset()
         task_local_storage()[:__TESTITEM_ACTIVE__] = prev
         @assert ts1 === ts
-        try
-            finish_test && Test.finish(ts) # This will throw an exception if any of the tests failed.
-        catch e
-            e isa TestSetException || rethrow()
+        if finish_test
+            if catch_test_error
+                try
+                    Test.finish(ts) # This will throw an exception if any of the tests failed.
+                catch e
+                    e isa TestSetException || rethrow()
+                end
+            else
+                Test.finish(ts)
+            end
         end
     end
     @debugv 1 "Test item $(repr(name)) done$(_on_worker())."

src/macros.jl

@@ -320,7 +320,7 @@ macro testitem(nm, exs...)
                 $store_test_item($ti)
                 $ti
             else # We are not in a `runtests` call, so we run the testitem immediately.
-                $runtestitem($ti)
+                $runtestitem($ti; catch_test_error=false)
                 $nothing
             end
         end

test/integrationtests.jl

@@ -1280,4 +1280,12 @@ end
     end
 end
 
+# see https://github.com/JuliaTesting/ReTestItems.jl/issues/177
+@testset "error code from running `@testitem` directly" begin
+    filename = joinpath(TEST_FILES_DIR, "_direct_testitem.jl")
+    cmd = `$(Base.julia_cmd()) --project $filename`
+    p = run(pipeline(ignorestatus(cmd); stdout, stderr), wait=true)
+    @test !success(p)
+end
+
 end # integrationtests.jl testset

test/testfiles/_direct_testitem.jl

@@ -0,0 +1,7 @@
+# This file is incorrect usage of ReTestItems
+# but we want to check that a failed `@testitem` used directly in a script
+# means the script exits with a non-zero exit code.
+using ReTestItems
+@testitem "should fail" begin
+    @test 1 == 2
+end