Flaky tests env var (#259)

* Initial flaky test case export

* Export BITRISE_FLAKY_TEST_CASES env var

* Update Exporter mock for tests

* Test BITRISE_FLAKY_TEST_CASES env var

* Add more tests for flaky test case export

* Add BITRISE_FLAKY_TEST_CASES step output

* Update README

* Update simulator version for test_test_plan e2e test

* Code cleanup and skipp test_test_plan on Xcode 26

* Add test bundle name to the exported flaky test cases

* Fix yaml lint issues

* Fix more yamllint issues

* Fix e2e test run_if

* Limit BITRISE_FLAKY_TEST_CASES env var value

Author: godrei

README.md

@@ -124,6 +124,7 @@ Run tests with custom xcconfig file path:
 | `BITRISE_XCODE_TEST_ATTACHMENTS_PATH` | This is the path of the test attachments zip. |
 | `BITRISE_XCODEBUILD_BUILD_LOG_PATH` | If `single_build` is set to false, the step runs `xcodebuild build` before the test, and exports the raw xcodebuild log. |
 | `BITRISE_XCODEBUILD_TEST_LOG_PATH` | The step exports the `xcodebuild test` command output log. |
+| `BITRISE_FLAKY_TEST_CASES` | A test case is considered flaky if it has failed at least once, but passed at least once as well.  The list contains the test cases in the following format: ``` - TestTarget_1.TestClass_1.TestMethod_1 - TestTarget_1.TestClass_1.TestMethod_2 - TestTarget_1.TestClass_2.TestMethod_1 - TestTarget_2.TestClass_1.TestMethod_1 ... ``` |
 </details>
 
 ## 🙋 Contributing

e2e/bitrise.yml

@@ -36,13 +36,27 @@ workflows:
 
   # Requires Xcode 11+
   test_test_plan:
+    description: |-
+        This test fails on Xcode 26 with: 'BullsEye (33951) encountered an error (Early unexpected exit, operation never finished bootstrapping - no restart will be attempted. (Underlying Error: Crash: BullsEye (33951) <external symbol>))' error.
+        Example build: https://app.bitrise.io/build/c71f9c3a-e164-423f-8779-d4c8db69827a
+    before_run:
+    - _expose_xcode_version
+    steps:
+    - bitrise-run:
+        run_if: |-
+          {{ getenv "XCODE_MAJOR_VERSION" | ne "26" }}
+        inputs:
+        - workflow_id: utility_test_test_plan
+        - bitrise_config_path: ./e2e/bitrise.yml
+
+  utility_test_test_plan:
     envs:
     - TEST_APP_URL: https://github.com/bitrise-io/sample-swift-project-with-parallel-ui-test.git
     - TEST_APP_BRANCH: master
     - BITRISE_PROJECT_PATH: BullsEye.xcworkspace
     - BITRISE_SCHEME: BullsEye
     - TEST_PLAN: UnitTests
-    - DESTINATION: platform=iOS Simulator,name=iPhone 12,OS=latest
+    - DESTINATION: platform=iOS Simulator,name=Bitrise iOS default,OS=latest
     - LOG_FORMATTER: xcbeautify
     - EXPECT_TEST_FAILURE: "false"
     - CACHE_LEVEL: swift_packages
@@ -263,3 +277,25 @@ workflows:
                 echo "BITRISE_CACHE_INCLUDE_PATHS is empty"
                 exit 1
             fi
+
+  _expose_xcode_version:
+    steps:
+    - script:
+        title: Expose Xcode major version
+        inputs:
+        - content: |-
+            #!/bin/env bash
+            set -e
+            if [[ ! -z "$XCODE_MAJOR_VERSION" ]]; then
+              echo "Xcode major version already exposed: $XCODE_MAJOR_VERSION"
+              exit 0
+            fi
+            version=`xcodebuild -version`
+            regex="Xcode ([0-9]*)."
+            if [[ ! $version =~ $regex ]]; then
+              echo "Failed to determine Xcode major version"
+              exit 1
+            fi
+            xcode_major_version=${BASH_REMATCH[1]}
+            echo "Xcode major version: $xcode_major_version"
+            envman add --key XCODE_MAJOR_VERSION --value $xcode_major_version

go.mod

@@ -10,6 +10,7 @@ require (
 	github.com/bitrise-io/go-utils v1.0.13
 	github.com/bitrise-io/go-utils/v2 v2.0.0-alpha.23
 	github.com/bitrise-io/go-xcode/v2 v2.0.0-alpha.53
+	github.com/bitrise-steplib/steps-deploy-to-bitrise-io v0.0.0-20250707145051-550b658b019b
 	github.com/hashicorp/go-version v1.7.0
 	github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51
 	github.com/stretchr/testify v1.10.0
@@ -18,9 +19,9 @@ require (
 require (
 	github.com/bitrise-io/envman v0.0.0-20240730123632-8066eeb61599 // indirect
 	github.com/bitrise-io/go-steputils v1.0.6 // indirect
+	github.com/bitrise-io/go-xcode v1.1.2 // indirect
 	github.com/bitrise-io/stepman v0.0.0-20240828074035-6ae1a5f5efde // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
-	github.com/google/go-cmp v0.6.0 // indirect
 	github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
 	github.com/hashicorp/go-retryablehttp v0.7.7 // indirect
 	github.com/kr/pretty v0.3.1 // indirect
@@ -34,4 +35,5 @@ require (
 	golang.org/x/text v0.21.0 // indirect
 	gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
+	howett.net/plist v1.0.1 // indirect
 )

go.sum

@@ -11,10 +11,14 @@ github.com/bitrise-io/go-utils v1.0.13 h1:1QENhTS/JlKH9F7+/nB+TtbTcor6jGrE6cQ4CJ
 github.com/bitrise-io/go-utils v1.0.13/go.mod h1:ZY1DI+fEpZuFpO9szgDeICM4QbqoWVt0RSY3tRI1heY=
 github.com/bitrise-io/go-utils/v2 v2.0.0-alpha.23 h1:Dfh4nyZPuEtilBisidejqxBrkx9cWvbOUrpq8VEION0=
 github.com/bitrise-io/go-utils/v2 v2.0.0-alpha.23/go.mod h1:3XUplo0dOWc3DqT2XA2SeHToDSg7+j1y1HTHibT2H68=
+github.com/bitrise-io/go-xcode v1.1.2 h1:7tit56lNPPzH714pBt083z/iZh7P44vgVsNrEdvkYG4=
+github.com/bitrise-io/go-xcode v1.1.2/go.mod h1:9OwsvrhZ4A2JxHVoEY7CPcABAKA+OE7FQqFfBfvbFuY=
 github.com/bitrise-io/go-xcode/v2 v2.0.0-alpha.53 h1:Ud59kqcvRulyEx8WsM+DpYlubRWhG4zAGE2ORHIcQYo=
 github.com/bitrise-io/go-xcode/v2 v2.0.0-alpha.53/go.mod h1:T4rhWQljdgH5As4Dq/RQWuazdScY0YB7uZAMuBUnxeY=
 github.com/bitrise-io/stepman v0.0.0-20240828074035-6ae1a5f5efde h1:LinFhZG5OdayDh1T1JO8QANsNwQWzqORZ0A9EGHQ0ps=
 github.com/bitrise-io/stepman v0.0.0-20240828074035-6ae1a5f5efde/go.mod h1:Lq9nEqKerBD35w3eSU8lf83F7uZPkXfmRSZEUDJN40w=
+github.com/bitrise-steplib/steps-deploy-to-bitrise-io v0.0.0-20250707145051-550b658b019b h1:/Gd2kLTzdcnmPyhoirrAPE66Dt4JV2Kh8iFRw+vcuGk=
+github.com/bitrise-steplib/steps-deploy-to-bitrise-io v0.0.0-20250707145051-550b658b019b/go.mod h1:p7uINAQkv4SXH7Mgrd/NDGYWZ65c/BT7GIqAfpCWvfQ=
 github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
@@ -34,6 +38,7 @@ github.com/hashicorp/go-retryablehttp v0.7.7 h1:C8hUCYzor8PIfXHa4UrZkU4VvK8o9ISH
 github.com/hashicorp/go-retryablehttp v0.7.7/go.mod h1:pkQpWZeYWskR+D1tR2O5OcBFOxfA7DoAO6xtkuQnHTk=
 github.com/hashicorp/go-version v1.7.0 h1:5tqGy27NaOTB8yJKUZELlFAS/LTKJkrmONwQKeRZfjY=
 github.com/hashicorp/go-version v1.7.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA=
+github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=
 github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 h1:Z9n2FFNUXsshfwJMBgNA0RU6/i7WVaAegv3PtuIHPMs=
 github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8=
 github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
@@ -85,9 +90,12 @@ golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGm
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
+gopkg.in/yaml.v1 v1.0.0-20140924161607-9f9df34309c0/go.mod h1:WDnlLJ4WF5VGsH/HVa3CI79GS0ol3YnhVnKP89i0kNg=
 gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
 gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+howett.net/plist v1.0.1 h1:37GdZ8tP09Q35o9ych3ehygcsL+HqKSwzctveSlarvM=
+howett.net/plist v1.0.1/go.mod h1:lqaXoTrLY4hg8tnEzNru53gicrbv7rrk+2xJA/7hw9g=