test: verify action verification logic consistency with schema (#261)

Part of #253.

---------

Co-authored-by: Piotr Krzemiński <3110813+krzema12@users.noreply.github.com>

Author: Vampire

src/jvmTest/kotlin/it/krzeminski/githubactionstyping/JsonSchemaValidatorSchemaValidationTest.kt

@@ -1,12 +1,8 @@
 package it.krzeminski.githubactionstyping
 
 import io.github.optimumcode.json.schema.JsonSchema
-import io.kotest.core.spec.style.FunSpec
-import io.kotest.datatest.withData
 import io.kotest.matchers.Matcher
 import io.kotest.matchers.MatcherResult
-import io.kotest.matchers.file.shouldBeADirectory
-import io.kotest.matchers.file.shouldNotBeEmptyDirectory
 import io.kotest.matchers.should
 import io.kotest.matchers.shouldNot
 import io.kotest.mpp.sysprop
@@ -19,66 +15,31 @@ import kotlinx.serialization.json.JsonPrimitive
 import java.io.File
 
 private val schemaFile = File(sysprop("schemaFile")!!)
-private val catalogDir = File(sysprop("catalogDir")!!)
-private val externalDir = File(sysprop("externalDir")!!)
-private val goodDir = File(sysprop("goodDir")!!)
-private val badDir = File(sysprop("badDir")!!)
 
 private lateinit var schema: JsonSchema
 
 /**
  * Validate good and bad typings against the Json Schema Validator schema validator.
  */
-class JsonSchemaValidatorSchemaValidationTest : FunSpec({
-    beforeSpec {
-        catalogDir.shouldBeADirectory()
-        catalogDir.shouldNotBeEmptyDirectory()
-
-        schema = schemaFile
-            .inputStream()
-            .use {
-                JsonSchema.fromDefinition(it.reader().readText())
-            }
-    }
-
-    context("catalog typings") {
-        withData(
-            nameFn = { it.name },
-            catalogDir.listFiles()!!.asSequence()
-        ) {
-            it.shouldBeValid()
-        }
-    }
-
-    if (externalDir.isDirectory) {
-        context("external typings") {
-            withData(
-                nameFn = { it.name },
-                externalDir.listFiles()!!.asSequence()
-            ) {
-                it.shouldBeValid()
-            }
+class JsonSchemaValidatorSchemaValidationTest : UseCaseTest() {
+    init {
+        beforeSpec {
+            schema = schemaFile
+                .inputStream()
+                .use {
+                    JsonSchema.fromDefinition(it.reader().readText())
+                }
         }
     }
 
-    context("valid typings") {
-        withData(
-            nameFn = { it.name },
-            goodDir.listFiles()!!.asSequence()
-        ) {
-            it.shouldBeValid()
-        }
+    override suspend fun testValid(typing: File) {
+        typing.shouldBeValid()
     }
 
-    context("invalid typings") {
-        withData(
-            nameFn = { it.name },
-            badDir.listFiles()!!.asSequence()
-        ) {
-            it.shouldNotBeValid()
-        }
+    override suspend fun testInvalid(typing: File) {
+        typing.shouldNotBeValid()
     }
-})
+}
 
 private fun beValid(): Matcher<File> {
     return Matcher { dataFile ->

src/jvmTest/kotlin/it/krzeminski/githubactionstyping/LogicConsistencyTest.kt

@@ -0,0 +1,42 @@
+package it.krzeminski.githubactionstyping
+
+import io.kotest.assertions.assertSoftly
+import io.kotest.assertions.withClue
+import io.kotest.matchers.should
+import io.kotest.matchers.shouldBe
+import io.kotest.matchers.types.beOfType
+import it.krzeminski.githubactionstyping.parsing.parseTypesManifest
+import it.krzeminski.githubactionstyping.reporting.toPlaintextReport
+import it.krzeminski.githubactionstyping.validation.ItemValidationResult
+import it.krzeminski.githubactionstyping.validation.validate
+import java.io.File
+
+/**
+ * Validate that the action validation logic agrees with the Json Schema.
+ */
+class LogicConsistencyTest : UseCaseTest() {
+    override suspend fun testValid(typing: File) {
+        val validationResult = parseTypesManifest(typing.readText()).validate(typing.toPath().fileName)
+        withClue(validationResult.toPlaintextReport()) {
+            validationResult.overallResult shouldBe ItemValidationResult.Valid
+        }
+    }
+
+    override suspend fun testInvalid(typing: File) {
+        val typesManifest = typing.readText()
+        val expectedValidationError = typesManifest
+            .split("\n# Expected validation error\n")[1]
+            .trimMargin("#")
+            .trim()
+
+        val validationResult = parseTypesManifest(typesManifest).validate(typing.toPath().fileName)
+        assertSoftly {
+            validationResult.overallResult should beOfType<ItemValidationResult.Invalid>()
+            validationResult
+                .toPlaintextReport()
+                .trim()
+                .replace("\u001B", "\\x1B")
+                .shouldBe(expectedValidationError)
+        }
+    }
+}

src/jvmTest/kotlin/it/krzeminski/githubactionstyping/UseCaseTest.kt

@@ -0,0 +1,60 @@
+package it.krzeminski.githubactionstyping
+
+import io.kotest.core.spec.style.FunSpec
+import io.kotest.datatest.withData
+import io.kotest.matchers.file.shouldBeADirectory
+import io.kotest.matchers.file.shouldNotBeEmptyDirectory
+import io.kotest.mpp.sysprop
+import java.io.File
+
+private val catalogDir = File(sysprop("catalogDir")!!)
+private val externalDir = File(sysprop("externalDir")!!)
+private val goodDir = File(sysprop("goodDir")!!)
+private val badDir = File(sysprop("badDir")!!)
+
+abstract class UseCaseTest : FunSpec({
+    beforeSpec {
+        catalogDir.shouldBeADirectory()
+        catalogDir.shouldNotBeEmptyDirectory()
+    }
+}) {
+    init {
+        context("catalog typings") {
+            withData(
+                nameFn = { it.name },
+                ts = catalogDir.listFiles()!!.asSequence(),
+                test = { testValid(it) },
+            )
+        }
+
+        if (externalDir.isDirectory) {
+            context("external typings") {
+                withData(
+                    nameFn = { it.name },
+                    ts = externalDir.listFiles()!!.asSequence(),
+                    test = { testValid(it) },
+                )
+            }
+        }
+
+        context("valid typings") {
+            withData(
+                nameFn = { it.name },
+                ts = goodDir.listFiles()!!.asSequence(),
+                test = { testValid(it) },
+            )
+        }
+
+        context("invalid typings") {
+            withData(
+                nameFn = { it.name },
+                ts = badDir.listFiles()!!.asSequence(),
+                test = { testInvalid(it) },
+            )
+        }
+    }
+
+    abstract suspend fun testValid(typing: File)
+
+    abstract suspend fun testInvalid(typing: File)
+}

src/test/resources/bad-typings/inputs_boolean_list_item_with_allowed_values.yml

@@ -9,3 +9,17 @@ inputs:
       allowed-values:
         - read
         - write
+
+# Expected validation error
+#
+#For action with manifest at 'inputs_boolean_list_item_with_allowed_values.yml':
+#Result:
+#\x1B[31m❌ INVALID: Some typing is invalid.\x1B[0m
+#
+#Inputs:
+#• list-of-boolean:
+#  \x1B[31m❌ INVALID: List item type: 'allowed-values' is not allowed for this type.\x1B[0m
+#
+#Outputs:
+#None.
+#

src/test/resources/bad-typings/inputs_boolean_list_item_with_list_item.yml

@@ -8,3 +8,17 @@ inputs:
       type: boolean
       list-item:
         type: integer
+
+# Expected validation error
+#
+#For action with manifest at 'inputs_boolean_list_item_with_list_item.yml':
+#Result:
+#\x1B[31m❌ INVALID: Some typing is invalid.\x1B[0m
+#
+#Inputs:
+#• list-of-boolean:
+#  \x1B[31m❌ INVALID: List item type: 'list-item' is not allowed for this type.\x1B[0m
+#
+#Outputs:
+#None.
+#

src/test/resources/bad-typings/inputs_boolean_list_item_with_name.yml

@@ -7,3 +7,17 @@ inputs:
     list-item:
       type: boolean
       name: AllowedValues
+
+# Expected validation error
+#
+#For action with manifest at 'inputs_boolean_list_item_with_name.yml':
+#Result:
+#\x1B[31m❌ INVALID: Some typing is invalid.\x1B[0m
+#
+#Inputs:
+#• list-of-boolean:
+#  \x1B[31m❌ INVALID: List item type: 'name' is not allowed for this type.\x1B[0m
+#
+#Outputs:
+#None.
+#

src/test/resources/bad-typings/inputs_boolean_list_item_with_named_values.yml

@@ -8,3 +8,17 @@ inputs:
       type: boolean
       named-values:
         infinite: 0
+
+# Expected validation error
+#
+#For action with manifest at 'inputs_boolean_list_item_with_named_values.yml':
+#Result:
+#\x1B[31m❌ INVALID: Some typing is invalid.\x1B[0m
+#
+#Inputs:
+#• list-of-boolean:
+#  \x1B[31m❌ INVALID: List item type: 'named-values' are currently supported only for integers.\x1B[0m
+#
+#Outputs:
+#None.
+#

src/test/resources/bad-typings/inputs_boolean_list_item_with_separator.yml

@@ -7,3 +7,17 @@ inputs:
     list-item:
       type: boolean
       separator: ','
+
+# Expected validation error
+#
+#For action with manifest at 'inputs_boolean_list_item_with_separator.yml':
+#Result:
+#\x1B[31m❌ INVALID: Some typing is invalid.\x1B[0m
+#
+#Inputs:
+#• list-of-boolean:
+#  \x1B[31m❌ INVALID: List item type: 'separator' is not allowed for this type.\x1B[0m
+#
+#Outputs:
+#None.
+#

src/test/resources/bad-typings/inputs_boolean_with_allowed_values.yml

@@ -6,3 +6,17 @@ inputs:
     allowed-values:
       - read
       - write
+
+# Expected validation error
+#
+#For action with manifest at 'inputs_boolean_with_allowed_values.yml':
+#Result:
+#\x1B[31m❌ INVALID: Some typing is invalid.\x1B[0m
+#
+#Inputs:
+#• verbose:
+#  \x1B[31m❌ INVALID: 'allowed-values' is not allowed for this type.\x1B[0m
+#
+#Outputs:
+#None.
+#

src/test/resources/bad-typings/inputs_boolean_with_list_item.yml

@@ -5,3 +5,17 @@ inputs:
     type: boolean
     list-item:
       type: boolean
+
+# Expected validation error
+#
+#For action with manifest at 'inputs_boolean_with_list_item.yml':
+#Result:
+#\x1B[31m❌ INVALID: Some typing is invalid.\x1B[0m
+#
+#Inputs:
+#• verbose:
+#  \x1B[31m❌ INVALID: 'list-item' is not allowed for this type.\x1B[0m
+#
+#Outputs:
+#None.
+#