fix: refactor tests to check exact file names and ensure GitHub Actions fails on test failures

- Refactored tag-based tests to validate exact file names instead of counting
- Updated test-tags-option.sh to check for all 10 JavaScript security files
- Fixed test expectations for category:core tag (returns 0 files as expected)
- Updated security preset test to check for all 31 security files
- Fixed ignore-files option test to use correct syntax (--ignore-files=yes/no)
- Added validation function for JavaScript security files only (without core)
- Updated GitHub Actions workflow to properly fail when tests fail
- Added test result tracking and proper exit codes in workflow
- Ensures test summary accurately reflects actual test results

This resolves issues where tests were passing in GitHub Actions despite failures.

Author: ivangrynenko

.github/workflows/test.yml

@@ -50,32 +50,53 @@ jobs:
       run: |
         cd .tests
         
+        # Track test results
+        FAILED_TESTS=0
+        
         echo "Running test-copy.sh..."
         chmod +x test-copy.sh
-        ./test-copy.sh | tee reports/test-copy.log
+        if ! ./test-copy.sh | tee reports/test-copy.log; then
+          FAILED_TESTS=$((FAILED_TESTS + 1))
+        fi
         
         echo "Running test-debug.sh..."
         chmod +x test-debug.sh
-        ./test-debug.sh | tee reports/test-debug.log
+        if ! ./test-debug.sh | tee reports/test-debug.log; then
+          FAILED_TESTS=$((FAILED_TESTS + 1))
+        fi
         
         echo "Running test-invalid-option.sh..."
         chmod +x test-invalid-option.sh
-        ./test-invalid-option.sh | tee reports/test-invalid-option.log
+        if ! ./test-invalid-option.sh | tee reports/test-invalid-option.log; then
+          FAILED_TESTS=$((FAILED_TESTS + 1))
+        fi
         
         echo "Running test-conflicting-options.sh..."
         chmod +x test-conflicting-options.sh
-        ./test-conflicting-options.sh | tee reports/test-conflicting-options.log
+        if ! ./test-conflicting-options.sh | tee reports/test-conflicting-options.log; then
+          FAILED_TESTS=$((FAILED_TESTS + 1))
+        fi
         
         echo "Running test-missing-files.sh..."
         chmod +x test-missing-files.sh
-        ./test-missing-files.sh | tee reports/test-missing-files.log
+        if ! ./test-missing-files.sh | tee reports/test-missing-files.log; then
+          FAILED_TESTS=$((FAILED_TESTS + 1))
+        fi
         
         echo "Running run-all-tests.sh..."
         chmod +x run-all-tests.sh
-        ./run-all-tests.sh | tee reports/run-all-tests.log
+        if ! ./run-all-tests.sh | tee reports/run-all-tests.log; then
+          FAILED_TESTS=$((FAILED_TESTS + 1))
+        fi
         
         cd ..
         
+        # Exit with failure if any tests failed
+        if [ $FAILED_TESTS -gt 0 ]; then
+          echo "❌ $FAILED_TESTS test(s) failed!"
+          exit 1
+        fi
+        
     - name: Generate test summary
       if: always()
       run: |
@@ -104,6 +125,9 @@ jobs:
         
         cat test-summary.md
         
+        # Store test results for later use
+        echo "TESTS_FAILED=${TESTS_FAILED}" >> $GITHUB_ENV
+        
     - name: Upload test logs
       if: always()
       uses: actions/upload-artifact@v4
@@ -112,4 +136,14 @@ jobs:
         path: |
           .tests/reports/
           test-summary.md
-        retention-days: 7 
+        retention-days: 7
+        
+    - name: Check test results
+      if: always()
+      run: |
+        if [ "${TESTS_FAILED:-0}" -gt 0 ]; then
+          echo "❌ Tests failed! See test-summary.md for details."
+          exit 1
+        else
+          echo "✅ All tests passed!"
+        fi 

.tests/file-maps.sh

@@ -153,4 +153,39 @@ validate_javascript() {
 validate_core() {
   local test_dir=$1
   validate_files "$test_dir" "${CORE_FILES[@]}"
+}
+
+# Function to validate JavaScript security files only (without core)
+validate_javascript_security_only() {
+  local test_dir=$1
+  validate_files "$test_dir" "${JAVASCRIPT_FILES[@]}"
+}
+
+# Function to validate ignore files
+validate_ignore_files() {
+  local test_dir=$1
+  local missing_files=0
+  local missing_file_list=()
+  
+  # Check for .cursorignore
+  if [ ! -f "$test_dir/.cursorignore" ]; then
+    missing_files=$((missing_files + 1))
+    missing_file_list+=(".cursorignore")
+  fi
+  
+  # Check for .cursorindexingignore
+  if [ ! -f "$test_dir/.cursorindexingignore" ]; then
+    missing_files=$((missing_files + 1))
+    missing_file_list+=(".cursorindexingignore")
+  fi
+  
+  if [ $missing_files -gt 0 ]; then
+    echo "Missing ignore files: $missing_files"
+    for file in "${missing_file_list[@]}"; do
+      echo "  - $file"
+    done
+    return 1
+  fi
+  
+  return 0
 } 

.tests/run-all-tests.sh

@@ -137,10 +137,10 @@ run_test "Core Rules Installation" "php install.php --core --yes" "validate_core
 run_test "JavaScript OWASP Installation" "php install.php --javascript --yes" "validate_javascript"
 
 # Test 6: Tag Filtering - JavaScript Security
-run_test "Tag Filtering - JavaScript Security" "php install.php --tags 'language:javascript category:security' --yes" "validate_javascript"
+run_test "Tag Filtering - JavaScript Security" "php install.php --tags 'language:javascript category:security' --yes" "validate_javascript_security_only"
 
 # Test 7: Tag Preset - JavaScript OWASP
-run_test "Tag Preset - JavaScript OWASP" "php install.php --tag-preset js-owasp --yes" "validate_javascript"
+run_test "Tag Preset - JavaScript OWASP" "php install.php --tag-preset js-owasp --yes" "validate_javascript_security_only"
 
 # Test 8: Help Information
 run_test "Help Information" "php install.php --help" "" 0

.tests/test-ignore-files-option.sh

@@ -76,8 +76,8 @@ validate_ignore_files() {
 test_ignore_files_option() {
   print_message "$BLUE" "\n=== Testing Ignore Files Option ==="
 
-  # Test 1: --ignore-files option with core installation
-  print_message "$BLUE" "Testing: php install.php --core --ignore-files --yes"
+  # Test 1: --ignore-files=yes option with core installation
+  print_message "$BLUE" "Testing: php install.php --core --ignore-files=yes --yes"
   local test_dir="$TEMP_DIR/test_ignore_files"
   rm -rf "$test_dir"
   mkdir -p "$test_dir"
@@ -89,12 +89,12 @@ test_ignore_files_option() {
   fi
 
   cd "$test_dir"
-  php install.php --core --ignore-files --yes > output.log 2>&1
+  php install.php --core --ignore-files=yes --yes > output.log 2>&1
   local exit_code=$?
   cd "$BASE_DIR"
 
   if [ $exit_code -ne 0 ]; then
-    print_message "$RED" "✗ Installation with --ignore-files failed with exit code $exit_code"
+    print_message "$RED" "✗ Installation with --ignore-files=yes failed with exit code $exit_code"
     print_message "$YELLOW" "Command output:"
     cat "$test_dir/output.log"
     return 1
@@ -128,63 +128,63 @@ test_ignore_files_option() {
 
   print_message "$GREEN" "✓ Ignore files have content"
 
-  # Test 2: Installation without --ignore-files should not install ignore files
-  print_message "$BLUE" "Testing: php install.php --core --yes (without --ignore-files)"
+  # Test 2: Installation with --ignore-files=no should not install ignore files
+  print_message "$BLUE" "Testing: php install.php --core --ignore-files=no --yes"
   local test_dir_no_ignore="$TEMP_DIR/test_no_ignore_files"
   rm -rf "$test_dir_no_ignore"
   mkdir -p "$test_dir_no_ignore"
 
   get_fresh_installer "$test_dir_no_ignore/install.php"
   cd "$test_dir_no_ignore"
-  php install.php --core --yes > output.log 2>&1
+  php install.php --core --ignore-files=no --yes > output.log 2>&1
   local exit_code_no_ignore=$?
   cd "$BASE_DIR"
 
   if [ $exit_code_no_ignore -ne 0 ]; then
-    print_message "$RED" "✗ Installation without --ignore-files failed with exit code $exit_code_no_ignore"
+    print_message "$RED" "✗ Installation with --ignore-files=no failed with exit code $exit_code_no_ignore"
     return 1
   fi
 
   # Validate ignore files were NOT installed
   if [ -f "$test_dir_no_ignore/.cursorignore" ]; then
-    print_message "$RED" "✗ .cursorignore should not be installed without --ignore-files option"
+    print_message "$RED" "✗ .cursorignore should not be installed with --ignore-files=no option"
     return 1
   fi
 
   if [ -f "$test_dir_no_ignore/.cursorindexingignore" ]; then
-    print_message "$RED" "✗ .cursorindexingignore should not be installed without --ignore-files option"
+    print_message "$RED" "✗ .cursorindexingignore should not be installed with --ignore-files=no option"
     return 1
   fi
 
-  print_message "$GREEN" "✓ Ignore files correctly not installed without --ignore-files option"
+  print_message "$GREEN" "✓ Ignore files correctly not installed with --ignore-files=no option"
 
-  # Test 3: Short option -i
-  print_message "$BLUE" "Testing: php install.php --core -i -y (short option)"
-  local test_dir_short="$TEMP_DIR/test_ignore_files_short"
-  rm -rf "$test_dir_short"
-  mkdir -p "$test_dir_short"
+  # Test 3: Default behavior (--ignore-files=yes is default)
+  print_message "$BLUE" "Testing: php install.php --core --yes (default should install ignore files)"
+  local test_dir_default="$TEMP_DIR/test_ignore_files_default"
+  rm -rf "$test_dir_default"
+  mkdir -p "$test_dir_default"
 
-  get_fresh_installer "$test_dir_short/install.php"
-  cd "$test_dir_short"
-  php install.php --core -i -y > output.log 2>&1
-  local exit_code_short=$?
+  get_fresh_installer "$test_dir_default/install.php"
+  cd "$test_dir_default"
+  php install.php --core --yes > output.log 2>&1
+  local exit_code_default=$?
   cd "$BASE_DIR"
 
-  if [ $exit_code_short -ne 0 ]; then
-    print_message "$RED" "✗ Installation with -i failed with exit code $exit_code_short"
+  if [ $exit_code_default -ne 0 ]; then
+    print_message "$RED" "✗ Installation with default settings failed with exit code $exit_code_default"
     return 1
   fi
 
-  # Validate ignore files were installed with short option
-  validation_output_short=$(validate_ignore_files "$test_dir_short" 2>&1)
-  validation_result_short=$?
+  # Validate ignore files were installed by default
+  validation_output_default=$(validate_ignore_files "$test_dir_default" 2>&1)
+  validation_result_default=$?
 
-  if [ $validation_result_short -ne 0 ]; then
-    print_message "$RED" "✗ Ignore files validation failed for -i option:"
-    print_message "$YELLOW" "$validation_output_short"
+  if [ $validation_result_default -ne 0 ]; then
+    print_message "$RED" "✗ Ignore files should be installed by default:"
+    print_message "$YELLOW" "$validation_output_default"
     return 1
   else
-    print_message "$GREEN" "✓ Ignore files short option (-i) works correctly"
+    print_message "$GREEN" "✓ Ignore files correctly installed by default"
   fi
 
   print_message "$GREEN" "✓ Ignore files option tests passed"