feat: add diff threshold (#201)

Author: vladopajic

.testcoverage.example.yml

@@ -41,11 +41,31 @@ exclude:
     - \.pb\.go$    # excludes all protobuf generated files
     - ^pkg/bar     # exclude package `pkg/bar`
 
-# File name of go-test-coverage breakdown file, which can be used to 
-# analyze coverage difference.
+# If specified, saves the current test coverage breakdown to this file.
+#
+# Typically, this breakdown is generated only for main (base) branches and 
+# stored as an artifact. Later, this file can be used in feature branches 
+# to compare test coverage against the base branch.
 breakdown-file-name: ''
 
 diff:
-  # File name of go-test-coverage breakdown file which will be used to 
-  # report coverage difference.
-  base-breakdown-file-name: ''
+  # Path to the test coverage breakdown file from the base branch.
+  #
+  # This file is usually generated and stored in the main (base) branch,
+  # controled via `breakdown-file-name` property.
+  # When set in a feature branch, it allows the tool to compute and report 
+  # the coverage difference between the current (feature) branch and the base.
+  base-breakdown-file-name: ''
+
+  # Allowed threshold for the test coverage difference (in percentage) 
+  # between the feature branch and the base branch.
+  #
+  # By default, this is disabled (set to nil). Valid values range from 
+  # -100.0 to +100.0.
+  #
+  # Example: 
+  #   If set to 0.5, an error will be reported if the feature branch has 
+  #   less than 0.5% more coverage than the base.
+  #
+  #   If set to -0.5, the check allows up to 0.5% less coverage than the base.
+  threshold: nil

README.md

@@ -125,14 +125,34 @@ exclude:
     - \.pb\.go$    # excludes all protobuf generated files
     - ^pkg/bar     # exclude package `pkg/bar`
 
-# File name of go-test-coverage breakdown file, which can be used to 
-# analyze coverage difference.
+# If specified, saves the current test coverage breakdown to this file.
+#
+# Typically, this breakdown is generated only for main (base) branches and 
+# stored as an artifact. Later, this file can be used in feature branches 
+# to compare test coverage against the base branch.
 breakdown-file-name: ''
 
 diff:
-  # File name of go-test-coverage breakdown file which will be used to 
-  # report coverage difference.
+  # Path to the test coverage breakdown file from the base branch.
+  #
+  # This file is usually generated and stored in the main (base) branch,
+  # controled via `breakdown-file-name` property.
+  # When set in a feature branch, it allows the tool to compute and report 
+  # the coverage difference between the current (feature) branch and the base.
   base-breakdown-file-name: ''
+
+  # Allowed threshold for the test coverage difference (in percentage) 
+  # between the feature branch and the base branch.
+  #
+  # By default, this is disabled (set to nil). Valid values range from 
+  # -100.0 to +100.0.
+  #
+  # Example: 
+  #   If set to 0.5, an error will be reported if the feature branch has 
+  #   less than 0.5% more coverage than the base.
+  #
+  #   If set to -0.5, the check allows up to 0.5% less coverage than the base.
+  threshold: nil
 ```
 
 ### Exclude Code from Coverage

pkg/testcoverage/check.go

@@ -105,6 +105,7 @@ func Analyze(cfg Config, current, base []coverage.Stats) AnalyzeResult {
 
 	return AnalyzeResult{
 		Threshold:           thr,
+		DiffThreshold:       cfg.Diff.Threshold,
 		HasFileOverrides:    hasFileOverrides,
 		HasPackageOverrides: hasPackageOverrides,
 		FilesBelowThreshold: checkCoverageStatsBelowThreshold(current, thr.File, overrideRules),
@@ -115,6 +116,7 @@ func Analyze(cfg Config, current, base []coverage.Stats) AnalyzeResult {
 		TotalStats:              coverage.StatsCalcTotal(current),
 		HasBaseBreakdown:        len(base) > 0,
 		Diff:                    calculateStatsDiff(current, base),
+		DiffPercentage:          TotalPercentageDiff(current, base),
 	}
 }
 

pkg/testcoverage/check_test.go

@@ -267,6 +267,109 @@ func TestCheck(t *testing.T) {
 	})
 }
 
+func TestCheckDiff(t *testing.T) {
+	t.Parallel()
+
+	if testing.Short() {
+		return
+	}
+
+	brakedownFile := t.TempDir() + "/breakdown.testcoverage"
+	brakedownCurrentFile := t.TempDir() + "/breakdown-current.testcoverage"
+	brakedownFileEdited := "breakdown-edit.testcoverage"
+
+	// run check to generate brakedown file
+	cfg := Config{
+		Profile:           profileOK,
+		BreakdownFileName: brakedownFile,
+		SourceDir:         sourceDir,
+	}
+	buf := &bytes.Buffer{}
+	pass, err := Check(buf, cfg)
+	assert.True(t, pass)
+	assert.NoError(t, err)
+
+	// should pass since brakedown is the same
+	cfg = Config{
+		Profile:   profileOK,
+		SourceDir: sourceDir,
+		Diff: Diff{
+			BaseBreakdownFileName: brakedownFile,
+			Threshold:             ptr(0.0),
+		},
+	}
+	buf = &bytes.Buffer{}
+	pass, err = Check(buf, cfg)
+	assert.True(t, pass)
+	assert.NoError(t, err)
+	assertDiffNoChange(t, buf.String())
+	assertDiffPercentage(t, buf.String(), 0.0)
+	assertDiffThreshold(t, buf.String(), *cfg.Diff.Threshold, true)
+
+	// should pass since diff is negative
+	cfg = Config{
+		Profile:   profileOK,
+		SourceDir: sourceDir,
+		Diff: Diff{
+			BaseBreakdownFileName: brakedownFile,
+			Threshold:             ptr(-0.001),
+		},
+	}
+	buf = &bytes.Buffer{}
+	pass, err = Check(buf, cfg)
+	assert.True(t, pass)
+	assert.NoError(t, err)
+	assertDiffNoChange(t, buf.String())
+	assertDiffPercentage(t, buf.String(), 0.0)
+	assertDiffThreshold(t, buf.String(), *cfg.Diff.Threshold, true)
+
+	// should NOT pass since brakedown is the same, and diff is positive
+	cfg = Config{
+		Profile:   profileOK,
+		SourceDir: sourceDir,
+		Diff: Diff{
+			BaseBreakdownFileName: brakedownFile,
+			Threshold:             ptr(0.1),
+		},
+	}
+	buf = &bytes.Buffer{}
+	pass, err = Check(buf, cfg)
+	assert.False(t, pass)
+	assert.NoError(t, err)
+	assertDiffNoChange(t, buf.String())
+	assertDiffPercentage(t, buf.String(), 0.0)
+	assertDiffThreshold(t, buf.String(), *cfg.Diff.Threshold, false)
+
+	// change brakedown file to have positive difference
+	base := readStats(t, brakedownFile)
+	base[0].Covered = 0
+	base[1].Covered = 0
+
+	tmpFile, err := os.CreateTemp(t.TempDir(), brakedownFileEdited)
+	assert.NoError(t, err)
+	_, err = tmpFile.Write(coverage.StatsSerialize(base))
+	assert.NoError(t, err)
+
+	// check should now pass since difference has increased
+	cfg = Config{
+		Profile:           profileOK,
+		SourceDir:         sourceDir,
+		BreakdownFileName: brakedownCurrentFile,
+		Diff: Diff{
+			BaseBreakdownFileName: tmpFile.Name(),
+			Threshold:             ptr(1.0),
+		},
+	}
+	buf = &bytes.Buffer{}
+	pass, err = Check(buf, cfg)
+	assert.True(t, pass)
+	assert.NoError(t, err)
+
+	diff := TotalPercentageDiff(readStats(t, brakedownCurrentFile), base)
+	assertDiffPercentage(t, buf.String(), diff)
+	assertDiffThreshold(t, buf.String(), *cfg.Diff.Threshold, true)
+}
+
 //nolint:paralleltest // must not be parallel because it uses env
 func TestCheckNoParallel(t *testing.T) {
 	if testing.Short() {
@@ -450,6 +553,65 @@ func Test_Analyze(t *testing.T) {
 		assert.False(t, result.Pass())
 		assertPrefix(t, result, prefix, true)
 	})
+
+	t.Run("diff stats", func(t *testing.T) {
+		t.Parallel()
+
+		stats := randStats(prefix, 10, 100)
+
+		cfg := Config{}
+		result := Analyze(cfg, stats, stats)
+		assert.Empty(t, result.Diff)
+		assert.True(t, result.Pass())
+		assert.Equal(t, 0.0, result.DiffPercentage) //nolint:testifylint //relax
+	})
+
+	t.Run("diff below threshold", func(t *testing.T) {
+		t.Parallel()
+
+		base := []coverage.Stats{{Name: "foo", Total: 10, Covered: 1}}
+		stats := []coverage.Stats{{Name: "foo", Total: 10, Covered: 8}}
+
+		cfg := Config{
+			Diff: Diff{Threshold: ptr(999.0)},
+		}
+		result := Analyze(cfg, stats, base)
+		assert.NotEmpty(t, result.Diff)
+		assert.False(t, result.Pass())
+		assert.False(t, result.MeetsDiffThreshold())
+		assert.Equal(t, 70.0, result.DiffPercentage) //nolint:testifylint //relax
+	})
+
+	t.Run("diff above threshold", func(t *testing.T) {
+		t.Parallel()
+
+		base := []coverage.Stats{{Name: "foo", Total: 10, Covered: 1}}
+		stats := []coverage.Stats{{Name: "foo", Total: 10, Covered: 8}}
+
+		cfg := Config{
+			Diff: Diff{Threshold: ptr(1.0)},
+		}
+		result := Analyze(cfg, stats, base)
+		assert.NotEmpty(t, result.Diff)
+		assert.True(t, result.Pass())
+		assert.True(t, result.MeetsDiffThreshold())
+		assert.Equal(t, 70.0, result.DiffPercentage) //nolint:testifylint //relax
+	})
+
+	t.Run("diff above threshold (small diff)", func(t *testing.T) {
+		t.Parallel()
+
+		base := []coverage.Stats{{Name: "foo", Total: 10000, Covered: 9999}}
+		stats := []coverage.Stats{{Name: "foo", Total: 10000, Covered: 10000}}
+
+		cfg := Config{
+			Diff: Diff{Threshold: ptr(0.0)},
+		}
+		result := Analyze(cfg, stats, base)
+		assert.True(t, result.Pass())
+		assert.True(t, result.MeetsDiffThreshold())
+		assert.Equal(t, 0.01, result.DiffPercentage) //nolint:testifylint //relax
+	})
 }
 
 func TestLoadBaseCoverageBreakdown(t *testing.T) {

pkg/testcoverage/config.go

@@ -53,7 +53,8 @@ type Exclude struct {
 }
 
 type Diff struct {
-	BaseBreakdownFileName string `yaml:"base-breakdown-file-name"`
+	BaseBreakdownFileName string   `yaml:"base-breakdown-file-name"`
+	Threshold             *float64 `yaml:"threshold,omitempty"`
 }
 
 type Badge struct {

pkg/testcoverage/config_test.go

@@ -240,6 +240,7 @@ func nonZeroConfig() Config {
 		BreakdownFileName: "breakdown.testcoverage",
 		Diff: Diff{
 			BaseBreakdownFileName: "breakdown.testcoverage",
+			Threshold:             ptr(-1.01),
 		},
 		GithubActionOutput: true,
 	}
@@ -261,7 +262,8 @@ exclude:
     - path2
 breakdown-file-name: 'breakdown.testcoverage'
 diff:
-  base-breakdown-file-name: 'breakdown.testcoverage'	
+  base-breakdown-file-name: 'breakdown.testcoverage'
+  threshold: -1.01
 github-action-output: true`
 }
 

pkg/testcoverage/coverage/types.go

@@ -27,7 +27,11 @@ func (s Stats) CoveredPercentage() int {
 }
 
 func (s Stats) CoveredPercentageF() float64 {
-	return coveredPercentageF(s.Total, s.Covered)
+	return coveredPercentageF(s.Total, s.Covered, true)
+}
+
+func (s Stats) CoveredPercentageFNR() float64 {
+	return coveredPercentageF(s.Total, s.Covered, false)
 }
 
 //nolint:mnd // relax
@@ -53,11 +57,11 @@ func StatsSearchMap(stats []Stats) map[string]Stats {
 }
 
 func CoveredPercentage(total, covered int64) int {
-	return int(coveredPercentageF(total, covered))
+	return int(coveredPercentageF(total, covered, true))
 }
 
 //nolint:mnd // relax
-func coveredPercentageF(total, covered int64) float64 {
+func coveredPercentageF(total, covered int64, round bool) float64 {
 	if total == 0 {
 		return 0
 	}
@@ -68,6 +72,10 @@ func coveredPercentageF(total, covered int64) float64 {
 
 	p := float64(covered*100) / float64(total)
 
+	if !round {
+		return p
+	}
+
 	// round to %.1f
 	return float64(int(math.Round(p*10))) / 10
 }

pkg/testcoverage/export_test.go

@@ -17,6 +17,7 @@ var (
 	LoadBaseCoverageBreakdown = loadBaseCoverageBreakdown
 	CompressUncoveredLines    = compressUncoveredLines
 	ReportUncoveredLines      = reportUncoveredLines
+	StatusStr                 = statusStr
 )
 
 type (