diff --git a/README.md b/README.md index 52a1d769..b33e072b 100644 --- a/README.md +++ b/README.md @@ -87,6 +87,29 @@ It provides the following information: - `(20/s)` (attempted) rate, - `avg: 72ns, min: 125ns, max: 27.590042ms` average, min and max iteration times. +### Chart Command + +The `chart` command allows you to plot a chart of the test scenarios that would be triggered over time with the provided run function. It supports various subcommands corresponding to different trigger modes. + +Usage: +``` +f1 chart +``` + +Flags: +- `--chart-start`: Optional start time for the chart (default: current time in RFC3339 format) +- `--chart-duration`: Duration for the chart (default: 10 minutes) +- `--filename`: Filename for optional detailed chart, e.g. `.png` + +The command generates an ASCII graph in the console and optionally creates a detailed PNG chart if a filename is provided. + +Example: +``` +f1 chart constant --chart-duration=5m --filename=constant_chart.png +``` + +This will generate an ASCII graph in the console and save a detailed PNG chart to `constant_chart.png`. + ### Environment variables | Name | Format | Default | Description | diff --git a/internal/chart/chart_cmd.go b/internal/chart/chart_cmd.go index c4b864f6..c511e570 100644 --- a/internal/chart/chart_cmd.go +++ b/internal/chart/chart_cmd.go @@ -70,7 +70,7 @@ func chartCmdExecute( return fmt.Errorf("creating builder: %w", err) } - if trig.DryRun == nil { + if trig.Rate == nil { return fmt.Errorf("%s does not support charting predicted load", cmd.Name()) } @@ -82,7 +82,7 @@ func chartCmdExecute( rates := []float64{0.0} times := []time.Time{current} for ; current.Add(sampleInterval).Before(end); current = current.Add(sampleInterval) { - rate := trig.DryRun(current) + rate := trig.Rate(current) rates = append(rates, float64(rate)) times = append(times, current) } @@ -95,26 +95,31 @@ func chartCmdExecute( return nil } graph := chart.Chart{ - Title: trig.Description, - TitleStyle: chart.StyleTextDefaults(), - Width: 1920, - Height: 1024, + Title: trig.Description, + TitleStyle: chart.Style{ + TextWrap: chart.TextWrapWord, + }, + Width: 1920, + Height: 1024, + Background: chart.Style{ + Padding: chart.Box{ + Top: 50, + }, + }, YAxis: chart.YAxis{ - Name: "Triggered Test Iterations", - NameStyle: chart.StyleTextDefaults(), - Style: chart.StyleTextDefaults(), - AxisType: chart.YAxisSecondary, + Name: "Triggered Test Iterations", + AxisType: chart.YAxisSecondary, + ValueFormatter: chart.IntValueFormatter, }, XAxis: chart.XAxis{ Name: "Time", - NameStyle: chart.StyleTextDefaults(), ValueFormatter: chart.TimeMinuteValueFormatter, - Style: chart.StyleTextDefaults(), }, Series: []chart.Series{ chart.TimeSeries{ Style: chart.Style{ - StrokeColor: chart.GetDefaultColor(0).WithAlpha(64), + StrokeColor: chart.GetDefaultColor(0), + StrokeWidth: 2.0, }, Name: "testing", XValues: times, diff --git a/internal/chart/chart_cmd_stage_test.go b/internal/chart/chart_cmd_stage_test.go index 108d35d9..2cd762c3 100644 --- a/internal/chart/chart_cmd_stage_test.go +++ b/internal/chart/chart_cmd_stage_test.go @@ -1,30 +1,41 @@ package chart_test import ( - "strconv" - "testing" - "time" - "github.com/stretchr/testify/assert" + "os" + "strings" + "testing" "github.com/form3tech-oss/f1/v2/internal/chart" + "github.com/form3tech-oss/f1/v2/internal/log" "github.com/form3tech-oss/f1/v2/internal/trigger" "github.com/form3tech-oss/f1/v2/internal/ui" ) type ChartTestStage struct { - t *testing.T - assert *assert.Assertions - err error - args []string + t *testing.T + assert *assert.Assertions + err error + args []string + output *ui.Output + results *lineStringBuilder + expectedOutput string } func NewChartTestStage(t *testing.T) (*ChartTestStage, *ChartTestStage, *ChartTestStage) { t.Helper() + sb := newLineStringBuilder() + p := ui.Printer{ + Writer: sb, + ErrWriter: sb, + } + stage := &ChartTestStage{ - t: t, - assert: assert.New(t), + t: t, + assert: assert.New(t), + output: ui.NewOutput(log.NewDiscardLogger(), &p, true, true), + results: sb, } return stage, stage, stage } @@ -34,8 +45,7 @@ func (s *ChartTestStage) and() *ChartTestStage { } func (s *ChartTestStage) i_execute_the_chart_command() *ChartTestStage { - outputer := ui.NewDiscardOutput() - cmd := chart.Cmd(trigger.GetBuilders(outputer), outputer) + cmd := chart.Cmd(trigger.GetBuilders(s.output), s.output) cmd.SetArgs(s.args) s.err = cmd.Execute() return s @@ -46,8 +56,16 @@ func (s *ChartTestStage) the_command_is_successful() *ChartTestStage { return s } +func (s *ChartTestStage) the_output_is_correct() *ChartTestStage { + s.assert.Equal(s.expectedOutput, s.results.String()) + return s +} + func (s *ChartTestStage) the_load_style_is_constant() *ChartTestStage { s.args = append(s.args, "constant", "--rate", "10/s", "--distribution", "none") + f, err := os.ReadFile("../testdata/expected-constant-chart-output.txt") + s.assert.NoError(err) + s.expectedOutput = string(f) return s } @@ -56,27 +74,55 @@ func (s *ChartTestStage) jitter_is_applied() *ChartTestStage { return s } -func (s *ChartTestStage) the_load_style_is_staged(stages string) *ChartTestStage { - s.args = append(s.args, "staged", "--stages", stages, "--distribution", "none") +func (s *ChartTestStage) the_load_style_is_staged() *ChartTestStage { + s.args = append(s.args, "staged", "--stages", "5m:100,2m:0,10s:100", "--distribution", "none") + f, err := os.ReadFile("../testdata/expected-staged-chart-output.txt") + s.assert.NoError(err) + s.expectedOutput = string(f) return s } func (s *ChartTestStage) the_load_style_is_ramp() *ChartTestStage { s.args = append(s.args, "ramp", "--start-rate", "0/s", "--end-rate", "10/s", "--ramp-duration", "10s", "--chart-duration", "10s", "--distribution", "none") + f, err := os.ReadFile("../testdata/expected-ramp-chart-output.txt") + s.assert.NoError(err) + s.expectedOutput = string(f) return s } -func (s *ChartTestStage) the_load_style_is_gaussian_with_a_volume_of(volume int) *ChartTestStage { - s.args = append(s.args, "gaussian", "--peak", "5m", "--repeat", "10m", "--volume", strconv.Itoa(volume), "--standard-deviation", "1m", "--distribution", "none") +func (s *ChartTestStage) the_load_style_is_gaussian_with_a_volume_of() *ChartTestStage { + s.args = append(s.args, "gaussian", "--peak", "5m", "--repeat", "10m", "--volume", "100000", "--standard-deviation", "1m", "--distribution", "none") + f, err := os.ReadFile("../testdata/expected-gaussian-chart-output.txt") + s.assert.NoError(err) + s.expectedOutput = string(f) return s } func (s *ChartTestStage) the_chart_starts_at_a_fixed_time() *ChartTestStage { - s.args = append(s.args, "--chart-start", time.Now().Truncate(10*time.Minute).Format(time.RFC3339)) + s.args = append(s.args, "--chart-start", "2024-09-19T17:00:00Z") return s } -func (s *ChartTestStage) the_load_style_is_defined_in_the_config_file(filename string) *ChartTestStage { - s.args = append(s.args, "file", filename, "--chart-duration", "5s") +func (s *ChartTestStage) the_load_style_is_defined_in_the_config_file() *ChartTestStage { + s.args = append(s.args, "file", "../testdata/config-file.yaml", "--chart-duration", "5s") + f, err := os.ReadFile("../testdata/expected-file-chart-output.txt") + s.assert.NoError(err) + s.expectedOutput = string(f) return s } + +type lineStringBuilder struct { + sb *strings.Builder +} + +func newLineStringBuilder() *lineStringBuilder { + return &lineStringBuilder{sb: &strings.Builder{}} +} + +func (l *lineStringBuilder) Write(p []byte) (n int, err error) { + return l.sb.Write(p) +} + +func (l *lineStringBuilder) String() string { + return strings.Replace(l.sb.String(), "\\n", "\n", -1) +} diff --git a/internal/chart/chart_cmd_test.go b/internal/chart/chart_cmd_test.go index 31154f1a..60278f41 100644 --- a/internal/chart/chart_cmd_test.go +++ b/internal/chart/chart_cmd_test.go @@ -32,7 +32,8 @@ func TestChartConstantNoJitter(t *testing.T) { i_execute_the_chart_command() then. - the_command_is_successful() + the_command_is_successful().and(). + the_output_is_correct() } func TestChartStaged(t *testing.T) { @@ -41,13 +42,14 @@ func TestChartStaged(t *testing.T) { given, when, then := NewChartTestStage(t) given. - the_load_style_is_staged("5m:100,2m:0,10s:100") + the_load_style_is_staged() when. i_execute_the_chart_command() then. - the_command_is_successful() + the_command_is_successful().and(). + the_output_is_correct() } func TestChartGaussian(t *testing.T) { @@ -56,14 +58,15 @@ func TestChartGaussian(t *testing.T) { given, when, then := NewChartTestStage(t) given. - the_load_style_is_gaussian_with_a_volume_of(100000).and(). + the_load_style_is_gaussian_with_a_volume_of().and(). the_chart_starts_at_a_fixed_time() when. i_execute_the_chart_command() then. - the_command_is_successful() + the_command_is_successful().and(). + the_output_is_correct() } func TestChartGaussianWithJitter(t *testing.T) { @@ -72,7 +75,7 @@ func TestChartGaussianWithJitter(t *testing.T) { given, when, then := NewChartTestStage(t) given. - the_load_style_is_gaussian_with_a_volume_of(100000).and(). + the_load_style_is_gaussian_with_a_volume_of().and(). jitter_is_applied().and(). the_chart_starts_at_a_fixed_time() @@ -95,7 +98,8 @@ func TestChartRamp(t *testing.T) { i_execute_the_chart_command() then. - the_command_is_successful() + the_command_is_successful().and(). + the_output_is_correct() } func TestChartFileConfig(t *testing.T) { @@ -104,11 +108,12 @@ func TestChartFileConfig(t *testing.T) { given, when, then := NewChartTestStage(t) given. - the_load_style_is_defined_in_the_config_file("../testdata/config-file.yaml") + the_load_style_is_defined_in_the_config_file() when. i_execute_the_chart_command() then. - the_command_is_successful() + the_command_is_successful().and(). + the_output_is_correct() } diff --git a/internal/testdata/expected-constant-chart-output.txt b/internal/testdata/expected-constant-chart-output.txt new file mode 100644 index 00000000..f8ff8408 --- /dev/null +++ b/internal/testdata/expected-constant-chart-output.txt @@ -0,0 +1,16 @@ + 10.00 ┤╭────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────── + 9.33 ┤│ + 8.67 ┤│ + 8.00 ┤│ + 7.33 ┤│ + 6.67 ┤│ + 6.00 ┤│ + 5.33 ┤│ + 4.67 ┤│ + 4.00 ┤│ + 3.33 ┤│ + 2.67 ┤│ + 2.00 ┤│ + 1.33 ┤│ + 0.67 ┤│ + 0.00 ┼╯ diff --git a/internal/testdata/expected-file-chart-output.txt b/internal/testdata/expected-file-chart-output.txt new file mode 100644 index 00000000..2d39018f --- /dev/null +++ b/internal/testdata/expected-file-chart-output.txt @@ -0,0 +1,16 @@ + 10.00 ┤╭────────────────╮ + 9.33 ┤│ │ ╭╮ + 8.67 ┤│ │ ││ + 8.00 ┤│ │ │╰╮ + 7.33 ┤│ │ ╭╯ │ + 6.67 ┤│ │ │ │ + 6.00 ┤│ │ ╭╯ │ + 5.33 ┤│ ╰────────────╮ │ ╰╮ + 4.67 ┤│ │ │ │ + 4.00 ┤│ │ ╭╯ │ + 3.33 ┤│ │ ╭╯ │ + 2.67 ┤│ │ │ │ + 2.00 ┤│ │ │ ╰╮ + 1.33 ┤│ │╭╯ │ ╭───╮ + 0.67 ┤│ ││ │ │ │ + 0.00 ┼╯ ╰╯ ╰───────────╯ ╰────────────────────────────────────────────────────────────────────────────────────────────────────── diff --git a/internal/testdata/expected-gaussian-chart-output.txt b/internal/testdata/expected-gaussian-chart-output.txt new file mode 100644 index 00000000..fb7d75ef --- /dev/null +++ b/internal/testdata/expected-gaussian-chart-output.txt @@ -0,0 +1,16 @@ + 665 ┤ ╭───────╮ + 621 ┤ ╭──╯ ╰──╮ + 576 ┤ ╭──╯ ╰──╮ + 532 ┤ ╭─╯ ╰─╮ + 488 ┤ ╭╯ ╰╮ + 443 ┤ ╭─╯ ╰─╮ + 399 ┤ ╭─╯ ╰─╮ + 355 ┤ ╭─╯ ╰─╮ + 310 ┤ ╭─╯ ╰─╮ + 266 ┤ ╭─╯ ╰─╮ + 222 ┤ ╭─╯ ╰─╮ + 177 ┤ ╭─╯ ╰─╮ + 133 ┤ ╭──╯ ╰──╮ + 89 ┤ ╭───╯ ╰───╮ + 44 ┤ ╭──────╯ ╰──────╮ + 0 ┼───────────────────────────────────────╯ ╰───────────────────────────────────── diff --git a/internal/testdata/expected-ramp-chart-output.txt b/internal/testdata/expected-ramp-chart-output.txt new file mode 100644 index 00000000..aefe1c7a --- /dev/null +++ b/internal/testdata/expected-ramp-chart-output.txt @@ -0,0 +1,16 @@ + 9.00 ┤ ╭────────────── + 8.40 ┤ │ + 7.80 ┤ ╭───────────────╯ + 7.20 ┤ ╭───────────────╯ + 6.60 ┤ │ + 6.00 ┤ ╭───────────────╯ + 5.40 ┤ │ + 4.80 ┤ ╭───────────────╯ + 4.20 ┤ ╭───────────────╯ + 3.60 ┤ │ + 3.00 ┤ ╭───────────────╯ + 2.40 ┤ │ + 1.80 ┤ ╭───────────────╯ + 1.20 ┤ ╭───────────────╯ + 0.60 ┤ │ + 0.00 ┼────────────────╯ diff --git a/internal/testdata/expected-staged-chart-output.txt b/internal/testdata/expected-staged-chart-output.txt new file mode 100644 index 00000000..c0057ffa --- /dev/null +++ b/internal/testdata/expected-staged-chart-output.txt @@ -0,0 +1,16 @@ + 99.00 ┤ ╭────╮ + 92.40 ┤ ╭────╯ ╰╮ + 85.80 ┤ ╭─────╯ ╰──╮ + 79.20 ┤ ╭────╯ ╰─╮ + 72.60 ┤ ╭────╯ ╰─╮ + 66.00 ┤ ╭────╯ ╰─╮ ╭╮ + 59.40 ┤ ╭────╯ ╰─╮ ││ + 52.80 ┤ ╭─────╯ ╰─╮ ││ + 46.20 ┤ ╭────╯ ╰─╮ ││ + 39.60 ┤ ╭────╯ ╰─╮ ││ + 33.00 ┤ ╭─────╯ ╰──╮ ││ + 26.40 ┤ ╭───╯ ╰╮ ╭╯│ + 19.80 ┤ ╭─────╯ ╰──╮ │ │ + 13.20 ┤ ╭─────╯ ╰─╮ │ │ + 6.60 ┤ ╭───╯ ╰─╮│ │ + 0.00 ┼────╯ ╰╯ ╰──────────────────────────────────────────── diff --git a/internal/trigger/api/api.go b/internal/trigger/api/api.go index e87a0862..bd772fc9 100644 --- a/internal/trigger/api/api.go +++ b/internal/trigger/api/api.go @@ -35,7 +35,7 @@ type Constructor func(*pflag.FlagSet) (*Trigger, error) type Trigger struct { Trigger WorkTriggerer - DryRun RateFunction + Rate RateFunction Description string Options Options Duration time.Duration @@ -54,7 +54,8 @@ type Options struct { } type Rates struct { - Rate RateFunction + IterationRate RateFunction IterationDuration time.Duration Duration time.Duration + Rate RateFunction } diff --git a/internal/trigger/constant/constant_rate.go b/internal/trigger/constant/constant_rate.go index dca65abc..7b72c23a 100644 --- a/internal/trigger/constant/constant_rate.go +++ b/internal/trigger/constant/constant_rate.go @@ -47,9 +47,9 @@ func Rate() api.Builder { } return &api.Trigger{ - Trigger: api.NewIterationWorker(rates.IterationDuration, rates.Rate), + Trigger: api.NewIterationWorker(rates.IterationDuration, rates.IterationRate), Description: fmt.Sprintf("%s constant rate, using distribution %s", rateArg, distributionTypeArg), - DryRun: rates.Rate, + Rate: rates.Rate, }, nil }, @@ -72,6 +72,7 @@ func CalculateConstantRate(jitterArg float64, rateArg, distributionTypeArg strin return &api.Rates{ IterationDuration: distributedIterationDuration, - Rate: distributedRateFn, + IterationRate: distributedRateFn, + Rate: rateFn, }, nil } diff --git a/internal/trigger/file/file_parser.go b/internal/trigger/file/file_parser.go index b76a87d6..a4c5feb2 100644 --- a/internal/trigger/file/file_parser.go +++ b/internal/trigger/file/file_parser.go @@ -115,7 +115,7 @@ func (s *Stage) parseStage(stageIdx int, defaults Stage) (*runnableStage, error) return &runnableStage{ StageDuration: *validatedConstantStage.Duration, IterationDuration: rates.IterationDuration, - Rate: rates.Rate, + Rate: rates.IterationRate, Params: *validatedConstantStage.Parameters, }, nil case "ramp": @@ -137,7 +137,7 @@ func (s *Stage) parseStage(stageIdx int, defaults Stage) (*runnableStage, error) return &runnableStage{ StageDuration: *validatedRampStage.Duration, IterationDuration: rates.IterationDuration, - Rate: rates.Rate, + Rate: rates.IterationRate, Params: *validatedRampStage.Parameters, }, nil case "staged": @@ -159,7 +159,7 @@ func (s *Stage) parseStage(stageIdx int, defaults Stage) (*runnableStage, error) return &runnableStage{ StageDuration: *validatedStagedStage.Duration, IterationDuration: rates.IterationDuration, - Rate: rates.Rate, + Rate: rates.IterationRate, Params: *validatedStagedStage.Parameters, }, nil case "gaussian": @@ -179,7 +179,7 @@ func (s *Stage) parseStage(stageIdx int, defaults Stage) (*runnableStage, error) return &runnableStage{ StageDuration: *validatedGaussianStage.Duration, IterationDuration: rates.IterationDuration, - Rate: rates.Rate, + Rate: rates.IterationRate, Params: *validatedGaussianStage.Parameters, }, nil case "users": diff --git a/internal/trigger/file/file_rate.go b/internal/trigger/file/file_rate.go index 46176181..6beb0e8d 100644 --- a/internal/trigger/file/file_rate.go +++ b/internal/trigger/file/file_rate.go @@ -53,7 +53,7 @@ func Rate(output *ui.Output) api.Builder { return &api.Trigger{ Trigger: newStagesWorker(runnableStages.Stages), - DryRun: newDryRun(runnableStages.Stages), + Rate: newDryRun(runnableStages.Stages), Description: fmt.Sprintf("%d different stages", len(runnableStages.Stages)), Duration: runnableStages.stagesTotalDuration, Options: api.Options{ diff --git a/internal/trigger/gaussian/gaussian_rate.go b/internal/trigger/gaussian/gaussian_rate.go index 78fc3f22..a926deec 100644 --- a/internal/trigger/gaussian/gaussian_rate.go +++ b/internal/trigger/gaussian/gaussian_rate.go @@ -137,8 +137,8 @@ func Rate(output *ui.Output) api.Builder { ) return &api.Trigger{ - Trigger: api.NewIterationWorker(rates.IterationDuration, rates.Rate), - DryRun: rates.Rate, + Trigger: api.NewIterationWorker(rates.IterationDuration, rates.IterationRate), + Rate: rates.Rate, Description: description, Duration: rates.Duration, }, @@ -191,8 +191,9 @@ func CalculateGaussianRate( return &api.Rates{ IterationDuration: distributedIterationDuration, - Rate: distributedRateFn, + IterationRate: distributedRateFn, Duration: time.Hour * 24 * 356, + Rate: rateFn, }, nil } diff --git a/internal/trigger/ramp/ramp_rate.go b/internal/trigger/ramp/ramp_rate.go index 09851237..a46b6ab8 100644 --- a/internal/trigger/ramp/ramp_rate.go +++ b/internal/trigger/ramp/ramp_rate.go @@ -68,10 +68,10 @@ func Rate() api.Builder { } return &api.Trigger{ - Trigger: api.NewIterationWorker(rates.IterationDuration, rates.Rate), + Trigger: api.NewIterationWorker(rates.IterationDuration, rates.IterationRate), Description: fmt.Sprintf("starting iterations from %s to %s during %v, using distribution %s", startRateArg, endRateArg, duration, distributionTypeArg), - DryRun: rates.Rate, + Rate: rates.Rate, }, nil }, } @@ -131,7 +131,8 @@ func CalculateRampRate( return &api.Rates{ IterationDuration: distributedIterationDuration, - Rate: distributedRateFn, + IterationRate: distributedRateFn, Duration: duration, + Rate: rateFn, }, nil } diff --git a/internal/trigger/staged/staged_rate.go b/internal/trigger/staged/staged_rate.go index 5c7edb61..764ce252 100644 --- a/internal/trigger/staged/staged_rate.go +++ b/internal/trigger/staged/staged_rate.go @@ -64,8 +64,8 @@ func Rate() api.Builder { } return &api.Trigger{ - Trigger: api.NewIterationWorker(rates.IterationDuration, rates.Rate), - DryRun: rates.Rate, + Trigger: api.NewIterationWorker(rates.IterationDuration, rates.IterationRate), + Rate: rates.Rate, Description: fmt.Sprintf( "Starting iterations every %s in numbers varying by time: %s, using distribution %s", frequency, stg, distributionTypeArg), @@ -99,7 +99,8 @@ func CalculateStagedRate( return &api.Rates{ IterationDuration: distributedIterationDuration, - Rate: distributedRateFn, + IterationRate: distributedRateFn, Duration: calculator.MaxDuration(), + Rate: rateFn, }, nil } diff --git a/internal/trigger/users/users_rate.go b/internal/trigger/users/users_rate.go index 2cf104fe..eb096b43 100644 --- a/internal/trigger/users/users_rate.go +++ b/internal/trigger/users/users_rate.go @@ -35,9 +35,7 @@ func Rate() api.Builder { Description: "Makes requests from a set of users specified by --concurrency", // The rate function used by the `users` mode, is actually dependent // on the number of users specified in the `--concurrency` flag. - // This flag is not required for the `chart` command, which uses the `DryRun` - // function, so its not possible to provide an accurate rate function here. - DryRun: func(time.Time) int { return 1 }, + Rate: func(time.Time) int { return 1 }, }, nil },