diff --git a/controllers/suite_test.go b/controllers/suite_test.go index ada8a07c..965e2f79 100644 --- a/controllers/suite_test.go +++ b/controllers/suite_test.go @@ -20,12 +20,11 @@ import ( "path/filepath" "testing" - . "github.com/onsi/ginkgo" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" "k8s.io/client-go/kubernetes/scheme" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/envtest" - "sigs.k8s.io/controller-runtime/pkg/envtest/printer" logf "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/log/zap" @@ -44,9 +43,7 @@ var ( func TestAPIs(t *testing.T) { RegisterFailHandler(Fail) - RunSpecsWithDefaultAndCustomReporters(t, - "Controller Suite", - []Reporter{printer.NewlineReporter{}}) + RunSpecs(t, "Controllers Suite") } var _ = BeforeSuite(func() { @@ -73,7 +70,7 @@ var _ = BeforeSuite(func() { k8sClient, err = client.New(cfg, client.Options{Scheme: scheme.Scheme}) Expect(err).NotTo(HaveOccurred()) Expect(k8sClient).NotTo(BeNil()) -}, 60) +}) var _ = AfterSuite(func() { By("tearing down the test environment") diff --git a/go.mod b/go.mod index d1f79a84..091f099d 100644 --- a/go.mod +++ b/go.mod @@ -12,7 +12,7 @@ require ( github.com/golang/glog v1.0.0 github.com/iancoleman/strcase v0.0.0-20190422225806-e506e3ef7365 github.com/imdario/mergo v0.3.12 - github.com/onsi/ginkgo v1.16.5 + github.com/onsi/ginkgo/v2 v2.0.0 github.com/onsi/gomega v1.17.0 github.com/presslabs/controller-util v0.3.0 github.com/spf13/cobra v1.1.3 diff --git a/go.sum b/go.sum index e74560b9..a45c6497 100644 --- a/go.sum +++ b/go.sum @@ -200,6 +200,7 @@ github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OI github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= @@ -246,6 +247,7 @@ github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpO github.com/iancoleman/strcase v0.0.0-20190422225806-e506e3ef7365 h1:ECW73yc9MY7935nNYXUkK7Dz17YuSUI9yqRqYS8aBww= github.com/iancoleman/strcase v0.0.0-20190422225806-e506e3ef7365/go.mod h1:SK73tn/9oHe+/Y0h39VT4UCxmurVJkR5NA7kMEAOgSE= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= github.com/imdario/mergo v0.3.12 h1:b6R2BslTbIEToALKP7LxUvijTsNI9TAe80pLWN2g/HU= github.com/imdario/mergo v0.3.12/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= @@ -324,9 +326,10 @@ github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+W github.com/onsi/ginkgo v1.11.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= github.com/onsi/ginkgo v1.16.2/go.mod h1:CObGmKUOKaSC0RjmoAK7tKyn4Azo5P2IWuoMnvwxz1E= +github.com/onsi/ginkgo v1.16.4 h1:29JGrr5oVBm5ulCWet69zQkzWipVXIol6ygQUe/EzNc= github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0= -github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= -github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= +github.com/onsi/ginkgo/v2 v2.0.0 h1:CcuG/HvWNkkaqCUpJifQY8z7qEMBJya6aLPx6ftGyjQ= +github.com/onsi/ginkgo/v2 v2.0.0/go.mod h1:vw5CSIxN1JObi/U8gcbwft7ZxR2dgaR70JSE3/PpL4c= github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= diff --git a/test/e2e/README.md b/test/e2e/README.md index 16860d99..d46741dc 100644 --- a/test/e2e/README.md +++ b/test/e2e/README.md @@ -2,7 +2,8 @@ ## Prerequisites -Prepare a client connected to K8S. +- Prepare a client connected to K8S. +- Make sure [Ginkgo V2](https://onsi.github.io/ginkgo/MIGRATING_TO_V2) is installed. ## Hands-on Lab @@ -14,48 +15,28 @@ export KUBECONFIG=$HOME/.kube/config ### Step 2: Run test +> The Ginkgo version of the following examples is V2. + +- Running all cases. + +``` +ginkgo test/e2e/ +``` + +- Running all cases labeled `simplecase`. + +``` +ginkgo --label-filter=simplecase test/e2e/ +``` + +- Skip the cases of describing information contains `Namespace`. + +``` +ginkgo --skip "list namespace" test/e2e/ +``` + +- Just run the description information contains `Namespace`'s cases. + ``` -make e2e-local -``` -Example output of simplecase: -``` -$ make e2e-local -=== RUN TestE2E -STEP: Creating framework with timeout: 1200 -Running Suite: MySQL Operator E2E Suite -======================================= -Random Seed: 1640785115 - Will randomize all specs -Will run 1 of 1 specs - -Namespece test - test list namespace - /home/runkecheng/goCode/src/radondb-mysql-kubernetes/test/e2e/simplecase/list_namespace.go:38 -[BeforeEach] Namespece test - /home/runkecheng/goCode/src/radondb-mysql-kubernetes/test/e2e/framework/framework.go:62 -STEP: creating a kubernetes client -STEP: create a namespace api object (e2e-mc-1-cnkbs) -[BeforeEach] Namespece test - /home/runkecheng/goCode/src/radondb-mysql-kubernetes/test/e2e/simplecase/list_namespace.go:34 -STEP: before each -[It] test list namespace - /home/runkecheng/goCode/src/radondb-mysql-kubernetes/test/e2e/simplecase/list_namespace.go:38 -default -kube-public -kube-system -kubesphere-controls-system -kubesphere-devops-system -kubesphere-devops-worker -kubesphere-monitoring-federated -kubesphere-monitoring-system -kubesphere-system -radondb-mysql -radondb-mysql-kubernetes-system -[AfterEach] Namespece test - /home/runkecheng/goCode/src/radondb-mysql-kubernetes/test/e2e/framework/framework.go:63 -STEP: Collecting logs -STEP: Run cleanup actions -STEP: Delete testing namespace -• -Ran 1 of 1 Specs in 0.743 seconds -SUCCESS! -- 1 Passed | 0 Failed | 0 Pending | 0 Skipped +ginkgo --focus "list namespace" test/e2e/ ``` diff --git a/test/e2e/e2e.go b/test/e2e/e2e.go deleted file mode 100644 index 26433a59..00000000 --- a/test/e2e/e2e.go +++ /dev/null @@ -1,102 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package e2e - -import ( - "fmt" - "os" - "path" - "testing" - - "github.com/golang/glog" - . "github.com/onsi/ginkgo" - "github.com/onsi/ginkgo/config" - "github.com/onsi/ginkgo/reporters" - . "github.com/onsi/gomega" - runtimeutils "k8s.io/apimachinery/pkg/util/runtime" - _ "k8s.io/client-go/plugin/pkg/client/auth/gcp" - - "github.com/radondb/radondb-mysql-kubernetes/test/e2e/framework" - "github.com/radondb/radondb-mysql-kubernetes/test/e2e/framework/ginkgowrapper" -) - -const ( - operatorNamespace = "mysql-operator" -) - -var _ = SynchronizedBeforeSuite(func() []byte { - // BeforeSuite logic. - return nil -}, func(data []byte) { - // all other nodes - framework.Logf("Running BeforeSuite actions on all node") -}) - -// Similar to SynchornizedBeforeSuite, we want to run some operations only once (such as collecting cluster logs). -// Here, the order of functions is reversed; first, the function which runs everywhere, -// and then the function that only runs on the first Ginkgo node. -var _ = SynchronizedAfterSuite(func() { - // AfterSuite logic. -}, func() { - // Run only Ginkgo on node 1 - framework.Logf("Running AfterSuite actions on node 1") -}) - -// RunE2ETests checks configuration parameters (specified through flags) and then runs -// E2E tests using the Ginkgo runner. -// If a "report directory" is specified, one or more JUnit test reports will be -// generated in this directory, and cluster logs will also be saved. -// This function is called on each Ginkgo node in parallel mode. -func RunE2ETests(t *testing.T) { - runtimeutils.ReallyCrash = true - - RegisterFailHandler(ginkgowrapper.Fail) - // Disable skipped tests unless they are explicitly requested. - if len(config.GinkgoConfig.FocusStrings) == 0 && len(config.GinkgoConfig.SkipStrings) == 0 { - config.GinkgoConfig.SkipStrings = []string{`\[Flaky\]`, `\[Feature:.+\]`} - } - - rps := func() (rps []Reporter) { - // Run tests through the Ginkgo runner with output to console + JUnit for Jenkins - if framework.TestContext.ReportDir != "" { - // TODO: we should probably only be trying to create this directory once - // rather than once-per-Ginkgo-node. - if err := os.MkdirAll(framework.TestContext.ReportDir, 0755); err != nil { - glog.Errorf("Failed creating report directory: %v", err) - return - } - // add junit report - rps = append(rps, reporters.NewJUnitReporter(path.Join(framework.TestContext.ReportDir, fmt.Sprintf("junit_%v%02d.xml", "mysql_o_", config.GinkgoConfig.ParallelNode)))) - - // add logs dumper - if framework.TestContext.DumpLogsOnFailure { - rps = append(rps, NewLogsPodReporter(operatorNamespace, path.Join(framework.TestContext.ReportDir, - fmt.Sprintf("pods_logs_%d_%d.txt", config.GinkgoConfig.RandomSeed, config.GinkgoConfig.ParallelNode)))) - } - } else { - // if reportDir is not specified then print logs to stdout - if framework.TestContext.DumpLogsOnFailure { - rps = append(rps, NewLogsPodReporter(operatorNamespace, "")) - } - } - return - }() - - glog.Infof("Starting e2e run on Ginkgo node %d", config.GinkgoConfig.ParallelNode) - - RunSpecsWithDefaultAndCustomReporters(t, "MySQL Operator E2E Suite", rps) -} diff --git a/test/e2e/e2e_test.go b/test/e2e/e2e_test.go index 36b846a2..35795203 100644 --- a/test/e2e/e2e_test.go +++ b/test/e2e/e2e_test.go @@ -17,14 +17,34 @@ limitations under the License. package e2e import ( + "context" + "fmt" + "os" + "path" + "time" + + "strings" "testing" + . "github.com/onsi/ginkgo/v2" + "github.com/onsi/ginkgo/v2/types" + . "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + runtimeutils "k8s.io/apimachinery/pkg/util/runtime" + clientset "k8s.io/client-go/kubernetes" + _ "k8s.io/client-go/plugin/pkg/client/auth/gcp" + "sigs.k8s.io/controller-runtime/pkg/client" + "github.com/radondb/radondb-mysql-kubernetes/test/e2e/framework" + "github.com/radondb/radondb-mysql-kubernetes/test/e2e/framework/ginkgowrapper" + + // Test case source. + // Comment out the package that you don't want to run. _ "github.com/radondb/radondb-mysql-kubernetes/test/e2e/simplecase" ) func init() { - // framework.ViperizeFlags() testing.Init() framework.RegisterParseFlags() } @@ -32,3 +52,134 @@ func init() { func TestE2E(t *testing.T) { RunE2ETests(t) } + +var _ = SynchronizedBeforeSuite(func() []byte { + kubeCfg, err := framework.LoadConfig() + Expect(err).To(Succeed()) + + c, err := client.New(kubeCfg, client.Options{}) + if err != nil { + Fail(fmt.Sprintf("can't instantiate k8s client: %s", err)) + } + + By("Create Namespace") + operatorNsObj := &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: framework.RadondbMysqlE2eNamespace, + }, + } + + if err := c.Create(context.TODO(), operatorNsObj); err != nil { + if !strings.Contains(err.Error(), "already exists") { + Fail(fmt.Sprintf("can't create mysql-operator namespace: %s", err)) + } + } + + if framework.TestContext.DumpLogs { + By("Create log dir") + os.MkdirAll(fmt.Sprintf("%s_%d", framework.TestContext.ReportDirPrefix, GinkgoRandomSeed()), 0777) + } + + By("Install RadonDB MySQL Operator") + framework.HelmInstallChart(framework.OperatorReleaseName, framework.RadondbMysqlE2eNamespace) + return nil +}, func(data []byte) { + framework.Logf("Running BeforeSuite actions on all node") +}) + +// Similar to SynchornizedBeforeSuite, we want to run some operations only once (such as collecting cluster logs). +// Here, the order of functions is reversed; first, the function which runs everywhere, +// and then the function that only runs on the first Ginkgo node. +var _ = SynchronizedAfterSuite(func() { + // Run on all Ginkgo nodes. + framework.Logf("Running AfterSuite actions on all node") + framework.RunCleanupActions() + + // Get the kubernetes client. + kubeCfg, err := framework.LoadConfig() + Expect(err).To(Succeed()) + + client, err := clientset.NewForConfig(kubeCfg) + Expect(err).NotTo(HaveOccurred()) + + By("Remove operator release") + framework.HelmPurgeRelease(framework.OperatorReleaseName, framework.RadondbMysqlE2eNamespace) + + By("Delete test namespace") + if err := framework.DeleteNS(client, framework.RadondbMysqlE2eNamespace, framework.DefaultNamespaceDeletionTimeout); err != nil { + framework.Failf(fmt.Sprintf("Can't delete namespace: %s", err)) + } +}, func() { + framework.Logf("Running AfterSuite actions on node 1") +}) + +var _ = ReportAfterSuite("Collect log", func(report Report) { + if framework.TestContext.DumpLogs { + f, err := os.OpenFile(path.Join(fmt.Sprintf("%s_%d", framework.TestContext.ReportDirPrefix, GinkgoRandomSeed()), "overview.txt"), os.O_RDWR|os.O_CREATE, 0644) + if err != nil { + fmt.Println(err) + return + } + // Get the kubernetes client. + kubeCfg, err := framework.LoadConfig() + if err != nil { + fmt.Println("Failed to get kubeconfig!") + return + } + client, err := clientset.NewForConfig(kubeCfg) + if err != nil { + fmt.Println("Failed to create k8s client!") + return + } + for _, specReport := range report.SpecReports { + // Collect the summary of all cases. + fmt.Fprintf(f, "%s | %s\n", specReport.FullText(), specReport.State) + // Collect the POD log of failure cases. + if specReport.State.Is(types.SpecStateFailed) { + fileName := fmt.Sprintf("%v.txt", specReport.ContainerHierarchyTexts[len(specReport.ContainerHierarchyTexts)-1]) + logFile, err := os.OpenFile(path.Join(fmt.Sprintf("%s_%d", framework.TestContext.ReportDirPrefix, GinkgoRandomSeed()), fileName), os.O_RDWR|os.O_CREATE, 0644) + if err != nil { + fmt.Printf("Failed to open file: %s with error: %s\n", fileName, err) + continue + } + fmt.Fprintf(logFile, "## Start test: %v\n", specReport.ContainerHierarchyTexts) + + framework.LogPodsWithLabels(client, framework.RadondbMysqlE2eNamespace, nil, time.Since(specReport.EndTime.Add(-1*time.Minute)), logFile) + + fmt.Fprintf(logFile, "## END test\n") + logFile.Close() + } + } + f.Close() + } +}) + +// RunE2ETests checks configuration parameters (specified through flags) and then runs +// E2E tests using the Ginkgo runner. +// If a "report directory" is specified, one or more JUnit test reports will be +// generated in this directory, and cluster logs will also be saved. +// This function is called on each Ginkgo node in parallel mode. +func RunE2ETests(t *testing.T) { + runtimeutils.ReallyCrash = true + + RegisterFailHandler(ginkgowrapper.Fail) + + // Fetch the current config. + suiteConfig, reporterConfig := GinkgoConfiguration() + // Whether printing FullTrace. + reporterConfig.FullTrace = true + // Whether printing more detail. + reporterConfig.Verbose = true + // Whether to display information of GinkgoWriter. + reporterConfig.AlwaysEmitGinkgoWriter = true + if framework.TestContext.DumpLogs { + if framework.TestContext.ReportDirPrefix == "" { + now := time.Now() + framework.TestContext.ReportDirPrefix = fmt.Sprintf("logs_%d%d_%d%d", now.Month(), now.Day(), now.Hour(), now.Minute()) + } + // Path of JUnitReport. + reporterConfig.JUnitReport = path.Join(fmt.Sprintf("%s_%d", framework.TestContext.ReportDirPrefix, GinkgoRandomSeed()), "junit.xml") + } + + RunSpecs(t, "MySQL Operator E2E Suite", Label("MySQL Operator"), suiteConfig, reporterConfig) +} diff --git a/test/e2e/framework/framework.go b/test/e2e/framework/framework.go index e864d15e..e98d3899 100644 --- a/test/e2e/framework/framework.go +++ b/test/e2e/framework/framework.go @@ -21,9 +21,10 @@ import ( "time" "github.com/go-logr/logr" - . "github.com/onsi/ginkgo" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" core "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" clientset "k8s.io/client-go/kubernetes" "k8s.io/client-go/kubernetes/scheme" "sigs.k8s.io/controller-runtime/pkg/client" @@ -31,11 +32,6 @@ import ( apis "github.com/radondb/radondb-mysql-kubernetes/api/v1alpha1" ) -const ( - maxKubectlExecRetries = 5 - DefaultNamespaceDeletionTimeout = 10 * time.Minute -) - type Framework struct { BaseName string Namespace *core.Namespace @@ -43,9 +39,6 @@ type Framework struct { Client client.Client ClientSet clientset.Interface - cleanupHandle CleanupActionHandle - SkipNamespaceCreation bool - Timeout time.Duration Log logr.Logger @@ -54,14 +47,9 @@ type Framework struct { func NewFramework(baseName string) *Framework { By(fmt.Sprintf("Creating framework with timeout: %v", TestContext.TimeoutSeconds)) f := &Framework{ - BaseName: baseName, - SkipNamespaceCreation: false, - Log: log, + BaseName: baseName, + Log: Log, } - - BeforeEach(f.BeforeEach) - AfterEach(f.AfterEach) - return f } @@ -69,7 +57,6 @@ func NewFramework(baseName string) *Framework { func (f *Framework) BeforeEach() { // The fact that we need this feels like a bug in ginkgo. // https://github.com/onsi/ginkgo/issues/222 - f.cleanupHandle = AddCleanupAction(f.AfterEach) f.Timeout = time.Duration(TestContext.TimeoutSeconds) * time.Second By("creating a kubernetes client") @@ -84,34 +71,10 @@ func (f *Framework) BeforeEach() { f.ClientSet, err = clientset.NewForConfig(cfg) Expect(err).NotTo(HaveOccurred()) - if !f.SkipNamespaceCreation { - namespace, err := f.CreateNamespace(map[string]string{ - "e2e-framework": f.BaseName, - }) - Expect(err).NotTo(HaveOccurred()) - By(fmt.Sprintf("create a namespace api object (%s)", namespace.Name)) - - f.Namespace = namespace - } - -} - -// AfterEach deletes the namespace, after reading its events. -func (f *Framework) AfterEach() { - By("Collecting logs") - if CurrentGinkgoTestDescription().Failed && TestContext.DumpLogsOnFailure { - logFunc := Logf - // TODO: log in file if ReportDir is set - LogPodsWithLabels(f.ClientSet, f.Namespace.Name, map[string]string{}, logFunc) - } - - By("Run cleanup actions") - RemoveCleanupAction(f.cleanupHandle) - - By("Delete testing namespace") - err := DeleteNS(f.ClientSet, f.Namespace.Name, DefaultNamespaceDeletionTimeout) - if err != nil { - Failf(fmt.Sprintf("Can't delete namespace: %s", err)) + f.Namespace = &core.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: RadondbMysqlE2eNamespace, + }, } } diff --git a/test/e2e/framework/ginkgowrapper/wrapper.go b/test/e2e/framework/ginkgowrapper/wrapper.go index 1cb3de1a..2313ab9b 100644 --- a/test/e2e/framework/ginkgowrapper/wrapper.go +++ b/test/e2e/framework/ginkgowrapper/wrapper.go @@ -26,7 +26,7 @@ import ( "runtime/debug" "strings" - "github.com/onsi/ginkgo" + "github.com/onsi/ginkgo/v2" ) // FailurePanic is the value that will be panicked from Fail. @@ -38,7 +38,7 @@ type FailurePanic struct { } // String makes FailurePanic look like the old Ginkgo panic when printed. -func (FailurePanic) String() string { return ginkgo.GINKGO_PANIC } +// func (FailurePanic) String() string { return ginkgo.GINKGO_PANIC } // Fail wraps ginkgo.Fail so that it panics with more useful // information about the failure. This function will panic with a @@ -76,7 +76,7 @@ type SkipPanic struct { } // String makes SkipPanic look like the old Ginkgo panic when printed. -func (SkipPanic) String() string { return ginkgo.GINKGO_PANIC } +// func (SkipPanic) String() string { return ginkgo.GINKGO_PANIC } // Skip wraps ginkgo.Skip so that it panics with more useful // information about why the test is being skipped. This function will @@ -107,7 +107,7 @@ func Skip(message string, callerSkip ...int) { // ginkgo adds a lot of test running infrastructure to the stack, so // we filter those out -var stackSkipPattern = regexp.MustCompile(`onsi/ginkgo`) +var stackSkipPattern = regexp.MustCompile("") func pruneStack(skip int) string { skip += 2 // one for pruneStack and one for debug.Stack diff --git a/test/e2e/framework/helm.go b/test/e2e/framework/helm.go index 084b77c8..2ae02282 100644 --- a/test/e2e/framework/helm.go +++ b/test/e2e/framework/helm.go @@ -30,11 +30,11 @@ func HelmInstallChart(release, ns string) { "--namespace", ns, "--values", TestContext.ChartValues, "--wait", "--kube-context", TestContext.KubeContext, + "--set", fmt.Sprintf("manager.image=%s", TestContext.OperatorImagePath), "--set", fmt.Sprintf("manager.tag=%s", TestContext.OperatorImageTag), } cmd := exec.Command("helm", args...) - cmd.Stdout = os.Stdout cmd.Stderr = os.Stderr Expect(cmd.Run()).Should(Succeed()) diff --git a/test/e2e/framework/reporter.go b/test/e2e/framework/reporter.go new file mode 100644 index 00000000..d818c598 --- /dev/null +++ b/test/e2e/framework/reporter.go @@ -0,0 +1,65 @@ +/* +Copyright 2015 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package framework + +import ( + "context" + "fmt" + "io" + "strconv" + "time" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + clientset "k8s.io/client-go/kubernetes" +) + +func LogPodsWithLabels(c clientset.Interface, ns string, match map[string]string, since time.Duration, out io.Writer) { + podList, err := c.CoreV1().Pods(ns).List(context.TODO(), metav1.ListOptions{LabelSelector: labels.SelectorFromSet(match).String()}) + if err != nil { + fmt.Fprintf(out, "error listing pods: %s", err) + return + } + + for _, pod := range podList.Items { + for _, container := range pod.Spec.Containers { + fmt.Fprintf(out, "\n\n===============\nSTART LOGS for %s (%s):\n", pod.Name, container.Name) + runLogs(c, ns, pod.Name, container.Name, false, since, out) + fmt.Fprintf(out, "\n\n===============\nSTOP LOGS for %s (%s):\n", pod.Name, container.Name) + } + } +} + +func runLogs(client clientset.Interface, namespace, name, container string, previous bool, sinceStart time.Duration, out io.Writer) error { + req := client.CoreV1().RESTClient().Get(). + Namespace(namespace). + Name(name). + Resource("pods"). + SubResource("log"). + Param("container", container). + Param("previous", strconv.FormatBool(previous)). + Param("since", strconv.FormatInt(int64(sinceStart.Round(time.Second).Seconds()), 10)) + + readCloser, err := req.Stream(context.TODO()) + if err != nil { + return err + } + + defer readCloser.Close() + _, err = io.Copy(out, readCloser) + return err +} diff --git a/test/e2e/framework/test_context.go b/test/e2e/framework/test_context.go index 511ba4b6..d9b95d4a 100644 --- a/test/e2e/framework/test_context.go +++ b/test/e2e/framework/test_context.go @@ -19,9 +19,44 @@ package framework import ( "flag" "os" + "time" - "github.com/onsi/ginkgo/config" "k8s.io/client-go/tools/clientcmd" + logf "sigs.k8s.io/controller-runtime/pkg/log" +) + +var Log = logf.Log.WithName("framework.util") + +const ( + // The namespace where the resource created by E2E. + RadondbMysqlE2eNamespace = "radondb-mysql-e2e" + // The name of the Operator to create. + OperatorReleaseName = "e2e-test" + // Export POD logs and test overview. + DumpLogs = true + // Optional directory to store junit and pod logs output in. + // If not specified, it will beset to the current date. + ReportDirPrefix = "" + + // Specify the directory that Helm Install will be executed. + ChartPath = "../../charts/mysql-operator" + // Image path of mysql operator. + OperatorImagePath = "radondb/mysql-operator" + // Image tag of mysql operator. + OperatorImageTag = "latest" + // Image path for mysql sidecar. + SidecarImage = "radondb/mysql-sidecar:latest" + + // How often to Poll pods, nodes and claims. + Poll = 2 * time.Second + // Time interval of checking cluster status. + POLLING = 2 * time.Second + // Timeout time of checking cluster status. + TIMEOUT = time.Minute + // Timeout of deleting namespace. + DefaultNamespaceDeletionTimeout = 10 * time.Minute + // Timeout of waiting for POD startup. + PodStartTimeout = 1 * time.Hour ) type TestContextType struct { @@ -29,43 +64,36 @@ type TestContextType struct { KubeConfig string KubeContext string - ReportDir string + ReportDirPrefix string ChartPath string ChartValues string - OperatorImageTag string - SidecarImage string + OperatorImagePath string + OperatorImageTag string + SidecarImage string - TimeoutSeconds int - DumpLogsOnFailure bool + TimeoutSeconds int + DumpLogs bool } var TestContext TestContextType // Register flags common to all e2e test suites. func RegisterCommonFlags() { - // Turn on verbose by default to get spec names - config.DefaultReporterConfig.Verbose = true - - // Turn on EmitSpecProgress to get spec progress (especially on interrupt) - config.GinkgoConfig.EmitSpecProgress = true - - // Randomize specs as well as suites - config.GinkgoConfig.RandomizeAllSpecs = true - flag.StringVar(&TestContext.KubeHost, "kubernetes-host", "", "The kubernetes host, or apiserver, to connect to") flag.StringVar(&TestContext.KubeConfig, "kubernetes-config", os.Getenv(clientcmd.RecommendedConfigPathEnvVar), "Path to config containing embedded authinfo for kubernetes. Default value is from environment variable "+clientcmd.RecommendedConfigPathEnvVar) flag.StringVar(&TestContext.KubeContext, "kubernetes-context", "", "config context to use for kuberentes. If unset, will use value from 'current-context'") - flag.StringVar(&TestContext.ReportDir, "report-dir", "", "Optional directory to store junit and pod logs output in. If not specified, no junit or logs files will be output") + flag.StringVar(&TestContext.ReportDirPrefix, "report-dir", ReportDirPrefix, "Optional directory to store logs output in.") - flag.StringVar(&TestContext.ChartPath, "operator-chart-path", "../../charts/mysql-operator", "The chart name or path for mysql operator") - flag.StringVar(&TestContext.OperatorImageTag, "operator-image-tag", "latest", "Image tag for mysql operator.") - flag.StringVar(&TestContext.SidecarImage, "sidecar-image", "radondb/mysql-sidecar:latest", "Image path for mysql sidecar.") + flag.StringVar(&TestContext.ChartPath, "operator-chart-path", ChartPath, "The chart name or path for mysql operator") + flag.StringVar(&TestContext.OperatorImagePath, "operator-image-path", OperatorImagePath, "Image tag of mysql operator.") + flag.StringVar(&TestContext.OperatorImageTag, "operator-image-tag", OperatorImageTag, "Image tag of mysql operator.") + flag.StringVar(&TestContext.SidecarImage, "sidecar-image", SidecarImage, "Image path of mysql sidecar.") flag.IntVar(&TestContext.TimeoutSeconds, "pod-wait-timeout", 1200, "Timeout to wait for a pod to be ready.") - flag.BoolVar(&TestContext.DumpLogsOnFailure, "dump-logs-on-failure", true, "Dump pods logs when a test fails.") + flag.BoolVar(&TestContext.DumpLogs, "dump-logs-on-failure", DumpLogs, "Dump logs.") } func RegisterParseFlags() { diff --git a/test/e2e/framework/util.go b/test/e2e/framework/util.go index 00264a11..4eac87b7 100644 --- a/test/e2e/framework/util.go +++ b/test/e2e/framework/util.go @@ -20,7 +20,6 @@ import ( "context" "errors" "fmt" - "math/rand" "strconv" "strings" "time" @@ -33,19 +32,10 @@ import ( restclient "k8s.io/client-go/rest" "k8s.io/client-go/tools/clientcmd" clientcmdapi "k8s.io/client-go/tools/clientcmd/api" - logf "sigs.k8s.io/controller-runtime/pkg/log" "github.com/radondb/radondb-mysql-kubernetes/test/e2e/framework/ginkgowrapper" ) -const ( - PodStartTimeout = 1 * time.Hour - // How often to Poll pods, nodes and claims. - Poll = 2 * time.Second -) - -var log = logf.Log.WithName("framework.util") - // CreateTestingNS should be used by every test, note that we append a common prefix to the provided test name. // Please see NewFramework instead of using this directly. func CreateTestingNS(baseName string, c clientset.Interface, labels map[string]string) (*corev1.Namespace, error) { @@ -137,33 +127,32 @@ func podRunningAndReadyByPhase(pod corev1.Pod) (bool, error) { return false, nil } -// deleteNS deletes the provided namespace, waits for it to be completely deleted, and then checks +// DeleteNS deletes the provided namespace, waits for it to be completely deleted, and then checks // whether there are any pods remaining in a non-terminating state. func DeleteNS(c clientset.Interface, namespace string, timeout time.Duration) error { - startTime := time.Now() if err := c.CoreV1().Namespaces().Delete(context.TODO(), namespace, metav1.DeleteOptions{}); err != nil { return err } - // wait for namespace to delete or timeout. - // err := wait.PollImmediate(2*time.Second, timeout, func() (bool, error) { - // if _, err := c.CoreV1().Namespaces().Get(namespace, metav1.GetOptions{}); err != nil { - // if apierrs.IsNotFound(err) { - // return true, nil - // } - // Logf("Error while waiting for namespace to be terminated: %v", err) - // return false, nil - // } - // return false, nil - // }) - - Logf("namespace %v deletion completed in %s", namespace, time.Since(startTime)) + // Wait for namespace to delete or timeout. + if err := wait.PollImmediate(2*time.Second, timeout, func() (bool, error) { + if _, err := c.CoreV1().Namespaces().Get(context.TODO(), namespace, metav1.GetOptions{}); err != nil { + if strings.Contains(err.Error(), "not found") { + return true, nil + } + Logf("Error while waiting for namespace to be terminated: %v", err) + return false, nil + } + return false, nil + }); err != nil { + return err + } return nil } func Logf(format string, args ...interface{}) { msg := fmt.Sprintf(format, args...) - log.Info(msg) + Log.Info(msg) } func Failf(format string, args ...interface{}) { @@ -183,7 +172,7 @@ func getPreviousPodLogs(c clientset.Interface, namespace, podName, containerName return getPodLogsInternal(c, namespace, podName, containerName, true) } -// utility function for gomega Eventually +// getPodLogsInternal is a utility function for gomega Eventually. func getPodLogsInternal(c clientset.Interface, namespace, podName, containerName string, previous bool) (string, error) { logs, err := c.CoreV1().RESTClient().Get(). Resource("pods"). @@ -221,18 +210,6 @@ func kubectlLogPod(c clientset.Interface, pod corev1.Pod, containerNameSubstr st } } -func LogPodsWithLabels(c clientset.Interface, ns string, match map[string]string, logFunc func(ftm string, args ...interface{})) { - podList, err := c.CoreV1().Pods(ns).List(context.TODO(), metav1.ListOptions{LabelSelector: labels.SelectorFromSet(match).String()}) - if err != nil { - logFunc("Error getting pods in namespace %q: %v", ns, err) - return - } - logFunc("Running kubectl logs on pods with labels %v in %v", match, ns) - for _, pod := range podList.Items { - kubectlLogPod(c, pod, "", logFunc) - } -} - func LogContainersInPodsWithLabels(c clientset.Interface, ns string, match map[string]string, containerSubstr string, logFunc func(ftm string, args ...interface{})) { podList, err := c.CoreV1().Pods(ns).List(context.TODO(), metav1.ListOptions{LabelSelector: labels.SelectorFromSet(match).String()}) if err != nil { @@ -243,14 +220,3 @@ func LogContainersInPodsWithLabels(c clientset.Interface, ns string, match map[s kubectlLogPod(c, pod, containerSubstr, logFunc) } } - -func RandStr(length int) string { - str := "abcdefghijklmnopqrstuvwxyz" - bytes := []byte(str) - result := []byte{} - rand.Seed(time.Now().UnixNano() + int64(rand.Intn(100))) - for i := 0; i < length; i++ { - result = append(result, bytes[rand.Intn(len(bytes))]) - } - return string(result) -} diff --git a/test/e2e/reporter.go b/test/e2e/reporter.go deleted file mode 100644 index bad5e5eb..00000000 --- a/test/e2e/reporter.go +++ /dev/null @@ -1,155 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package e2e - -import ( - "context" - "fmt" - "io" - "os" - "strconv" - "time" - - "github.com/onsi/ginkgo" - "github.com/onsi/ginkgo/config" - "github.com/onsi/ginkgo/reporters" - "github.com/onsi/ginkgo/types" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/labels" - clientset "k8s.io/client-go/kubernetes" - - "github.com/radondb/radondb-mysql-kubernetes/test/e2e/framework" -) - -type podLogReporter struct { - namespace string - - logPath string - logFile *os.File - - out io.Writer -} - -var radondbMysqlTestLabel = map[string]string{ - "app.kubernetes.io/managed-by": "mysql.radondb.com", -} - -// NewLogsPodReporter writes the logs for all pods in the specified namespace. -// if path is specified then the logs are written to that path, else logs are -// written to GinkgoWriter -func NewLogsPodReporter(ns, path string) reporters.Reporter { - return &podLogReporter{ - namespace: ns, - logPath: path, - out: ginkgo.GinkgoWriter, - } -} - -// called when suite starts -func (r *podLogReporter) SpecSuiteWillBegin(config config.GinkgoConfigType, s *types.SuiteSummary) { - if r.logPath != "" { - var err error - r.logFile, err = os.OpenFile(r.logPath, os.O_RDWR|os.O_CREATE, 0644) - if err != nil { - fmt.Printf("Failed to open file: %s with error: %s\n", r.logPath, err) - return - } - - r.out = r.logFile - } -} - -// called before BeforeSuite before starting tests -func (r *podLogReporter) BeforeSuiteDidRun(setupSummary *types.SetupSummary) {} - -// called before every test -func (r *podLogReporter) SpecWillRun(specSummary *types.SpecSummary) {} - -// called after every test -func (r *podLogReporter) SpecDidComplete(specSummary *types.SpecSummary) { - // don't output logs if test didn't failed - if specSummary.State <= types.SpecStatePassed { - return - } - - // get the kubernetes client - kubeCfg, err := framework.LoadConfig() - if err != nil { - fmt.Println("Failed to get kubeconfig!") - return - } - - client, err := clientset.NewForConfig(kubeCfg) - if err != nil { - fmt.Println("Failed to create k8s client!") - return - } - - fmt.Fprintf(r.out, "## Start test: %v\n", specSummary.ComponentTexts) - - LogPodsWithLabels(client, r.namespace, radondbMysqlTestLabel, specSummary.RunTime, r.out) - - fmt.Fprintf(r.out, "## END test\n") - -} - -// called before AfterSuite runs -func (r *podLogReporter) AfterSuiteDidRun(setupSummary *types.SetupSummary) {} - -// caleed at the end -func (r *podLogReporter) SpecSuiteDidEnd(summary *types.SuiteSummary) { - if r.logFile != nil { - r.logFile.Close() - } -} - -func LogPodsWithLabels(c clientset.Interface, ns string, match map[string]string, since time.Duration, out io.Writer) { - podList, err := c.CoreV1().Pods(ns).List(context.TODO(), metav1.ListOptions{LabelSelector: labels.SelectorFromSet(match).String()}) - if err != nil { - fmt.Fprintf(out, "error listing pods: %s", err) - return - } - - for _, pod := range podList.Items { - for _, container := range pod.Spec.Containers { - fmt.Fprintf(out, "\n\n===============\nSTART LOGS for %s (%s):\n", pod.Name, container.Name) - runLogs(c, ns, pod.Name, container.Name, false, since, out) - fmt.Fprintf(out, "\n\n===============\nSTOP LOGS for %s (%s):\n", pod.Name, container.Name) - } - } -} - -func runLogs(client clientset.Interface, namespace, name, container string, previous bool, sinceStart time.Duration, out io.Writer) error { - req := client.CoreV1().RESTClient().Get(). - Namespace(namespace). - Name(name). - Resource("pods"). - SubResource("log"). - Param("container", container). - Param("previous", strconv.FormatBool(previous)). - Param("since", strconv.FormatInt(int64(sinceStart.Round(time.Second).Seconds()), 10)) - - readCloser, err := req.Stream(context.TODO()) - if err != nil { - return err - } - - defer readCloser.Close() - _, err = io.Copy(out, readCloser) - return err - -} diff --git a/test/e2e/simplecase/list_namespace.go b/test/e2e/simplecase/list_namespace.go index 790763aa..703b649b 100644 --- a/test/e2e/simplecase/list_namespace.go +++ b/test/e2e/simplecase/list_namespace.go @@ -20,7 +20,7 @@ import ( "context" "fmt" - . "github.com/onsi/ginkgo" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -28,14 +28,17 @@ import ( ) // Simple and quick test case, used to verify that the E2E framework is available. -var _ = Describe("Namespece test", func() { - f := framework.NewFramework("mc-1") - +var _ = Describe("Namespace test", Label("simplecase"), Ordered, func() { + var f *framework.Framework BeforeEach(func() { - By("before each") + f = &framework.Framework{ + BaseName: "mysqlcluster-e2e", + Log: framework.Log, + } + f.BeforeEach() }) - It("test list namespace", func() { + It("test list namespace", Label("list ns"), func() { namespaces, err := f.ClientSet.CoreV1().Namespaces().List(context.TODO(), metav1.ListOptions{}) Expect(err).Should(BeNil()) Expect(len(namespaces.Items)).ShouldNot(BeZero())