test: example code for e2e.

go.mod

@@ -6,19 +6,20 @@ require (
 	bou.ke/monkey v1.0.2
 	github.com/blang/semver v3.5.1+incompatible
 	github.com/go-ini/ini v1.62.0
+	github.com/go-logr/logr v0.4.0
 	github.com/go-sql-driver/mysql v1.6.0
 	github.com/go-test/deep v1.0.7
+	github.com/golang/glog v1.0.0
 	github.com/iancoleman/strcase v0.0.0-20190422225806-e506e3ef7365
 	github.com/imdario/mergo v0.3.12
-	github.com/onsi/ginkgo v1.16.4
-	github.com/onsi/gomega v1.13.0
+	github.com/onsi/ginkgo v1.16.5
+	github.com/onsi/gomega v1.17.0
 	github.com/presslabs/controller-util v0.3.0
 	github.com/spf13/cobra v1.1.1
 	github.com/stretchr/testify v1.7.0
 	k8s.io/api v0.21.2
 	k8s.io/apimachinery v0.21.2
 	k8s.io/client-go v0.21.2
-	k8s.io/component-base v0.21.2
 	k8s.io/klog/v2 v2.8.0
 	sigs.k8s.io/controller-runtime v0.9.2
 )

go.sum

@@ -152,6 +152,8 @@ github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zV
 github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
 github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
+github.com/golang/glog v1.0.0 h1:nfP3RFugxnNRyKgeWd4oI1nYvXpxrx8ck8ZrcizshdQ=
+github.com/golang/glog v1.0.0/go.mod h1:EWib/APOK0SL3dFbYqvxE3UYd8E6s1ouQ7iEp/0LWV4=
 github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
@@ -322,14 +324,16 @@ github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+W
 github.com/onsi/ginkgo v1.11.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk=
 github.com/onsi/ginkgo v1.16.2/go.mod h1:CObGmKUOKaSC0RjmoAK7tKyn4Azo5P2IWuoMnvwxz1E=
-github.com/onsi/ginkgo v1.16.4 h1:29JGrr5oVBm5ulCWet69zQkzWipVXIol6ygQUe/EzNc=
 github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0=
+github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE=
+github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU=
 github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA=
 github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
 github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY=
 github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo=
-github.com/onsi/gomega v1.13.0 h1:7lLHu94wT9Ij0o6EWWclhu0aOh32VxhkwEJvzuWPeak=
 github.com/onsi/gomega v1.13.0/go.mod h1:lRk9szgn8TxENtWd0Tp4c3wjlRfMTMH27I+3Je41yGY=
+github.com/onsi/gomega v1.17.0 h1:9Luw4uT5HTjHTN8+aNcSThgH1vdXnmdJ8xIfZ4wyTRE=
+github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY=
 github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
 github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
 github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU=

test/e2e/cluster/cluster.go

@@ -0,0 +1,141 @@
+/*
+Copyright 2018 Pressinfra SRL
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package cluster
+
+import (
+	"context"
+	"fmt"
+	"math/rand"
+	"time"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/types"
+
+	api "github.com/radondb/radondb-mysql-kubernetes/api/v1alpha1"
+	"github.com/radondb/radondb-mysql-kubernetes/test/e2e/framework"
+	"github.com/radondb/radondb-mysql-kubernetes/utils"
+)
+
+const (
+	POLLING = 2 * time.Second
+)
+
+var _ = Describe("MySQL Cluster E2E Tests", func() {
+	f := framework.NewFramework("mc-1")
+	two := int32(2)
+	three := int32(3)
+	five := int32(5)
+
+	sysbenchOptions := framework.SysbenchOptions{
+		Timeout:   10 * time.Minute,
+		Threads:   8,
+		Tables:    4,
+		TableSize: 10000,
+	}
+
+	var (
+		cluster    *api.MysqlCluster
+		clusterKey types.NamespacedName
+		name       string
+	)
+
+	BeforeEach(func() {
+		// be careful, mysql allowed hostname lenght is <63
+		name = fmt.Sprintf("cl-%d", rand.Int31()/1000)
+
+		By("creating a new cluster")
+		cluster = framework.NewCluster(name, f.Namespace.Name)
+		clusterKey = types.NamespacedName{Name: cluster.Name, Namespace: cluster.Namespace}
+		Expect(f.Client.Create(context.TODO(), cluster)).To(Succeed(), "failed to create cluster '%s'", cluster.Name)
+
+		By("testing the cluster readiness")
+		testClusterReadiness(f, cluster)
+
+		Expect(f.Client.Get(context.TODO(), clusterKey, cluster)).To(Succeed(), "failed to get cluster %s", cluster.Name)
+
+		f.PrepareData(cluster, &sysbenchOptions)
+		// By("testing the data readiness")
+		// f.WaitDataReady(cluster, sysbenchOptions.Tables, sysbenchOptions.TableSize)
+	})
+
+	It("scale out/in a cluster, 2 -> 3 -> 5 -> 3 -> 2", func() {
+		f.Client.Get(context.TODO(), types.NamespacedName{Name: cluster.Name, Namespace: cluster.Namespace}, cluster)
+
+		// start oltp test before testing scale in/out
+		f.RunOltpTest(cluster, &sysbenchOptions)
+
+		By("test cluster is ready after scale out 2 -> 3")
+		cluster.Spec.Replicas = &three
+		Expect(f.Client.Update(context.TODO(), cluster)).To(Succeed())
+		testClusterReadiness(f, cluster)
+		testXenonReadiness(cluster)
+
+		By("test cluster is ready after scale out 3 -> 5")
+		cluster.Spec.Replicas = &five
+		Expect(f.Client.Update(context.TODO(), cluster)).To(Succeed())
+		testClusterReadiness(f, cluster)
+		testXenonReadiness(cluster)
+
+		By("test cluster is ready after scale in 5 -> 3")
+		cluster.Spec.Replicas = &three
+		Expect(f.Client.Update(context.TODO(), cluster)).To(Succeed())
+		testClusterReadiness(f, cluster)
+		testXenonReadiness(cluster)
+
+		By("test cluster is ready after scale in 3 -> 2")
+		cluster.Spec.Replicas = &two
+		Expect(f.Client.Update(context.TODO(), cluster)).To(Succeed())
+		testClusterReadiness(f, cluster)
+		testXenonReadiness(cluster)
+	})
+})
+
+// testClusterReadiness determine whether the cluster is ready.
+func testClusterReadiness(f *framework.Framework, cluster *api.MysqlCluster) {
+	timeout := f.Timeout
+	if *cluster.Spec.Replicas > 0 {
+		timeout = time.Duration(*cluster.Spec.Replicas) * f.Timeout
+	}
+
+	// wait for pods to be ready
+	Eventually(func() int {
+		cl := &api.MysqlCluster{}
+		f.Client.Get(context.TODO(), types.NamespacedName{Name: cluster.Name, Namespace: cluster.Namespace}, cl)
+		return cl.Status.ReadyNodes
+	}, timeout, POLLING).Should(Equal(int(*cluster.Spec.Replicas)), "Not ready replicas of cluster '%s'", cluster.Name)
+
+	f.ClusterEventuallyCondition(cluster, api.ConditionReady, corev1.ConditionTrue, f.Timeout)
+}
+
+// testXenonReadiness determine whether the role of the cluster is normal.
+func testXenonReadiness(cluster *api.MysqlCluster) {
+	leader := []string{}
+	follower := []string{}
+	for _, node := range cluster.Status.Nodes {
+		if node.RaftStatus.Role == string(utils.Leader) {
+			leader = append(leader, node.Name)
+		} else if node.RaftStatus.Role == string(utils.Follower) {
+			follower = append(follower, node.Name)
+		} else {
+			Expect(node).Should(BeEmpty(), "some nodes not ready")
+		}
+	}
+	Expect(len(leader)).Should(Equal(1), "cluster need have only one leader")
+	Expect(len(follower)).Should(Equal(len(cluster.Status.Nodes)-len(leader)), "in addition to the leader node, the cluster has only follower nodes")
+}

test/e2e/e2e.go

@@ -17,9 +17,136 @@ limitations under the License.
 package e2e
 
 import (
+	"context"
+	"fmt"
+	"os"
+	"path"
+	"strings"
 	"testing"
+
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+
+	"github.com/golang/glog"
+	"github.com/onsi/ginkgo"
+	"github.com/onsi/ginkgo/config"
+	"github.com/onsi/ginkgo/reporters"
+	"github.com/onsi/gomega"
+	runtimeutils "k8s.io/apimachinery/pkg/util/runtime"
+	clientset "k8s.io/client-go/kubernetes"
+
+	"github.com/radondb/radondb-mysql-kubernetes/test/e2e/framework"
+	"github.com/radondb/radondb-mysql-kubernetes/test/e2e/framework/ginkgowrapper"
+
+	_ "k8s.io/client-go/plugin/pkg/client/auth/gcp"
+)
+
+const (
+	operatorNamespace = "mysql-operator"
+	releaseName       = "demo"
 )
 
+var _ = ginkgo.SynchronizedBeforeSuite(func() []byte {
+	kubeCfg, err := framework.LoadConfig()
+	gomega.Expect(err).To(gomega.Succeed())
+	// restClient := core.NewForConfigOrDie(kubeCfg).RESTClient()
+
+	c, err := client.New(kubeCfg, client.Options{})
+	if err != nil {
+		ginkgo.Fail(fmt.Sprintf("can't instantiate k8s client: %s", err))
+	}
+
+	// ginkgo node 1
+	ginkgo.By("Install operator")
+	operatorNsObj := &corev1.Namespace{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: operatorNamespace,
+		},
+	}
+	if err := c.Create(context.TODO(), operatorNsObj); err != nil {
+		if !strings.Contains(err.Error(), "already exists") {
+			ginkgo.Fail(fmt.Sprintf("can't create mysql-operator namespace: %s", err))
+		}
+	}
+	framework.HelmInstallChart(releaseName, operatorNamespace)
+
+	return nil
+
+}, func(data []byte) {
+	// all other nodes
+	framework.Logf("Running BeforeSuite actions on all node")
+})
+
+// Similar to SynchornizedBeforeSuite, we want to run some operations only once (such as collecting cluster logs).
+// Here, the order of functions is reversed; first, the function which runs everywhere,
+// and then the function that only runs on the first Ginkgo node.
+var _ = ginkgo.SynchronizedAfterSuite(func() {
+	// Run on all Ginkgo nodes
+	framework.Logf("Running AfterSuite actions on all node")
+	framework.RunCleanupActions()
+
+	// get the kubernetes client
+	kubeCfg, err := framework.LoadConfig()
+	gomega.Expect(err).To(gomega.Succeed())
+
+	client, err := clientset.NewForConfig(kubeCfg)
+	gomega.Expect(err).NotTo(gomega.HaveOccurred())
+
+	ginkgo.By("Remove operator release")
+	framework.HelmPurgeRelease(releaseName, operatorNamespace)
+
+	ginkgo.By("Delete operator namespace")
+
+	if err := framework.DeleteNS(client, operatorNamespace, framework.DefaultNamespaceDeletionTimeout); err != nil {
+		framework.Failf(fmt.Sprintf("Can't delete namespace: %s", err))
+	}
+}, func() {
+	// Run only Ginkgo on node 1
+	framework.Logf("Running AfterSuite actions on node 1")
+})
+
+// RunE2ETests checks configuration parameters (specified through flags) and then runs
+// E2E tests using the Ginkgo runner.
+// If a "report directory" is specified, one or more JUnit test reports will be
+// generated in this directory, and cluster logs will also be saved.
+// This function is called on each Ginkgo node in parallel mode.
 func RunE2ETests(t *testing.T) {
-	// empty now
+	runtimeutils.ReallyCrash = true
+
+	gomega.RegisterFailHandler(ginkgowrapper.Fail)
+	// Disable skipped tests unless they are explicitly requested.
+	if len(config.GinkgoConfig.FocusStrings) == 0 && len(config.GinkgoConfig.SkipStrings) == 0 {
+		config.GinkgoConfig.SkipStrings = []string{`\[Flaky\]`, `\[Feature:.+\]`}
+	}
+
+	rps := func() (rps []ginkgo.Reporter) {
+		// Run tests through the Ginkgo runner with output to console + JUnit for Jenkins
+		if framework.TestContext.ReportDir != "" {
+			// TODO: we should probably only be trying to create this directory once
+			// rather than once-per-Ginkgo-node.
+			if err := os.MkdirAll(framework.TestContext.ReportDir, 0755); err != nil {
+				glog.Errorf("Failed creating report directory: %v", err)
+				return
+			}
+			// add junit report
+			rps = append(rps, reporters.NewJUnitReporter(path.Join(framework.TestContext.ReportDir, fmt.Sprintf("junit_%v%02d.xml", "mysql_o_", config.GinkgoConfig.ParallelNode))))
+
+			// add logs dumper
+			if framework.TestContext.DumpLogsOnFailure {
+				rps = append(rps, NewLogsPodReporter(operatorNamespace, path.Join(framework.TestContext.ReportDir,
+					fmt.Sprintf("pods_logs_%d_%d.txt", config.GinkgoConfig.RandomSeed, config.GinkgoConfig.ParallelNode))))
+			}
+		} else {
+			// if reportDir is not specified then print logs to stdout
+			if framework.TestContext.DumpLogsOnFailure {
+				rps = append(rps, NewLogsPodReporter(operatorNamespace, ""))
+			}
+		}
+		return
+	}()
+
+	glog.Infof("Starting e2e run on Ginkgo node %d", config.GinkgoConfig.ParallelNode)
+
+	ginkgo.RunSpecsWithDefaultAndCustomReporters(t, "MySQL Operator E2E Suite", rps)
 }

test/e2e/e2e_test.go

@@ -17,38 +17,17 @@ limitations under the License.
 package e2e
 
 import (
-	"flag"
-	"fmt"
-	"math/rand"
-	"os"
 	"testing"
-	"time"
-
-	"k8s.io/component-base/version"
 
 	"github.com/radondb/radondb-mysql-kubernetes/test/e2e/framework"
-)
-
-// handleFlags sets up all flags and parses the command line.
-func handleFlags() {
-	framework.RegisterCommonFlags(flag.CommandLine)
-	flag.Parse()
-}
 
-func TestMain(m *testing.M) {
-	var versionFlag bool
-	flag.CommandLine.BoolVar(&versionFlag, "version", false, "Displays version information.")
-
-	// Register test flags, then parse flags.
-	handleFlags()
-
-	if versionFlag {
-		fmt.Printf("%s\n", version.Get())
-		os.Exit(0)
-	}
+	_ "github.com/radondb/radondb-mysql-kubernetes/test/e2e/cluster"
+)
 
-	rand.Seed(time.Now().UnixNano())
-	os.Exit(m.Run())
+func init() {
+	// framework.ViperizeFlags()
+	testing.Init()
+	framework.RegisterParseFlags()
 }
 
 func TestE2E(t *testing.T) {