Merge pull request radondb#386 from acekingke/CronJOBONE

*: support the cronjob to backup radondb#215

Author: andyli029

PROJECT

@@ -30,6 +30,14 @@ resources:
   kind: Backup
   path: github.com/radondb/radondb-mysql-kubernetes/api/v1alpha1
   version: v1alpha1
+- api:
+    crdVersion: v1
+    namespaced: true
+  controller: true
+  domain: radondb.com
+  group: mysql
+  kind: BackupCron
+  version: v1alpha1
 - api:
     crdVersion: v1
     namespaced: true

api/v1alpha1/mysqlcluster_types.go

@@ -85,6 +85,17 @@ type MysqlClusterSpec struct {
 	// Represents NFS ip address where cluster restore from.
 	// +optional
 	NFSServerAddress string `json:"nfsServerAddress,omitempty"`
+
+	// Specify under crontab format interval to take backups
+	// leave it empty to deactivate the backup process
+	// Defaults to ""
+	// +optional
+	BackupSchedule string `json:"backupSchedule,omitempty"`
+
+	// If set keeps last BackupScheduleJobsHistoryLimit Backups
+	// +optional
+	// +kubebuilder:default:=6
+	BackupScheduleJobsHistoryLimit *int `json:"backupScheduleJobsHistoryLimit,omitempty"`
 }
 
 // MysqlOpts defines the options of MySQL container.

api/v1alpha1/zz_generated.deepcopy.go

@@ -0,0 +1,130 @@
+package backup
+
+import (
+	"context"
+	"fmt"
+	"sort"
+	"time"
+
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+
+	"github.com/go-logr/logr"
+	apiv1alpha1 "github.com/radondb/radondb-mysql-kubernetes/api/v1alpha1"
+)
+
+// The job structure contains the context to schedule a backup
+type CronJob struct {
+	ClusterName string
+	Namespace   string
+
+	// kubernetes client
+	Client client.Client
+
+	BackupScheduleJobsHistoryLimit *int
+	Image                          string
+	Log                            logr.Logger
+}
+
+func (j *CronJob) Run() {
+	// nolint: govet
+	log := j.Log
+	log.Info("scheduled backup job started")
+
+	// run garbage collector if needed
+	if j.BackupScheduleJobsHistoryLimit != nil {
+		defer j.backupGC()
+	}
+
+	// check if a backup is running
+	if j.scheduledBackupsRunningCount() > 0 {
+		log.Info("at least a backup is running", "running_backups_count", j.scheduledBackupsRunningCount())
+		return
+	}
+
+	// create the backup
+	if _, err := j.createBackup(); err != nil {
+		log.Error(err, "failed to create backup")
+	}
+}
+
+func (j *CronJob) scheduledBackupsRunningCount() int {
+	log := j.Log
+	backupsList := &apiv1alpha1.BackupList{}
+	// select all backups with labels recurrent=true and and not completed of the cluster
+	selector := j.backupSelector()
+	client.MatchingFields{"status.completed": "false"}.ApplyToList(selector)
+
+	if err := j.Client.List(context.TODO(), backupsList, selector); err != nil {
+		log.Error(err, "failed getting backups", "selector", selector)
+		return 0
+	}
+
+	return len(backupsList.Items)
+}
+
+func (j *CronJob) backupSelector() *client.ListOptions {
+	selector := &client.ListOptions{}
+
+	client.InNamespace(j.Namespace).ApplyToList(selector)
+	client.MatchingLabels(j.recurrentBackupLabels()).ApplyToList(selector)
+
+	return selector
+}
+
+func (j *CronJob) recurrentBackupLabels() map[string]string {
+	return map[string]string{
+		"recurrent": "true",
+		"cluster":   j.ClusterName,
+	}
+}
+
+func (j *CronJob) backupGC() {
+	var err error
+	log := j.Log
+	backupsList := &apiv1alpha1.BackupList{}
+	if err = j.Client.List(context.TODO(), backupsList, j.backupSelector()); err != nil {
+		log.Error(err, "failed getting backups", "selector", j.backupSelector())
+		return
+	}
+
+	// sort backups by creation time before removing extra backups
+	sort.Sort(byTimestamp(backupsList.Items))
+
+	for i, backup := range backupsList.Items {
+		if i >= *j.BackupScheduleJobsHistoryLimit {
+			// delete the backup
+			if err = j.Client.Delete(context.TODO(), &backup); err != nil {
+				log.Error(err, "failed to delete a backup", "backup", backup)
+			}
+		}
+	}
+}
+
+func (j *CronJob) createBackup() (*apiv1alpha1.Backup, error) {
+	backupName := fmt.Sprintf("%s-auto-%s", j.ClusterName, time.Now().Format("2006-01-02t15-04-05"))
+
+	backup := &apiv1alpha1.Backup{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      backupName,
+			Namespace: j.Namespace,
+			Labels:    j.recurrentBackupLabels(),
+		},
+		Spec: apiv1alpha1.BackupSpec{
+			ClusterName: j.ClusterName,
+			//TODO modify to cluster sidecar image
+			Image: j.Image,
+			//RemoteDeletePolicy: j.BackupRemoteDeletePolicy,
+			HostName: fmt.Sprintf("%s-mysql-0", j.ClusterName),
+		},
+	}
+	return backup, j.Client.Create(context.TODO(), backup)
+}
+
+type byTimestamp []apiv1alpha1.Backup
+
+func (a byTimestamp) Len() int      { return len(a) }
+func (a byTimestamp) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
+func (a byTimestamp) Less(i, j int) bool {
+	return a[j].ObjectMeta.CreationTimestamp.Before(&a[i].ObjectMeta.CreationTimestamp)
+}

backup/cronbackup.go

@@ -58,6 +58,14 @@ spec:
           spec:
             description: MysqlClusterSpec defines the desired state of MysqlCluster
             properties:
+              backupSchedule:
+                description: Specify under crontab format interval to take backups
+                  leave it empty to deactivate the backup process Defaults to ""
+                type: string
+              backupScheduleJobsHistoryLimit:
+                default: 6
+                description: If set keeps last BackupScheduleJobsHistoryLimit Backups
+                type: integer
               backupSecretName:
                 description: Represents the name of the secret that contains credentials
                   to connect to the storage provider to store backups.

charts/mysql-operator/crds/mysql.radondb.com_mysqlclusters.yaml

@@ -19,6 +19,7 @@ package main
 import (
 	"flag"
 	"os"
+	"sync"
 
 	// Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.)
 	// to ensure that exec-entrypoint and run can make use of them.
@@ -34,6 +35,7 @@ import (
 	mysqlv1alpha1 "github.com/radondb/radondb-mysql-kubernetes/api/v1alpha1"
 	"github.com/radondb/radondb-mysql-kubernetes/controllers"
 	"github.com/radondb/radondb-mysql-kubernetes/internal"
+	"github.com/wgliang/cron"
 	//+kubebuilder:scaffold:imports
 )
 
@@ -116,6 +118,16 @@ func main() {
 		setupLog.Error(err, "unable to create controller", "controller", "MysqlUser")
 		os.Exit(1)
 	}
+	if err = (&controllers.BackupCronReconciler{
+		Client:          mgr.GetClient(),
+		Scheme:          mgr.GetScheme(),
+		Recorder:        mgr.GetEventRecorderFor("controller.BackupCron"),
+		Cron:            cron.New(),
+		LockJobRegister: new(sync.Mutex),
+	}).SetupWithManager(mgr); err != nil {
+		setupLog.Error(err, "unable to create controller", "controller", "BackupCron")
+		os.Exit(1)
+	}
 	//+kubebuilder:scaffold:builder
 	if os.Getenv("ENABLE_WEBHOOKS") != "false" {
 		if err = (&mysqlv1alpha1.MysqlCluster{}).SetupWebhookWithManager(mgr); err != nil {

cmd/manager/main.go

@@ -58,6 +58,14 @@ spec:
           spec:
             description: MysqlClusterSpec defines the desired state of MysqlCluster
             properties:
+              backupSchedule:
+                description: Specify under crontab format interval to take backups
+                  leave it empty to deactivate the backup process Defaults to ""
+                type: string
+              backupScheduleJobsHistoryLimit:
+                default: 6
+                description: If set keeps last BackupScheduleJobsHistoryLimit Backups
+                type: integer
               backupSecretName:
                 description: Represents the name of the secret that contains credentials
                   to connect to the storage provider to store backups.

config/crd/bases/mysql.radondb.com_mysqlclusters.yaml

@@ -0,0 +1,88 @@
+apiVersion: mysql.radondb.com/v1alpha1
+kind: MysqlCluster
+metadata:
+  name: sample
+spec:
+  replicas: 3
+  mysqlVersion: "5.7"
+  
+  # the backupSecretName specify the secret file name which store S3 information,
+  # if you want S3 backup or restore, please create backup_secret.yaml, uncomment below and fill secret name:
+  backupSecretName: sample-backup-secret
+  
+  # if you want create mysqlcluster from S3, uncomment and fill the directory in S3 bucket below:
+  # restoreFrom: 
+  BackupSchedule: "0 50 * * * *" 
+  mysqlOpts:
+    rootPassword: "RadonDB@123"
+    rootHost: localhost
+    user: radondb_usr
+    password: RadonDB@123
+    database: radondb
+    initTokuDB: true
+
+    # A simple map between string and string.
+    # Such as:
+    #    mysqlConf:
+    #      expire_logs_days: "7"
+    mysqlConf: {}
+
+    resources:
+      requests:
+        cpu: 100m
+        memory: 256Mi
+      limits:
+        cpu: 500m
+        memory: 1Gi
+
+  xenonOpts:
+    image: radondb/xenon:1.1.5-alpha
+    admitDefeatHearbeatCount: 5
+    electionTimeout: 10000
+
+    resources:
+      requests:
+        cpu: 50m
+        memory: 128Mi
+      limits:
+        cpu: 100m
+        memory: 256Mi
+
+  metricsOpts:
+    enabled: false
+    image: prom/mysqld-exporter:v0.12.1
+
+    resources:
+      requests:
+        cpu: 10m
+        memory: 32Mi
+      limits:
+        cpu: 100m
+        memory: 128Mi
+
+  podPolicy:
+    imagePullPolicy: IfNotPresent
+    sidecarImage: radondb/mysql-sidecar:latest
+    busyboxImage: busybox:1.32
+
+    slowLogTail: false
+    auditLogTail: false
+
+    labels: {}
+    annotations: {}
+    affinity: {}
+    priorityClassName: ""
+    tolerations: []
+    schedulerName: ""
+    # extraResources defines quotas for containers other than mysql or xenon.
+    extraResources:
+      requests:
+        cpu: 10m
+        memory: 32Mi
+
+  persistence:
+    enabled: true
+    accessModes:
+    - ReadWriteOnce
+    #storageClass: ""
+    size: 20Gi