Merge pull request #197 from acekingke/backupS3

*: backup to S3 and restore from S3 #116

Author: andyli029

.github/workflows/auto_build_operator.yml

@@ -10,6 +10,7 @@ on:
      - controllers/*
      - internal/*
      - utils/*
+     - backup/*
   workflow_dispatch:
 
 jobs:

Dockerfile

@@ -15,6 +15,7 @@ COPY cmd/manager/main.go cmd/manager/main.go
 COPY api/ api/
 COPY cluster/ cluster/
 COPY controllers/ controllers/
+COPY backup/ backup/
 COPY internal/ internal/
 COPY utils/ utils/
 

Dockerfile.sidecar

@@ -8,6 +8,7 @@ WORKDIR /workspace
 # Copy the Go Modules manifests
 COPY go.mod go.mod
 COPY go.sum go.sum
+
 # cache deps before building and copying source so that we don't need to re-download as much
 # and so that source changes don't invalidate our downloaded layer
 RUN go env -w GOPROXY=https://goproxy.cn,direct; \
@@ -24,12 +25,28 @@ RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -a -o bin/sidecar cmd/sidecar
 ###############################################################################
 #  Docker image for Sidecar
 ###############################################################################
-FROM alpine:3.13
+FROM ubuntu:focal
+
+RUN set -ex; \
+    groupadd --gid 1001 --system mysql; \
+    useradd \
+    --uid 1001 \
+    --system \
+    --home-dir /var/lib/mysql \
+    --no-create-home \
+    --gid mysql \
+    mysql;
 
-# Create a group and user
-RUN addgroup -g 1001 mysql && adduser -u 1001 -g 1001 -S mysql
+ARG XTRABACKUP_PKG=percona-xtrabackup-24
+RUN set -ex; \
+    apt-get update; \
+    apt-get install -y --no-install-recommends gnupg2 wget lsb-release curl; \
+    wget -P /tmp --no-check-certificate https://repo.percona.com/apt/percona-release_latest.$(lsb_release -sc)_all.deb; \
+    dpkg -i /tmp/percona-release_latest.$(lsb_release -sc)_all.deb; \
+    apt-get update; \
+    apt-get install -y --no-install-recommends ${XTRABACKUP_PKG}; \
+    rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
 
 WORKDIR /
 COPY --from=builder /workspace/bin/sidecar /usr/local/bin/sidecar
-
 ENTRYPOINT ["sidecar"]

PROJECT

@@ -18,4 +18,13 @@ resources:
   group: mysql
   kind: Status
   version: v1alpha1
+- api:
+    crdVersion: v1
+    namespaced: true
+  controller: true
+  domain: radondb.com
+  group: mysql
+  kind: Backup
+  path: github.com/radondb/radondb-mysql-kubernetes/api/v1alpha1
+  version: v1alpha1
 version: "3"

api/v1alpha1/backup_types.go

@@ -0,0 +1,109 @@
+/*
+Copyright 2021 RadonDB.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package v1alpha1
+
+import (
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+// EDIT THIS FILE!  THIS IS SCAFFOLDING FOR YOU TO OWN!
+// NOTE: json tags are required.  Any new fields you add must have json tags for the fields to be serialized.
+
+// BackupSpec defines the desired state of Backup
+type BackupSpec struct {
+	// INSERT ADDITIONAL SPEC FIELDS - desired state of cluster
+	// Important: Run "make" to regenerate code after modifying this file
+
+	// To specify the image that will be used for sidecar container.
+	// +optional
+	// +kubebuilder:default:="radondb/mysql-sidecar:0.1.88"
+	Image string `json:"image"`
+
+	// HostName represents the host for which to take backup
+	HostName string `json:"hostname"`
+
+	// Cluster represents the cluster name to backup
+	ClusterName string `json:"clustname"`
+
+	// History Limit of job
+	// +optional
+	// +kubebuilder:default:=3
+	HistoryLimit *int32 `json:"historyLimit,omitempty"`
+}
+
+// BackupStatus defines the observed state of Backup
+type BackupStatus struct {
+	// INSERT ADDITIONAL STATUS FIELD - define observed state of cluster
+	// Important: Run "make" to regenerate code after modifying this file
+
+	// Completed indicates whether the backup is in a final state,
+	// no matter whether its' corresponding job failed or succeeded
+	Completed bool `json:"completed,omitempty"`
+
+	// Conditions represents the backup resource conditions list.
+	Conditions []BackupCondition `json:"conditions,omitempty"`
+}
+
+//+kubebuilder:object:root=true
+//+kubebuilder:subresource:status
+
+// Backup is the Schema for the backups API
+type Backup struct {
+	metav1.TypeMeta   `json:",inline"`
+	metav1.ObjectMeta `json:"metadata,omitempty"`
+
+	Spec   BackupSpec   `json:"spec,omitempty"`
+	Status BackupStatus `json:"status,omitempty"`
+}
+
+// BackupCondition defines condition struct for backup resource
+type BackupCondition struct {
+	// type of cluster condition, values in (\"Ready\")
+	Type BackupConditionType `json:"type"`
+	// Status of the condition, one of (\"True\", \"False\", \"Unknown\")
+	Status corev1.ConditionStatus `json:"status"`
+	// LastTransitionTime
+	LastTransitionTime metav1.Time `json:"lastTransitionTime"`
+	// Reason
+	Reason string `json:"reason"`
+	// Message
+	Message string `json:"message"`
+}
+
+// BackupConditionType defines condition types of a backup resources
+type BackupConditionType string
+
+const (
+	// BackupComplete means the backup has finished his execution
+	BackupComplete BackupConditionType = "Complete"
+	// BackupFailed means backup has failed
+	BackupFailed BackupConditionType = "Failed"
+)
+
+//+kubebuilder:object:root=true
+
+// BackupList contains a list of Backup
+type BackupList struct {
+	metav1.TypeMeta `json:",inline"`
+	metav1.ListMeta `json:"metadata,omitempty"`
+	Items           []Backup `json:"items"`
+}
+
+func init() {
+	SchemeBuilder.Register(&Backup{}, &BackupList{})
+}

api/v1alpha1/cluster_types.go

@@ -72,6 +72,15 @@ type ClusterSpec struct {
 	// +optional
 	// +kubebuilder:default:={enabled: true, accessModes: {"ReadWriteOnce"}, size: "10Gi"}
 	Persistence Persistence `json:"persistence,omitempty"`
+
+	// Represents the name of the secret that contains credentials to connect to
+	// the storage provider to store backups.
+	// +optional
+	BackupSecretName string `json:"backupSecretName,omitempty"`
+
+	// Represents the name of the cluster restore from backup path
+	// +optional
+	RestoreFrom string `json:"restoreFrom,omitempty"`
 }
 
 // MysqlOpts defines the options of MySQL container.

api/v1alpha1/zz_generated.deepcopy.go

@@ -0,0 +1,55 @@
+/*
+Copyright 2021 RadonDB.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package backup
+
+import (
+	"fmt"
+
+	"github.com/radondb/radondb-mysql-kubernetes/utils"
+	logf "sigs.k8s.io/controller-runtime/pkg/log"
+
+	v1alhpa1 "github.com/radondb/radondb-mysql-kubernetes/api/v1alpha1"
+)
+
+var log = logf.Log.WithName("backup")
+
+// Backup is a type wrapper over Backup that contains the Business logic
+type Backup struct {
+	*v1alhpa1.Backup
+}
+
+// New returns a wraper object over Backup
+func New(backup *v1alhpa1.Backup) *Backup {
+	return &Backup{
+		Backup: backup,
+	}
+}
+
+// Unwrap returns the api backup object
+func (b *Backup) Unwrap() *v1alhpa1.Backup {
+	return b.Backup
+}
+
+// GetNameForJob returns the name of the job
+func (b *Backup) GetNameForJob() string {
+	return fmt.Sprintf("%s-backup", b.Name)
+}
+
+// Create the backup Domain Name.
+func (b *Backup) GetBackupURL(cluster_name string, hostname string) string {
+	return fmt.Sprintf("%s.%s-mysql.%s:%v", hostname, cluster_name, b.Namespace, utils.XBackupPort)
+}