diff --git a/cmd/clickhouse/check_and_finalize.go b/cmd/clickhouse/check_and_finalize.go index e86365e..74b258b 100644 --- a/cmd/clickhouse/check_and_finalize.go +++ b/cmd/clickhouse/check_and_finalize.go @@ -2,10 +2,10 @@ package clickhouse import ( "fmt" - "os" "time" "github.com/spf13/cobra" + "github.com/stackvista/stackstate-backup-cli/cmd/cmdutils" "github.com/stackvista/stackstate-backup-cli/internal/app" "github.com/stackvista/stackstate-backup-cli/internal/clients/clickhouse" "github.com/stackvista/stackstate-backup-cli/internal/foundation/config" @@ -34,15 +34,7 @@ func checkAndFinalizeCmd(globalFlags *config.CLIGlobalFlags) *cobra.Command { This command is useful when a restore was started without --wait flag or was interrupted. It will check the restore status and if complete, execute post-restore tasks and scale up resources.`, Run: func(_ *cobra.Command, _ []string) { - appCtx, err := app.NewContext(globalFlags) - if err != nil { - _, _ = fmt.Fprintf(os.Stderr, "error: %v\n", err) - os.Exit(1) - } - if err := runCheckAndFinalize(appCtx); err != nil { - _, _ = fmt.Fprintf(os.Stderr, "error: %v\n", err) - os.Exit(1) - } + cmdutils.Run(globalFlags, runCheckAndFinalize, cmdutils.MinioIsRequired) }, } diff --git a/cmd/clickhouse/list.go b/cmd/clickhouse/list.go index a2d0352..05b0e4c 100644 --- a/cmd/clickhouse/list.go +++ b/cmd/clickhouse/list.go @@ -2,10 +2,10 @@ package clickhouse import ( "fmt" - "os" "sort" "github.com/spf13/cobra" + "github.com/stackvista/stackstate-backup-cli/cmd/cmdutils" "github.com/stackvista/stackstate-backup-cli/internal/app" "github.com/stackvista/stackstate-backup-cli/internal/foundation/config" "github.com/stackvista/stackstate-backup-cli/internal/foundation/output" @@ -18,15 +18,7 @@ func listCmd(globalFlags *config.CLIGlobalFlags) *cobra.Command { Short: "List available Clickhouse backups", Long: `List all Clickhouse backups from the ClickHouse Backup API.`, Run: func(_ *cobra.Command, _ []string) { - appCtx, err := app.NewContext(globalFlags) - if err != nil { - _, _ = fmt.Fprintf(os.Stderr, "error: %v\n", err) - os.Exit(1) - } - if err := runList(appCtx); err != nil { - _, _ = fmt.Fprintf(os.Stderr, "error: %v\n", err) - os.Exit(1) - } + cmdutils.Run(globalFlags, runList, cmdutils.MinioIsRequired) }, } } diff --git a/cmd/clickhouse/restore.go b/cmd/clickhouse/restore.go index 5fb4d27..b92acc7 100644 --- a/cmd/clickhouse/restore.go +++ b/cmd/clickhouse/restore.go @@ -2,10 +2,10 @@ package clickhouse import ( "fmt" - "os" "sort" "github.com/spf13/cobra" + "github.com/stackvista/stackstate-backup-cli/cmd/cmdutils" "github.com/stackvista/stackstate-backup-cli/internal/app" "github.com/stackvista/stackstate-backup-cli/internal/foundation/config" "github.com/stackvista/stackstate-backup-cli/internal/orchestration/portforward" @@ -27,15 +27,7 @@ func restoreCmd(globalFlags *config.CLIGlobalFlags) *cobra.Command { Short: "Restore ClickHouse from a backup archive", Long: `Restore ClickHouse data from a backup archive via ClickHouse Backup API. Waits for completion by default; use --background to run asynchronously.`, Run: func(_ *cobra.Command, _ []string) { - appCtx, err := app.NewContext(globalFlags) - if err != nil { - _, _ = fmt.Fprintf(os.Stderr, "error: %v\n", err) - os.Exit(1) - } - if err := runRestore(appCtx); err != nil { - _, _ = fmt.Fprintf(os.Stderr, "error: %v\n", err) - os.Exit(1) - } + cmdutils.Run(globalFlags, runRestore, cmdutils.MinioIsRequired) }, } diff --git a/cmd/cmdutils/common.go b/cmd/cmdutils/common.go new file mode 100644 index 0000000..d511ef2 --- /dev/null +++ b/cmd/cmdutils/common.go @@ -0,0 +1,36 @@ +package cmdutils + +import ( + "errors" + "fmt" + "io" + "os" + + "github.com/stackvista/stackstate-backup-cli/internal/app" + "github.com/stackvista/stackstate-backup-cli/internal/foundation/config" +) + +const ( + MinioIsRequired bool = true + MinioIsNotRequired bool = false +) + +func Run(globalFlags *config.CLIGlobalFlags, runFunc func(ctx *app.Context) error, minioRequired bool) { + appCtx, err := app.NewContext(globalFlags) + if err != nil { + exitWithError(err, os.Stderr) + } + if minioRequired && !appCtx.Config.Minio.Enabled { + exitWithError(errors.New("commands that interact with Minio require SUSE Observability to be deployed with .Values.global.backup.enabled=true"), os.Stderr) + } + if err := runFunc(appCtx); err != nil { + exitWithError(err, os.Stderr) + } +} + +// ExitWithError prints an error message to the writer and exits with status code 1. +// This is a helper function to avoid repeating error handling code in commands. +func exitWithError(err error, w io.Writer) { + _, _ = fmt.Fprintf(w, "error: %v\n", err) + os.Exit(1) +} diff --git a/cmd/elasticsearch/check_and_finalize.go b/cmd/elasticsearch/check_and_finalize.go index 08863a7..13960c9 100644 --- a/cmd/elasticsearch/check_and_finalize.go +++ b/cmd/elasticsearch/check_and_finalize.go @@ -2,9 +2,9 @@ package elasticsearch import ( "fmt" - "os" "github.com/spf13/cobra" + "github.com/stackvista/stackstate-backup-cli/cmd/cmdutils" "github.com/stackvista/stackstate-backup-cli/internal/app" "github.com/stackvista/stackstate-backup-cli/internal/foundation/config" "github.com/stackvista/stackstate-backup-cli/internal/orchestration/portforward" @@ -25,15 +25,7 @@ func checkAndFinalizeCmd(globalFlags *config.CLIGlobalFlags) *cobra.Command { Long: `Check the status of a restore operation and perform finalization (scale up deployments) if complete. If the restore is still running and --wait is specified, wait for completion before finalizing.`, Run: func(_ *cobra.Command, _ []string) { - appCtx, err := app.NewContext(globalFlags) - if err != nil { - _, _ = fmt.Fprintf(os.Stderr, "error: %v\n", err) - os.Exit(1) - } - if err := runCheckAndFinalize(appCtx); err != nil { - _, _ = fmt.Fprintf(os.Stderr, "error: %v\n", err) - os.Exit(1) - } + cmdutils.Run(globalFlags, runCheckAndFinalize, cmdutils.MinioIsRequired) }, } diff --git a/cmd/elasticsearch/configure.go b/cmd/elasticsearch/configure.go index 52a81c1..0d00199 100644 --- a/cmd/elasticsearch/configure.go +++ b/cmd/elasticsearch/configure.go @@ -2,9 +2,9 @@ package elasticsearch import ( "fmt" - "os" "github.com/spf13/cobra" + "github.com/stackvista/stackstate-backup-cli/cmd/cmdutils" "github.com/stackvista/stackstate-backup-cli/internal/app" "github.com/stackvista/stackstate-backup-cli/internal/foundation/config" "github.com/stackvista/stackstate-backup-cli/internal/orchestration/portforward" @@ -16,15 +16,7 @@ func configureCmd(globalFlags *config.CLIGlobalFlags) *cobra.Command { Short: "Configure Elasticsearch snapshot repository and SLM policy", Long: `Configure Elasticsearch snapshot repository and Snapshot Lifecycle Management (SLM) policy for automated backups.`, Run: func(_ *cobra.Command, _ []string) { - appCtx, err := app.NewContext(globalFlags) - if err != nil { - _, _ = fmt.Fprintf(os.Stderr, "error: %v\n", err) - os.Exit(1) - } - if err := runConfigure(appCtx); err != nil { - _, _ = fmt.Fprintf(os.Stderr, "error: %v\n", err) - os.Exit(1) - } + cmdutils.Run(globalFlags, runConfigure, cmdutils.MinioIsRequired) }, } } diff --git a/cmd/elasticsearch/list-indices.go b/cmd/elasticsearch/list-indices.go index 2367705..cc931dc 100644 --- a/cmd/elasticsearch/list-indices.go +++ b/cmd/elasticsearch/list-indices.go @@ -2,9 +2,9 @@ package elasticsearch import ( "fmt" - "os" "github.com/spf13/cobra" + "github.com/stackvista/stackstate-backup-cli/cmd/cmdutils" "github.com/stackvista/stackstate-backup-cli/internal/app" "github.com/stackvista/stackstate-backup-cli/internal/foundation/config" "github.com/stackvista/stackstate-backup-cli/internal/foundation/output" @@ -16,15 +16,7 @@ func listIndicesCmd(globalFlags *config.CLIGlobalFlags) *cobra.Command { Use: "list-indices", Short: "List Elasticsearch indices", Run: func(_ *cobra.Command, _ []string) { - appCtx, err := app.NewContext(globalFlags) - if err != nil { - _, _ = fmt.Fprintf(os.Stderr, "error: %v\n", err) - os.Exit(1) - } - if err := runListIndices(appCtx); err != nil { - _, _ = fmt.Fprintf(os.Stderr, "error: %v\n", err) - os.Exit(1) - } + cmdutils.Run(globalFlags, runListIndices, cmdutils.MinioIsRequired) }, } } diff --git a/cmd/elasticsearch/list.go b/cmd/elasticsearch/list.go index 42c7864..0c548c6 100644 --- a/cmd/elasticsearch/list.go +++ b/cmd/elasticsearch/list.go @@ -2,10 +2,10 @@ package elasticsearch import ( "fmt" - "os" "sort" "github.com/spf13/cobra" + "github.com/stackvista/stackstate-backup-cli/cmd/cmdutils" "github.com/stackvista/stackstate-backup-cli/internal/app" "github.com/stackvista/stackstate-backup-cli/internal/foundation/config" "github.com/stackvista/stackstate-backup-cli/internal/foundation/output" @@ -17,15 +17,7 @@ func listCmd(globalFlags *config.CLIGlobalFlags) *cobra.Command { Use: "list", Short: "List available Elasticsearch snapshots", Run: func(_ *cobra.Command, _ []string) { - appCtx, err := app.NewContext(globalFlags) - if err != nil { - _, _ = fmt.Fprintf(os.Stderr, "error: %v\n", err) - os.Exit(1) - } - if err := runListSnapshots(appCtx); err != nil { - _, _ = fmt.Fprintf(os.Stderr, "error: %v\n", err) - os.Exit(1) - } + cmdutils.Run(globalFlags, runListSnapshots, cmdutils.MinioIsRequired) }, } } diff --git a/cmd/elasticsearch/list_test.go b/cmd/elasticsearch/list_test.go index b09ad41..81684d6 100644 --- a/cmd/elasticsearch/list_test.go +++ b/cmd/elasticsearch/list_test.go @@ -24,6 +24,7 @@ const ( // minimalMinioStackgraphConfig provides the required Minio and Stackgraph configuration for tests const minimalMinioStackgraphConfig = ` minio: + enabled: true service: name: minio port: 9000 @@ -79,6 +80,7 @@ settings: receiverBaseUrl: "http://receiver:7077" platformVersion: "5.2.0" zookeeperQuorum: "zookeeper:2181" + pvc: "suse-observability-settings-backup-data" job: image: settings-backup:latest waitImage: wait:latest diff --git a/cmd/elasticsearch/restore.go b/cmd/elasticsearch/restore.go index ef5a31f..2e42dd0 100644 --- a/cmd/elasticsearch/restore.go +++ b/cmd/elasticsearch/restore.go @@ -2,12 +2,12 @@ package elasticsearch import ( "fmt" - "os" "sort" "strings" "time" "github.com/spf13/cobra" + "github.com/stackvista/stackstate-backup-cli/cmd/cmdutils" "github.com/stackvista/stackstate-backup-cli/internal/app" es "github.com/stackvista/stackstate-backup-cli/internal/clients/elasticsearch" "github.com/stackvista/stackstate-backup-cli/internal/foundation/config" @@ -38,15 +38,7 @@ func restoreCmd(globalFlags *config.CLIGlobalFlags) *cobra.Command { Short: "Restore Elasticsearch from a snapshot", Long: `Restore Elasticsearch indices from a snapshot. Deletes existing STS indices before restore. Waits for completion by default; use --background to run asynchronously.`, Run: func(_ *cobra.Command, _ []string) { - appCtx, err := app.NewContext(globalFlags) - if err != nil { - _, _ = fmt.Fprintf(os.Stderr, "error: %v\n", err) - os.Exit(1) - } - if err := runRestore(appCtx); err != nil { - _, _ = fmt.Fprintf(os.Stderr, "error: %v\n", err) - os.Exit(1) - } + cmdutils.Run(globalFlags, runRestore, cmdutils.MinioIsRequired) }} cmd.Flags().StringVarP(&snapshotName, "snapshot", "s", "", "Snapshot name to restore (mutually exclusive with --latest)") diff --git a/cmd/settings/check_and_finalize.go b/cmd/settings/check_and_finalize.go index 9ceac4c..97ea29c 100644 --- a/cmd/settings/check_and_finalize.go +++ b/cmd/settings/check_and_finalize.go @@ -1,10 +1,8 @@ package settings import ( - "fmt" - "os" - "github.com/spf13/cobra" + "github.com/stackvista/stackstate-backup-cli/cmd/cmdutils" "github.com/stackvista/stackstate-backup-cli/internal/app" "github.com/stackvista/stackstate-backup-cli/internal/foundation/config" "github.com/stackvista/stackstate-backup-cli/internal/orchestration/restore" @@ -33,15 +31,7 @@ Examples: # Wait for job completion and cleanup sts-backup settings check-and-finalize --job settings-restore-20250128t143000 --wait -n my-namespace`, Run: func(_ *cobra.Command, _ []string) { - appCtx, err := app.NewContext(globalFlags) - if err != nil { - _, _ = fmt.Fprintf(os.Stderr, "error: %v\n", err) - os.Exit(1) - } - if err := runCheckAndFinalize(appCtx); err != nil { - _, _ = fmt.Fprintf(os.Stderr, "error: %v\n", err) - os.Exit(1) - } + cmdutils.Run(globalFlags, runCheckAndFinalize, cmdutils.MinioIsNotRequired) }, } diff --git a/cmd/settings/list.go b/cmd/settings/list.go index dda81f4..a6ccd1c 100644 --- a/cmd/settings/list.go +++ b/cmd/settings/list.go @@ -1,23 +1,34 @@ package settings import ( + "bufio" "context" + "errors" "fmt" - "os" + "slices" "sort" + "strconv" + "strings" + "time" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/s3" "github.com/spf13/cobra" + "github.com/stackvista/stackstate-backup-cli/cmd/cmdutils" "github.com/stackvista/stackstate-backup-cli/internal/app" + "github.com/stackvista/stackstate-backup-cli/internal/clients/k8s" s3client "github.com/stackvista/stackstate-backup-cli/internal/clients/s3" "github.com/stackvista/stackstate-backup-cli/internal/foundation/config" "github.com/stackvista/stackstate-backup-cli/internal/foundation/output" "github.com/stackvista/stackstate-backup-cli/internal/orchestration/portforward" + "github.com/stackvista/stackstate-backup-cli/internal/orchestration/restore" + corev1 "k8s.io/api/core/v1" ) const ( - isMultiPartArchive = false + isMultiPartArchive = false + expectedListJobPodCount = 1 + expectedListJobContainerCount = 1 ) func listCmd(globalFlags *config.CLIGlobalFlags) *cobra.Command { @@ -25,20 +36,87 @@ func listCmd(globalFlags *config.CLIGlobalFlags) *cobra.Command { Use: "list", Short: "List available Settings backups from S3/Minio", Run: func(_ *cobra.Command, _ []string) { - appCtx, err := app.NewContext(globalFlags) - if err != nil { - _, _ = fmt.Fprintf(os.Stderr, "error: %v\n", err) - os.Exit(1) - } - if err := runList(appCtx); err != nil { - _, _ = fmt.Fprintf(os.Stderr, "error: %v\n", err) - os.Exit(1) - } + cmdutils.Run(globalFlags, runList, cmdutils.MinioIsNotRequired) }, } } func runList(appCtx *app.Context) error { + backups, err := getAllBackups(appCtx) + if err != nil { + return err + } + + if len(backups) == 0 { + appCtx.Formatter.PrintMessage("No backups found") + return nil + } + table := output.Table{ + Headers: []string{"NAME", "LAST MODIFIED", "SIZE"}, + Rows: make([][]string, 0, len(backups)), + } + + for _, obj := range backups { + row := []string{ + obj.Filename, + obj.LastModified.Format("2006-01-02 15:04:05 MST"), + output.FormatBytes(obj.Size), + } + table.Rows = append(table.Rows, row) + } + + return appCtx.Formatter.PrintTable(table) +} + +// getAllBackups retrieves backups from all sources (S3 and PVC), deduplicates and sorts them by LastModified time (most recent first) +func getAllBackups(appCtx *app.Context) ([]BackupFileInfo, error) { + var backups []BackupFileInfo + var err error + + // Get backups from S3 if enabled + if appCtx.Config.Minio.Enabled { + if backups, err = getBackupListFromS3(appCtx); err != nil { + return nil, fmt.Errorf("failed to get list of backups from Minio: %v", err) + } + } + + // Get backups from PVC + backupsFromPvc, err := getBackupListFromPVC(appCtx) + if err != nil { + return nil, fmt.Errorf("failed to get list of backups from PVC: %v", err) + } + backups = append(backups, backupsFromPvc...) + + if len(backups) == 0 { + return []BackupFileInfo{}, nil + } + + // Sort by name for deduplication + sort.Slice(backups, func(i, j int) bool { + return backups[i].Filename < backups[j].Filename + }) + + // Deduplicate by filename + backups = slices.CompactFunc(backups, func(i, j BackupFileInfo) bool { + return i.Filename == j.Filename + }) + + // Sort by LastModified time (most recent first) + sort.Slice(backups, func(i, j int) bool { + return backups[i].LastModified.After(backups[j].LastModified) + }) + + return backups, nil +} + +// BackupFileInfo represents metadata for a backup file +type BackupFileInfo struct { + LastModified time.Time // Unix timestamp with nanoseconds + Filename string // Name of the backup file + Size int64 // File size in bytes +} + +func getBackupListFromS3(appCtx *app.Context) ([]BackupFileInfo, error) { // Setup port-forward to Minio serviceName := appCtx.Config.Minio.Service.Name localPort := appCtx.Config.Minio.Service.LocalPortForwardPort @@ -46,7 +124,7 @@ func runList(appCtx *app.Context) error { pf, err := portforward.SetupPortForward(appCtx.K8sClient, appCtx.Namespace, serviceName, localPort, remotePort, appCtx.Logger) if err != nil { - return err + return nil, err } defer close(pf.StopChan) @@ -63,35 +141,149 @@ func runList(appCtx *app.Context) error { result, err := appCtx.S3Client.ListObjectsV2(context.Background(), input) if err != nil { - return fmt.Errorf("failed to list S3 objects: %w", err) + return nil, fmt.Errorf("failed to list S3 objects: %w", err) } // Filter objects based on whether the archive is split or not filteredObjects := s3client.FilterBackupObjects(result.Contents, isMultiPartArchive) - // Sort by LastModified time (most recent first) - sort.Slice(filteredObjects, func(i, j int) bool { - return filteredObjects[i].LastModified.After(filteredObjects[j].LastModified) - }) + var backups []BackupFileInfo + for _, obj := range filteredObjects { + row := BackupFileInfo{ + Filename: obj.Key, + LastModified: obj.LastModified, + Size: obj.Size, + } + backups = append(backups, row) + } + return backups, nil +} - if len(filteredObjects) == 0 { - appCtx.Formatter.PrintMessage("No backups found") - return nil +func getBackupListFromPVC(appCtx *app.Context) ([]BackupFileInfo, error) { + // Create list job + appCtx.Logger.Println() + appCtx.Logger.Infof("Creating job to list Settings backups stored on PVC...") + + jobName := fmt.Sprintf("%s-%s", listJobNameTemplate, time.Now().Format("20060102t150405")) + + if err := createListJob(appCtx.K8sClient, appCtx.Namespace, jobName, appCtx.Config); err != nil { + return nil, fmt.Errorf("failed to create list job: %w", err) } - table := output.Table{ - Headers: []string{"NAME", "LAST MODIFIED", "SIZE"}, - Rows: make([][]string, 0, len(filteredObjects)), + appCtx.Logger.Successf("List job created: %s", jobName) + + defer func() { + err := restore.CleanupResources(appCtx.K8sClient, appCtx.Namespace, jobName, "", appCtx.Logger, false) + if err != nil { + appCtx.Logger.Errorf("failed to clean up resources: %s", err) + } + }() + + if err := restore.WaitForJobCompletion(appCtx.K8sClient, appCtx.Namespace, jobName, appCtx.Logger); err != nil { + return nil, err } - for _, obj := range filteredObjects { - row := []string{ - obj.Key, - obj.LastModified.Format("2006-01-02 15:04:05 MST"), - output.FormatBytes(obj.Size), + appCtx.Logger.Println() + appCtx.Logger.Successf("List job completed successfully") + + jobPodLogs, err := appCtx.K8sClient.GetJobLogs(appCtx.Namespace, jobName) + if err != nil { + return nil, err + } + + podLogsCount := len(jobPodLogs) + if podLogsCount != expectedListJobPodCount { + appCtx.Logger.Errorf("Expected exactly 1 pod log from the list job, got %d", podLogsCount) + return nil, errors.New("fail to get backups from the list job") + } + + containerLogsCount := len(jobPodLogs[0].ContainerLogs) + if containerLogsCount != expectedListJobContainerCount { + appCtx.Logger.Errorf("Expected exactly 2 container log from the list job, got %d", containerLogsCount) + return nil, errors.New("fail to get backups from the list job") + } + + files, err := ParseListJobOutput(jobPodLogs[0].ContainerLogs[0].Logs) + if err != nil { + fmt.Printf("Error parsing files: %v\n", err) + return nil, fmt.Errorf("failed to parse list job output: %w", err) + } + + return files, nil +} + +// createListJob creates a Kubernetes Job and PVC for listing Settings backups from PVC +func createListJob(k8sClient *k8s.Client, namespace string, jobName string, config *config.Config) error { + defaultMode := int32(configMapDefaultFileMode) + + // Merge common labels with resource-specific labels + jobLabels := k8s.MergeLabels(config.Kubernetes.CommonLabels, config.Settings.Restore.Job.Labels) + + listEnvVar := buildEnvVar([]corev1.EnvVar{}, config) + + // Build job spec using configuration + spec := k8s.JobSpec{ + Name: jobName, + Labels: jobLabels, + ImagePullSecrets: k8s.ConvertImagePullSecrets(config.Settings.Restore.Job.ImagePullSecrets), + SecurityContext: k8s.ConvertPodSecurityContext(&config.Settings.Restore.Job.SecurityContext), + NodeSelector: config.Settings.Restore.Job.NodeSelector, + Tolerations: k8s.ConvertTolerations(config.Settings.Restore.Job.Tolerations), + Affinity: k8s.ConvertAffinity(config.Settings.Restore.Job.Affinity), + Containers: []corev1.Container{buildContainer(listEnvVar, []string{"bash", "-c", "find /settings-backup-data/ -maxdepth 1 -type f -printf '%T@ %f %s\n'"}, config)}, + Volumes: buildVolumes(config, defaultMode), + } + + // Create job + if _, err := k8sClient.CreateJob(namespace, spec); err != nil { + return fmt.Errorf("failed to create job: %w", err) + } + + return nil +} + +// ParseListJobOutput parses a multiline string containing backup file information +func ParseListJobOutput(input string) ([]BackupFileInfo, error) { + var files []BackupFileInfo + + scanner := bufio.NewScanner(strings.NewReader(input)) + for scanner.Scan() { + line := strings.TrimSpace(scanner.Text()) + if line == "" { + continue } - table.Rows = append(table.Rows, row) + + // Split the line into parts + parts := strings.Fields(line) + if len(parts) != 3 { //nolint:mnd + return nil, fmt.Errorf("invalid line format: expected 3 fields, got %d", len(parts)) + } + + // Parse Unix timestamp (ignore fractional part) + timestampFloat, err := strconv.ParseFloat(parts[0], 64) + if err != nil { + return nil, fmt.Errorf("failed to parse timestamp: %w", err) + } + + // Convert to time.Time (truncate to seconds) + timestamp := time.Unix(int64(timestampFloat), 0).UTC() + + // Parse file size + size, err := strconv.ParseInt(parts[2], 10, 64) + if err != nil { + return nil, fmt.Errorf("failed to parse file size: %w", err) + } + + files = append(files, BackupFileInfo{ + LastModified: timestamp, + Filename: parts[1], + Size: size, + }) } - return appCtx.Formatter.PrintTable(table) + if err := scanner.Err(); err != nil { + return nil, fmt.Errorf("error reading input: %w", err) + } + + return files, nil } diff --git a/cmd/settings/restore.go b/cmd/settings/restore.go index 5e553fd..68131a2 100644 --- a/cmd/settings/restore.go +++ b/cmd/settings/restore.go @@ -1,28 +1,24 @@ package settings import ( - "context" "fmt" - "os" - "sort" + "strconv" "time" - "github.com/aws/aws-sdk-go-v2/aws" - "github.com/aws/aws-sdk-go-v2/service/s3" "github.com/spf13/cobra" + "github.com/stackvista/stackstate-backup-cli/cmd/cmdutils" "github.com/stackvista/stackstate-backup-cli/internal/app" "github.com/stackvista/stackstate-backup-cli/internal/clients/k8s" - s3client "github.com/stackvista/stackstate-backup-cli/internal/clients/s3" "github.com/stackvista/stackstate-backup-cli/internal/foundation/config" "github.com/stackvista/stackstate-backup-cli/internal/foundation/logger" - "github.com/stackvista/stackstate-backup-cli/internal/orchestration/portforward" "github.com/stackvista/stackstate-backup-cli/internal/orchestration/restore" "github.com/stackvista/stackstate-backup-cli/internal/orchestration/scale" corev1 "k8s.io/api/core/v1" ) const ( - jobNameTemplate = "settings-restore" + restoreJobNameTemplate = "settings-restore" + listJobNameTemplate = "settings-list" configMapDefaultFileMode = 0755 ) @@ -40,15 +36,7 @@ func restoreCmd(globalFlags *config.CLIGlobalFlags) *cobra.Command { Short: "Restore Settings from a backup archive", Long: `Restore Settings data from a backup archive stored in S3/Minio. Can use --latest or --archive to specify which backup to restore.`, Run: func(_ *cobra.Command, _ []string) { - appCtx, err := app.NewContext(globalFlags) - if err != nil { - _, _ = fmt.Fprintf(os.Stderr, "error: %v\n", err) - os.Exit(1) - } - if err := runRestore(appCtx); err != nil { - _, _ = fmt.Fprintf(os.Stderr, "error: %v\n", err) - os.Exit(1) - } + cmdutils.Run(globalFlags, runRestore, cmdutils.MinioIsNotRequired) }, } @@ -67,7 +55,7 @@ func runRestore(appCtx *app.Context) error { backupFile := archiveName if useLatest { appCtx.Logger.Infof("Finding latest backup...") - latest, err := getLatestBackup(appCtx.K8sClient, appCtx.Namespace, appCtx.Config, appCtx.Logger) + latest, err := getLatestBackup(appCtx) if err != nil { return err } @@ -118,7 +106,7 @@ func runRestore(appCtx *app.Context) error { appCtx.Logger.Println() appCtx.Logger.Infof("Creating restore job for backup: %s", backupFile) - jobName := fmt.Sprintf("%s-%s", jobNameTemplate, time.Now().Format("20060102t150405")) + jobName := fmt.Sprintf("%s-%s", restoreJobNameTemplate, time.Now().Format("20060102t150405")) if err = createRestoreJob(appCtx.K8sClient, appCtx.Namespace, jobName, backupFile, appCtx.Config); err != nil { return fmt.Errorf("failed to create restore job: %w", err) @@ -137,56 +125,22 @@ func runRestore(appCtx *app.Context) error { // waitAndCleanupRestoreJob waits for job completion and cleans up resources func waitAndCleanupRestoreJob(k8sClient *k8s.Client, namespace, jobName string, log *logger.Logger) error { restore.PrintWaitingMessage(log, "settings", jobName, namespace) - return restore.WaitAndCleanup(k8sClient, namespace, jobName, log, true) + return restore.WaitAndCleanup(k8sClient, namespace, jobName, log, false) } -// getLatestBackup retrieves the most recent backup from S3 -func getLatestBackup(k8sClient *k8s.Client, namespace string, config *config.Config, log *logger.Logger) (string, error) { - // Setup port-forward to Minio - serviceName := config.Minio.Service.Name - localPort := config.Minio.Service.LocalPortForwardPort - remotePort := config.Minio.Service.Port - - pf, err := portforward.SetupPortForward(k8sClient, namespace, serviceName, localPort, remotePort, log) - if err != nil { - return "", err - } - defer close(pf.StopChan) - - // Create S3 client - endpoint := fmt.Sprintf("http://localhost:%d", pf.LocalPort) - s3Client, err := s3client.NewClient(endpoint, config.Minio.AccessKey, config.Minio.SecretKey) +// getLatestBackup retrieves the most recent backup from all sources (S3 and PVC) +func getLatestBackup(appCtx *app.Context) (string, error) { + backups, err := getAllBackups(appCtx) if err != nil { return "", err } - // List objects in bucket - bucket := config.Settings.Bucket - prefix := config.Settings.S3Prefix - - input := &s3.ListObjectsV2Input{ - Bucket: aws.String(bucket), - Prefix: aws.String(prefix), + if len(backups) == 0 { + return "", fmt.Errorf("no backups found") } - result, err := s3Client.ListObjectsV2(context.Background(), input) - if err != nil { - return "", fmt.Errorf("failed to list S3 objects: %w", err) - } - - // Filter objects based on whether the archive is split or not - filteredObjects := s3client.FilterBackupObjects(result.Contents, isMultiPartArchive) - - if len(filteredObjects) == 0 { - return "", fmt.Errorf("no backups found in bucket %s", bucket) - } - - // Sort by LastModified time (most recent first) - sort.Slice(filteredObjects, func(i, j int) bool { - return filteredObjects[i].LastModified.After(filteredObjects[j].LastModified) - }) - - return filteredObjects[0].Key, nil + // getAllBackups returns backups sorted by LastModified (most recent first) + return backups[0].Filename, nil } // createRestoreJob creates a Kubernetes Job and PVC for restoring from backup @@ -196,8 +150,10 @@ func createRestoreJob(k8sClient *k8s.Client, namespace, jobName, backupFile stri // Merge common labels with resource-specific labels jobLabels := k8s.MergeLabels(config.Kubernetes.CommonLabels, config.Settings.Restore.Job.Labels) + restoreEnvVar := buildEnvVar([]corev1.EnvVar{{Name: "BACKUP_FILE", Value: backupFile}}, config) + // Build job spec using configuration - spec := k8s.BackupJobSpec{ + spec := k8s.JobSpec{ Name: jobName, Labels: jobLabels, ImagePullSecrets: k8s.ConvertImagePullSecrets(config.Settings.Restore.Job.ImagePullSecrets), @@ -205,23 +161,21 @@ func createRestoreJob(k8sClient *k8s.Client, namespace, jobName, backupFile stri NodeSelector: config.Settings.Restore.Job.NodeSelector, Tolerations: k8s.ConvertTolerations(config.Settings.Restore.Job.Tolerations), Affinity: k8s.ConvertAffinity(config.Settings.Restore.Job.Affinity), - Containers: buildRestoreContainers(backupFile, config), - InitContainers: buildRestoreInitContainers(config), - Volumes: buildRestoreVolumes(config, defaultMode), + Containers: []corev1.Container{buildContainer(restoreEnvVar, []string{"/backup-restore-scripts/restore-settings-backup.sh"}, config)}, + Volumes: buildVolumes(config, defaultMode), } // Create job - if _, err := k8sClient.CreateBackupJob(namespace, spec); err != nil { + if _, err := k8sClient.CreateJob(namespace, spec); err != nil { return fmt.Errorf("failed to create job: %w", err) } return nil } -// buildRestoreEnvVars constructs environment variables for the restore job -func buildRestoreEnvVars(backupFile string, config *config.Config) []corev1.EnvVar { - return []corev1.EnvVar{ - {Name: "BACKUP_FILE", Value: backupFile}, +// buildEnvVar constructs environment variables for the container spec +func buildEnvVar(extraEnvVar []corev1.EnvVar, config *config.Config) []corev1.EnvVar { + commonVar := []corev1.EnvVar{ {Name: "BACKUP_CONFIGURATION_BUCKET_NAME", Value: config.Settings.Bucket}, {Name: "BACKUP_CONFIGURATION_S3_PREFIX", Value: config.Settings.S3Prefix}, {Name: "MINIO_ENDPOINT", Value: fmt.Sprintf("%s:%d", config.Minio.Service.Name, config.Minio.Service.Port)}, @@ -229,38 +183,25 @@ func buildRestoreEnvVars(backupFile string, config *config.Config) []corev1.EnvV {Name: "RECEIVER_BASE_URL", Value: config.Settings.Restore.ReceiverBaseURL}, {Name: "PLATFORM_VERSION", Value: config.Settings.Restore.PlatformVersion}, {Name: "ZOOKEEPER_QUORUM", Value: config.Settings.Restore.ZookeeperQuorum}, + {Name: "BACKUP_CONFIGURATION_UPLOAD_REMOTE", Value: strconv.FormatBool(config.Minio.Enabled)}, } + commonVar = append(commonVar, extraEnvVar...) + return commonVar } -// buildRestoreVolumeMounts constructs volume mounts for the restore job container -func buildRestoreVolumeMounts() []corev1.VolumeMount { +// buildVolumeMounts constructs volume mounts for the restore job container +func buildVolumeMounts() []corev1.VolumeMount { return []corev1.VolumeMount{ {Name: "backup-log", MountPath: "/opt/docker/etc_log"}, {Name: "backup-restore-scripts", MountPath: "/backup-restore-scripts"}, {Name: "minio-keys", MountPath: "/aws-keys"}, {Name: "tmp-data", MountPath: "/tmp-data"}, + {Name: "settings-backup-data", MountPath: "/settings-backup-data"}, } } -// buildRestoreInitContainers constructs init containers for the restore job -func buildRestoreInitContainers(config *config.Config) []corev1.Container { - return []corev1.Container{ - { - Name: "wait", - Image: config.Settings.Restore.Job.WaitImage, - ImagePullPolicy: corev1.PullIfNotPresent, - Command: []string{ - "sh", - "-c", - fmt.Sprintf("/entrypoint -c %s:%d -t 300", config.Minio.Service.Name, config.Minio.Service.Port), - }, - SecurityContext: k8s.ConvertSecurityContext(config.Settings.Restore.Job.ContainerSecurityContext), - }, - } -} - -// buildRestoreVolumes constructs volumes for the restore job pod -func buildRestoreVolumes(config *config.Config, defaultMode int32) []corev1.Volume { +// buildVolumes constructs volumes for the restore job pod +func buildVolumes(config *config.Config, defaultMode int32) []corev1.Volume { return []corev1.Volume{ { Name: "backup-log", @@ -297,21 +238,27 @@ func buildRestoreVolumes(config *config.Config, defaultMode int32) []corev1.Volu EmptyDir: &corev1.EmptyDirVolumeSource{}, }, }, + { + Name: "settings-backup-data", + VolumeSource: corev1.VolumeSource{ + PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ + ClaimName: config.Settings.Restore.PVC, + }, + }, + }, } } -// buildRestoreContainers constructs containers for the restore job -func buildRestoreContainers(backupFile string, config *config.Config) []corev1.Container { - return []corev1.Container{ - { - Name: "restore", - Image: config.Settings.Restore.Job.Image, - ImagePullPolicy: corev1.PullIfNotPresent, - SecurityContext: k8s.ConvertSecurityContext(config.Settings.Restore.Job.ContainerSecurityContext), - Command: []string{"/backup-restore-scripts/restore-settings-backup.sh"}, - Env: buildRestoreEnvVars(backupFile, config), - Resources: k8s.ConvertResources(config.Settings.Restore.Job.Resources), - VolumeMounts: buildRestoreVolumeMounts(), - }, +// buildContainers constructs containers for the restore job +func buildContainer(envVar []corev1.EnvVar, command []string, config *config.Config) corev1.Container { + return corev1.Container{ + Name: "settings", + Image: config.Settings.Restore.Job.Image, + ImagePullPolicy: corev1.PullIfNotPresent, + SecurityContext: k8s.ConvertSecurityContext(config.Settings.Restore.Job.ContainerSecurityContext), + Command: command, + Env: envVar, + Resources: k8s.ConvertResources(config.Settings.Restore.Job.Resources), + VolumeMounts: buildVolumeMounts(), } } diff --git a/cmd/stackgraph/check_and_finalize.go b/cmd/stackgraph/check_and_finalize.go index 8139270..8ab92e1 100644 --- a/cmd/stackgraph/check_and_finalize.go +++ b/cmd/stackgraph/check_and_finalize.go @@ -1,10 +1,8 @@ package stackgraph import ( - "fmt" - "os" - "github.com/spf13/cobra" + "github.com/stackvista/stackstate-backup-cli/cmd/cmdutils" "github.com/stackvista/stackstate-backup-cli/internal/app" "github.com/stackvista/stackstate-backup-cli/internal/foundation/config" "github.com/stackvista/stackstate-backup-cli/internal/orchestration/restore" @@ -33,15 +31,7 @@ Examples: # Wait for job completion and cleanup sts-backup stackgraph check-and-finalize --job stackgraph-restore-20250128t143000 --wait -n my-namespace`, Run: func(_ *cobra.Command, _ []string) { - appCtx, err := app.NewContext(globalFlags) - if err != nil { - _, _ = fmt.Fprintf(os.Stderr, "error: %v\n", err) - os.Exit(1) - } - if err := runCheckAndFinalize(appCtx); err != nil { - _, _ = fmt.Fprintf(os.Stderr, "error: %v\n", err) - os.Exit(1) - } + cmdutils.Run(globalFlags, runCheckAndFinalize, cmdutils.MinioIsRequired) }, } diff --git a/cmd/stackgraph/list.go b/cmd/stackgraph/list.go index 0d45f45..73a81de 100644 --- a/cmd/stackgraph/list.go +++ b/cmd/stackgraph/list.go @@ -3,12 +3,12 @@ package stackgraph import ( "context" "fmt" - "os" "sort" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/s3" "github.com/spf13/cobra" + "github.com/stackvista/stackstate-backup-cli/cmd/cmdutils" "github.com/stackvista/stackstate-backup-cli/internal/app" s3client "github.com/stackvista/stackstate-backup-cli/internal/clients/s3" "github.com/stackvista/stackstate-backup-cli/internal/foundation/config" @@ -21,15 +21,7 @@ func listCmd(globalFlags *config.CLIGlobalFlags) *cobra.Command { Use: "list", Short: "List available Stackgraph backups from S3/Minio", Run: func(_ *cobra.Command, _ []string) { - appCtx, err := app.NewContext(globalFlags) - if err != nil { - _, _ = fmt.Fprintf(os.Stderr, "error: %v\n", err) - os.Exit(1) - } - if err := runList(appCtx); err != nil { - _, _ = fmt.Fprintf(os.Stderr, "error: %v\n", err) - os.Exit(1) - } + cmdutils.Run(globalFlags, runList, cmdutils.MinioIsRequired) }, } } diff --git a/cmd/stackgraph/restore.go b/cmd/stackgraph/restore.go index cc49992..d6036f4 100644 --- a/cmd/stackgraph/restore.go +++ b/cmd/stackgraph/restore.go @@ -3,7 +3,6 @@ package stackgraph import ( "context" "fmt" - "os" "sort" "strconv" "time" @@ -11,6 +10,7 @@ import ( "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/s3" "github.com/spf13/cobra" + "github.com/stackvista/stackstate-backup-cli/cmd/cmdutils" "github.com/stackvista/stackstate-backup-cli/internal/app" "github.com/stackvista/stackstate-backup-cli/internal/clients/k8s" s3client "github.com/stackvista/stackstate-backup-cli/internal/clients/s3" @@ -42,15 +42,7 @@ func restoreCmd(globalFlags *config.CLIGlobalFlags) *cobra.Command { Short: "Restore Stackgraph from a backup archive", Long: `Restore Stackgraph data from a backup archive stored in S3/Minio. Can use --latest or --archive to specify which backup to restore.`, Run: func(_ *cobra.Command, _ []string) { - appCtx, err := app.NewContext(globalFlags) - if err != nil { - _, _ = fmt.Fprintf(os.Stderr, "error: %v\n", err) - os.Exit(1) - } - if err := runRestore(appCtx); err != nil { - _, _ = fmt.Fprintf(os.Stderr, "error: %v\n", err) - os.Exit(1) - } + cmdutils.Run(globalFlags, runRestore, cmdutils.MinioIsRequired) }, } @@ -236,7 +228,7 @@ func createRestoreJob(k8sClient *k8s.Client, namespace, jobName, backupFile stri } // Build job spec using configuration - spec := k8s.BackupJobSpec{ + spec := k8s.JobSpec{ Name: jobName, Labels: jobLabels, ImagePullSecrets: k8s.ConvertImagePullSecrets(config.Stackgraph.Restore.Job.ImagePullSecrets), @@ -250,7 +242,7 @@ func createRestoreJob(k8sClient *k8s.Client, namespace, jobName, backupFile stri } // Create job - _, err = k8sClient.CreateBackupJob(namespace, spec) + _, err = k8sClient.CreateJob(namespace, spec) if err != nil { // Cleanup PVC if job creation fails _ = k8sClient.DeletePVC(namespace, pvc.Name) diff --git a/cmd/victoriametrics/check_and_finalize.go b/cmd/victoriametrics/check_and_finalize.go index 768a5f8..50875d7 100644 --- a/cmd/victoriametrics/check_and_finalize.go +++ b/cmd/victoriametrics/check_and_finalize.go @@ -1,10 +1,8 @@ package victoriametrics import ( - "fmt" - "os" - "github.com/spf13/cobra" + "github.com/stackvista/stackstate-backup-cli/cmd/cmdutils" "github.com/stackvista/stackstate-backup-cli/internal/app" "github.com/stackvista/stackstate-backup-cli/internal/foundation/config" "github.com/stackvista/stackstate-backup-cli/internal/orchestration/restore" @@ -33,15 +31,7 @@ Examples: # Wait for job completion and cleanup sts-backup victoriametrics check-and-finalize --job victoriametrics-restore-20250128t143000 --wait -n my-namespace`, Run: func(_ *cobra.Command, _ []string) { - appCtx, err := app.NewContext(globalFlags) - if err != nil { - _, _ = fmt.Fprintf(os.Stderr, "error: %v\n", err) - os.Exit(1) - } - if err := runCheckAndFinalize(appCtx); err != nil { - _, _ = fmt.Fprintf(os.Stderr, "error: %v\n", err) - os.Exit(1) - } + cmdutils.Run(globalFlags, runCheckAndFinalize, cmdutils.MinioIsRequired) }, } diff --git a/cmd/victoriametrics/list.go b/cmd/victoriametrics/list.go index ea5d8d7..839db9e 100644 --- a/cmd/victoriametrics/list.go +++ b/cmd/victoriametrics/list.go @@ -3,13 +3,13 @@ package victoriametrics import ( "context" "fmt" - "os" "sort" "time" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/s3" "github.com/spf13/cobra" + "github.com/stackvista/stackstate-backup-cli/cmd/cmdutils" "github.com/stackvista/stackstate-backup-cli/internal/app" s3client "github.com/stackvista/stackstate-backup-cli/internal/clients/s3" "github.com/stackvista/stackstate-backup-cli/internal/foundation/config" @@ -27,15 +27,7 @@ func listCmd(globalFlags *config.CLIGlobalFlags) *cobra.Command { Use: "list", Short: "List available VictoriaMetrics backups from S3/Minio", Run: func(_ *cobra.Command, _ []string) { - appCtx, err := app.NewContext(globalFlags) - if err != nil { - _, _ = fmt.Fprintf(os.Stderr, "error: %v\n", err) - os.Exit(1) - } - if err := runList(appCtx); err != nil { - _, _ = fmt.Fprintf(os.Stderr, "error: %v\n", err) - os.Exit(1) - } + cmdutils.Run(globalFlags, runList, cmdutils.MinioIsRequired) }, } } diff --git a/cmd/victoriametrics/restore.go b/cmd/victoriametrics/restore.go index fb1c007..725be16 100644 --- a/cmd/victoriametrics/restore.go +++ b/cmd/victoriametrics/restore.go @@ -3,13 +3,13 @@ package victoriametrics import ( "context" "fmt" - "os" "sort" "time" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/s3" "github.com/spf13/cobra" + "github.com/stackvista/stackstate-backup-cli/cmd/cmdutils" "github.com/stackvista/stackstate-backup-cli/internal/app" "github.com/stackvista/stackstate-backup-cli/internal/clients/k8s" s3client "github.com/stackvista/stackstate-backup-cli/internal/clients/s3" @@ -40,15 +40,7 @@ func restoreCmd(globalFlags *config.CLIGlobalFlags) *cobra.Command { Short: "Restore VictoriaMetrics from a backup archive", Long: `Restore VictoriaMetrics data from a backup archive stored in S3/Minio. Can use --latest or --archive to specify which backup to restore.`, Run: func(_ *cobra.Command, _ []string) { - appCtx, err := app.NewContext(globalFlags) - if err != nil { - _, _ = fmt.Fprintf(os.Stderr, "error: %v\n", err) - os.Exit(1) - } - if err := runRestore(appCtx); err != nil { - _, _ = fmt.Fprintf(os.Stderr, "error: %v\n", err) - os.Exit(1) - } + cmdutils.Run(globalFlags, runRestore, cmdutils.MinioIsRequired) }, } @@ -205,7 +197,7 @@ func createRestoreJob(k8sClient *k8s.Client, namespace, jobName, backupFile stri jobLabels := k8s.MergeLabels(config.Kubernetes.CommonLabels, config.VictoriaMetrics.Restore.Job.Labels) // Build job spec using configuration - spec := k8s.BackupJobSpec{ + spec := k8s.JobSpec{ Name: jobName, Labels: jobLabels, ImagePullSecrets: k8s.ConvertImagePullSecrets(config.VictoriaMetrics.Restore.Job.ImagePullSecrets), @@ -219,7 +211,7 @@ func createRestoreJob(k8sClient *k8s.Client, namespace, jobName, backupFile stri } // Create job - _, err := k8sClient.CreateBackupJob(namespace, spec) + _, err := k8sClient.CreateJob(namespace, spec) if err != nil { return fmt.Errorf("failed to create job: %w", err) } diff --git a/internal/clients/k8s/job.go b/internal/clients/k8s/job.go index 4e8b0c6..202b515 100644 --- a/internal/clients/k8s/job.go +++ b/internal/clients/k8s/job.go @@ -15,8 +15,8 @@ const ( defaultJobTTLSeconds = 86400 ) -// BackupJobSpec contains all parameters needed to create a backup/restore job -type BackupJobSpec struct { +// JobSpec contains all parameters needed to create a backup/restore job +type JobSpec struct { // Job metadata Name string Labels map[string]string @@ -84,10 +84,10 @@ func (c *Client) CreatePVC(namespace string, spec PVCSpec) (*corev1.PersistentVo return createdPVC, nil } -// CreateBackupJob creates a Kubernetes Job for backup/restore operations +// CreateJob creates a Kubernetes Job for backup/restore operations // Note: PVC must be created separately if needed using CreatePVC // Returns the created Job and any error -func (c *Client) CreateBackupJob(namespace string, spec BackupJobSpec) (*batchv1.Job, error) { +func (c *Client) CreateJob(namespace string, spec JobSpec) (*batchv1.Job, error) { ctx := context.Background() // Build Job spec diff --git a/internal/clients/k8s/job_test.go b/internal/clients/k8s/job_test.go index 0dbde62..cc06420 100644 --- a/internal/clients/k8s/job_test.go +++ b/internal/clients/k8s/job_test.go @@ -102,21 +102,21 @@ func TestClient_CreatePVC(t *testing.T) { } } -// TestClient_CreateBackupJob tests Job creation for backup/restore operations +// TestClient_CreateJob tests Job creation for backup/restore operations // //nolint:funlen -func TestClient_CreateBackupJob(t *testing.T) { +func TestClient_CreateJob(t *testing.T) { tests := []struct { name string namespace string - spec BackupJobSpec + spec JobSpec expectError bool validateFunc func(*testing.T, *batchv1.Job) }{ { name: "create minimal job", namespace: "test-ns", - spec: BackupJobSpec{ + spec: JobSpec{ Name: "backup-job", Labels: map[string]string{"app": "backup"}, Containers: []corev1.Container{ @@ -138,7 +138,7 @@ func TestClient_CreateBackupJob(t *testing.T) { { name: "create job with environment variables", namespace: "test-ns", - spec: BackupJobSpec{ + spec: JobSpec{ Name: "restore-job", Containers: []corev1.Container{ { @@ -161,7 +161,7 @@ func TestClient_CreateBackupJob(t *testing.T) { { name: "create job with resource requirements", namespace: "test-ns", - spec: BackupJobSpec{ + spec: JobSpec{ Name: "resource-job", Containers: []corev1.Container{ { @@ -190,7 +190,7 @@ func TestClient_CreateBackupJob(t *testing.T) { { name: "create job with init containers", namespace: "test-ns", - spec: BackupJobSpec{ + spec: JobSpec{ Name: "init-job", Containers: []corev1.Container{ { @@ -215,7 +215,7 @@ func TestClient_CreateBackupJob(t *testing.T) { { name: "create job with volumes and mounts", namespace: "test-ns", - spec: BackupJobSpec{ + spec: JobSpec{ Name: "volume-job", Containers: []corev1.Container{ { @@ -257,7 +257,7 @@ func TestClient_CreateBackupJob(t *testing.T) { { name: "create job with security context", namespace: "test-ns", - spec: BackupJobSpec{ + spec: JobSpec{ Name: "secure-job", Containers: []corev1.Container{ { @@ -286,7 +286,7 @@ func TestClient_CreateBackupJob(t *testing.T) { { name: "create job with node selector and tolerations", namespace: "test-ns", - spec: BackupJobSpec{ + spec: JobSpec{ Name: "scheduled-job", Containers: []corev1.Container{ { @@ -317,7 +317,7 @@ func TestClient_CreateBackupJob(t *testing.T) { { name: "create job with image pull secrets", namespace: "test-ns", - spec: BackupJobSpec{ + spec: JobSpec{ Name: "private-image-job", Containers: []corev1.Container{ { @@ -342,7 +342,7 @@ func TestClient_CreateBackupJob(t *testing.T) { fakeClient := fake.NewSimpleClientset() client := &Client{clientset: fakeClient} - job, err := client.CreateBackupJob(tt.namespace, tt.spec) + job, err := client.CreateJob(tt.namespace, tt.spec) if tt.expectError { assert.Error(t, err) diff --git a/internal/foundation/config/config.go b/internal/foundation/config/config.go index 3814a24..7c8ada9 100644 --- a/internal/foundation/config/config.go +++ b/internal/foundation/config/config.go @@ -79,6 +79,7 @@ type ServiceConfig struct { // MinioConfig holds Minio-specific configuration type MinioConfig struct { + Enabled bool `yaml:"enabled" validate:"boolean"` Service ServiceConfig `yaml:"service" validate:"required"` AccessKey string `yaml:"accessKey" validate:"required"` // From secret SecretKey string `yaml:"secretKey" validate:"required"` // From secret @@ -133,6 +134,7 @@ type SettingsRestoreConfig struct { PlatformVersion string `yaml:"platformVersion" validate:"required"` ZookeeperQuorum string `yaml:"zookeeperQuorum" validate:"required"` Job JobConfig `yaml:"job" validate:"required"` + PVC string `yaml:"pvc" validate:"required"` } // ClickhouseConfig holds Clickhouse-specific configuration diff --git a/internal/foundation/config/config_test.go b/internal/foundation/config/config_test.go index c2201eb..48a13a1 100644 --- a/internal/foundation/config/config_test.go +++ b/internal/foundation/config/config_test.go @@ -359,6 +359,7 @@ func TestConfig_StructValidation(t *testing.T) { }, }, Minio: MinioConfig{ + Enabled: true, Service: ServiceConfig{ Name: "minio", Port: 9000, @@ -435,6 +436,7 @@ func TestConfig_StructValidation(t *testing.T) { ReceiverBaseURL: "http://receiver:7077", PlatformVersion: "5.2.0", ZookeeperQuorum: "zookeeper:2181", + PVC: "suse-observability-settings-backup-data", Job: JobConfig{ Image: "settings-backup:latest", WaitImage: "wait:latest", diff --git a/internal/foundation/config/testdata/validConfigMapConfig.yaml b/internal/foundation/config/testdata/validConfigMapConfig.yaml index af41ced..02c68a3 100644 --- a/internal/foundation/config/testdata/validConfigMapConfig.yaml +++ b/internal/foundation/config/testdata/validConfigMapConfig.yaml @@ -63,6 +63,7 @@ elasticsearch: # Minio configuration for S3-compatible storage minio: + enabled: true # Minio service connection details service: name: suse-observability-minio @@ -148,6 +149,7 @@ settings: receiverBaseUrl: "http://suse-observability-receiver:7077" platformVersion: "5.2.0" zookeeperQuorum: "suse-observability-zookeeper:2181" + pvc: "suse-observability-settings-backup-data" job: labels: app: settings-restore diff --git a/internal/foundation/config/testdata/validConfigMapOnly.yaml b/internal/foundation/config/testdata/validConfigMapOnly.yaml index f7eeb1b..c6d11ae 100644 --- a/internal/foundation/config/testdata/validConfigMapOnly.yaml +++ b/internal/foundation/config/testdata/validConfigMapOnly.yaml @@ -70,6 +70,7 @@ elasticsearch: # Minio configuration for S3-compatible storage minio: + enabled: true service: name: suse-observability-minio port: 9000 @@ -138,6 +139,7 @@ settings: receiverBaseUrl: "http://suse-observability-receiver:7077" platformVersion: "5.2.0" zookeeperQuorum: "suse-observability-zookeeper:2181" + pvc: "suse-observability-settings-backup-data" job: labels: app: settings-restore diff --git a/internal/scripts/scripts/restore-settings-backup.sh b/internal/scripts/scripts/restore-settings-backup.sh index 3502181..ed6531f 100644 --- a/internal/scripts/scripts/restore-settings-backup.sh +++ b/internal/scripts/scripts/restore-settings-backup.sh @@ -6,14 +6,16 @@ export TMP_DIR=/tmp-data RESTORE_FILE="${BACKUP_DIR}/${BACKUP_FILE}" -export AWS_ACCESS_KEY_ID -AWS_ACCESS_KEY_ID="$(cat /aws-keys/accesskey)" -export AWS_SECRET_ACCESS_KEY -AWS_SECRET_ACCESS_KEY="$(cat /aws-keys/secretkey)" +if [ "$BACKUP_CONFIGURATION_UPLOAD_REMOTE" == "true" ] && [ ! -f "${RESTORE_FILE}" ]; then + export AWS_ACCESS_KEY_ID + AWS_ACCESS_KEY_ID="$(cat /aws-keys/accesskey)" + export AWS_SECRET_ACCESS_KEY + AWS_SECRET_ACCESS_KEY="$(cat /aws-keys/secretkey)" -echo "=== Downloading Settings backup \"${BACKUP_FILE}\" from bucket \"${BACKUP_CONFIGURATION_BUCKET_NAME}\"..." -sts-toolbox aws s3 --endpoint "http://${MINIO_ENDPOINT}" --region minio cp "s3://${BACKUP_CONFIGURATION_BUCKET_NAME}/${BACKUP_CONFIGURATION_S3_PREFIX}${BACKUP_FILE}" "${TMP_DIR}/${BACKUP_FILE}" -RESTORE_FILE="${TMP_DIR}/${BACKUP_FILE}" + echo "=== Downloading Settings backup \"${BACKUP_FILE}\" from bucket \"${BACKUP_CONFIGURATION_BUCKET_NAME}\"..." + sts-toolbox aws s3 --endpoint "http://${MINIO_ENDPOINT}" --region minio cp "s3://${BACKUP_CONFIGURATION_BUCKET_NAME}/${BACKUP_CONFIGURATION_S3_PREFIX}${BACKUP_FILE}" "${TMP_DIR}/${BACKUP_FILE}" + RESTORE_FILE="${TMP_DIR}/${BACKUP_FILE}" +fi if [ ! -f "${RESTORE_FILE}" ]; then echo "=== Backup file \"${RESTORE_FILE}\" not found, exiting..."