feat: add sbs-migration tool (#68)

Author: Tomy2e

Dockerfile

@@ -1,4 +1,4 @@
-FROM golang:1.22-alpine as builder
+FROM golang:1.22-alpine AS builder
 
 RUN apk update && apk add --no-cache git ca-certificates && update-ca-certificates
 
@@ -15,9 +15,11 @@ ARG TAG
 ARG COMMIT_SHA
 ARG BUILD_DATE
 RUN CGO_ENABLED=0 GOOS=${TARGETOS} GOARCH=${TARGETARCH} go build -a -ldflags "-w -s -X github.com/scaleway/scaleway-csi/driver.driverVersion=${TAG} -X github.com/scaleway/scaleway-csi/driver.buildDate=${BUILD_DATE} -X github.com/scaleway/scaleway-csi/driver.gitCommit=${COMMIT_SHA} " -o scaleway-csi ./cmd/scaleway-csi
+RUN CGO_ENABLED=0 GOOS=${TARGETOS} GOARCH=${TARGETARCH} go build -a -ldflags "-w -s -X github.com/scaleway/scaleway-csi/driver.driverVersion=${TAG} -X github.com/scaleway/scaleway-csi/driver.buildDate=${BUILD_DATE} -X github.com/scaleway/scaleway-csi/driver.gitCommit=${COMMIT_SHA} " -o sbs-migration ./cmd/sbs-migration
 
 FROM alpine:3.15
 RUN apk update && apk add --no-cache e2fsprogs e2fsprogs-extra xfsprogs xfsprogs-extra cryptsetup ca-certificates blkid && update-ca-certificates
 WORKDIR /
 COPY --from=builder /go/src/github.com/scaleway/scaleway-csi/scaleway-csi .
+COPY --from=builder /go/src/github.com/scaleway/scaleway-csi/sbs-migration .
 ENTRYPOINT ["/scaleway-csi"]

cmd/sbs-migration/README.md

@@ -0,0 +1,37 @@
+# sbs-migration
+
+The `sbs-migration` tool migrates your Kubernetes `PersistentVolumes` from [Instance](https://console.scaleway.com/instance/volumes)
+to the new [Block Storage](https://console.scaleway.com/block-storage/volumes) product. There is no downtime during the migration.
+Your Snapshots will also be migrated.
+
+**You must run this tool during the `v0.{1,2}.X` to `v0.3.X` upgrade of the Scaleway CSI.**
+
+> [!CAUTION]
+> This tool is intended to be used by customers who manage their own Kubernetes cluster.
+>
+> **DO NOT USE THIS TOOL IF YOUR CLUSTER IS MANAGED BY SCALEWAY (e.g. Kapsule / Kosmos)**.
+
+## Requirements
+
+- The kubeconfig of a Kubernetes cluster that you manage, with the Scaleway CSI installed (version `v0.1.X` or `v0.2.X`).
+- [Go 1.20+](https://go.dev/dl/).
+- Kubectl CLI.
+
+## Usage
+
+Please read all the steps before executing any command.
+
+1. Stop the CSI controller: `$ kubectl scale deployment scaleway-csi-controller -n kube-system --replicas=0`.
+2. Set the following environment variables:
+
+   ```bash
+   export SCW_DEFAULT_ZONE=fr-par-1
+   export SCW_DEFAULT_PROJECT_ID=11111111-1111-1111-1111-111111111111
+   export SCW_ACCESS_KEY=SCW123456789ABCDE
+   export SCW_SECRET_KEY=11111111-1111-1111-1111-111111111111
+   ```
+
+3. Run the `sbs-migration` tool with dry-run enabled: `$ go run cmd/sbs-migration/main.go -kubeconfig=<path to your kubeconfig> -dry-run`.
+4. If you are happy with the dry-run result, run the `sbs-migration` with dry-run disabled
+   to effectively migrate your volumes and snapshots: `$ go run cmd/sbs-migration/main.go -kubeconfig=<path to your kubeconfig>`.
+5. Upgrade the CSI to `v0.3.1` or higher.

cmd/sbs-migration/main.go

@@ -0,0 +1,60 @@
+package main
+
+import (
+	"context"
+	"flag"
+
+	"github.com/scaleway/scaleway-csi/pkg/driver"
+	"github.com/scaleway/scaleway-csi/pkg/migration"
+	"github.com/scaleway/scaleway-csi/pkg/scaleway"
+	"k8s.io/client-go/dynamic"
+	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/tools/clientcmd"
+	"k8s.io/klog/v2"
+)
+
+func main() {
+	var (
+		ctx = context.Background()
+
+		// Flags
+		kubeconfig               = flag.String("kubeconfig", "", "absolute path to the kubeconfig file")
+		disableVolumeMigration   = flag.Bool("disable-volume-migration", false, "Disables listing volumes and migrating them")
+		disableSnapshotMigration = flag.Bool("disable-snapshot-migration", false, "Disables listing snapshots and migrating them")
+		dryRun                   = flag.Bool("dry-run", false, "Simulates the volume and snapshot migration process")
+	)
+	flag.Parse()
+
+	// Create Kubernetes client.
+	config, err := clientcmd.BuildConfigFromFlags("", *kubeconfig)
+	if err != nil {
+		klog.Fatal(err)
+	}
+
+	clientset, err := kubernetes.NewForConfig(config)
+	if err != nil {
+		klog.Fatal(err)
+	}
+
+	dynClient, err := dynamic.NewForConfig(config)
+	if err != nil {
+		klog.Fatal(err)
+	}
+
+	// Create Scaleway client.
+	scw, err := scaleway.New(driver.UserAgent())
+	if err != nil {
+		klog.Fatal(err)
+	}
+
+	// Migrate volumes and snapshots from Instance to Block API.
+	opts := &migration.Options{
+		DryRun:                   *dryRun,
+		DisableVolumeMigration:   *disableVolumeMigration,
+		DisableSnapshotMigration: *disableSnapshotMigration,
+	}
+
+	if err := migration.New(clientset, dynClient, scw, opts).Do(ctx); err != nil {
+		klog.Fatal(err)
+	}
+}

go.mod

@@ -3,6 +3,7 @@ module github.com/scaleway/scaleway-csi
 go 1.22.2
 
 require (
+	github.com/avast/retry-go/v4 v4.6.0
 	github.com/container-storage-interface/spec v1.9.0
 	github.com/golang/protobuf v1.5.4
 	github.com/google/uuid v1.6.0
@@ -13,27 +14,55 @@ require (
 	github.com/scaleway/scaleway-sdk-go v1.0.0-beta.26
 	golang.org/x/crypto v0.22.0
 	golang.org/x/exp v0.0.0-20240416160154-fe59bbe5cc7f
+	golang.org/x/sync v0.7.0
 	golang.org/x/sys v0.19.0
 	google.golang.org/grpc v1.63.2
 	google.golang.org/protobuf v1.33.0
+	k8s.io/apimachinery v0.30.0
+	k8s.io/client-go v0.30.0
 	k8s.io/klog/v2 v2.120.1
 	k8s.io/mount-utils v0.30.0
 	k8s.io/utils v0.0.0-20240310230437-4693a0247e57
 	oya.to/namedlocker v1.0.0
 )
 
 require (
+	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/emicklei/go-restful/v3 v3.11.0 // indirect
 	github.com/go-logr/logr v1.4.1 // indirect
+	github.com/go-openapi/jsonpointer v0.19.6 // indirect
+	github.com/go-openapi/jsonreference v0.20.2 // indirect
+	github.com/go-openapi/swag v0.22.3 // indirect
 	github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 // indirect
+	github.com/gogo/protobuf v1.3.2 // indirect
+	github.com/google/gnostic-models v0.6.8 // indirect
 	github.com/google/go-cmp v0.6.0 // indirect
+	github.com/google/gofuzz v1.2.0 // indirect
 	github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 // indirect
+	github.com/imdario/mergo v0.3.6 // indirect
+	github.com/josharian/intern v1.0.0 // indirect
+	github.com/json-iterator/go v1.1.12 // indirect
 	github.com/kr/fs v0.1.0 // indirect
-	github.com/kr/text v0.2.0 // indirect
+	github.com/mailru/easyjson v0.7.7 // indirect
 	github.com/moby/sys/mountinfo v0.6.2 // indirect
+	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
+	github.com/modern-go/reflect2 v1.0.2 // indirect
+	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
+	github.com/spf13/pflag v1.0.5 // indirect
 	golang.org/x/net v0.24.0 // indirect
+	golang.org/x/oauth2 v0.17.0 // indirect
+	golang.org/x/term v0.19.0 // indirect
 	golang.org/x/text v0.14.0 // indirect
+	golang.org/x/time v0.3.0 // indirect
 	golang.org/x/tools v0.20.0 // indirect
+	google.golang.org/appengine v1.6.8 // indirect
 	google.golang.org/genproto/googleapis/rpc v0.0.0-20240227224415-6ceb2ff114de // indirect
+	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
+	k8s.io/api v0.30.0 // indirect
+	k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 // indirect
+	sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect
+	sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect
+	sigs.k8s.io/yaml v1.3.0 // indirect
 )