diff --git a/.github/workflows/pr-check.yaml b/.github/workflows/pr-check.yaml index 580fa9f..a446d66 100644 --- a/.github/workflows/pr-check.yaml +++ b/.github/workflows/pr-check.yaml @@ -6,28 +6,28 @@ on: jobs: lint: name: Lint - runs-on: ubuntu-20.04 + runs-on: ubuntu-24.04 steps: - - name: Setup up Go 1.x + - name: Setup up Go 1.23 uses: actions/setup-go@v5 with: - go-version: "^1.15" + go-version: "1.23" - name: Check out code uses: actions/checkout@v4 - name: golangci-lint uses: golangci/golangci-lint-action@v6 with: - version: v1.58.2 + version: v1.63.4 args: --timeout=5m build: name: Test & Build - runs-on: ubuntu-20.04 + runs-on: ubuntu-24.04 steps: - - name: Setup up Go 1.x + - name: Setup up Go 1.23 uses: actions/setup-go@v5 with: - go-version: "^1.15" + go-version: "1.23" - name: Check out code uses: actions/checkout@v4 diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 67c9d93..bfb3ac2 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -14,7 +14,7 @@ env: jobs: push: name: Push images - runs-on: ubuntu-20.04 + runs-on: ubuntu-24.04 steps: - name: Check out code @@ -67,7 +67,7 @@ jobs: release: name: Release - runs-on: ubuntu-20.04 + runs-on: ubuntu-24.04 # Run only if previous job has succeeded needs: [push] @@ -87,9 +87,11 @@ jobs: echo "---" >> manifest.yaml cat deploy/k8s/csidriver.yaml >> manifest.yaml echo "---" >> manifest.yaml - sed -E "s|image: +cloudstack-csi-driver|image: ${REGISTRY_NAME}/cloudstack-csi-driver:${VERSION}|" deploy/k8s/controller-deployment.yaml >> manifest.yaml + sed -E "s|(image: +${REGISTRY_NAME}/cloudstack-csi-driver)(:[^ ]+)?|\\1:${VERSION}|" deploy/k8s/controller-deployment.yaml >> manifest.yaml echo "---" >> manifest.yaml - sed -E "s|image: +cloudstack-csi-driver|image: ${REGISTRY_NAME}/cloudstack-csi-driver:${VERSION}|" deploy/k8s/node-daemonset.yaml >> manifest.yaml + sed -E "s|(image: +${REGISTRY_NAME}/cloudstack-csi-driver)(:[^ ]+)?|\\1:${VERSION}|" deploy/k8s/node-daemonset.yaml >> manifest.yaml + echo "---" >> manifest.yaml + cat deploy/k8s/volume-snapshot-class.yaml >> manifest.yaml - name: Create Release id: create_release @@ -102,6 +104,16 @@ jobs: draft: false prerelease: false + - name: Upload Snapshot CRDs Asset + uses: actions/upload-release-asset@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + upload_url: ${{ steps.create_release.outputs.upload_url }} + asset_path: deploy/k8s/00-snapshot-crds.yaml + asset_name: snapshot-crds.yaml + asset_content_type: application/x-yaml + - name: Upload Release Asset uses: actions/upload-release-asset@v1 env: diff --git a/Makefile b/Makefile index 7c64ee1..8ac5ce6 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ CMDS=cloudstack-csi-driver cloudstack-csi-sc-syncer -PKG=github.com/shapeblue/cloudstack-csi-driver +PKG=github.com/cloudstack/cloudstack-csi-driver # Revision that gets built into each binary via the main.version # string. Uses the `git describe` output based on the most recent # version tag with a short revision suffix or, if nothing has been diff --git a/README.md b/README.md index 55641bb..6a57e02 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,7 @@ +**Fork Notice:** + +This repo is a fork of the [Leaseweb's] (https://github.com/leaseweb/cloudstack-csi-driver) maitained cloudstack-csi-driver, which is in-turn a fork of [Apalia's](https://github.com/apalia/cloudstack-csi-driver) cloudstack-csi-driver + # CloudStack CSI Driver [![Go Reference](https://pkg.go.dev/badge/github.com/cloudstack/cloudstack-csi-driver.svg)](https://pkg.go.dev/github.com/cloudstack/cloudstack-csi-driver) @@ -76,6 +80,11 @@ The storage class must also have a parameter named `csi.cloudstack.apache.org/disk-offering-id` whose value is the CloudStack disk offering ID. +**Reclaim Policy**: Storage classes can have a `reclaimPolicy` of either `Delete` or `Retain`. If no `reclaimPolicy` is specified, it defaults to `Delete`. + +- `Delete`: When a PVC is deleted or a CKS cluster (Managed Kubernetes Cluster in CloudStack) is deleted, the associated persistent volumes and their underlying CloudStack disk volumes will be automatically removed. +- `Retain`: Persistent volumes and their underlying CloudStack disk volumes will be preserved even after PVC deletion or cluster deletion, allowing for manual recovery or data preservation. + #### Using cloudstack-csi-sc-syncer The tool `cloudstack-csi-sc-syncer` may also be used to synchronize CloudStack @@ -83,6 +92,16 @@ disk offerings to Kubernetes storage classes. [More info...](./cmd/cloudstack-csi-sc-syncer/README.md) +> **Note:** The VolumeSnapshot CRDs (CustomResourceDefinitions) of version 8.3.0 are installed in this deployment. If you use a different version, please ensure compatibility with your Kubernetes cluster and CSI sidecars. + + +``` +kubectl apply -f https://raw.githubusercontent.com/kubernetes-csi/external-snapshotter/v8.3.0/client/config/crd/snapshot.storage.k8s.io_volumesnapshotclasses.yaml +kubectl apply -f https://raw.githubusercontent.com/kubernetes-csi/external-snapshotter/v8.3.0/client/config/crd/snapshot.storage.k8s.io_volumesnapshots.yaml +kubectl apply -f https://raw.githubusercontent.com/kubernetes-csi/external-snapshotter/v8.3.0/client/config/crd/snapshot.storage.k8s.io_volumesnapshotcontents.yaml + +``` + ### Usage Example: @@ -106,6 +125,115 @@ To build the container images: make container ``` + +## Volume Snapshots + +**NOTE:** To create volume snapshots in KVM, make sure to set the `kvm.snapshot.enabled` global setting to true and restart the Management Server + +### Volume snapshot creation +For Volume snapshots to be created, the following configurations need to be applied: + +``` +kubectl apply -f deploy/k8s/00-snapshot-crds.yaml # Installs the VolumeSnapshotClass, VolumeSnapshotContent and VolumeSnapshtot CRDs +kubectl apply -f deploy/k8s/volume-snapshot-class.yaml # Defines VolumeSnapshotClass for CloudStack CSI driver +``` + +Once the CRDs are installed, the snapshot can be taken by applying: +``` +kubectl apply ./examples/k8s/snapshot/snapshot.yaml +``` + +In order to take the snapshot of a volume, `persistentVolumeClaimName` should be set to the right PVC name that is bound to the volume whose snapshot is to be taken. + +You can check CloudStack volume snapshots if the snapshot was successfully created. If for any reason there was an issue, it can be investgated by checking the logs of the cloudstack-csi-controller pods: cloudstack-csi-controller, csi-snapshotter and snapshot-controller containers + +``` +kubectl logs -f -n kube-system # defaults to tailing logs of cloudstack-csi-controller +kubectl logs -f -n kube-system -c csi-snapshotter +kubectl logs -f -n kube-system -c snapshot-controller +``` + +### Restoring a Volume snapshot + +To restore a volume snapshot: +1. Restore a snapshot and Use it in a pod +* Create a PVC from the snapshot - for example ./examples/k8s/snapshot/pvc-from-snapshot.yaml +* Apply the configuration: +``` +kubectl apply -f ./examples/k8s/snapshot/pvc-from-snapshot.yaml +``` +* Create a pod that uses the restored PVC; example pod config ./examples/k8s/snapshot/restore-pod.yaml +``` +kubectl apply -f ./examples/k8s/snapshot/restore-pod.yaml +``` +2. To restore a snapshot when using a deployment +Update the deployment to point to the restored PVC + +``` +spec: + volumes: + - name: app-volume + persistentVolumeClaim: + claimName: pvc-from-snapshot +``` + + +### Deletion of a volume snapshot + +To delete a volume snapshot +One can simlpy delete the volume snapshot created in kubernetes using + +``` +kubectl delete volumesnapshot snapshot-1 # here, snapshot-1 is the name of the snapshot created +``` + +#### Troubleshooting issues with volume snapshot deletion +If for whatever reason, snapshot deletion gets stuck, one can troubleshoot the issue doing the following: + +* Inspect the snapshot + +``` +kubectl get volumesnapshot [-n ] -o yaml +``` + +Look for the following section: +``` +metadata: + finalizers: + - snapshot.storage.kubernetes.io/volumesnapshot-as-source +``` + +If finalizers are present, Kubernetes will not delete the resource until they are removed or resolved. + +* Patch to Remove Finalizers + +``` +kubectl patch volumesnapshot [-n ] --type=merge -p '{"metadata":{"finalizers":[]}}' +``` + +**Caution:** This bypasses cleanup logic. Use only if you're certain the snapshot is no longer needed at the CSI/backend level + +### What happens when you restore a volume from a snapshot +* The CSI external-provisioner (a container in the cloudstack-csi-controller pod) sees the new PVC and notices it references a snapshot +* The CSI driver's `CreateVolume` method is called with a `VolumeContentSource` that contains the snapshot ID +* The CSI driver creates a new volume from the snapshot (using the CloudStack's createVolume API) +* The new volume is now available as a PV (persistent volume) and is bound to the new PVC +* The volume is NOT attached to any node just by restoring from a snapshot, the volume is only attached to a node when a Pod that uses the new PVC is scheduled on a node +* The CSI driver's `ControllerPublishVolume` and `NodePublishVolume` methods are called to attach and mount the volume to the node where the Pod is running + +Hence to debug any issues during restoring a snapshot, check the logs of the cloudstack-csi-controller, external-provisioner containers + +``` +kubectl logs -f -n kube-system # defaults to tailing logs of cloudstack-csi-controller +kubectl logs -f -n kube-system -c external-provisioner +``` + +## Additional General Notes: + +**Node Scheduling Best Practices**: When deploying applications that require specific node placement, use `nodeSelector` or `nodeAffinity` instead of `nodeName`. The `nodeName` field bypasses the Kubernetes scheduler, which can cause issues with storage provisioning. When a StorageClass has `volumeBindingMode: WaitForFirstConsumer`, the CSI controller relies on scheduler decisions to properly bind PVCs. Using `nodeName` prevents this scheduling integration, potentially causing PVC binding failures. + +**Network CIDR Considerations**: When deploying CKS (CloudStack Kubernetes Service) clusters on pre-existing networks, avoid using the `10.0.0.0/16` CIDR range as it conflicts with Calico's default pod network configuration. This overlap can prevent proper CSI driver initialization and may cause networking issues within the cluster. + ## See also - [CloudStack Kubernetes Provider](https://github.com/apache/cloudstack-kubernetes-provider) - Kubernetes Cloud Controller Manager for Apache CloudStack diff --git a/charts/cloudstack-csi/Chart.yaml b/charts/cloudstack-csi/Chart.yaml index e187a38..48ccd0d 100644 --- a/charts/cloudstack-csi/Chart.yaml +++ b/charts/cloudstack-csi/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: cloudstack-csi description: A Helm chart for CloudStack CSI driver type: application -version: 2.0.3 +version: 3.0.0 appVersion: 0.6.1 sources: - https://github.com/cloudstack/cloudstack-csi-driver diff --git a/cmd/cloudstack-csi-driver/Dockerfile b/cmd/cloudstack-csi-driver/Dockerfile index 8bf16e2..97ee99f 100644 --- a/cmd/cloudstack-csi-driver/Dockerfile +++ b/cmd/cloudstack-csi-driver/Dockerfile @@ -14,7 +14,9 @@ RUN apk add --no-cache \ # blkid, mount and umount are required by k8s.io/mount-utils \ blkid \ mount \ - umount + umount \ + # Provides udevadm for device path detection \ + udev COPY ./bin/cloudstack-csi-driver /cloudstack-csi-driver -ENTRYPOINT ["/cloudstack-csi-driver"] \ No newline at end of file +ENTRYPOINT ["/cloudstack-csi-driver"] diff --git a/cmd/cloudstack-csi-driver/main.go b/cmd/cloudstack-csi-driver/main.go index 576cd69..eb17261 100644 --- a/cmd/cloudstack-csi-driver/main.go +++ b/cmd/cloudstack-csi-driver/main.go @@ -1,3 +1,22 @@ +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// + // cloudstack-csi-driver binary. // // To get usage information: diff --git a/cmd/cloudstack-csi-sc-syncer/main.go b/cmd/cloudstack-csi-sc-syncer/main.go index 9ac113e..14415bf 100644 --- a/cmd/cloudstack-csi-sc-syncer/main.go +++ b/cmd/cloudstack-csi-sc-syncer/main.go @@ -1,3 +1,22 @@ +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// + // Small utility to synchronize CloudStack disk offerings to // Kubernetes storage classes. package main diff --git a/deploy/k8s/00-snapshot-crds.yaml b/deploy/k8s/00-snapshot-crds.yaml new file mode 100644 index 0000000..b6a402b --- /dev/null +++ b/deploy/k8s/00-snapshot-crds.yaml @@ -0,0 +1,954 @@ +## CRD for VolumeSnapshotClass from https://github.com/kubernetes-csi/external-snapshotter/blob/v8.3.0/client/config/crd/groupsnapshot.storage.k8s.io_volumegroupsnapshotclasses.yaml +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + api-approved.kubernetes.io: "https://github.com/kubernetes-csi/external-snapshotter/pull/814" + controller-gen.kubebuilder.io/version: v0.15.0 + name: volumesnapshotclasses.snapshot.storage.k8s.io +spec: + group: snapshot.storage.k8s.io + names: + kind: VolumeSnapshotClass + listKind: VolumeSnapshotClassList + plural: volumesnapshotclasses + shortNames: + - vsclass + - vsclasses + singular: volumesnapshotclass + scope: Cluster + versions: + - additionalPrinterColumns: + - jsonPath: .driver + name: Driver + type: string + - description: Determines whether a VolumeSnapshotContent created through the + VolumeSnapshotClass should be deleted when its bound VolumeSnapshot is deleted. + jsonPath: .deletionPolicy + name: DeletionPolicy + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1 + schema: + openAPIV3Schema: + description: |- + VolumeSnapshotClass specifies parameters that a underlying storage system uses when + creating a volume snapshot. A specific VolumeSnapshotClass is used by specifying its + name in a VolumeSnapshot object. + VolumeSnapshotClasses are non-namespaced + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + deletionPolicy: + description: |- + deletionPolicy determines whether a VolumeSnapshotContent created through + the VolumeSnapshotClass should be deleted when its bound VolumeSnapshot is deleted. + Supported values are "Retain" and "Delete". + "Retain" means that the VolumeSnapshotContent and its physical snapshot on underlying storage system are kept. + "Delete" means that the VolumeSnapshotContent and its physical snapshot on underlying storage system are deleted. + Required. + enum: + - Delete + - Retain + type: string + driver: + description: |- + driver is the name of the storage driver that handles this VolumeSnapshotClass. + Required. + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + parameters: + additionalProperties: + type: string + description: |- + parameters is a key-value map with storage driver specific parameters for creating snapshots. + These values are opaque to Kubernetes. + type: object + required: + - deletionPolicy + - driver + type: object + served: true + storage: true + subresources: {} + - additionalPrinterColumns: + - jsonPath: .driver + name: Driver + type: string + - description: Determines whether a VolumeSnapshotContent created through the VolumeSnapshotClass should be deleted when its bound VolumeSnapshot is deleted. + jsonPath: .deletionPolicy + name: DeletionPolicy + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta1 + # This indicates the v1beta1 version of the custom resource is deprecated. + # API requests to this version receive a warning in the server response. + deprecated: true + # This overrides the default warning returned to clients making v1beta1 API requests. + deprecationWarning: "snapshot.storage.k8s.io/v1beta1 VolumeSnapshotClass is deprecated; use snapshot.storage.k8s.io/v1 VolumeSnapshotClass" + schema: + openAPIV3Schema: + description: VolumeSnapshotClass specifies parameters that a underlying storage system uses when creating a volume snapshot. A specific VolumeSnapshotClass is used by specifying its name in a VolumeSnapshot object. VolumeSnapshotClasses are non-namespaced + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + deletionPolicy: + description: deletionPolicy determines whether a VolumeSnapshotContent created through the VolumeSnapshotClass should be deleted when its bound VolumeSnapshot is deleted. Supported values are "Retain" and "Delete". "Retain" means that the VolumeSnapshotContent and its physical snapshot on underlying storage system are kept. "Delete" means that the VolumeSnapshotContent and its physical snapshot on underlying storage system are deleted. Required. + enum: + - Delete + - Retain + type: string + driver: + description: driver is the name of the storage driver that handles this VolumeSnapshotClass. Required. + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + parameters: + additionalProperties: + type: string + description: parameters is a key-value map with storage driver specific parameters for creating snapshots. These values are opaque to Kubernetes. + type: object + required: + - deletionPolicy + - driver + type: object + served: false + storage: false + subresources: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +## VolumeSnapshotContent CRD - https://github.com/kubernetes-csi/external-snapshotter/blob/v8.3.0/client/config/crd/snapshot.storage.k8s.io_volumesnapshotcontents.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.15.0 + api-approved.kubernetes.io: "https://github.com/kubernetes-csi/external-snapshotter/pull/955" + name: volumesnapshotcontents.snapshot.storage.k8s.io +spec: + group: snapshot.storage.k8s.io + names: + kind: VolumeSnapshotContent + listKind: VolumeSnapshotContentList + plural: volumesnapshotcontents + shortNames: + - vsc + - vscs + singular: volumesnapshotcontent + scope: Cluster + versions: + - additionalPrinterColumns: + - description: Indicates if the snapshot is ready to be used to restore a volume. + jsonPath: .status.readyToUse + name: ReadyToUse + type: boolean + - description: Represents the complete size of the snapshot in bytes + jsonPath: .status.restoreSize + name: RestoreSize + type: integer + - description: Determines whether this VolumeSnapshotContent and its physical + snapshot on the underlying storage system should be deleted when its bound + VolumeSnapshot is deleted. + jsonPath: .spec.deletionPolicy + name: DeletionPolicy + type: string + - description: Name of the CSI driver used to create the physical snapshot on + the underlying storage system. + jsonPath: .spec.driver + name: Driver + type: string + - description: Name of the VolumeSnapshotClass to which this snapshot belongs. + jsonPath: .spec.volumeSnapshotClassName + name: VolumeSnapshotClass + type: string + - description: Name of the VolumeSnapshot object to which this VolumeSnapshotContent + object is bound. + jsonPath: .spec.volumeSnapshotRef.name + name: VolumeSnapshot + type: string + - description: Namespace of the VolumeSnapshot object to which this VolumeSnapshotContent + object is bound. + jsonPath: .spec.volumeSnapshotRef.namespace + name: VolumeSnapshotNamespace + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1 + schema: + openAPIV3Schema: + description: |- + VolumeSnapshotContent represents the actual "on-disk" snapshot object in the + underlying storage system + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: |- + spec defines properties of a VolumeSnapshotContent created by the underlying storage system. + Required. + properties: + deletionPolicy: + description: |- + deletionPolicy determines whether this VolumeSnapshotContent and its physical snapshot on + the underlying storage system should be deleted when its bound VolumeSnapshot is deleted. + Supported values are "Retain" and "Delete". + "Retain" means that the VolumeSnapshotContent and its physical snapshot on underlying storage system are kept. + "Delete" means that the VolumeSnapshotContent and its physical snapshot on underlying storage system are deleted. + For dynamically provisioned snapshots, this field will automatically be filled in by the + CSI snapshotter sidecar with the "DeletionPolicy" field defined in the corresponding + VolumeSnapshotClass. + For pre-existing snapshots, users MUST specify this field when creating the + VolumeSnapshotContent object. + Required. + enum: + - Delete + - Retain + type: string + driver: + description: |- + driver is the name of the CSI driver used to create the physical snapshot on + the underlying storage system. + This MUST be the same as the name returned by the CSI GetPluginName() call for + that driver. + Required. + type: string + source: + description: |- + source specifies whether the snapshot is (or should be) dynamically provisioned + or already exists, and just requires a Kubernetes object representation. + This field is immutable after creation. + Required. + properties: + snapshotHandle: + description: |- + snapshotHandle specifies the CSI "snapshot_id" of a pre-existing snapshot on + the underlying storage system for which a Kubernetes object representation + was (or should be) created. + This field is immutable. + type: string + x-kubernetes-validations: + - message: snapshotHandle is immutable + rule: self == oldSelf + volumeHandle: + description: |- + volumeHandle specifies the CSI "volume_id" of the volume from which a snapshot + should be dynamically taken from. + This field is immutable. + type: string + x-kubernetes-validations: + - message: volumeHandle is immutable + rule: self == oldSelf + type: object + x-kubernetes-validations: + - message: volumeHandle is required once set + rule: '!has(oldSelf.volumeHandle) || has(self.volumeHandle)' + - message: snapshotHandle is required once set + rule: '!has(oldSelf.snapshotHandle) || has(self.snapshotHandle)' + - message: exactly one of volumeHandle and snapshotHandle must be + set + rule: (has(self.volumeHandle) && !has(self.snapshotHandle)) || (!has(self.volumeHandle) + && has(self.snapshotHandle)) + sourceVolumeMode: + description: |- + SourceVolumeMode is the mode of the volume whose snapshot is taken. + Can be either “Filesystem” or “Block”. + If not specified, it indicates the source volume's mode is unknown. + This field is immutable. + This field is an alpha field. + type: string + x-kubernetes-validations: + - message: sourceVolumeMode is immutable + rule: self == oldSelf + volumeSnapshotClassName: + description: |- + name of the VolumeSnapshotClass from which this snapshot was (or will be) + created. + Note that after provisioning, the VolumeSnapshotClass may be deleted or + recreated with different set of values, and as such, should not be referenced + post-snapshot creation. + type: string + volumeSnapshotRef: + description: |- + volumeSnapshotRef specifies the VolumeSnapshot object to which this + VolumeSnapshotContent object is bound. + VolumeSnapshot.Spec.VolumeSnapshotContentName field must reference to + this VolumeSnapshotContent's name for the bidirectional binding to be valid. + For a pre-existing VolumeSnapshotContent object, name and namespace of the + VolumeSnapshot object MUST be provided for binding to happen. + This field is immutable after creation. + Required. + properties: + apiVersion: + description: API version of the referent. + type: string + fieldPath: + description: |- + If referring to a piece of an object instead of an entire object, this string + should contain a valid JSON/Go field access statement, such as desiredState.manifest.containers[2]. + For example, if the object reference is to a container within a pod, this would take on a value like: + "spec.containers{name}" (where "name" refers to the name of the container that triggered + the event) or if no container name is specified "spec.containers[2]" (container with + index 2 in this pod). This syntax is chosen only to have some well-defined way of + referencing a part of an object. + TODO: this design is not final and this field is subject to change in the future. + type: string + kind: + description: |- + Kind of the referent. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + name: + description: |- + Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + namespace: + description: |- + Namespace of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/ + type: string + resourceVersion: + description: |- + Specific resourceVersion to which this reference is made, if any. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency + type: string + uid: + description: |- + UID of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids + type: string + type: object + x-kubernetes-map-type: atomic + x-kubernetes-validations: + - message: both spec.volumeSnapshotRef.name and spec.volumeSnapshotRef.namespace + must be set + rule: has(self.name) && has(self.__namespace__) + required: + - deletionPolicy + - driver + - source + - volumeSnapshotRef + type: object + x-kubernetes-validations: + - message: sourceVolumeMode is required once set + rule: '!has(oldSelf.sourceVolumeMode) || has(self.sourceVolumeMode)' + status: + description: status represents the current information of a snapshot. + properties: + creationTime: + description: |- + creationTime is the timestamp when the point-in-time snapshot is taken + by the underlying storage system. + In dynamic snapshot creation case, this field will be filled in by the + CSI snapshotter sidecar with the "creation_time" value returned from CSI + "CreateSnapshot" gRPC call. + For a pre-existing snapshot, this field will be filled with the "creation_time" + value returned from the CSI "ListSnapshots" gRPC call if the driver supports it. + If not specified, it indicates the creation time is unknown. + The format of this field is a Unix nanoseconds time encoded as an int64. + On Unix, the command `date +%s%N` returns the current time in nanoseconds + since 1970-01-01 00:00:00 UTC. + format: int64 + type: integer + error: + description: |- + error is the last observed error during snapshot creation, if any. + Upon success after retry, this error field will be cleared. + properties: + message: + description: |- + message is a string detailing the encountered error during snapshot + creation if specified. + NOTE: message may be logged, and it should not contain sensitive + information. + type: string + time: + description: time is the timestamp when the error was encountered. + format: date-time + type: string + type: object + readyToUse: + description: |- + readyToUse indicates if a snapshot is ready to be used to restore a volume. + In dynamic snapshot creation case, this field will be filled in by the + CSI snapshotter sidecar with the "ready_to_use" value returned from CSI + "CreateSnapshot" gRPC call. + For a pre-existing snapshot, this field will be filled with the "ready_to_use" + value returned from the CSI "ListSnapshots" gRPC call if the driver supports it, + otherwise, this field will be set to "True". + If not specified, it means the readiness of a snapshot is unknown. + type: boolean + restoreSize: + description: |- + restoreSize represents the complete size of the snapshot in bytes. + In dynamic snapshot creation case, this field will be filled in by the + CSI snapshotter sidecar with the "size_bytes" value returned from CSI + "CreateSnapshot" gRPC call. + For a pre-existing snapshot, this field will be filled with the "size_bytes" + value returned from the CSI "ListSnapshots" gRPC call if the driver supports it. + When restoring a volume from this snapshot, the size of the volume MUST NOT + be smaller than the restoreSize if it is specified, otherwise the restoration will fail. + If not specified, it indicates that the size is unknown. + format: int64 + minimum: 0 + type: integer + snapshotHandle: + description: |- + snapshotHandle is the CSI "snapshot_id" of a snapshot on the underlying storage system. + If not specified, it indicates that dynamic snapshot creation has either failed + or it is still in progress. + type: string + volumeGroupSnapshotHandle: + description: |- + VolumeGroupSnapshotHandle is the CSI "group_snapshot_id" of a group snapshot + on the underlying storage system. + type: string + type: object + required: + - spec + type: object + served: true + storage: true + subresources: + status: {} + - additionalPrinterColumns: + - description: Indicates if the snapshot is ready to be used to restore a volume. + jsonPath: .status.readyToUse + name: ReadyToUse + type: boolean + - description: Represents the complete size of the snapshot in bytes + jsonPath: .status.restoreSize + name: RestoreSize + type: integer + - description: Determines whether this VolumeSnapshotContent and its physical snapshot on the underlying storage system should be deleted when its bound VolumeSnapshot is deleted. + jsonPath: .spec.deletionPolicy + name: DeletionPolicy + type: string + - description: Name of the CSI driver used to create the physical snapshot on the underlying storage system. + jsonPath: .spec.driver + name: Driver + type: string + - description: Name of the VolumeSnapshotClass to which this snapshot belongs. + jsonPath: .spec.volumeSnapshotClassName + name: VolumeSnapshotClass + type: string + - description: Name of the VolumeSnapshot object to which this VolumeSnapshotContent object is bound. + jsonPath: .spec.volumeSnapshotRef.name + name: VolumeSnapshot + type: string + - description: Namespace of the VolumeSnapshot object to which this VolumeSnapshotContent object is bound. + jsonPath: .spec.volumeSnapshotRef.namespace + name: VolumeSnapshotNamespace + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta1 + # This indicates the v1beta1 version of the custom resource is deprecated. + # API requests to this version receive a warning in the server response. + deprecated: true + # This overrides the default warning returned to clients making v1beta1 API requests. + deprecationWarning: "snapshot.storage.k8s.io/v1beta1 VolumeSnapshotContent is deprecated; use snapshot.storage.k8s.io/v1 VolumeSnapshotContent" + schema: + openAPIV3Schema: + description: VolumeSnapshotContent represents the actual "on-disk" snapshot object in the underlying storage system + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + spec: + description: spec defines properties of a VolumeSnapshotContent created by the underlying storage system. Required. + properties: + deletionPolicy: + description: deletionPolicy determines whether this VolumeSnapshotContent and its physical snapshot on the underlying storage system should be deleted when its bound VolumeSnapshot is deleted. Supported values are "Retain" and "Delete". "Retain" means that the VolumeSnapshotContent and its physical snapshot on underlying storage system are kept. "Delete" means that the VolumeSnapshotContent and its physical snapshot on underlying storage system are deleted. For dynamically provisioned snapshots, this field will automatically be filled in by the CSI snapshotter sidecar with the "DeletionPolicy" field defined in the corresponding VolumeSnapshotClass. For pre-existing snapshots, users MUST specify this field when creating the VolumeSnapshotContent object. Required. + enum: + - Delete + - Retain + type: string + driver: + description: driver is the name of the CSI driver used to create the physical snapshot on the underlying storage system. This MUST be the same as the name returned by the CSI GetPluginName() call for that driver. Required. + type: string + source: + description: source specifies whether the snapshot is (or should be) dynamically provisioned or already exists, and just requires a Kubernetes object representation. This field is immutable after creation. Required. + properties: + snapshotHandle: + description: snapshotHandle specifies the CSI "snapshot_id" of a pre-existing snapshot on the underlying storage system for which a Kubernetes object representation was (or should be) created. This field is immutable. + type: string + volumeHandle: + description: volumeHandle specifies the CSI "volume_id" of the volume from which a snapshot should be dynamically taken from. This field is immutable. + type: string + type: object + volumeSnapshotClassName: + description: name of the VolumeSnapshotClass from which this snapshot was (or will be) created. Note that after provisioning, the VolumeSnapshotClass may be deleted or recreated with different set of values, and as such, should not be referenced post-snapshot creation. + type: string + volumeSnapshotRef: + description: volumeSnapshotRef specifies the VolumeSnapshot object to which this VolumeSnapshotContent object is bound. VolumeSnapshot.Spec.VolumeSnapshotContentName field must reference to this VolumeSnapshotContent's name for the bidirectional binding to be valid. For a pre-existing VolumeSnapshotContent object, name and namespace of the VolumeSnapshot object MUST be provided for binding to happen. This field is immutable after creation. Required. + properties: + apiVersion: + description: API version of the referent. + type: string + fieldPath: + description: 'If referring to a piece of an object instead of an entire object, this string should contain a valid JSON/Go field access statement, such as desiredState.manifest.containers[2]. For example, if the object reference is to a container within a pod, this would take on a value like: "spec.containers{name}" (where "name" refers to the name of the container that triggered the event) or if no container name is specified "spec.containers[2]" (container with index 2 in this pod). This syntax is chosen only to have some well-defined way of referencing a part of an object. TODO: this design is not final and this field is subject to change in the future.' + type: string + kind: + description: 'Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + type: string + namespace: + description: 'Namespace of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/' + type: string + resourceVersion: + description: 'Specific resourceVersion to which this reference is made, if any. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency' + type: string + uid: + description: 'UID of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids' + type: string + type: object + required: + - deletionPolicy + - driver + - source + - volumeSnapshotRef + type: object + status: + description: status represents the current information of a snapshot. + properties: + creationTime: + description: creationTime is the timestamp when the point-in-time snapshot is taken by the underlying storage system. In dynamic snapshot creation case, this field will be filled in by the CSI snapshotter sidecar with the "creation_time" value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing snapshot, this field will be filled with the "creation_time" value returned from the CSI "ListSnapshots" gRPC call if the driver supports it. If not specified, it indicates the creation time is unknown. The format of this field is a Unix nanoseconds time encoded as an int64. On Unix, the command `date +%s%N` returns the current time in nanoseconds since 1970-01-01 00:00:00 UTC. + format: int64 + type: integer + error: + description: error is the last observed error during snapshot creation, if any. Upon success after retry, this error field will be cleared. + properties: + message: + description: 'message is a string detailing the encountered error during snapshot creation if specified. NOTE: message may be logged, and it should not contain sensitive information.' + type: string + time: + description: time is the timestamp when the error was encountered. + format: date-time + type: string + type: object + readyToUse: + description: readyToUse indicates if a snapshot is ready to be used to restore a volume. In dynamic snapshot creation case, this field will be filled in by the CSI snapshotter sidecar with the "ready_to_use" value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing snapshot, this field will be filled with the "ready_to_use" value returned from the CSI "ListSnapshots" gRPC call if the driver supports it, otherwise, this field will be set to "True". If not specified, it means the readiness of a snapshot is unknown. + type: boolean + restoreSize: + description: restoreSize represents the complete size of the snapshot in bytes. In dynamic snapshot creation case, this field will be filled in by the CSI snapshotter sidecar with the "size_bytes" value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing snapshot, this field will be filled with the "size_bytes" value returned from the CSI "ListSnapshots" gRPC call if the driver supports it. When restoring a volume from this snapshot, the size of the volume MUST NOT be smaller than the restoreSize if it is specified, otherwise the restoration will fail. If not specified, it indicates that the size is unknown. + format: int64 + minimum: 0 + type: integer + snapshotHandle: + description: snapshotHandle is the CSI "snapshot_id" of a snapshot on the underlying storage system. If not specified, it indicates that dynamic snapshot creation has either failed or it is still in progress. + type: string + type: object + required: + - spec + type: object + served: false + storage: false + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +## VolumeSnapshot CRD - https://github.com/kubernetes-csi/external-snapshotter/blob/v8.3.0/client/config/crd/snapshot.storage.k8s.io_volumesnapshots.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.15.0 + api-approved.kubernetes.io: "https://github.com/kubernetes-csi/external-snapshotter/pull/814" + name: volumesnapshots.snapshot.storage.k8s.io +spec: + group: snapshot.storage.k8s.io + names: + kind: VolumeSnapshot + listKind: VolumeSnapshotList + plural: volumesnapshots + shortNames: + - vs + singular: volumesnapshot + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: Indicates if the snapshot is ready to be used to restore a volume. + jsonPath: .status.readyToUse + name: ReadyToUse + type: boolean + - description: If a new snapshot needs to be created, this contains the name of + the source PVC from which this snapshot was (or will be) created. + jsonPath: .spec.source.persistentVolumeClaimName + name: SourcePVC + type: string + - description: If a snapshot already exists, this contains the name of the existing + VolumeSnapshotContent object representing the existing snapshot. + jsonPath: .spec.source.volumeSnapshotContentName + name: SourceSnapshotContent + type: string + - description: Represents the minimum size of volume required to rehydrate from + this snapshot. + jsonPath: .status.restoreSize + name: RestoreSize + type: string + - description: The name of the VolumeSnapshotClass requested by the VolumeSnapshot. + jsonPath: .spec.volumeSnapshotClassName + name: SnapshotClass + type: string + - description: Name of the VolumeSnapshotContent object to which the VolumeSnapshot + object intends to bind to. Please note that verification of binding actually + requires checking both VolumeSnapshot and VolumeSnapshotContent to ensure + both are pointing at each other. Binding MUST be verified prior to usage of + this object. + jsonPath: .status.boundVolumeSnapshotContentName + name: SnapshotContent + type: string + - description: Timestamp when the point-in-time snapshot was taken by the underlying + storage system. + jsonPath: .status.creationTime + name: CreationTime + type: date + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1 + schema: + openAPIV3Schema: + description: |- + VolumeSnapshot is a user's request for either creating a point-in-time + snapshot of a persistent volume, or binding to a pre-existing snapshot. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: |- + spec defines the desired characteristics of a snapshot requested by a user. + More info: https://kubernetes.io/docs/concepts/storage/volume-snapshots#volumesnapshots + Required. + properties: + source: + description: |- + source specifies where a snapshot will be created from. + This field is immutable after creation. + Required. + properties: + persistentVolumeClaimName: + description: |- + persistentVolumeClaimName specifies the name of the PersistentVolumeClaim + object representing the volume from which a snapshot should be created. + This PVC is assumed to be in the same namespace as the VolumeSnapshot + object. + This field should be set if the snapshot does not exists, and needs to be + created. + This field is immutable. + type: string + x-kubernetes-validations: + - message: persistentVolumeClaimName is immutable + rule: self == oldSelf + volumeSnapshotContentName: + description: |- + volumeSnapshotContentName specifies the name of a pre-existing VolumeSnapshotContent + object representing an existing volume snapshot. + This field should be set if the snapshot already exists and only needs a representation in Kubernetes. + This field is immutable. + type: string + x-kubernetes-validations: + - message: volumeSnapshotContentName is immutable + rule: self == oldSelf + type: object + x-kubernetes-validations: + - message: persistentVolumeClaimName is required once set + rule: '!has(oldSelf.persistentVolumeClaimName) || has(self.persistentVolumeClaimName)' + - message: volumeSnapshotContentName is required once set + rule: '!has(oldSelf.volumeSnapshotContentName) || has(self.volumeSnapshotContentName)' + - message: exactly one of volumeSnapshotContentName and persistentVolumeClaimName + must be set + rule: (has(self.volumeSnapshotContentName) && !has(self.persistentVolumeClaimName)) + || (!has(self.volumeSnapshotContentName) && has(self.persistentVolumeClaimName)) + volumeSnapshotClassName: + description: |- + VolumeSnapshotClassName is the name of the VolumeSnapshotClass + requested by the VolumeSnapshot. + VolumeSnapshotClassName may be left nil to indicate that the default + SnapshotClass should be used. + A given cluster may have multiple default Volume SnapshotClasses: one + default per CSI Driver. If a VolumeSnapshot does not specify a SnapshotClass, + VolumeSnapshotSource will be checked to figure out what the associated + CSI Driver is, and the default VolumeSnapshotClass associated with that + CSI Driver will be used. If more than one VolumeSnapshotClass exist for + a given CSI Driver and more than one have been marked as default, + CreateSnapshot will fail and generate an event. + Empty string is not allowed for this field. + type: string + x-kubernetes-validations: + - message: volumeSnapshotClassName must not be the empty string when + set + rule: size(self) > 0 + required: + - source + type: object + status: + description: |- + status represents the current information of a snapshot. + Consumers must verify binding between VolumeSnapshot and + VolumeSnapshotContent objects is successful (by validating that both + VolumeSnapshot and VolumeSnapshotContent point at each other) before + using this object. + properties: + boundVolumeSnapshotContentName: + description: |- + boundVolumeSnapshotContentName is the name of the VolumeSnapshotContent + object to which this VolumeSnapshot object intends to bind to. + If not specified, it indicates that the VolumeSnapshot object has not been + successfully bound to a VolumeSnapshotContent object yet. + NOTE: To avoid possible security issues, consumers must verify binding between + VolumeSnapshot and VolumeSnapshotContent objects is successful (by validating that + both VolumeSnapshot and VolumeSnapshotContent point at each other) before using + this object. + type: string + creationTime: + description: |- + creationTime is the timestamp when the point-in-time snapshot is taken + by the underlying storage system. + In dynamic snapshot creation case, this field will be filled in by the + snapshot controller with the "creation_time" value returned from CSI + "CreateSnapshot" gRPC call. + For a pre-existing snapshot, this field will be filled with the "creation_time" + value returned from the CSI "ListSnapshots" gRPC call if the driver supports it. + If not specified, it may indicate that the creation time of the snapshot is unknown. + format: date-time + type: string + error: + description: |- + error is the last observed error during snapshot creation, if any. + This field could be helpful to upper level controllers(i.e., application controller) + to decide whether they should continue on waiting for the snapshot to be created + based on the type of error reported. + The snapshot controller will keep retrying when an error occurs during the + snapshot creation. Upon success, this error field will be cleared. + properties: + message: + description: |- + message is a string detailing the encountered error during snapshot + creation if specified. + NOTE: message may be logged, and it should not contain sensitive + information. + type: string + time: + description: time is the timestamp when the error was encountered. + format: date-time + type: string + type: object + readyToUse: + description: |- + readyToUse indicates if the snapshot is ready to be used to restore a volume. + In dynamic snapshot creation case, this field will be filled in by the + snapshot controller with the "ready_to_use" value returned from CSI + "CreateSnapshot" gRPC call. + For a pre-existing snapshot, this field will be filled with the "ready_to_use" + value returned from the CSI "ListSnapshots" gRPC call if the driver supports it, + otherwise, this field will be set to "True". + If not specified, it means the readiness of a snapshot is unknown. + type: boolean + restoreSize: + type: string + description: |- + restoreSize represents the minimum size of volume required to create a volume + from this snapshot. + In dynamic snapshot creation case, this field will be filled in by the + snapshot controller with the "size_bytes" value returned from CSI + "CreateSnapshot" gRPC call. + For a pre-existing snapshot, this field will be filled with the "size_bytes" + value returned from the CSI "ListSnapshots" gRPC call if the driver supports it. + When restoring a volume from this snapshot, the size of the volume MUST NOT + be smaller than the restoreSize if it is specified, otherwise the restoration will fail. + If not specified, it indicates that the size is unknown. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + volumeGroupSnapshotName: + description: |- + VolumeGroupSnapshotName is the name of the VolumeGroupSnapshot of which this + VolumeSnapshot is a part of. + type: string + type: object + required: + - spec + type: object + served: true + storage: true + subresources: + status: {} + - additionalPrinterColumns: + - description: Indicates if the snapshot is ready to be used to restore a volume. + jsonPath: .status.readyToUse + name: ReadyToUse + type: boolean + - description: If a new snapshot needs to be created, this contains the name of the source PVC from which this snapshot was (or will be) created. + jsonPath: .spec.source.persistentVolumeClaimName + name: SourcePVC + type: string + - description: If a snapshot already exists, this contains the name of the existing VolumeSnapshotContent object representing the existing snapshot. + jsonPath: .spec.source.volumeSnapshotContentName + name: SourceSnapshotContent + type: string + - description: Represents the minimum size of volume required to rehydrate from this snapshot. + jsonPath: .status.restoreSize + name: RestoreSize + type: string + - description: The name of the VolumeSnapshotClass requested by the VolumeSnapshot. + jsonPath: .spec.volumeSnapshotClassName + name: SnapshotClass + type: string + - description: Name of the VolumeSnapshotContent object to which the VolumeSnapshot object intends to bind to. Please note that verification of binding actually requires checking both VolumeSnapshot and VolumeSnapshotContent to ensure both are pointing at each other. Binding MUST be verified prior to usage of this object. + jsonPath: .status.boundVolumeSnapshotContentName + name: SnapshotContent + type: string + - description: Timestamp when the point-in-time snapshot was taken by the underlying storage system. + jsonPath: .status.creationTime + name: CreationTime + type: date + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta1 + # This indicates the v1beta1 version of the custom resource is deprecated. + # API requests to this version receive a warning in the server response. + deprecated: true + # This overrides the default warning returned to clients making v1beta1 API requests. + deprecationWarning: "snapshot.storage.k8s.io/v1beta1 VolumeSnapshot is deprecated; use snapshot.storage.k8s.io/v1 VolumeSnapshot" + schema: + openAPIV3Schema: + description: VolumeSnapshot is a user's request for either creating a point-in-time snapshot of a persistent volume, or binding to a pre-existing snapshot. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + spec: + description: 'spec defines the desired characteristics of a snapshot requested by a user. More info: https://kubernetes.io/docs/concepts/storage/volume-snapshots#volumesnapshots Required.' + properties: + source: + description: source specifies where a snapshot will be created from. This field is immutable after creation. Required. + properties: + persistentVolumeClaimName: + description: persistentVolumeClaimName specifies the name of the PersistentVolumeClaim object representing the volume from which a snapshot should be created. This PVC is assumed to be in the same namespace as the VolumeSnapshot object. This field should be set if the snapshot does not exists, and needs to be created. This field is immutable. + type: string + volumeSnapshotContentName: + description: volumeSnapshotContentName specifies the name of a pre-existing VolumeSnapshotContent object representing an existing volume snapshot. This field should be set if the snapshot already exists and only needs a representation in Kubernetes. This field is immutable. + type: string + type: object + volumeSnapshotClassName: + description: 'VolumeSnapshotClassName is the name of the VolumeSnapshotClass requested by the VolumeSnapshot. VolumeSnapshotClassName may be left nil to indicate that the default SnapshotClass should be used. A given cluster may have multiple default Volume SnapshotClasses: one default per CSI Driver. If a VolumeSnapshot does not specify a SnapshotClass, VolumeSnapshotSource will be checked to figure out what the associated CSI Driver is, and the default VolumeSnapshotClass associated with that CSI Driver will be used. If more than one VolumeSnapshotClass exist for a given CSI Driver and more than one have been marked as default, CreateSnapshot will fail and generate an event. Empty string is not allowed for this field.' + type: string + required: + - source + type: object + status: + description: status represents the current information of a snapshot. Consumers must verify binding between VolumeSnapshot and VolumeSnapshotContent objects is successful (by validating that both VolumeSnapshot and VolumeSnapshotContent point at each other) before using this object. + properties: + boundVolumeSnapshotContentName: + description: 'boundVolumeSnapshotContentName is the name of the VolumeSnapshotContent object to which this VolumeSnapshot object intends to bind to. If not specified, it indicates that the VolumeSnapshot object has not been successfully bound to a VolumeSnapshotContent object yet. NOTE: To avoid possible security issues, consumers must verify binding between VolumeSnapshot and VolumeSnapshotContent objects is successful (by validating that both VolumeSnapshot and VolumeSnapshotContent point at each other) before using this object.' + type: string + creationTime: + description: creationTime is the timestamp when the point-in-time snapshot is taken by the underlying storage system. In dynamic snapshot creation case, this field will be filled in by the snapshot controller with the "creation_time" value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing snapshot, this field will be filled with the "creation_time" value returned from the CSI "ListSnapshots" gRPC call if the driver supports it. If not specified, it may indicate that the creation time of the snapshot is unknown. + format: date-time + type: string + error: + description: error is the last observed error during snapshot creation, if any. This field could be helpful to upper level controllers(i.e., application controller) to decide whether they should continue on waiting for the snapshot to be created based on the type of error reported. The snapshot controller will keep retrying when an error occurs during the snapshot creation. Upon success, this error field will be cleared. + properties: + message: + description: 'message is a string detailing the encountered error during snapshot creation if specified. NOTE: message may be logged, and it should not contain sensitive information.' + type: string + time: + description: time is the timestamp when the error was encountered. + format: date-time + type: string + type: object + readyToUse: + description: readyToUse indicates if the snapshot is ready to be used to restore a volume. In dynamic snapshot creation case, this field will be filled in by the snapshot controller with the "ready_to_use" value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing snapshot, this field will be filled with the "ready_to_use" value returned from the CSI "ListSnapshots" gRPC call if the driver supports it, otherwise, this field will be set to "True". If not specified, it means the readiness of a snapshot is unknown. + type: boolean + restoreSize: + type: string + description: restoreSize represents the minimum size of volume required to create a volume from this snapshot. In dynamic snapshot creation case, this field will be filled in by the snapshot controller with the "size_bytes" value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing snapshot, this field will be filled with the "size_bytes" value returned from the CSI "ListSnapshots" gRPC call if the driver supports it. When restoring a volume from this snapshot, the size of the volume MUST NOT be smaller than the restoreSize if it is specified, otherwise the restoration will fail. If not specified, it indicates that the size is unknown. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + required: + - spec + type: object + served: false + storage: false + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] diff --git a/deploy/k8s/controller-deployment.yaml b/deploy/k8s/controller-deployment.yaml index 44adffc..174696c 100644 --- a/deploy/k8s/controller-deployment.yaml +++ b/deploy/k8s/controller-deployment.yaml @@ -25,14 +25,16 @@ spec: serviceAccountName: cloudstack-csi-controller affinity: podAntiAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - - labelSelector: - matchExpressions: - - key: "app.kubernetes.io/name" - operator: In - values: - - cloudstack-csi-controller - topologyKey: "kubernetes.io/hostname" + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + podAffinityTerm: + labelSelector: + matchExpressions: + - key: "app.kubernetes.io/name" + operator: In + values: + - cloudstack-csi-controller + topologyKey: "kubernetes.io/hostname" nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: @@ -59,7 +61,7 @@ spec: containers: - name: cloudstack-csi-controller - image: cloudstack-csi-driver + image: ghcr.io/cloudstack/cloudstack-csi-driver:main imagePullPolicy: Always args: - "controller" @@ -176,8 +178,45 @@ spec: type: RuntimeDefault readOnlyRootFilesystem: true allowPrivilegeEscalation: false - + - name: csi-snapshotter + image: registry.k8s.io/sig-storage/csi-snapshotter:v6.3.0 + args: + - "--v=5" + - "--csi-address=$(CSI_ADDRESS)" + - "--leader-election" + - "--leader-election-lease-duration=30s" + - "--leader-election-renew-deadline=20s" + - "--leader-election-retry-period=10s" + env: + - name: CSI_ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + volumeMounts: + - name: socket-dir + mountPath: /var/lib/csi/sockets/pluginproxy/ + resources: + limits: + cpu: 400m + memory: 200Mi + requests: + cpu: 10m + memory: 20Mi + - name: snapshot-controller + image: registry.k8s.io/sig-storage/snapshot-controller:v6.3.0 + args: + - "--v=5" + - "--leader-election" + - "--leader-election-lease-duration=30s" + - "--leader-election-renew-deadline=20s" + - "--leader-election-retry-period=10s" + resources: + limits: + cpu: 400m + memory: 200Mi + requests: + cpu: 10m + memory: 20Mi - name: liveness-probe + imagePullPolicy: IfNotPresent image: registry.k8s.io/sig-storage/livenessprobe:v2.12.0 args: - "--v=4" diff --git a/deploy/k8s/node-daemonset.yaml b/deploy/k8s/node-daemonset.yaml index 665312b..d3831c2 100644 --- a/deploy/k8s/node-daemonset.yaml +++ b/deploy/k8s/node-daemonset.yaml @@ -36,7 +36,7 @@ spec: containers: - name: cloudstack-csi-node - image: cloudstack-csi-driver + image: ghcr.io/cloudstack/cloudstack-csi-driver:main imagePullPolicy: IfNotPresent args: - "node" @@ -64,6 +64,8 @@ spec: mountPath: /dev - name: cloud-init-dir mountPath: /run/cloud-init/ + - name: sys-dir + mountPath: /sys # Comment the above 2 lines and uncomment the next 2 lines for Ignition support # - name: ignition-dir # mountPath: /run/metadata @@ -177,6 +179,10 @@ spec: hostPath: path: /run/cloud-init/ type: Directory + - name: sys-dir + hostPath: + path: /sys + type: Directory # Comment the above 4 lines and uncomment the next 4 lines for Ignition support # - name: ignition-dir # hostPath: diff --git a/deploy/k8s/rbac.yaml b/deploy/k8s/rbac.yaml index 664e97c..64fc918 100644 --- a/deploy/k8s/rbac.yaml +++ b/deploy/k8s/rbac.yaml @@ -36,6 +36,9 @@ rules: - apiGroups: ["storage.k8s.io"] resources: ["volumeattachments/status"] verbs: ["patch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshots", "volumesnapshots/status", "volumesnapshotclasses", "volumesnapshotcontents", "volumesnapshotcontents/status"] + verbs: ["get", "list", "watch", "update", "create", "patch", "delete"] --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding @@ -83,3 +86,35 @@ roleRef: kind: ClusterRole name: cloudstack-csi-node-role apiGroup: rbac.authorization.k8s.io +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: cloudstack-csi-snapshotter-role +rules: +- apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotclasses"] + verbs: ["get", "list", "watch"] +- apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotcontents"] + verbs: ["create", "get", "list", "watch", "update", "delete"] +- apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshots"] + verbs: ["get", "list", "watch", "update"] +- apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotcontents/status"] + verbs: ["update"] + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: cloudstack-csi-snapshotter-binding +subjects: +- kind: ServiceAccount + name: cloudstack-csi-controller + namespace: kube-system +roleRef: + kind: ClusterRole + name: cloudstack-csi-snapshotter-role + apiGroup: rbac.authorization.k8s.io \ No newline at end of file diff --git a/deploy/k8s/volume-snapshot-class.yaml b/deploy/k8s/volume-snapshot-class.yaml new file mode 100644 index 0000000..4d14a7d --- /dev/null +++ b/deploy/k8s/volume-snapshot-class.yaml @@ -0,0 +1,6 @@ +apiVersion: snapshot.storage.k8s.io/v1 +kind: VolumeSnapshotClass +metadata: + name: cloudstack-snapshot +driver: csi.cloudstack.apache.org +deletionPolicy: Delete # Deleting the snapshot object in Kubernetes with delete the snapshot in CloudStack; You can use policy Retain if the snapshot shouldn't be deleted from CloudStack \ No newline at end of file diff --git a/examples/k8s/snapshot/pvc-from-snapshot.yaml b/examples/k8s/snapshot/pvc-from-snapshot.yaml new file mode 100644 index 0000000..89a8182 --- /dev/null +++ b/examples/k8s/snapshot/pvc-from-snapshot.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: pvc-from-snapshot +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi + dataSource: + name: snapshot-1 + kind: VolumeSnapshot + apiGroup: snapshot.storage.k8s.io + storageClassName: cloudstack-custom diff --git a/examples/k8s/snapshot/pvc.yaml b/examples/k8s/snapshot/pvc.yaml new file mode 100644 index 0000000..0b42906 --- /dev/null +++ b/examples/k8s/snapshot/pvc.yaml @@ -0,0 +1,11 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: my-pvc +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 10Gi + storageClassName: cloudstack-custom diff --git a/examples/k8s/snapshot/restore-pod.yaml b/examples/k8s/snapshot/restore-pod.yaml new file mode 100644 index 0000000..f8e9c22 --- /dev/null +++ b/examples/k8s/snapshot/restore-pod.yaml @@ -0,0 +1,16 @@ +apiVersion: v1 +kind: Pod +metadata: + name: restore-pod +spec: + containers: + - name: app + image: busybox + command: [ "sleep", "3600" ] + volumeMounts: + - mountPath: /data + name: data + volumes: + - name: data + persistentVolumeClaim: + claimName: pvc-from-snapshot \ No newline at end of file diff --git a/examples/k8s/snapshot/snapshot.yaml b/examples/k8s/snapshot/snapshot.yaml new file mode 100644 index 0000000..0d71b43 --- /dev/null +++ b/examples/k8s/snapshot/snapshot.yaml @@ -0,0 +1,8 @@ +apiVersion: snapshot.storage.k8s.io/v1 +kind: VolumeSnapshot +metadata: + name: snapshot-1 +spec: + volumeSnapshotClassName: cloudstack-snapshot + source: + persistentVolumeClaimName: example-pvc diff --git a/go.mod b/go.mod index e08d930..2449efa 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,8 @@ module github.com/cloudstack/cloudstack-csi-driver -go 1.21 +go 1.23 + +toolchain go1.23.5 require ( github.com/apache/cloudstack-go/v2 v2.16.1 @@ -12,6 +14,7 @@ require ( golang.org/x/sys v0.20.0 golang.org/x/text v0.16.0 google.golang.org/grpc v1.65.0 + google.golang.org/protobuf v1.34.1 gopkg.in/gcfg.v1 v1.2.3 k8s.io/api v0.29.7 k8s.io/apimachinery v0.29.7 @@ -67,7 +70,6 @@ require ( golang.org/x/time v0.5.0 // indirect golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20240528184218-531527333157 // indirect - google.golang.org/protobuf v1.34.1 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/warnings.v0 v0.1.2 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect diff --git a/pkg/cloud/cloud.go b/pkg/cloud/cloud.go index a8b5417..19109a3 100644 --- a/pkg/cloud/cloud.go +++ b/pkg/cloud/cloud.go @@ -1,3 +1,22 @@ +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// + // Package cloud contains CloudStack related // functions. package cloud @@ -10,6 +29,8 @@ import ( ) // Interface is the CloudStack client interface. + +//nolint:interfacebloat type Interface interface { GetNodeInfo(ctx context.Context, vmName string) (*VM, error) GetVMByID(ctx context.Context, vmID string) (*VM, error) @@ -23,6 +44,13 @@ type Interface interface { AttachVolume(ctx context.Context, volumeID, vmID string) (string, error) DetachVolume(ctx context.Context, volumeID string) error ExpandVolume(ctx context.Context, volumeID string, newSizeInGB int64) error + + CreateVolumeFromSnapshot(ctx context.Context, zoneID, name, projectID, snapshotID string, sizeInGB int64) (*Volume, error) + GetSnapshotByID(ctx context.Context, snapshotID string) (*Snapshot, error) + GetSnapshotByName(ctx context.Context, name string) (*Snapshot, error) + CreateSnapshot(ctx context.Context, volumeID, name string) (*Snapshot, error) + DeleteSnapshot(ctx context.Context, snapshotID string) error + ListSnapshots(ctx context.Context, volumeID, snapshotID string) ([]*Snapshot, error) } // Volume represents a CloudStack volume. @@ -34,12 +62,27 @@ type Volume struct { Size int64 DiskOfferingID string + DomainID string + ProjectID string ZoneID string VirtualMachineID string DeviceID string } +type Snapshot struct { + ID string + Name string + Size int64 + + DomainID string + ProjectID string + ZoneID string + + VolumeID string + CreatedAt string +} + // VM represents a CloudStack Virtual Machine. type VM struct { ID string @@ -50,16 +93,18 @@ type VM struct { var ( ErrNotFound = errors.New("not found") ErrTooManyResults = errors.New("too many results") + ErrAlreadyExists = errors.New("already exists") ) // client is the implementation of Interface. type client struct { *cloudstack.CloudStackClient + projectID string } // New creates a new cloud connector, given its configuration. func New(config *Config) Interface { csClient := cloudstack.NewAsyncClient(config.APIURL, config.APIKey, config.SecretKey, config.VerifySSL) - return &client{csClient} + return &client{csClient, config.ProjectID} } diff --git a/pkg/cloud/config.go b/pkg/cloud/config.go index 86e8c57..d669aa2 100644 --- a/pkg/cloud/config.go +++ b/pkg/cloud/config.go @@ -1,9 +1,28 @@ +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// + package cloud import ( "fmt" - "gopkg.in/gcfg.v1" + gcfg "gopkg.in/gcfg.v1" ) // Config holds CloudStack connection configuration. @@ -12,6 +31,7 @@ type Config struct { APIKey string SecretKey string VerifySSL bool + ProjectID string } // csConfig wraps the config for the CloudStack cloud provider. @@ -40,6 +60,7 @@ func ReadConfig(configFilePath string) (*Config, error) { return &Config{ APIURL: cfg.Global.APIURL, APIKey: cfg.Global.APIKey, + ProjectID: cfg.Global.ProjectID, SecretKey: cfg.Global.SecretKey, VerifySSL: !cfg.Global.SSLNoVerify, }, nil diff --git a/pkg/cloud/fake/fake.go b/pkg/cloud/fake/fake.go index f3e0052..84b38b2 100644 --- a/pkg/cloud/fake/fake.go +++ b/pkg/cloud/fake/fake.go @@ -1,9 +1,29 @@ +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// + // Package fake provides a fake implementation of the cloud // connector interface, to be used in tests. package fake import ( "context" + "errors" "github.com/hashicorp/go-uuid" @@ -14,9 +34,11 @@ import ( const zoneID = "a1887604-237c-4212-a9cd-94620b7880fa" type fakeConnector struct { - node *cloud.VM - volumesByID map[string]cloud.Volume - volumesByName map[string]cloud.Volume + node *cloud.VM + volumesByID map[string]cloud.Volume + volumesByName map[string]cloud.Volume + snapshotsByID map[string]*cloud.Snapshot + snapshotsByName map[string][]*cloud.Snapshot } // New returns a new fake implementation of the @@ -36,10 +58,15 @@ func New() cloud.Interface { ZoneID: zoneID, } + snapshotsByID := make(map[string]*cloud.Snapshot) + snapshotsByName := make(map[string][]*cloud.Snapshot) + return &fakeConnector{ - node: node, - volumesByID: map[string]cloud.Volume{volume.ID: volume}, - volumesByName: map[string]cloud.Volume{volume.Name: volume}, + node: node, + volumesByID: map[string]cloud.Volume{volume.ID: volume}, + volumesByName: map[string]cloud.Volume{volume.Name: volume}, + snapshotsByID: snapshotsByID, + snapshotsByName: snapshotsByName, } } @@ -60,6 +87,9 @@ func (f *fakeConnector) ListZonesID(_ context.Context) ([]string, error) { } func (f *fakeConnector) GetVolumeByID(_ context.Context, volumeID string) (*cloud.Volume, error) { + if volumeID == "" { + return nil, errors.New("invalid volume ID: empty string") + } vol, ok := f.volumesByID[volumeID] if ok { return &vol, nil @@ -69,6 +99,9 @@ func (f *fakeConnector) GetVolumeByID(_ context.Context, volumeID string) (*clou } func (f *fakeConnector) GetVolumeByName(_ context.Context, name string) (*cloud.Volume, error) { + if name == "" { + return nil, errors.New("invalid volume name: empty string") + } vol, ok := f.volumesByName[name] if ok { return &vol, nil @@ -124,3 +157,121 @@ func (f *fakeConnector) ExpandVolume(_ context.Context, volumeID string, newSize return cloud.ErrNotFound } + +func (f *fakeConnector) CreateVolumeFromSnapshot(_ context.Context, zoneID, name, _, _ string, sizeInGB int64) (*cloud.Volume, error) { + vol := &cloud.Volume{ + ID: "fake-vol-from-snap-" + name, + Name: name, + Size: util.GigaBytesToBytes(sizeInGB), + DiskOfferingID: "fake-disk-offering", + ZoneID: zoneID, + } + f.volumesByID[vol.ID] = *vol + f.volumesByName[vol.Name] = *vol + + return vol, nil +} + +func (f *fakeConnector) CreateSnapshot(_ context.Context, volumeID, name string) (*cloud.Snapshot, error) { + if name == "" { + return nil, errors.New("invalid snapshot name: empty string") + } + for _, snap := range f.snapshotsByName[name] { + if snap.VolumeID == volumeID { + // Allow multiple snapshots with the same name for the same volume + continue + } + + // Name conflict: same name, different volume + return nil, cloud.ErrAlreadyExists + } + id, _ := uuid.GenerateUUID() + newSnap := &cloud.Snapshot{ + ID: id, + Name: name, + DomainID: "fake-domain", + ZoneID: zoneID, + VolumeID: volumeID, + CreatedAt: "2025-07-07T16:13:06-0700", + } + f.snapshotsByID[newSnap.ID] = newSnap + f.snapshotsByName[name] = append(f.snapshotsByName[name], newSnap) + + return newSnap, nil +} + +func (f *fakeConnector) GetSnapshotByID(_ context.Context, snapshotID string) (*cloud.Snapshot, error) { + snap, ok := f.snapshotsByID[snapshotID] + if ok { + return snap, nil + } + + return nil, cloud.ErrNotFound +} + +func (f *fakeConnector) GetSnapshotByName(_ context.Context, name string) (*cloud.Snapshot, error) { + if name == "" { + return nil, errors.New("invalid snapshot name: empty string") + } + snaps, ok := f.snapshotsByName[name] + if ok && len(snaps) > 0 { + return snaps[0], nil // Return the first for compatibility + } + + return nil, cloud.ErrNotFound +} + +// ListSnapshots returns all matching snapshots; pagination must be handled by the controller. +func (f *fakeConnector) ListSnapshots(_ context.Context, volumeID, snapshotID string) ([]*cloud.Snapshot, error) { + if snapshotID != "" { + result := make([]*cloud.Snapshot, 0, 1) + if snap, ok := f.snapshotsByID[snapshotID]; ok { + result = append(result, snap) + } + + return result, nil + } + if volumeID != "" { + count := 0 + for _, snap := range f.snapshotsByID { + if snap.VolumeID == volumeID { + count++ + } + } + result := make([]*cloud.Snapshot, 0, count) + for _, snap := range f.snapshotsByID { + if snap.VolumeID == volumeID { + result = append(result, snap) + } + } + + return result, nil + } + result := make([]*cloud.Snapshot, 0, len(f.snapshotsByID)) + for _, snap := range f.snapshotsByID { + result = append(result, snap) + } + + return result, nil +} + +func (f *fakeConnector) DeleteSnapshot(_ context.Context, snapshotID string) error { + snap, ok := f.snapshotsByID[snapshotID] + if !ok { + return cloud.ErrNotFound + } + + delete(f.snapshotsByID, snapshotID) + + name := snap.Name + snaps := f.snapshotsByName[name] + for i, s := range snaps { + if s.ID == snapshotID { + f.snapshotsByName[name] = append(snaps[:i], snaps[i+1:]...) + + break + } + } + + return nil +} diff --git a/pkg/cloud/metadata.go b/pkg/cloud/metadata.go index 17a0c80..35ec6f8 100644 --- a/pkg/cloud/metadata.go +++ b/pkg/cloud/metadata.go @@ -1,3 +1,22 @@ +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// + package cloud import ( diff --git a/pkg/cloud/node.go b/pkg/cloud/node.go index c3eab50..ff63c29 100644 --- a/pkg/cloud/node.go +++ b/pkg/cloud/node.go @@ -1,3 +1,22 @@ +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// + package cloud import ( diff --git a/pkg/cloud/snapshots.go b/pkg/cloud/snapshots.go new file mode 100644 index 0000000..fe2451f --- /dev/null +++ b/pkg/cloud/snapshots.go @@ -0,0 +1,186 @@ +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// + +package cloud + +import ( + "context" + "strings" + + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + "k8s.io/klog/v2" +) + +func (c *client) GetSnapshotByID(ctx context.Context, snapshotID string) (*Snapshot, error) { + logger := klog.FromContext(ctx) + p := c.Snapshot.NewListSnapshotsParams() + if snapshotID != "" { + p.SetId(snapshotID) + } + if c.projectID != "" { + p.SetProjectid(c.projectID) + } + logger.V(2).Info("CloudStack API call", "command", "ListSnapshots", "params", map[string]string{ + "id": snapshotID, + "projectid": c.projectID, + }) + l, err := c.Snapshot.ListSnapshots(p) + if err != nil { + return nil, err + } + if l.Count == 0 { + return nil, ErrNotFound + } + if l.Count > 1 { + return nil, ErrTooManyResults + } + snapshot := l.Snapshots[0] + s := Snapshot{ + ID: snapshot.Id, + Name: snapshot.Name, + DomainID: snapshot.Domainid, + ProjectID: snapshot.Projectid, + ZoneID: snapshot.Zoneid, + VolumeID: snapshot.Volumeid, + } + + return &s, nil +} + +func (c *client) CreateSnapshot(ctx context.Context, volumeID, name string) (*Snapshot, error) { + logger := klog.FromContext(ctx) + p := c.Snapshot.NewCreateSnapshotParams(volumeID) + if name != "" { + p.SetName(name) + } + logger.V(2).Info("CloudStack API call", "command", "CreateSnapshot", "params", map[string]string{ + "volumeid": volumeID, + "name": name, + }) + + snapshot, err := c.Snapshot.CreateSnapshot(p) + if err != nil { + return nil, status.Errorf(codes.Internal, "Error %v", err) + } + + snap := Snapshot{ + ID: snapshot.Id, + Name: snapshot.Name, + Size: snapshot.Virtualsize, + DomainID: snapshot.Domainid, + ProjectID: snapshot.Projectid, + ZoneID: snapshot.Zoneid, + VolumeID: snapshot.Volumeid, + CreatedAt: snapshot.Created, + } + + return &snap, nil +} + +func (c *client) DeleteSnapshot(_ context.Context, snapshotID string) error { + p := c.Snapshot.NewDeleteSnapshotParams(snapshotID) + _, err := c.Snapshot.DeleteSnapshot(p) + if err != nil && strings.Contains(err.Error(), "4350") { + // CloudStack error InvalidParameterValueException + return ErrNotFound + } + + return err +} + +func (c *client) GetSnapshotByName(ctx context.Context, name string) (*Snapshot, error) { + logger := klog.FromContext(ctx) + if name == "" { + return nil, ErrNotFound + } + p := c.Snapshot.NewListSnapshotsParams() + p.SetName(name) + if c.projectID != "" { + p.SetProjectid(c.projectID) + } + logger.V(2).Info("CloudStack API call", "command", "ListSnapshots", "params", map[string]string{ + "name": name, + "projectid": c.projectID, + }) + l, err := c.Snapshot.ListSnapshots(p) + if err != nil { + return nil, err + } + if l.Count == 0 { + return nil, ErrNotFound + } + if l.Count > 1 { + return nil, ErrTooManyResults + } + snapshot := l.Snapshots[0] + s := Snapshot{ + ID: snapshot.Id, + Name: snapshot.Name, + DomainID: snapshot.Domainid, + ProjectID: snapshot.Projectid, + ZoneID: snapshot.Zoneid, + VolumeID: snapshot.Volumeid, + CreatedAt: snapshot.Created, + } + + return &s, nil +} + +func (c *client) ListSnapshots(ctx context.Context, volumeID, snapshotID string) ([]*Snapshot, error) { + logger := klog.FromContext(ctx) + p := c.Snapshot.NewListSnapshotsParams() + if snapshotID != "" { + p.SetId(snapshotID) + } + if volumeID != "" { + p.SetVolumeid(volumeID) + } + if c.projectID != "" { + p.SetProjectid(c.projectID) + } + logger.V(2).Info("CloudStack API call", "command", "ListSnapshots", "params", map[string]string{ + "id": snapshotID, + "volumeid": volumeID, + "projectid": c.projectID, + }) + l, err := c.Snapshot.ListSnapshots(p) + if err != nil { + return nil, err + } + if l.Count == 0 { + return []*Snapshot{}, nil + } + result := make([]*Snapshot, 0, l.Count) + for _, snapshot := range l.Snapshots { + s := &Snapshot{ + ID: snapshot.Id, + Name: snapshot.Name, + Size: snapshot.Virtualsize, + DomainID: snapshot.Domainid, + ProjectID: snapshot.Projectid, + ZoneID: snapshot.Zoneid, + VolumeID: snapshot.Volumeid, + CreatedAt: snapshot.Created, + } + result = append(result, s) + } + + return result, nil +} diff --git a/pkg/cloud/vms.go b/pkg/cloud/vms.go index 68a0505..2df3c7f 100644 --- a/pkg/cloud/vms.go +++ b/pkg/cloud/vms.go @@ -1,3 +1,22 @@ +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// + package cloud import ( @@ -10,8 +29,12 @@ func (c *client) GetVMByID(ctx context.Context, vmID string) (*VM, error) { logger := klog.FromContext(ctx) p := c.VirtualMachine.NewListVirtualMachinesParams() p.SetId(vmID) + if c.projectID != "" { + p.SetProjectid(c.projectID) + } logger.V(2).Info("CloudStack API call", "command", "ListVirtualMachines", "params", map[string]string{ - "id": vmID, + "id": vmID, + "projectID": c.projectID, }) l, err := c.VirtualMachine.ListVirtualMachines(p) if err != nil { @@ -24,6 +47,7 @@ func (c *client) GetVMByID(ctx context.Context, vmID string) (*VM, error) { return nil, ErrTooManyResults } vm := l.VirtualMachines[0] + logger.V(2).Info("Returning VM", "vmID", vm.Id, "zoneID", vm.Zoneid) return &VM{ ID: vm.Id, diff --git a/pkg/cloud/volumes.go b/pkg/cloud/volumes.go index 2d64b58..caaa7a3 100644 --- a/pkg/cloud/volumes.go +++ b/pkg/cloud/volumes.go @@ -1,3 +1,22 @@ +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// + package cloud import ( @@ -29,6 +48,8 @@ func (c *client) listVolumes(p *cloudstack.ListVolumesParams) (*Volume, error) { Name: vol.Name, Size: vol.Size, DiskOfferingID: vol.Diskofferingid, + DomainID: vol.Domainid, + ProjectID: vol.Projectid, ZoneID: vol.Zoneid, VirtualMachineID: vol.Virtualmachineid, DeviceID: strconv.FormatInt(vol.Deviceid, 10), @@ -41,8 +62,12 @@ func (c *client) GetVolumeByID(ctx context.Context, volumeID string) (*Volume, e logger := klog.FromContext(ctx) p := c.Volume.NewListVolumesParams() p.SetId(volumeID) + if c.projectID != "" { + p.SetProjectid(c.projectID) + } logger.V(2).Info("CloudStack API call", "command", "ListVolumes", "params", map[string]string{ - "id": volumeID, + "id": volumeID, + "projectid": c.projectID, }) return c.listVolumes(p) @@ -66,11 +91,15 @@ func (c *client) CreateVolume(ctx context.Context, diskOfferingID, zoneID, name p.SetZoneid(zoneID) p.SetName(name) p.SetSize(sizeInGB) + if c.projectID != "" { + p.SetProjectid(c.projectID) + } logger.V(2).Info("CloudStack API call", "command", "CreateVolume", "params", map[string]string{ "diskofferingid": diskOfferingID, "zoneid": zoneID, "name": name, "size": strconv.FormatInt(sizeInGB, 10), + "projectid": c.projectID, }) vol, err := c.Volume.CreateVolume(p) if err != nil { @@ -153,3 +182,44 @@ func (c *client) ExpandVolume(ctx context.Context, volumeID string, newSizeInGB return nil } + +func (c *client) CreateVolumeFromSnapshot(ctx context.Context, zoneID, name, projectID, snapshotID string, sizeInGB int64) (*Volume, error) { + logger := klog.FromContext(ctx) + + p := c.Volume.NewCreateVolumeParams() + p.SetZoneid(zoneID) + if projectID != "" { + p.SetProjectid(projectID) + } + p.SetName(name) + p.SetSize(sizeInGB) + p.SetSnapshotid(snapshotID) + + logger.V(2).Info("CloudStack API call", "command", "CreateVolume", "params", map[string]string{ + "name": name, + "size": strconv.FormatInt(sizeInGB, 10), + "snapshotid": snapshotID, + "projectid": projectID, + "zoneid": zoneID, + }) + // Execute the API call to create volume from snapshot + vol, err := c.Volume.CreateVolume(p) + if err != nil { + // Handle the error accordingly + return nil, fmt.Errorf("failed to create volume from snapshot '%s': %w", snapshotID, err) + } + + v := Volume{ + ID: vol.Id, + Name: vol.Name, + Size: vol.Size, + DiskOfferingID: vol.Diskofferingid, + DomainID: vol.Domainid, + ProjectID: vol.Projectid, + ZoneID: vol.Zoneid, + VirtualMachineID: vol.Virtualmachineid, + DeviceID: strconv.FormatInt(vol.Deviceid, 10), + } + + return &v, nil +} diff --git a/pkg/cloud/zones.go b/pkg/cloud/zones.go index c137677..8cbb84d 100644 --- a/pkg/cloud/zones.go +++ b/pkg/cloud/zones.go @@ -1,3 +1,22 @@ +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// + package cloud import ( diff --git a/pkg/driver/constants.go b/pkg/driver/constants.go index 7775695..64a77ad 100644 --- a/pkg/driver/constants.go +++ b/pkg/driver/constants.go @@ -1,3 +1,22 @@ +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// + package driver // DriverName is the name of the CSI plugin. diff --git a/pkg/driver/controller.go b/pkg/driver/controller.go index 506eb96..9ca1185 100644 --- a/pkg/driver/controller.go +++ b/pkg/driver/controller.go @@ -1,15 +1,38 @@ +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// + package driver import ( "context" + "encoding/json" "errors" "fmt" "math/rand" + "strconv" + "time" "github.com/container-storage-interface/spec/lib/go/csi" "github.com/kubernetes-csi/csi-lib-utils/protosanitizer" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" + "google.golang.org/protobuf/types/known/timestamppb" "k8s.io/klog/v2" "github.com/cloudstack/cloudstack-csi-driver/pkg/cloud" @@ -46,6 +69,7 @@ func NewControllerServer(connector cloud.Interface) csi.ControllerServer { } } +//nolint:gocognit func (cs *controllerServer) CreateVolume(ctx context.Context, req *csi.CreateVolumeRequest) (*csi.CreateVolumeResponse, error) { logger := klog.FromContext(ctx) logger.V(6).Info("CreateVolume: called", "args", *req) @@ -108,6 +132,14 @@ func (cs *controllerServer) CreateVolume(ctx context.Context, req *csi.CreateVol return resp, nil } + // Check if this is a volume from snapshot + var snapshotID string + if src := req.GetVolumeContentSource(); src != nil { + if snap := src.GetSnapshot(); snap != nil { + snapshotID = snap.GetSnapshotId() + } + } + // We have to create the volume. // Determine volume size using requested capacity range. @@ -116,6 +148,47 @@ func (cs *controllerServer) CreateVolume(ctx context.Context, req *csi.CreateVol return nil, status.Error(codes.InvalidArgument, err.Error()) } + // If creating from snapshot, get the snapshot size + var snapshotSizeGiB int64 + if snapshotID != "" { + logger.Info("Creating volume from snapshot", "snapshotID", snapshotID) + // Call the cloud connector's CreateVolumeFromSnapshot if implemented + printVolumeAsJSON(req) + snapshot, err := cs.connector.GetSnapshotByID(ctx, snapshotID) + if errors.Is(err, cloud.ErrNotFound) { + return nil, status.Errorf(codes.NotFound, "Snapshot %v not found", snapshotID) + } else if err != nil { + // Error with CloudStack + return nil, status.Errorf(codes.Internal, "Error %v", err) + } + + logger.Info("PVC created with", "size", sizeInGB) + snapshotSizeGiB = util.RoundUpBytesToGB(snapshot.Size) + if snapshotSizeGiB > sizeInGB { + logger.Info("Snapshot size is greater than the request PVC, creating volume from snapshot of size", "snapshot size:", snapshotSizeGiB) + sizeInGB = snapshotSizeGiB + } + + volFromSnapshot, err := cs.connector.CreateVolumeFromSnapshot(ctx, snapshot.ZoneID, name, snapshot.ProjectID, snapshotID, sizeInGB) + if err != nil { + return nil, status.Errorf(codes.Internal, "Cannot create volume from snapshot %s: %v", snapshotID, err.Error()) + } + + resp := &csi.CreateVolumeResponse{ + Volume: &csi.Volume{ + VolumeId: volFromSnapshot.ID, + CapacityBytes: volFromSnapshot.Size, + VolumeContext: req.GetParameters(), + ContentSource: req.GetVolumeContentSource(), + AccessibleTopology: []*csi.Topology{ + Topology{ZoneID: volFromSnapshot.ZoneID}.ToCSI(), + }, + }, + } + + return resp, nil + } + // Determine zone using topology constraints. var zoneID string topologyRequirement := req.GetAccessibilityRequirements() @@ -159,7 +232,7 @@ func (cs *controllerServer) CreateVolume(ctx context.Context, req *csi.CreateVol VolumeId: volID, CapacityBytes: util.GigaBytesToBytes(sizeInGB), VolumeContext: req.GetParameters(), - // ContentSource: req.GetVolumeContentSource(), TODO: snapshot support. + ContentSource: req.GetVolumeContentSource(), AccessibleTopology: []*csi.Topology{ Topology{ZoneID: zoneID}.ToCSI(), }, @@ -169,6 +242,16 @@ func (cs *controllerServer) CreateVolume(ctx context.Context, req *csi.CreateVol return resp, nil } +func printVolumeAsJSON(vol *csi.CreateVolumeRequest) { + b, err := json.MarshalIndent(vol, "", " ") + if err != nil { + klog.Errorf("Failed to marshal CreateVolumeRequest to JSON: %v", err) + + return + } + klog.V(5).Infof("CreateVolumeRequest as JSON:\n%s", string(b)) +} + func checkVolumeSuitable(vol *cloud.Volume, diskOfferingID string, capRange *csi.CapacityRange, topologyRequirement *csi.TopologyRequirement, ) (bool, string) { @@ -230,7 +313,7 @@ func determineSize(req *csi.CreateVolumeRequest) (int64, error) { func (cs *controllerServer) DeleteVolume(ctx context.Context, req *csi.DeleteVolumeRequest) (*csi.DeleteVolumeResponse, error) { logger := klog.FromContext(ctx) - logger.V(6).Info("DeleteVolume: called", "args", *req) + logger.V(4).Info("DeleteVolume: called", "args", *req) if req.GetVolumeId() == "" { return nil, status.Error(codes.InvalidArgument, "Volume ID missing in request") @@ -265,6 +348,122 @@ func (cs *controllerServer) DeleteVolume(ctx context.Context, req *csi.DeleteVol return &csi.DeleteVolumeResponse{}, nil } +func (cs *controllerServer) CreateSnapshot(ctx context.Context, req *csi.CreateSnapshotRequest) (*csi.CreateSnapshotResponse, error) { + klog.V(4).Infof("CreateSnapshot") + + if req.GetName() == "" { + return nil, status.Error(codes.InvalidArgument, "Snapshot name missing in request") + } + + volumeID := req.GetSourceVolumeId() + if volumeID == "" { + return nil, status.Error(codes.InvalidArgument, "SourceVolumeId missing in request") + } + + volume, err := cs.connector.GetVolumeByID(ctx, volumeID) + if err != nil { + if err.Error() == "invalid volume ID: empty string" { + return nil, status.Error(codes.InvalidArgument, "Invalid volume ID") + } + if errors.Is(err, cloud.ErrNotFound) { + return nil, status.Errorf(codes.NotFound, "Volume %v not found", volumeID) + } + + return nil, status.Errorf(codes.Internal, "Error %v", err) + } + + klog.V(4).Infof("CreateSnapshot of volume: %s", volume.ID) + snapshot, err := cs.connector.CreateSnapshot(ctx, volume.ID, req.GetName()) + if errors.Is(err, cloud.ErrAlreadyExists) { + return nil, status.Errorf(codes.AlreadyExists, "Snapshot name conflict: already exists for a different source volume") + } else if err != nil { + return nil, status.Errorf(codes.Internal, "Failed to create snapshot for volume %s: %v", volume.ID, err.Error()) + } + + t, err := time.Parse("2006-01-02T15:04:05-0700", snapshot.CreatedAt) + if err != nil { + return nil, status.Errorf(codes.Internal, "Failed to parse snapshot creation time: %v", err) + } + + ts := timestamppb.New(t) + + resp := &csi.CreateSnapshotResponse{ + Snapshot: &csi.Snapshot{ + SnapshotId: snapshot.ID, + SourceVolumeId: volume.ID, + CreationTime: ts, + ReadyToUse: true, + }, + } + + return resp, nil +} + +func (cs *controllerServer) ListSnapshots(ctx context.Context, req *csi.ListSnapshotsRequest) (*csi.ListSnapshotsResponse, error) { + entries := []*csi.ListSnapshotsResponse_Entry{} + + snapshots, err := cs.connector.ListSnapshots(ctx, req.GetSourceVolumeId(), req.GetSnapshotId()) + if err != nil { + return nil, status.Errorf(codes.Internal, "Failed to list snapshots: %v", err) + } + + // Pagination logic + start := 0 + if req.GetStartingToken() != "" { + var err error + start, err = strconv.Atoi(req.GetStartingToken()) + if err != nil || start < 0 || start > len(snapshots) { + return nil, status.Error(codes.Aborted, "Invalid startingToken") + } + } + maxEntries := int(req.GetMaxEntries()) + end := len(snapshots) + if maxEntries > 0 && start+maxEntries < end { + end = start + maxEntries + } + nextToken := "" + if end < len(snapshots) { + nextToken = strconv.Itoa(end) + } + + for i := start; i < end; i++ { + snap := snapshots[i] + t, _ := time.Parse("2006-01-02T15:04:05-0700", snap.CreatedAt) + ts := timestamppb.New(t) + entry := &csi.ListSnapshotsResponse_Entry{ + Snapshot: &csi.Snapshot{ + SnapshotId: snap.ID, + SourceVolumeId: snap.VolumeID, + CreationTime: ts, + ReadyToUse: true, + }, + } + entries = append(entries, entry) + } + + return &csi.ListSnapshotsResponse{Entries: entries, NextToken: nextToken}, nil +} + +func (cs *controllerServer) DeleteSnapshot(ctx context.Context, req *csi.DeleteSnapshotRequest) (*csi.DeleteSnapshotResponse, error) { + snapshotID := req.GetSnapshotId() + + if snapshotID == "" { + return nil, status.Error(codes.InvalidArgument, "Snapshot ID missing in request") + } + + klog.V(4).Infof("DeleteSnapshot for snapshotID: %s", snapshotID) + + err := cs.connector.DeleteSnapshot(ctx, snapshotID) + if errors.Is(err, cloud.ErrNotFound) { + // Per CSI spec, return OK if snapshot does not exist + return &csi.DeleteSnapshotResponse{}, nil + } else if err != nil { + return nil, status.Errorf(codes.Internal, "Error %v", err) + } + + return &csi.DeleteSnapshotResponse{}, nil +} + func (cs *controllerServer) ControllerPublishVolume(ctx context.Context, req *csi.ControllerPublishVolumeRequest) (*csi.ControllerPublishVolumeResponse, error) { logger := klog.FromContext(ctx) logger.V(6).Info("ControllerPublishVolume: called", "args", *req) @@ -557,6 +756,20 @@ func (cs *controllerServer) ControllerGetCapabilities(ctx context.Context, req * }, }, }, + { + Type: &csi.ControllerServiceCapability_Rpc{ + Rpc: &csi.ControllerServiceCapability_RPC{ + Type: csi.ControllerServiceCapability_RPC_CREATE_DELETE_SNAPSHOT, + }, + }, + }, + { + Type: &csi.ControllerServiceCapability_Rpc{ + Rpc: &csi.ControllerServiceCapability_RPC{ + Type: csi.ControllerServiceCapability_RPC_LIST_SNAPSHOTS, + }, + }, + }, }, } diff --git a/pkg/driver/controller_test.go b/pkg/driver/controller_test.go index bf9f3d1..a939f28 100644 --- a/pkg/driver/controller_test.go +++ b/pkg/driver/controller_test.go @@ -1,3 +1,22 @@ +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// + package driver import ( diff --git a/pkg/driver/driver.go b/pkg/driver/driver.go index 9420d34..136b282 100644 --- a/pkg/driver/driver.go +++ b/pkg/driver/driver.go @@ -1,3 +1,22 @@ +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// + // Package driver provides the implementation of the CSI plugin. // // It contains the gRPC server implementation of CSI specification. diff --git a/pkg/driver/identity.go b/pkg/driver/identity.go index e265275..7d064da 100644 --- a/pkg/driver/identity.go +++ b/pkg/driver/identity.go @@ -1,3 +1,22 @@ +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// + package driver import ( diff --git a/pkg/driver/node.go b/pkg/driver/node.go index e764162..4a416e1 100644 --- a/pkg/driver/node.go +++ b/pkg/driver/node.go @@ -1,3 +1,22 @@ +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// + package driver import ( diff --git a/pkg/driver/options.go b/pkg/driver/options.go index f714f30..d0beb35 100644 --- a/pkg/driver/options.go +++ b/pkg/driver/options.go @@ -1,3 +1,22 @@ +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// + package driver import ( diff --git a/pkg/driver/topology.go b/pkg/driver/topology.go index bd6bfa7..e58caf6 100644 --- a/pkg/driver/topology.go +++ b/pkg/driver/topology.go @@ -1,3 +1,22 @@ +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// + package driver import ( diff --git a/pkg/mount/fake.go b/pkg/mount/fake.go index 8129033..c80ce8a 100644 --- a/pkg/mount/fake.go +++ b/pkg/mount/fake.go @@ -1,3 +1,22 @@ +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// + package mount import ( diff --git a/pkg/mount/mount.go b/pkg/mount/mount.go index 6e45aaf..9b7c8f6 100644 --- a/pkg/mount/mount.go +++ b/pkg/mount/mount.go @@ -1,3 +1,22 @@ +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// + // Package mount provides utilities to detect, // format and mount storage devices. package mount @@ -79,20 +98,22 @@ func (m *mounter) GetBlockSizeBytes(devicePath string) (int64, error) { } func (m *mounter) GetDevicePath(ctx context.Context, volumeID string) (string, error) { + logger := klog.FromContext(ctx) backoff := wait.Backoff{ - Duration: 1 * time.Second, - Factor: 1.1, - Steps: 15, + Duration: 2 * time.Second, + Factor: 1.5, + Steps: 20, } var devicePath string err := wait.ExponentialBackoffWithContext(ctx, backoff, func(context.Context) (bool, error) { - path, err := m.getDevicePathBySerialID(volumeID) + path, err := m.getDevicePathBySerialID(ctx, volumeID) if err != nil { return false, err } if path != "" { devicePath = path + logger.V(4).Info("Device path found", "volumeID", volumeID, "devicePath", path) return true, nil } @@ -110,7 +131,27 @@ func (m *mounter) GetDevicePath(ctx context.Context, volumeID string) (string, e return devicePath, nil } -func (m *mounter) getDevicePathBySerialID(volumeID string) (string, error) { +func (m *mounter) getDevicePathBySerialID(ctx context.Context, volumeID string) (string, error) { + logger := klog.FromContext(ctx) + + // First try XenServer device paths + xenDevicePath, err := m.getDevicePathForXenServer(ctx, volumeID) + if err != nil { + logger.V(4).Info("Failed to get XenServer device path", "volumeID", volumeID, "error", err) + } + if xenDevicePath != "" { + return xenDevicePath, nil + } + + // Try VMware device paths + vmwareDevicePath, err := m.getDevicePathForVMware(ctx, volumeID) + if err != nil { + logger.V(4).Info("Failed to get VMware device path", "volumeID", volumeID, "error", err) + } + if vmwareDevicePath != "" { + return vmwareDevicePath, nil + } + // Fall back to standard device paths (for KVM) sourcePathPrefixes := []string{"virtio-", "scsi-", "scsi-0QEMU_QEMU_HARDDISK_"} serial := diskUUIDToSerial(volumeID) for _, prefix := range sourcePathPrefixes { @@ -120,6 +161,8 @@ func (m *mounter) getDevicePathBySerialID(volumeID string) (string, error) { return source, nil } if !os.IsNotExist(err) { + logger.Error(err, "Failed to stat device path", "path", source) + return "", err } } @@ -127,6 +170,118 @@ func (m *mounter) getDevicePathBySerialID(volumeID string) (string, error) { return "", nil } +func (m *mounter) getDevicePathForXenServer(ctx context.Context, volumeID string) (string, error) { + logger := klog.FromContext(ctx) + + for i := 'b'; i <= 'z'; i++ { + devicePath := fmt.Sprintf("/dev/xvd%c", i) + logger.V(5).Info("Checking XenServer device path", "devicePath", devicePath, "volumeID", volumeID) + + if _, err := os.Stat(devicePath); err == nil { + isBlock, err := m.IsBlockDevice(devicePath) + if err == nil && isBlock { + if m.verifyDevice(ctx, devicePath, volumeID) { + logger.V(4).Info("Found and verified XenServer device", "devicePath", devicePath, "volumeID", volumeID) + + return devicePath, nil + } + } + } + } + + return "", fmt.Errorf("device not found for volume %s", volumeID) +} + +func (m *mounter) getDevicePathForVMware(ctx context.Context, volumeID string) (string, error) { + logger := klog.FromContext(ctx) + + // Loop through /dev/sdb to /dev/sdz (/dev/sda -> the root disk) + for i := 'b'; i <= 'z'; i++ { + devicePath := fmt.Sprintf("/dev/sd%c", i) + logger.V(5).Info("Checking VMware device path", "devicePath", devicePath, "volumeID", volumeID) + + if _, err := os.Stat(devicePath); err == nil { + isBlock, err := m.IsBlockDevice(devicePath) + if err == nil && isBlock { + if m.verifyDevice(ctx, devicePath, volumeID) { + logger.V(4).Info("Found and verified VMware device", "devicePath", devicePath, "volumeID", volumeID) + + return devicePath, nil + } + } + } + } + + return "", fmt.Errorf("device not found for volume %s", volumeID) +} + +func (m *mounter) verifyDevice(ctx context.Context, devicePath string, volumeID string) bool { + logger := klog.FromContext(ctx) + + size, err := m.GetBlockSizeBytes(devicePath) + if err != nil { + logger.V(4).Info("Failed to get device size", "devicePath", devicePath, "volumeID", volumeID, "error", err) + + return false + } + logger.V(5).Info("Device size retrieved", "devicePath", devicePath, "volumeID", volumeID, "sizeBytes", size) + + mounted, err := m.isDeviceMounted(devicePath) + if err != nil { + logger.V(4).Info("Failed to check if device is mounted", "devicePath", devicePath, "volumeID", volumeID, "error", err) + + return false + } + if mounted { + logger.V(4).Info("Device is already mounted", "devicePath", devicePath, "volumeID", volumeID) + + return false + } + + props, err := m.getDeviceProperties(devicePath) + if err != nil { + logger.V(4).Info("Failed to get device properties", "devicePath", devicePath, "volumeID", volumeID, "error", err) + + return false + } + logger.V(5).Info("Device properties retrieved", "devicePath", devicePath, "volumeID", volumeID, "properties", props) + + return true +} + +func (m *mounter) isDeviceMounted(devicePath string) (bool, error) { + output, err := m.Exec.Command("grep", devicePath, "/proc/mounts").Output() + if err != nil { + if strings.Contains(err.Error(), "exit status 1") { + return false, nil + } + + return false, err + } + + return len(output) > 0, nil +} + +func (m *mounter) getDeviceProperties(devicePath string) (map[string]string, error) { + output, err := m.Exec.Command("udevadm", "info", "--query=property", devicePath).Output() + if err != nil { + return nil, err + } + + props := make(map[string]string) + for _, line := range strings.Split(string(output), "\n") { + if line == "" { + continue + } + parts := strings.Split(line, "=") + if len(parts) == 2 { + props[parts[0]] = parts[1] + } + } + + return props, nil +} + func (m *mounter) probeVolume(ctx context.Context) { logger := klog.FromContext(ctx) logger.V(2).Info("Scanning SCSI host") @@ -242,13 +397,13 @@ func (m *mounter) GetStatistics(volumePath string) (volumeStatistics, error) { } volStats := volumeStatistics{ - AvailableBytes: int64(statfs.Bavail) * int64(statfs.Bsize), //nolint:unconvert - TotalBytes: int64(statfs.Blocks) * int64(statfs.Bsize), //nolint:unconvert - UsedBytes: (int64(statfs.Blocks) - int64(statfs.Bfree)) * int64(statfs.Bsize), //nolint:unconvert + AvailableBytes: int64(statfs.Bavail) * int64(statfs.Bsize), //nolint:gosec,unconvert + TotalBytes: int64(statfs.Blocks) * int64(statfs.Bsize), //nolint:gosec,unconvert + UsedBytes: (int64(statfs.Blocks) - int64(statfs.Bfree)) * int64(statfs.Bsize), //nolint:gosec,unconvert - AvailableInodes: int64(statfs.Ffree), - TotalInodes: int64(statfs.Files), - UsedInodes: int64(statfs.Files) - int64(statfs.Ffree), + AvailableInodes: int64(statfs.Ffree), //nolint:gosec + TotalInodes: int64(statfs.Files), //nolint:gosec + UsedInodes: int64(statfs.Files) - int64(statfs.Ffree), //nolint:gosec } return volStats, nil diff --git a/pkg/syncer/error.go b/pkg/syncer/error.go index 4328fcc..46abb1c 100644 --- a/pkg/syncer/error.go +++ b/pkg/syncer/error.go @@ -1,3 +1,22 @@ +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// + package syncer import "fmt" diff --git a/pkg/syncer/name.go b/pkg/syncer/name.go index e6b001d..de12e0a 100644 --- a/pkg/syncer/name.go +++ b/pkg/syncer/name.go @@ -1,3 +1,22 @@ +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// + package syncer import ( diff --git a/pkg/syncer/name_test.go b/pkg/syncer/name_test.go index 82480d0..2cf3058 100644 --- a/pkg/syncer/name_test.go +++ b/pkg/syncer/name_test.go @@ -1,3 +1,22 @@ +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// + package syncer import ( diff --git a/pkg/syncer/run.go b/pkg/syncer/run.go index 60c1704..32562c0 100644 --- a/pkg/syncer/run.go +++ b/pkg/syncer/run.go @@ -1,3 +1,22 @@ +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// + package syncer import ( diff --git a/pkg/syncer/syncer.go b/pkg/syncer/syncer.go index 52b3a52..ed17400 100644 --- a/pkg/syncer/syncer.go +++ b/pkg/syncer/syncer.go @@ -1,3 +1,22 @@ +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// + // Package syncer provides the logic used by command line tool cloudstack-csi-sc-syncer. // // It provides functions to synchronize CloudStack disk offerings diff --git a/pkg/util/doc.go b/pkg/util/doc.go index e1beef7..af89a03 100644 --- a/pkg/util/doc.go +++ b/pkg/util/doc.go @@ -1,2 +1,21 @@ +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// + // Package util provides shared utility functions. package util diff --git a/pkg/util/endpoint.go b/pkg/util/endpoint.go index cb46ddf..f7a7269 100644 --- a/pkg/util/endpoint.go +++ b/pkg/util/endpoint.go @@ -1,3 +1,22 @@ +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// + package util import ( diff --git a/pkg/util/endpoint_test.go b/pkg/util/endpoint_test.go index 93e2878..ec9f949 100644 --- a/pkg/util/endpoint_test.go +++ b/pkg/util/endpoint_test.go @@ -1,3 +1,22 @@ +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// + package util import ( diff --git a/pkg/util/gb.go b/pkg/util/gb.go index 1523872..7920bfa 100644 --- a/pkg/util/gb.go +++ b/pkg/util/gb.go @@ -1,3 +1,22 @@ +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// + package util // RoundUpBytesToGB converts a size given in bytes to GB with diff --git a/pkg/util/gb_test.go b/pkg/util/gb_test.go index bde5ad8..8cb6c96 100644 --- a/pkg/util/gb_test.go +++ b/pkg/util/gb_test.go @@ -1,3 +1,22 @@ +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// + package util import ( diff --git a/test/e2e/run.sh b/test/e2e/run.sh index e4aa5df..6c3684c 100755 --- a/test/e2e/run.sh +++ b/test/e2e/run.sh @@ -1,3 +1,22 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + #!/bin/bash cd "$(dirname "$0")" || exit diff --git a/test/sanity/sanity_test.go b/test/sanity/sanity_test.go index e4c7a96..5d624d6 100644 --- a/test/sanity/sanity_test.go +++ b/test/sanity/sanity_test.go @@ -1,3 +1,22 @@ +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// + //go:build sanity package sanity