diff --git a/.github/workflows/object-storage-adapter-check.yaml b/.github/workflows/object-storage-adapter-check.yaml index a757e7c39c..f0bd5865ea 100644 --- a/.github/workflows/object-storage-adapter-check.yaml +++ b/.github/workflows/object-storage-adapter-check.yaml @@ -91,6 +91,7 @@ jobs: run: | container_id=$(docker create "container-registry.oracle.com/java/jdk:${{ env.INT_TEST_JAVA_RUNTIME_VERSION }}") docker cp -L "$container_id:/usr/java/default" /usr/lib/jvm/oracle-jdk && docker rm "$container_id" + - name: Setup Gradle uses: gradle/actions/setup-gradle@v5 @@ -145,11 +146,18 @@ jobs: run: | container_id=$(docker create "container-registry.oracle.com/java/jdk:${{ env.INT_TEST_JAVA_RUNTIME_VERSION }}") docker cp -L "$container_id:/usr/java/default" /usr/lib/jvm/oracle-jdk && docker rm "$container_id" + - name: Setup Gradle uses: gradle/actions/setup-gradle@v5 + - name: Prepare Google Cloud Credentials + run: | + echo '${{ env.CLOUD_STORAGE_SERVICE_ACCOUNT_KEY }}' > ${{ runner.temp }}/gcloud_service_account.json + - name: Execute Gradle 'integrationTestObjectStorage' task - run: ./gradlew integrationTestObjectStorage -Dscalardb.object_storage.storage=cloud-storage -Dscalardb.object_storage.endpoint=scalardb-test-bucket -Dscalardb.object_storage.username=${{ env.CLOUD_STORAGE_PROJECT_ID }} -Dscalardb.object_storage.password=${{ env.CLOUD_STORAGE_SERVICE_ACCOUNT_KEY }} ${{ matrix.mode.group_commit_enabled && env.INT_TEST_GRADLE_OPTIONS_FOR_GROUP_COMMIT || '' }} + env: + GOOGLE_APPLICATION_CREDENTIALS: ${{ runner.temp }}/gcloud_service_account.json + run: ./gradlew integrationTestObjectStorage -Dscalardb.object_storage.storage=cloud-storage -Dscalardb.object_storage.endpoint=scalardb-test-bucket -Dscalardb.object_storage.username=${{ env.CLOUD_STORAGE_PROJECT_ID }} ${{ matrix.mode.group_commit_enabled && env.INT_TEST_GRADLE_OPTIONS_FOR_GROUP_COMMIT || '' }} - name: Upload Gradle test reports if: always() diff --git a/core/src/main/java/com/scalar/db/common/CoreError.java b/core/src/main/java/com/scalar/db/common/CoreError.java index 8473597b65..87779cd038 100644 --- a/core/src/main/java/com/scalar/db/common/CoreError.java +++ b/core/src/main/java/com/scalar/db/common/CoreError.java @@ -931,18 +931,6 @@ public enum CoreError implements ScalarDbError { "Conditions on indexed columns in cross-partition scan operations are not allowed in the SERIALIZABLE isolation level", "", ""), - OBJECT_STORAGE_CLOUD_STORAGE_SERVICE_ACCOUNT_KEY_NOT_FOUND( - Category.USER_ERROR, - "0263", - "The service account key for Cloud Storage was not found.", - "", - ""), - OBJECT_STORAGE_CLOUD_STORAGE_SERVICE_ACCOUNT_KEY_LOAD_FAILED( - Category.USER_ERROR, - "0264", - "Failed to load the service account key for Cloud Storage.", - "", - ""), // // Errors for the concurrency error category diff --git a/core/src/main/java/com/scalar/db/storage/objectstorage/ObjectStorageConfig.java b/core/src/main/java/com/scalar/db/storage/objectstorage/ObjectStorageConfig.java index 3866e66f43..aa22b37adc 100644 --- a/core/src/main/java/com/scalar/db/storage/objectstorage/ObjectStorageConfig.java +++ b/core/src/main/java/com/scalar/db/storage/objectstorage/ObjectStorageConfig.java @@ -9,13 +9,6 @@ public interface ObjectStorageConfig { */ String getStorageName(); - /** - * Returns the password for authentication. - * - * @return the password - */ - String getPassword(); - /** * Returns the bucket name. * diff --git a/core/src/main/java/com/scalar/db/storage/objectstorage/blobstorage/BlobStorageConfig.java b/core/src/main/java/com/scalar/db/storage/objectstorage/blobstorage/BlobStorageConfig.java index edb6f769f7..47df2f8332 100644 --- a/core/src/main/java/com/scalar/db/storage/objectstorage/blobstorage/BlobStorageConfig.java +++ b/core/src/main/java/com/scalar/db/storage/objectstorage/blobstorage/BlobStorageConfig.java @@ -78,11 +78,6 @@ public String getStorageName() { return STORAGE_NAME; } - @Override - public String getPassword() { - return password; - } - @Override public String getBucket() { return bucket; @@ -101,6 +96,10 @@ public String getUsername() { return username; } + public String getPassword() { + return password; + } + public Optional getParallelUploadBlockSizeInBytes() { return Optional.ofNullable(parallelUploadBlockSizeInBytes); } diff --git a/core/src/main/java/com/scalar/db/storage/objectstorage/cloudstorage/CloudStorageConfig.java b/core/src/main/java/com/scalar/db/storage/objectstorage/cloudstorage/CloudStorageConfig.java index 41f2929ef0..7324865c36 100644 --- a/core/src/main/java/com/scalar/db/storage/objectstorage/cloudstorage/CloudStorageConfig.java +++ b/core/src/main/java/com/scalar/db/storage/objectstorage/cloudstorage/CloudStorageConfig.java @@ -2,14 +2,8 @@ import static com.scalar.db.config.ConfigUtils.getInt; -import com.google.auth.Credentials; -import com.google.auth.oauth2.ServiceAccountCredentials; -import com.scalar.db.common.CoreError; import com.scalar.db.config.DatabaseConfig; import com.scalar.db.storage.objectstorage.ObjectStorageConfig; -import java.io.ByteArrayInputStream; -import java.io.IOException; -import java.nio.charset.StandardCharsets; import java.util.Optional; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -22,7 +16,6 @@ public class CloudStorageConfig implements ObjectStorageConfig { PREFIX + "parallel_upload_block_size_in_bytes"; private static final Logger logger = LoggerFactory.getLogger(CloudStorageConfig.class); - private final String password; private final String bucket; private final String metadataNamespace; private final String projectId; @@ -39,7 +32,6 @@ public CloudStorageConfig(DatabaseConfig databaseConfig) { } bucket = databaseConfig.getContactPoints().get(0); projectId = databaseConfig.getUsername().orElse(null); - password = databaseConfig.getPassword().orElse(null); metadataNamespace = databaseConfig.getSystemNamespaceName(); if (databaseConfig.getScanFetchSize() != DatabaseConfig.DEFAULT_SCAN_FETCH_SIZE) { @@ -58,11 +50,6 @@ public String getStorageName() { return STORAGE_NAME; } - @Override - public String getPassword() { - return password; - } - @Override public String getBucket() { return bucket; @@ -77,21 +64,6 @@ public String getProjectId() { return projectId; } - public Credentials getCredentials() { - String serviceAccountJson = getPassword(); - if (serviceAccountJson == null) { - throw new IllegalArgumentException( - CoreError.OBJECT_STORAGE_CLOUD_STORAGE_SERVICE_ACCOUNT_KEY_NOT_FOUND.buildMessage()); - } - try (ByteArrayInputStream keyStream = - new ByteArrayInputStream(serviceAccountJson.getBytes(StandardCharsets.UTF_8))) { - return ServiceAccountCredentials.fromStream(keyStream); - } catch (IOException e) { - throw new IllegalArgumentException( - CoreError.OBJECT_STORAGE_CLOUD_STORAGE_SERVICE_ACCOUNT_KEY_LOAD_FAILED.buildMessage()); - } - } - public Optional getParallelUploadBlockSizeInBytes() { return Optional.ofNullable(parallelUploadBlockSizeInBytes); } diff --git a/core/src/main/java/com/scalar/db/storage/objectstorage/cloudstorage/CloudStorageWrapper.java b/core/src/main/java/com/scalar/db/storage/objectstorage/cloudstorage/CloudStorageWrapper.java index ab89310ed7..cda5b786dc 100644 --- a/core/src/main/java/com/scalar/db/storage/objectstorage/cloudstorage/CloudStorageWrapper.java +++ b/core/src/main/java/com/scalar/db/storage/objectstorage/cloudstorage/CloudStorageWrapper.java @@ -34,12 +34,7 @@ public class CloudStorageWrapper implements ObjectStorageWrapper { private final Integer parallelUploadBlockSizeInBytes; public CloudStorageWrapper(CloudStorageConfig config) { - storage = - StorageOptions.newBuilder() - .setProjectId(config.getProjectId()) - .setCredentials(config.getCredentials()) - .build() - .getService(); + storage = StorageOptions.newBuilder().setProjectId(config.getProjectId()).build().getService(); bucket = config.getBucket(); parallelUploadBlockSizeInBytes = config.getParallelUploadBlockSizeInBytes().orElse(null); } diff --git a/core/src/main/java/com/scalar/db/storage/objectstorage/s3/S3Config.java b/core/src/main/java/com/scalar/db/storage/objectstorage/s3/S3Config.java index 82a2a70f01..a699ff3cd8 100644 --- a/core/src/main/java/com/scalar/db/storage/objectstorage/s3/S3Config.java +++ b/core/src/main/java/com/scalar/db/storage/objectstorage/s3/S3Config.java @@ -25,8 +25,6 @@ public class S3Config implements ObjectStorageConfig { public static final String REQUEST_TIMEOUT_IN_SECONDS = PREFIX + "request_timeout_in_seconds"; private static final Logger logger = LoggerFactory.getLogger(S3Config.class); - private final String username; - private final String password; private final String bucket; private final String metadataNamespace; private final String region; @@ -56,8 +54,6 @@ public S3Config(DatabaseConfig databaseConfig) { throw new IllegalArgumentException( "Invalid contact points format. Expected: S3_REGION/BUCKET_NAME"); } - username = databaseConfig.getUsername().orElse(null); - password = databaseConfig.getPassword().orElse(null); metadataNamespace = databaseConfig.getSystemNamespaceName(); if (databaseConfig.getScanFetchSize() != DatabaseConfig.DEFAULT_SCAN_FETCH_SIZE) { @@ -82,11 +78,6 @@ public String getStorageName() { return STORAGE_NAME; } - @Override - public String getPassword() { - return password; - } - @Override public String getBucket() { return bucket; @@ -101,10 +92,6 @@ public String getRegion() { return region; } - public String getUsername() { - return username; - } - public Optional getParallelUploadBlockSizeInBytes() { return Optional.ofNullable(parallelUploadBlockSizeInBytes); } diff --git a/core/src/main/java/com/scalar/db/storage/objectstorage/s3/S3Wrapper.java b/core/src/main/java/com/scalar/db/storage/objectstorage/s3/S3Wrapper.java index 2f7342d2b4..0d0e4d712a 100644 --- a/core/src/main/java/com/scalar/db/storage/objectstorage/s3/S3Wrapper.java +++ b/core/src/main/java/com/scalar/db/storage/objectstorage/s3/S3Wrapper.java @@ -13,8 +13,6 @@ import java.util.Optional; import java.util.Set; import javax.annotation.concurrent.ThreadSafe; -import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; -import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; import software.amazon.awssdk.core.ResponseBytes; import software.amazon.awssdk.core.async.AsyncRequestBody; import software.amazon.awssdk.core.async.AsyncResponseTransformer; @@ -61,9 +59,6 @@ public S3Wrapper(S3Config config) { this.client = S3AsyncClient.builder() .region(Region.of(config.getRegion())) - .credentialsProvider( - StaticCredentialsProvider.create( - AwsBasicCredentials.create(config.getUsername(), config.getPassword()))) .httpClientBuilder(httpClientBuilder) .multipartConfiguration(multipartConfigBuilder.build()) .overrideConfiguration(overrideConfigBuilder.build()) diff --git a/core/src/test/java/com/scalar/db/storage/objectstorage/cloudstorage/CloudStorageConfigTest.java b/core/src/test/java/com/scalar/db/storage/objectstorage/cloudstorage/CloudStorageConfigTest.java index 254e695427..266b13fe78 100644 --- a/core/src/test/java/com/scalar/db/storage/objectstorage/cloudstorage/CloudStorageConfigTest.java +++ b/core/src/test/java/com/scalar/db/storage/objectstorage/cloudstorage/CloudStorageConfigTest.java @@ -35,7 +35,6 @@ public void constructor_AllPropertiesGiven_ShouldLoadProperly() { // Assert assertThat(config.getProjectId()).isEqualTo(ANY_PROJECT_ID); assertThat(config.getBucket()).isEqualTo(ANY_BUCKET); - assertThat(config.getPassword()).isEqualTo(ANY_PASSWORD); assertThat(config.getMetadataNamespace()).isEqualTo(ANY_TABLE_METADATA_NAMESPACE); assertThat(config.getParallelUploadBlockSizeInBytes()).isNotEmpty(); assertThat(config.getParallelUploadBlockSizeInBytes().get()).isEqualTo(5242880); @@ -56,7 +55,6 @@ public void constructor_PropertiesWithoutNonMandatoryOptionsGiven_ShouldLoadProp // Assert assertThat(config.getProjectId()).isEqualTo(ANY_PROJECT_ID); assertThat(config.getBucket()).isEqualTo(ANY_BUCKET); - assertThat(config.getPassword()).isEqualTo(ANY_PASSWORD); assertThat(config.getMetadataNamespace()) .isEqualTo(DatabaseConfig.DEFAULT_SYSTEM_NAMESPACE_NAME); assertThat(config.getParallelUploadBlockSizeInBytes()).isEmpty(); diff --git a/core/src/test/java/com/scalar/db/storage/objectstorage/s3/S3ConfigTest.java b/core/src/test/java/com/scalar/db/storage/objectstorage/s3/S3ConfigTest.java index 0efb0c81a8..7f119c1b62 100644 --- a/core/src/test/java/com/scalar/db/storage/objectstorage/s3/S3ConfigTest.java +++ b/core/src/test/java/com/scalar/db/storage/objectstorage/s3/S3ConfigTest.java @@ -43,8 +43,6 @@ public void constructor_AllPropertiesGiven_ShouldLoadProperly() { // Assert assertThat(config.getRegion()).isEqualTo(ANY_REGION); assertThat(config.getBucket()).isEqualTo(ANY_BUCKET); - assertThat(config.getUsername()).isEqualTo(ANY_USERNAME); - assertThat(config.getPassword()).isEqualTo(ANY_PASSWORD); assertThat(config.getMetadataNamespace()).isEqualTo(ANY_TABLE_METADATA_NAMESPACE); assertThat(config.getParallelUploadBlockSizeInBytes()).isNotEmpty(); assertThat(config.getParallelUploadBlockSizeInBytes().get()).isEqualTo(5242880); @@ -71,8 +69,6 @@ public void constructor_PropertiesWithoutNonMandatoryOptionsGiven_ShouldLoadProp // Assert assertThat(config.getRegion()).isEqualTo(ANY_REGION); assertThat(config.getBucket()).isEqualTo(ANY_BUCKET); - assertThat(config.getUsername()).isEqualTo(ANY_USERNAME); - assertThat(config.getPassword()).isEqualTo(ANY_PASSWORD); assertThat(config.getMetadataNamespace()) .isEqualTo(DatabaseConfig.DEFAULT_SYSTEM_NAMESPACE_NAME); assertThat(config.getParallelUploadBlockSizeInBytes()).isEmpty();