Skip to content

Commit 9146ea0

Browse files
feeblefakieKodaiDbrfrn169
authored
Backport to branch(3) : Add support for administrative operations in Blob adapter (#3123)
Co-authored-by: Kodai Doki <52027276+KodaiD@users.noreply.github.com> Co-authored-by: Toshihiro Suzuki <brfrn169@gmail.com> Co-authored-by: Kodai Doki <kodai.doki@scalar-labs.com>
1 parent bd99ccf commit 9146ea0

File tree

35 files changed

+3630
-38
lines changed

35 files changed

+3630
-38
lines changed

‎.github/workflows/ci.yaml‎

Lines changed: 70 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2122,6 +2122,76 @@ jobs:
21222122
name: tidb_8_5_integration_test_reports_${{ matrix.mode.label }}
21232123
path: core/build/reports/tests/integrationTestJdbc
21242124

2125+
integration-test-for-blob-storage:
2126+
name: Blob Storage integration test (${{ matrix.mode.label }})
2127+
runs-on: ubuntu-latest
2128+
2129+
strategy:
2130+
fail-fast: false
2131+
matrix:
2132+
mode:
2133+
- label: default
2134+
group_commit_enabled: false
2135+
- label: with_group_commit
2136+
group_commit_enabled: true
2137+
2138+
services:
2139+
postgres:
2140+
image: mcr.microsoft.com/azure-storage/azurite
2141+
env:
2142+
AZURITE_ACCOUNTS: "test:test"
2143+
ports:
2144+
- 10000:10000
2145+
2146+
steps:
2147+
- uses: actions/checkout@v5
2148+
2149+
- name: Set up JDK ${{ env.JAVA_VERSION }} (${{ env.JAVA_VENDOR }})
2150+
uses: actions/setup-java@v5
2151+
with:
2152+
java-version: ${{ env.JAVA_VERSION }}
2153+
distribution: ${{ env.JAVA_VENDOR }}
2154+
2155+
- name: Set up JDK ${{ env.INT_TEST_JAVA_RUNTIME_VERSION }} (${{ env.INT_TEST_JAVA_RUNTIME_VENDOR }}) to run integration test
2156+
uses: actions/setup-java@v5
2157+
if: ${{ env.SET_UP_INT_TEST_RUNTIME_NON_ORACLE_JDK == 'true'}}
2158+
with:
2159+
java-version: ${{ env.INT_TEST_JAVA_RUNTIME_VERSION }}
2160+
distribution: ${{ env.INT_TEST_JAVA_RUNTIME_VENDOR }}
2161+
2162+
- name: Login to Oracle container registry
2163+
uses: docker/login-action@v3
2164+
if: ${{ env.INT_TEST_JAVA_RUNTIME_VENDOR == 'oracle' }}
2165+
with:
2166+
registry: container-registry.oracle.com
2167+
username: ${{ secrets.OCR_USERNAME }}
2168+
password: ${{ secrets.OCR_TOKEN }}
2169+
2170+
- name: Set up JDK ${{ env.INT_TEST_JAVA_RUNTIME_VERSION }} (oracle) to run the integration test
2171+
if: ${{ env.INT_TEST_JAVA_RUNTIME_VENDOR == 'oracle' }}
2172+
run: |
2173+
container_id=$(docker create "container-registry.oracle.com/java/jdk:${{ env.INT_TEST_JAVA_RUNTIME_VERSION }}")
2174+
docker cp -L "$container_id:/usr/java/default" /usr/lib/jvm/oracle-jdk && docker rm "$container_id"
2175+
2176+
- name: Setup Gradle
2177+
uses: gradle/actions/setup-gradle@v5
2178+
2179+
- name: Create Blob Storage container
2180+
run: |
2181+
az storage container create \
2182+
--name test-container \
2183+
--connection-string "DefaultEndpointsProtocol=http;AccountName=test;AccountKey=test;BlobEndpoint=http://localhost:10000/test;"
2184+
2185+
- name: Execute Gradle 'integrationTestObjectStorage' task
2186+
run: ./gradlew integrationTestObjectStorage -Dscalardb.object_storage.endpoint=http://localhost:10000/test/test-container -Dscalardb.object_storage.username=test -Dscalardb.object_storage.password=test ${{ matrix.mode.group_commit_enabled && env.INT_TEST_GRADLE_OPTIONS_FOR_GROUP_COMMIT || '' }}
2187+
2188+
- name: Upload Gradle test reports
2189+
if: always()
2190+
uses: actions/upload-artifact@v4
2191+
with:
2192+
name: blob_storage_integration_test_reports_${{ matrix.mode.label }}
2193+
path: core/build/reports/tests/integrationTestObjectStorage
2194+
21252195
integration-test-for-multi-storage:
21262196
name: Multi-storage integration test (${{ matrix.mode.label }})
21272197
runs-on: ubuntu-latest

‎build.gradle‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@ subprojects {
2828
slf4jVersion = '1.7.36'
2929
cassandraDriverVersion = '3.11.5'
3030
azureCosmosVersion = '4.75.0'
31+
azureBlobStorageVersion = '12.31.3'
3132
jooqVersion = '3.14.16'
3233
awssdkVersion = '2.37.3'
3334
commonsDbcp2Version = '2.13.0'

‎core/build.gradle‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -64,6 +64,16 @@ sourceSets {
6464
}
6565
resources.srcDir file('src/integration-test/resources')
6666
}
67+
integrationTestObjectStorage {
68+
java {
69+
compileClasspath += main.output + test.output
70+
runtimeClasspath += main.output + test.output
71+
srcDir file('src/integration-test/java')
72+
include '**/com/scalar/db/common/*.java'
73+
include '**/com/scalar/db/storage/objectstorage/*.java'
74+
}
75+
resources.srcDir file('src/integration-test/resources')
76+
}
6777
integrationTestMultiStorage {
6878
java {
6979
compileClasspath += main.output + test.output
@@ -136,6 +146,9 @@ configurations {
136146
integrationTestJdbcImplementation.extendsFrom testImplementation
137147
integrationTestJdbcRuntimeOnly.extendsFrom testRuntimeOnly
138148
integrationTestJdbcCompileOnly.extendsFrom testCompileOnly
149+
integrationTestObjectStorageImplementation.extendsFrom testImplementation
150+
integrationTestObjectStorageRuntimeOnly.extendsFrom testRuntimeOnly
151+
integrationTestObjectStorageCompileOnly.extendsFrom testCompileOnly
139152
integrationTestMultiStorageImplementation.extendsFrom testImplementation
140153
integrationTestMultiStorageRuntimeOnly.extendsFrom testRuntimeOnly
141154
integrationTestMultiStorageCompileOnly.extendsFrom testCompileOnly
@@ -156,6 +169,7 @@ dependencies {
156169
implementation "org.slf4j:slf4j-api:${slf4jVersion}"
157170
implementation "com.datastax.cassandra:cassandra-driver-core:${cassandraDriverVersion}"
158171
implementation "com.azure:azure-cosmos:${azureCosmosVersion}"
172+
implementation "com.azure:azure-storage-blob:${azureBlobStorageVersion}"
159173
implementation "org.jooq:jooq:${jooqVersion}"
160174
implementation platform("software.amazon.awssdk:bom:${awssdkVersion}")
161175
implementation 'software.amazon.awssdk:applicationautoscaling'
@@ -254,6 +268,17 @@ task integrationTestJdbc(type: Test) {
254268
maxHeapSize = "4g"
255269
}
256270

271+
task integrationTestObjectStorage(type: Test) {
272+
description = 'Runs the integration tests for Object Storages.'
273+
group = 'verification'
274+
testClassesDirs = sourceSets.integrationTestObjectStorage.output.classesDirs
275+
classpath = sourceSets.integrationTestObjectStorage.runtimeClasspath
276+
outputs.upToDateWhen { false } // ensures integration tests are run every time when called
277+
options {
278+
systemProperties(System.getProperties().findAll{it.key.toString().startsWith("scalardb")})
279+
}
280+
}
281+
257282
task integrationTestMultiStorage(type: Test) {
258283
description = 'Runs the integration tests for multi-storage.'
259284
group = 'verification'

‎core/src/integration-test/java/com/scalar/db/storage/objectstorage/ConsensusCommitAdminIntegrationTestWithObjectStorage.java‎

Lines changed: 143 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,143 @@
1+
package com.scalar.db.storage.objectstorage;
2+
3+
import com.scalar.db.config.DatabaseConfig;
4+
import com.scalar.db.transaction.consensuscommit.ConsensusCommitAdminIntegrationTestBase;
5+
import com.scalar.db.transaction.consensuscommit.ConsensusCommitConfig;
6+
import com.scalar.db.transaction.consensuscommit.Coordinator;
7+
import java.util.Properties;
8+
import org.junit.jupiter.api.Disabled;
9+
10+
public class ConsensusCommitAdminIntegrationTestWithObjectStorage
11+
extends ConsensusCommitAdminIntegrationTestBase {
12+
13+
@Override
14+
protected Properties getProps(String testName) {
15+
return ObjectStorageEnv.getProperties(testName);
16+
}
17+
18+
@Override
19+
protected String getSystemNamespaceName(Properties properties) {
20+
return ObjectStorageUtils.getObjectStorageConfig(new DatabaseConfig(properties))
21+
.getMetadataNamespace();
22+
}
23+
24+
@Override
25+
protected String getCoordinatorNamespaceName(String testName) {
26+
return new ConsensusCommitConfig(new DatabaseConfig(getProperties(testName)))
27+
.getCoordinatorNamespace()
28+
.orElse(Coordinator.NAMESPACE);
29+
}
30+
31+
@Override
32+
@Disabled("Temporarily disabled because it includes DML operations")
33+
public void truncateTable_ShouldTruncateProperly() {}
34+
35+
@Override
36+
@Disabled("Object Storage does not support index-related operations")
37+
public void createIndex_ForAllDataTypesWithExistingData_ShouldCreateIndexesCorrectly() {}
38+
39+
@Override
40+
@Disabled("Object Storage does not support index-related operations")
41+
public void createIndex_ForNonExistingTable_ShouldThrowIllegalArgumentException() {}
42+
43+
@Override
44+
@Disabled("Object Storage does not support index-related operations")
45+
public void createIndex_ForNonExistingColumn_ShouldThrowIllegalArgumentException() {}
46+
47+
@Override
48+
@Disabled("Object Storage does not support index-related operations")
49+
public void createIndex_ForAlreadyExistingIndex_ShouldThrowIllegalArgumentException() {}
50+
51+
@Override
52+
@Disabled("Object Storage does not support index-related operations")
53+
public void createIndex_IfNotExists_ForAlreadyExistingIndex_ShouldNotThrowAnyException() {}
54+
55+
@Override
56+
@Disabled("Object Storage does not support index-related operations")
57+
public void dropIndex_ForAllDataTypesWithExistingData_ShouldDropIndexCorrectly() {}
58+
59+
@Override
60+
@Disabled("Object Storage does not support index-related operations")
61+
public void dropIndex_ForNonExistingTable_ShouldThrowIllegalArgumentException() {}
62+
63+
@Override
64+
@Disabled("Object Storage does not support index-related operations")
65+
public void dropIndex_ForNonExistingIndex_ShouldThrowIllegalArgumentException() {}
66+
67+
@Override
68+
@Disabled("Object Storage does not support index-related operations")
69+
public void dropIndex_IfExists_ForNonExistingIndex_ShouldNotThrowAnyException() {}
70+
71+
@Override
72+
@Disabled("Object Storage does not support dropping columns")
73+
public void dropColumnFromTable_DropColumnForEachExistingDataType_ShouldDropColumnsCorrectly() {}
74+
75+
@Override
76+
@Disabled("Object Storage does not support dropping columns")
77+
public void dropColumnFromTable_ForNonExistingTable_ShouldThrowIllegalArgumentException() {}
78+
79+
@Override
80+
@Disabled("Object Storage does not support dropping columns")
81+
public void dropColumnFromTable_ForNonExistingColumn_ShouldThrowIllegalArgumentException() {}
82+
83+
@Override
84+
@Disabled("Object Storage does not support dropping columns")
85+
public void dropColumnFromTable_ForPrimaryKeyColumn_ShouldThrowIllegalArgumentException() {}
86+
87+
@Override
88+
@Disabled("Object Storage does not support dropping columns")
89+
public void dropColumnFromTable_ForIndexedColumn_ShouldDropColumnAndIndexCorrectly() {}
90+
91+
@Override
92+
@Disabled("Object Storage does not support renaming columns")
93+
public void renameColumn_ShouldRenameColumnCorrectly() {}
94+
95+
@Override
96+
@Disabled("Object Storage does not support renaming columns")
97+
public void renameColumn_ForNonExistingTable_ShouldThrowIllegalArgumentException() {}
98+
99+
@Override
100+
@Disabled("Object Storage does not support renaming columns")
101+
public void renameColumn_ForNonExistingColumn_ShouldThrowIllegalArgumentException() {}
102+
103+
@Override
104+
@Disabled("Object Storage does not support renaming columns")
105+
public void renameColumn_ForPrimaryKeyColumn_ShouldRenameColumnCorrectly() {}
106+
107+
@Override
108+
@Disabled("Object Storage does not support renaming columns")
109+
public void renameColumn_ForIndexKeyColumn_ShouldRenameColumnAndIndexCorrectly() {}
110+
111+
@Override
112+
@Disabled("Object Storage does not support altering column types")
113+
public void
114+
alterColumnType_AlterColumnTypeFromEachExistingDataTypeToText_ShouldAlterColumnTypesCorrectly() {}
115+
116+
@Override
117+
@Disabled("Object Storage does not support altering column types")
118+
public void alterColumnType_WideningConversion_ShouldAlterColumnTypesCorrectly() {}
119+
120+
@Override
121+
@Disabled("Object Storage does not support altering column types")
122+
public void alterColumnType_ForPrimaryKeyOrIndexKeyColumn_ShouldThrowIllegalArgumentException() {}
123+
124+
@Override
125+
@Disabled("Object Storage does not support renaming tables")
126+
public void renameTable_ForExistingTable_ShouldRenameTableCorrectly() {}
127+
128+
@Override
129+
@Disabled("Object Storage does not support renaming tables")
130+
public void renameTable_ForNonExistingTable_ShouldThrowIllegalArgumentException() {}
131+
132+
@Override
133+
@Disabled("Object Storage does not support renaming tables")
134+
public void renameTable_IfNewTableNameAlreadyExists_ShouldThrowIllegalArgumentException() {}
135+
136+
@Override
137+
@Disabled("Object Storage does not support renaming tables")
138+
public void renameTable_ForExistingTableWithIndexes_ShouldRenameTableAndIndexesCorrectly() {}
139+
140+
@Override
141+
@Disabled("Object Storage does not support renaming tables")
142+
public void renameTable_IfOnlyOneTableExists_ShouldRenameTableCorrectly() {}
143+
}

‎core/src/integration-test/java/com/scalar/db/storage/objectstorage/ConsensusCommitAdminRepairTableIntegrationTestWithObjectStorage.java‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
package com.scalar.db.storage.objectstorage;
2+
3+
import com.scalar.db.transaction.consensuscommit.ConsensusCommitAdminRepairTableIntegrationTestBase;
4+
import com.scalar.db.util.AdminTestUtils;
5+
import java.util.Properties;
6+
import org.junit.jupiter.api.Disabled;
7+
8+
public class ConsensusCommitAdminRepairTableIntegrationTestWithObjectStorage
9+
extends ConsensusCommitAdminRepairTableIntegrationTestBase {
10+
11+
@Override
12+
protected Properties getProps(String testName) {
13+
return ObjectStorageEnv.getProperties(testName);
14+
}
15+
16+
@Override
17+
protected AdminTestUtils getAdminTestUtils(String testName) {
18+
return new ObjectStorageAdminTestUtils(getProperties(testName));
19+
}
20+
21+
@Override
22+
@Disabled("Object Storage recreates missing coordinator tables")
23+
public void
24+
repairTableAndCoordinatorTable_CoordinatorTablesDoNotExist_ShouldThrowIllegalArgumentException() {}
25+
26+
@Override
27+
@Disabled("Object Storage recreates missing coordinator tables")
28+
public void repairTable_ForNonExistingTable_ShouldThrowIllegalArgument() {}
29+
}

0 commit comments

Comments
 (0)