Skip to content

Commit a055d0e

Browse files
DzkolDzkol
authored
Merge pull request #222 from tencentyun/dev/checkPointUpload
Dev/check point upload
2 parents ce35bec + 18758b6 commit a055d0e

28 files changed

+1139
-171
lines changed

‎pom.xml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@
44
<modelVersion>4.0.0</modelVersion>
55
<groupId>com.qcloud</groupId>
66
<artifactId>cos_api</artifactId>
7-
<version>5.6.233</version>
7+
<version>5.6.234</version>
88
<packaging>jar</packaging>
99
<name>cos-java-sdk</name>
1010
<description>java sdk for qcloud cos</description>
@@ -110,7 +110,7 @@
110110
<dependency>
111111
<groupId>org.bouncycastle</groupId>
112112
<artifactId>bcprov-jdk15on</artifactId>
113-
<version>1.67</version>
113+
<version>1.70</version>
114114
</dependency>
115115

116116
<dependency>

‎src/main/java/com/qcloud/cos/COSClient.java‎

Lines changed: 195 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,7 @@
3939
import java.util.Map;
4040
import java.util.Map.Entry;
4141
import java.util.Objects;
42+
import java.util.concurrent.ConcurrentHashMap;
4243

4344
import com.fasterxml.jackson.core.Version;
4445
import com.fasterxml.jackson.databind.ObjectMapper;
@@ -67,6 +68,7 @@
6768
import com.qcloud.cos.internal.XmlResponsesSaxParser.CompleteMultipartUploadHandler;
6869
import com.qcloud.cos.internal.XmlResponsesSaxParser.CopyObjectResultHandler;
6970
import com.qcloud.cos.model.*;
71+
import com.qcloud.cos.model.IntelligentTiering.BucketIntelligentTieringConfiguration;
7072
import com.qcloud.cos.model.bucketcertificate.BucketDomainCertificateRequest;
7173
import com.qcloud.cos.model.bucketcertificate.BucketGetDomainCertificate;
7274
import com.qcloud.cos.model.bucketcertificate.BucketPutDomainCertificate;
@@ -164,6 +166,8 @@ public class COSClient implements COS {
164166

165167
private CosHttpClient cosHttpClient;
166168

169+
private ConcurrentHashMap<String, Long> preflightBuckets = new ConcurrentHashMap<>();
170+
167171
public COSClient(COSCredentials cred, ClientConfig clientConfig) {
168172
this(new COSStaticCredentialsProvider(cred), clientConfig);
169173
}
@@ -839,6 +843,17 @@ ObjectMetadata uploadObjectInternal(UploadMode uploadMode, UploadObjectRequest u
839843
rejectNull(bucketName,
840844
"The bucket name parameter must be specified when uploading an object");
841845
rejectNull(key, "The key parameter must be specified when uploading an object");
846+
847+
try {
848+
preflightObj(uploadObjectRequest);
849+
} catch (CosServiceException cse) {
850+
log.warn("fail to do the preflight request due to the service exception, will not do the upload obj request", cse);
851+
throw cse;
852+
} catch (CosClientException cce) {
853+
log.warn("fail to do the preflight request due to the client exception, will not do the upload obj request", cce);
854+
throw cce;
855+
}
856+
842857
// If a file is specified for upload, we need to pull some additional
843858
// information from it to auto-configure a few options
844859
if (file == null) {
@@ -978,6 +993,22 @@ ObjectMetadata uploadObjectInternal(UploadMode uploadMode, UploadObjectRequest u
978993
CosDataSource.Utils.cleanupDataSource(uploadObjectRequest, file, isOrig, input, log);
979994
}
980995

996+
if (returnedMetadata.isNeedPreflight()) {
997+
Long currentTime = System.currentTimeMillis();
998+
if ((preflightBuckets.get(bucketName) == null) || ((currentTime - preflightBuckets.get(bucketName)) > clientConfig.getPreflightStatusUpdateInterval())) {
999+
String reqMsg = String.format("will update preflight status, bucket[%s]", bucketName);
1000+
log.info(reqMsg);
1001+
preflightBuckets.put(bucketName, currentTime);
1002+
}
1003+
} else {
1004+
Long currentTime = System.currentTimeMillis();
1005+
if ((preflightBuckets.get(bucketName) != null) && ((currentTime - preflightBuckets.get(bucketName)) > clientConfig.getPreflightStatusUpdateInterval())) {
1006+
String reqMsg = String.format("will remove bucket[%s] from preflight lists", bucketName);
1007+
log.info(reqMsg);
1008+
preflightBuckets.remove(bucketName);
1009+
}
1010+
}
1011+
9811012
String contentMd5 = metadata.getContentMD5();
9821013
if (md5DigestStream != null) {
9831014
contentMd5 = Base64.encodeAsString(md5DigestStream.getMd5Digest());
@@ -1542,11 +1573,25 @@ public List<Bucket> listBuckets() throws CosClientException, CosServiceException
15421573
@Override
15431574
public List<Bucket> listBuckets(ListBucketsRequest listBucketsRequest)
15441575
throws CosClientException, CosServiceException {
1576+
ListBucketsResult result = getService(listBucketsRequest);
1577+
return result.getBuckets();
1578+
}
1579+
1580+
public ListBucketsResult getService(ListBucketsRequest listBucketsRequest)
1581+
throws CosClientException, CosServiceException {
15451582
rejectNull(listBucketsRequest,
15461583
"The request object parameter listBucketsRequest must be specified.");
15471584
CosHttpRequest<ListBucketsRequest> request =
15481585
createRequest(null, null, listBucketsRequest, HttpMethodName.GET);
1549-
return invoke(request, new Unmarshallers.ListBucketsUnmarshaller());
1586+
if (!listBucketsRequest.getMarker().isEmpty()) {
1587+
request.addParameter("marker", listBucketsRequest.getMarker());
1588+
}
1589+
1590+
if (listBucketsRequest.getMaxKeys() != null && listBucketsRequest.getMaxKeys() > 0) {
1591+
request.addParameter("max-keys", listBucketsRequest.getMaxKeys().toString());
1592+
}
1593+
1594+
return invoke(request, new Unmarshallers.GetServiceUnmarshaller());
15501595
}
15511596

15521597
@Override
@@ -1627,7 +1672,8 @@ public InitiateMultipartUploadResult initiateMultipartUpload(
16271672
// xml payload unmarshaller
16281673
new Unmarshallers.InitiateMultipartUploadResultUnmarshaller(),
16291674
// header handlers
1630-
new ServerSideEncryptionHeaderHandler<InitiateMultipartUploadResult>());
1675+
new ServerSideEncryptionHeaderHandler<InitiateMultipartUploadResult>(),
1676+
new VIDResultHandler<InitiateMultipartUploadResult>());
16311677
return invoke(request, responseHandler);
16321678
}
16331679

@@ -1740,6 +1786,7 @@ private UploadPartResult doUploadPart(final String bucketName, final String key,
17401786
result.setSSECustomerAlgorithm(metadata.getSSECustomerAlgorithm());
17411787
result.setSSECustomerKeyMd5(metadata.getSSECustomerKeyMd5());
17421788
result.setCrc64Ecma(metadata.getCrc64Ecma());
1789+
result.setRequestId(metadata.getRequestId());
17431790

17441791
return result;
17451792
} catch (Throwable t) {
@@ -2226,7 +2273,8 @@ public CopyPartResult copyPart(CopyPartRequest copyPartRequest)
22262273
new Unmarshallers.CopyObjectUnmarshaller(),
22272274
// header handlers
22282275
new ServerSideEncryptionHeaderHandler<CopyObjectResultHandler>(),
2229-
new COSVersionHeaderHandler());
2276+
new COSVersionHeaderHandler(),
2277+
new VIDResultHandler<CopyObjectResultHandler>());
22302278
copyObjectResultHandler = invoke(request, handler);
22312279
} catch (CosServiceException cse) {
22322280
/*
@@ -2287,6 +2335,7 @@ public CopyPartResult copyPart(CopyPartRequest copyPartRequest)
22872335
copyPartResult.setSSECustomerAlgorithm(copyObjectResultHandler.getSSECustomerAlgorithm());
22882336
copyPartResult.setSSECustomerKeyMd5(copyObjectResultHandler.getSSECustomerKeyMd5());
22892337
copyPartResult.setCrc64Ecma(copyObjectResultHandler.getCrc64Ecma());
2338+
copyPartResult.setRequestId(copyObjectResultHandler.getRequestId());
22902339

22912340
return copyPartResult;
22922341
}
@@ -3874,6 +3923,24 @@ public void setBucketIntelligentTieringConfiguration(SetBucketIntelligentTierCon
38743923
invoke(request, voidCosResponseHandler);
38753924
}
38763925

3926+
public List<BucketIntelligentTieringConfiguration> listBucketIntelligentTieringConfiguration(String bucketName) throws CosServiceException, CosClientException {
3927+
rejectEmpty(bucketName,
3928+
"The bucket name parameter must be specified when listing bucket IntelligentTieringConfiguration");
3929+
CosHttpRequest<CosServiceRequest> request = createRequest(bucketName, null, new CosServiceRequest(), HttpMethodName.GET);
3930+
request.addParameter("intelligent-tiering", null);
3931+
3932+
try {
3933+
return invoke(request, new Unmarshallers.ListBucketTieringConfigurationUnmarshaller());
3934+
} catch (CosServiceException cse) {
3935+
switch (cse.getStatusCode()) {
3936+
case 404:
3937+
return null;
3938+
default:
3939+
throw cse;
3940+
}
3941+
}
3942+
}
3943+
38773944
@Override
38783945
public MediaJobResponse createMediaJobs(MediaJobsRequest req) {
38793946
this.rejectStartWith(req.getCallBack(),"http","The CallBack parameter mush start with http or https");
@@ -5367,5 +5434,130 @@ private String buildPrivateM3U8Url(CosHttpRequest<PrivateM3U8Request> request)
53675434
URL url = generatePresignedUrl(request.getBucketName(), request.getResourcePath(), expiredTime, HttpMethodName.GET, new HashMap<String, String>(), params, false, false);
53685435
return url.toString();
53695436
}
5437+
5438+
public void putBucketEncryptionConfiguration(String bucketName, BucketEncryptionConfiguration bucketEncryptionConfiguration)
5439+
throws CosClientException, CosServiceException {
5440+
rejectEmpty(bucketName,
5441+
"The bucket name parameter must be specified when putting bucket encryption");
5442+
5443+
CosHttpRequest<CosServiceRequest> request = createRequest(bucketName, null, new CosServiceRequest(), HttpMethodName.PUT);
5444+
request.addParameter("encryption", null);
5445+
5446+
rejectEmpty(bucketEncryptionConfiguration.getSseAlgorithm(),
5447+
"The SseAlgorithm parameter must be specified when putting bucket encryption");
5448+
5449+
byte[] bytes = new BucketConfigurationXmlFactory().convertToXmlByteArray(bucketEncryptionConfiguration);
5450+
request.setContent(new ByteArrayInputStream(bytes));
5451+
request.addHeader(Headers.CONTENT_LENGTH, String.valueOf(bytes.length));
5452+
5453+
invoke(request, voidCosResponseHandler);
5454+
}
5455+
5456+
public BucketEncryptionConfiguration getBucketEncryptionConfiguration(String bucketName) throws CosClientException, CosServiceException {
5457+
rejectEmpty(bucketName,
5458+
"The bucket name parameter must be specified when getting bucket encryption");
5459+
5460+
CosHttpRequest<CosServiceRequest> request = createRequest(bucketName, null, new CosServiceRequest(), HttpMethodName.GET);
5461+
request.addParameter("encryption", null);
5462+
5463+
return invoke(request, new Unmarshallers.BucketEncryptionConfigurationUnmarshaller());
5464+
}
5465+
5466+
public void deleteBucketEncryptionConfiguration(String bucketName) throws CosClientException, CosServiceException {
5467+
rejectEmpty(bucketName,
5468+
"The bucket name parameter must be specified when deleting bucket encryption");
5469+
5470+
CosHttpRequest<CosServiceRequest> request = createRequest(bucketName, null, new CosServiceRequest(), HttpMethodName.DELETE);
5471+
request.addParameter("encryption", null);
5472+
5473+
invoke(request, voidCosResponseHandler);
5474+
}
5475+
5476+
public BucketObjectLockConfiguration getBucketObjectLockConfiguration(String bucketName) throws CosClientException, CosServiceException {
5477+
rejectEmpty(bucketName,
5478+
"The bucket name parameter must be specified when getting bucket object lock configuration");
5479+
5480+
CosHttpRequest<CosServiceRequest> request = createRequest(bucketName, null, new CosServiceRequest(), HttpMethodName.GET);
5481+
request.addParameter("object-lock", null);
5482+
5483+
try {
5484+
return invoke(request, new Unmarshallers.BucketObjectLockConfigurationUnmarshaller());
5485+
} catch (CosServiceException cse) {
5486+
switch (cse.getStatusCode()) {
5487+
case 404:
5488+
return null;
5489+
default:
5490+
throw cse;
5491+
}
5492+
}
5493+
}
5494+
5495+
public BucketGetMetadataResult getBucketMetadata(String bucketName) throws CosClientException, CosServiceException {
5496+
rejectEmpty(bucketName,
5497+
"The bucket name parameter must be specified when getting bucket metadata");
5498+
BucketGetMetadataResult result = new BucketGetMetadataResult();
5499+
HeadBucketResult headBucketResult = headBucket(new HeadBucketRequest(bucketName));
5500+
result.bucketName = bucketName;
5501+
StringBuilder strBuilder = new StringBuilder();
5502+
strBuilder.append(clientConfig.getHttpProtocol().toString()).append("://");
5503+
strBuilder.append(clientConfig.getEndpointBuilder()
5504+
.buildGeneralApiEndpoint(formatBucket(bucketName, null)));
5505+
result.bucketUrl = strBuilder.toString();
5506+
result.isMaz = headBucketResult.isMazBucket();
5507+
result.isOFS = headBucketResult.isMergeBucket();
5508+
result.location = headBucketResult.getBucketRegion();
5509+
5510+
try {
5511+
BucketEncryptionConfiguration encryptionConfiguration = getBucketEncryptionConfiguration(bucketName);
5512+
result.encryptionConfiguration = encryptionConfiguration;
5513+
} catch (CosServiceException cse) {
5514+
if (cse.getStatusCode() != 404) {
5515+
throw cse;
5516+
}
5517+
}
5518+
5519+
result.accessControlList = getBucketAcl(bucketName);
5520+
result.websiteConfiguration = getBucketWebsiteConfiguration(bucketName);
5521+
result.loggingConfiguration = getBucketLoggingConfiguration(bucketName);
5522+
result.crossOriginConfiguration = getBucketCrossOriginConfiguration(bucketName);
5523+
result.versioningConfiguration = getBucketVersioningConfiguration(bucketName);
5524+
result.lifecycleConfiguration = getBucketLifecycleConfiguration(bucketName);
5525+
5526+
try {
5527+
result.replicationConfiguration = getBucketReplicationConfiguration(bucketName);
5528+
} catch (CosServiceException cse) {
5529+
if (cse.getStatusCode() != 404) {
5530+
throw cse;
5531+
}
5532+
}
5533+
5534+
result.taggingConfiguration = getBucketTaggingConfiguration(bucketName);
5535+
result.tieringConfigurations = listBucketIntelligentTieringConfiguration(bucketName);
5536+
result.bucketObjectLockConfiguration = getBucketObjectLockConfiguration(bucketName);
5537+
5538+
return result;
5539+
}
5540+
5541+
private void preflightObj(PutObjectRequest putObjectRequest) throws CosClientException, CosServiceException {
5542+
String bucketName = putObjectRequest.getBucketName();
5543+
String key = putObjectRequest.getKey();
5544+
rejectEmpty(bucketName,
5545+
"The bucket name parameter must be specified when doing preflight request");
5546+
rejectEmpty(key,
5547+
"The key parameter must be specified when doing preflight request");
5548+
if (clientConfig.isCheckPreflightStatus() && preflightBuckets.containsKey(bucketName)) {
5549+
String reqMsg = String.format("will do preflight request for put object[%s] to the bucket[%s]", key, bucketName);
5550+
log.debug(reqMsg);
5551+
CosServiceRequest serviceRequest = new CosServiceRequest();
5552+
CosHttpRequest<CosServiceRequest> request = createRequest(bucketName, key, serviceRequest, HttpMethodName.HEAD);
5553+
if (putObjectRequest.getFixedEndpointAddr() != null) {
5554+
request.setEndpoint(putObjectRequest.getFixedEndpointAddr());
5555+
}
5556+
request.addParameter("preflight", null);
5557+
request.addHeader("x-cos-next-action", "PutObject");
5558+
5559+
invoke(request, voidCosResponseHandler);
5560+
}
5561+
}
53705562
}
53715563

‎src/main/java/com/qcloud/cos/ClientConfig.java‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -127,6 +127,10 @@ public class ClientConfig {
127127

128128
private boolean isRefreshEndpointAddr = false;
129129

130+
private boolean checkPreflightStatus = true;
131+
132+
private long preflightStatusUpdateInterval = 10 * 1000L;
133+
130134
// 不传入region 用于后续调用List Buckets(获取所有的bucket信息)
131135
public ClientConfig() {
132136
super();
@@ -451,4 +455,16 @@ public void turnOnRefreshEndpointAddrSwitch() {
451455
public boolean IsRefreshEndpointAddr() {
452456
return isRefreshEndpointAddr;
453457
}
458+
459+
public boolean isCheckPreflightStatus() {
460+
return checkPreflightStatus;
461+
}
462+
463+
public void setCheckPreflightStatus(boolean checkPreflightStatus) {
464+
this.checkPreflightStatus = checkPreflightStatus;
465+
}
466+
467+
public long getPreflightStatusUpdateInterval() {
468+
return preflightStatusUpdateInterval;
469+
}
454470
}

‎src/main/java/com/qcloud/cos/Headers.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -109,6 +109,8 @@ public interface Headers {
109109
*/
110110
public static final String BUCKET_ARCH = "x-cos-bucket-arch";
111111

112+
public static final String BUCKET_AZ_TYPE = "x-cos-bucket-az-type";
113+
112114
/**
113115
* COS request header indicating how to handle metadata when copying an object
114116
*/

‎src/main/java/com/qcloud/cos/demo/BucketDemo.java‎

Lines changed: 26 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
package com.qcloud.cos.demo;
22

3+
import java.util.ArrayList;
34
import java.util.List;
45

56
import com.qcloud.cos.COSClient;
@@ -11,17 +12,19 @@
1112
import com.qcloud.cos.model.Bucket;
1213
import com.qcloud.cos.model.CannedAccessControlList;
1314
import com.qcloud.cos.model.CreateBucketRequest;
15+
import com.qcloud.cos.model.ListBucketsRequest;
16+
import com.qcloud.cos.model.ListBucketsResult;
1417
import com.qcloud.cos.region.Region;
1518

1619
/**
1720
* 展示了创建bucket, 删除bucket, 查询bucket是否存在的demo
1821
*
1922
*/
2023
public class BucketDemo {
21-
private static String secretId = "AKIDXXXXXXXX";
22-
private static String secretKey = "1A2Z3YYYYYYYYYY";
23-
private static String cosRegion = "ap-guangzhou";
24-
private static String bucketName = "examplebucket-12500000000";
24+
private static String secretId = System.getenv("SECRETID");
25+
private static String secretKey = System.getenv("SECRETKEY");
26+
private static String bucketName = System.getenv("BUCKET_NAME");
27+
private static String cosRegion = System.getenv("REGION");
2528
private static COSClient cosClient = createCli();
2629

2730
public static void main(String[] args) {
@@ -87,6 +90,25 @@ private static void listBuckets() {
8790
}
8891
}
8992

93+
private static void listAllBucket() {
94+
ListBucketsRequest listBucketsRequest = new ListBucketsRequest();
95+
List<Bucket> buckets = new ArrayList<>();
96+
while (true) {
97+
ListBucketsResult result = cosClient.getService(listBucketsRequest);
98+
buckets.addAll(result.getBuckets());
99+
if (result.isTruncated()) {
100+
listBucketsRequest.setMarker(result.getNextMarker());
101+
} else {
102+
break;
103+
}
104+
}
105+
for (Bucket bucket : buckets) {
106+
System.out.println(bucket.getName());
107+
System.out.println(bucket.getLocation());
108+
}
109+
System.out.println("finish list all buckets, totally " + buckets.size() + " buckets.");
110+
}
111+
90112
//创多AZ桶
91113
private static void createMAZBucketDemo() {
92114
CreateBucketRequest createBucketRequest = new CreateBucketRequest(bucketName);

0 commit comments

Comments
 (0)