Merge pull request #207 from tencentyun/dev/checkPointUpload

Dzkol · web-flow · commit 4e15c6d59775 · 2024-07-18T14:53:03.000+08:00
delete check &amp; range set
diff --git a/pom.xml b/pom.xml
@@ -4,7 +4,7 @@
     <modelVersion>4.0.0</modelVersion>
     <groupId>com.qcloud</groupId>
     <artifactId>cos_api</artifactId>
-    <version>5.6.219.1</version>
+    <version>5.6.220</version>
     <packaging>jar</packaging>
     <name>cos-java-sdk</name>
     <description>java sdk for qcloud cos</description>
diff --git a/src/main/java/com/qcloud/cos/COSClient.java b/src/main/java/com/qcloud/cos/COSClient.java
@@ -38,6 +38,7 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Map.Entry;
+import java.util.Objects;
 
 import com.fasterxml.jackson.core.Version;
 import com.fasterxml.jackson.databind.ObjectMapper;
@@ -143,6 +144,7 @@
 import com.qcloud.cos.utils.*;
 import com.qcloud.cos.http.TimeOutCosHttpClient;
 
+import com.qcloud.cos.utils.Base64;
 import org.apache.commons.codec.DecoderException;
 import org.apache.http.client.methods.HttpRequestBase;
 import org.apache.http.entity.ContentType;
@@ -1072,8 +1074,16 @@ public COSObject getObject(GetObjectRequest getObjectRequest)
         // Range
         long[] range = getObjectRequest.getRange();
         if (range != null) {
-            request.addHeader(Headers.RANGE,
-                    "bytes=" + Long.toString(range[0]) + "-" + Long.toString(range[1]));
+            if (range[0] == -1) {
+                request.addHeader(Headers.RANGE,
+                        "bytes=" + "-" + Long.toString(range[1]));
+            } else if (range[1] == -1) {
+                request.addHeader(Headers.RANGE,
+                        "bytes=" + Long.toString(range[0]) + "-");
+            } else {
+                request.addHeader(Headers.RANGE,
+                        "bytes=" + Long.toString(range[0]) + "-" + Long.toString(range[1]));
+            }
         }
         addResponseHeaderParameters(request, getObjectRequest.getResponseHeaders());
 
@@ -1274,6 +1284,11 @@ public void deleteObject(DeleteObjectRequest deleteObjectRequest)
 
         rejectEmpty(deleteObjectRequest.getKey(),
                                 "The length of the key must be greater than 0");
+
+        if (Objects.equals(deleteObjectRequest.getKey(), "/")) {
+            throw new IllegalArgumentException("The specified key should not be /");
+        }
+
         CosHttpRequest<DeleteObjectRequest> request =
                 createRequest(deleteObjectRequest.getBucketName(), deleteObjectRequest.getKey(),
                         deleteObjectRequest, HttpMethodName.DELETE);
diff --git a/src/main/java/com/qcloud/cos/demo/TransferManagerDemo.java b/src/main/java/com/qcloud/cos/demo/TransferManagerDemo.java
@@ -35,10 +35,30 @@
 // TransferManager提供异步的上传文件, 下载文件，copy文件的高级API接口
 // 可以根据文件大小自动的选择上传接口或者copy接口,方便用户使用, 无需自行封装较复杂的分块上传或者分块copy
 public class TransferManagerDemo {
+    private static String secretId = "AKIDXXXXXXXX";
+    private static String secretKey = "1A2Z3YYYYYYYYYY";
+    private static String cosRegion = "ap-guangzhou";
+    private static String bucketName = "examplebucket-12500000000";
     public static void main(String[] args) {
         //multipartUploadWithMetaData();
         resumableDownloadFile();
     }
+
+    private static TransferManager createTransferManager() {
+        // 1 初始化用户身份信息(secretId, secretKey)
+        COSCredentials cred = new BasicCOSCredentials(secretId, secretKey);
+        // 2 设置bucket的区域, COS地域的简称请参照 https://www.qcloud.com/document/product/436/6224
+        ClientConfig clientConfig = new ClientConfig(new Region(cosRegion));
+        clientConfig.setHttpProtocol(HttpProtocol.https);
+        // 3 生成cos客户端
+        COSClient cosclient = new COSClient(cred, clientConfig);
+
+        ExecutorService threadPool = Executors.newFixedThreadPool(32);
+        // 传入一个threadpool, 若不传入线程池, 默认TransferManager中会生成一个单线程的线程池。
+        TransferManager transferManager = new TransferManager(cosclient, threadPool);
+        return transferManager;
+    }
+
     // Prints progress while waiting for the transfer to finish.
     private static void showTransferProgress(Transfer transfer) {
         System.out.println(transfer.getDescription());
@@ -59,19 +79,7 @@ private static void showTransferProgress(Transfer transfer) {
 
     // 上传文件, 根据文件大小自动选择简单上传或者分块上传。
     private static void uploadFile() {
-        // 1 初始化用户身份信息(secretId, secretKey)
-        COSCredentials cred = new BasicCOSCredentials("AKIDXXXXXXXX", "1A2Z3YYYYYYYYYY");
-        // 2 设置bucket的区域, COS地域的简称请参照 https://www.qcloud.com/document/product/436/6224
-        ClientConfig clientConfig = new ClientConfig(new Region("ap-beijing-1"));
-        clientConfig.setHttpProtocol(HttpProtocol.https);
-        // 3 生成cos客户端
-        COSClient cosclient = new COSClient(cred, clientConfig);
-        // bucket名需包含appid
-        String bucketName = "mybucket-12500000000";
-
-        ExecutorService threadPool = Executors.newFixedThreadPool(32);
-        // 传入一个threadpool, 若不传入线程池, 默认TransferManager中会生成一个单线程的线程池。
-        TransferManager transferManager = new TransferManager(cosclient, threadPool);
+        TransferManager transferManager = createTransferManager();
 
         String key = "aaa/bbb.txt";
         File localFile = new File("src/test/resources/len30M.txt");
@@ -96,23 +104,11 @@ private static void uploadFile() {
         }
 
         transferManager.shutdownNow();
-        cosclient.shutdown();
     }
 
     // 上传文件（上传过程中暂停, 并继续上传)
     private static void pauseUploadFileAndResume() {
-        // 1 初始化用户身份信息(secretId, secretKey)
-        COSCredentials cred = new BasicCOSCredentials("AKIDXXXXXXXX", "1A2Z3YYYYYYYYYY");
-        // 2 设置bucket的区域, COS地域的简称请参照 https://www.qcloud.com/document/product/436/6224
-        ClientConfig clientConfig = new ClientConfig(new Region("ap-beijing-1"));
-        // 3 生成cos客户端
-        COSClient cosclient = new COSClient(cred, clientConfig);
-        // bucket名需包含appid
-        String bucketName = "mybucket-12500000000";
-
-        ExecutorService threadPool = Executors.newFixedThreadPool(4);
-        // 传入一个threadpool, 若不传入线程池, 默认TransferManager中会生成一个单线程的线程池。
-        TransferManager transferManager = new TransferManager(cosclient, threadPool);
+        TransferManager transferManager = createTransferManager();
 
         String key = "aaa/bbb.txt";
         File localFile = new File("src/test/resources/len30M.txt");
@@ -135,22 +131,10 @@ private static void pauseUploadFileAndResume() {
         }
 
         transferManager.shutdownNow();
-        cosclient.shutdown();
     }
 
     private static void multipartUploadWithMetaData() {
-        // 1 初始化用户身份信息(secretId, secretKey)
-        COSCredentials cred = new BasicCOSCredentials("AKIDXXXXXXXX", "1A2Z3YYYYYYYYYY");
-        // 2 设置bucket的区域, COS地域的简称请参照 https://www.qcloud.com/document/product/436/6224
-        ClientConfig clientConfig = new ClientConfig(new Region("ap-shanghai"));
-        // 3 生成cos客户端
-        COSClient cosclient = new COSClient(cred, clientConfig);
-        // bucket名需包含appid
-        String bucketName = "mybucket-12500000000";
-
-        ExecutorService threadPool = Executors.newFixedThreadPool(2);
-        // 传入一个threadpool, 若不传入线程池, 默认TransferManager中会生成一个单线程的线程池。
-        TransferManager transferManager = new TransferManager(cosclient, threadPool);
+        TransferManager transferManager = createTransferManager();
 
         String key = "aaa/bbb.txt";
         File localFile = new File("src/test/resources/len20M.txt");
@@ -181,23 +165,11 @@ private static void multipartUploadWithMetaData() {
         }
 
         transferManager.shutdownNow();
-        cosclient.shutdown();
     }
 
     // 批量上传
     private static void uploadDirectory() {
-        // 1 初始化用户身份信息(secretId, secretKey)
-        COSCredentials cred = new BasicCOSCredentials("AKIDXXXXXXXX", "1A2Z3YYYYYYYYYY");
-        // 2 设置bucket的区域, COS地域的简称请参照 https://www.qcloud.com/document/product/436/6224
-        ClientConfig clientConfig = new ClientConfig(new Region("ap-beijing-1"));
-        // 3 生成cos客户端
-        COSClient cosclient = new COSClient(cred, clientConfig);
-        // bucket名需包含appid
-        String bucketName = "mybucket-12500000000";
-
-        ExecutorService threadPool = Executors.newFixedThreadPool(4);
-        // 传入一个threadpool, 若不传入线程池, 默认TransferManager中会生成一个单线程的线程池。
-        TransferManager transferManager = new TransferManager(cosclient, threadPool);
+        TransferManager transferManager = createTransferManager();
 
         // 设置文件上传到 bucket 之后的前缀目录，设置为 “”，表示上传到 bucket 的根目录
         String cos_path = "/prefix";
@@ -226,23 +198,11 @@ private static void uploadDirectory() {
         }
 
         transferManager.shutdownNow();
-        cosclient.shutdown();
     }
 
     // 批量下载
     private static void downloadDirectory() {
-        // 1 初始化用户身份信息(secretId, secretKey)
-        COSCredentials cred = new BasicCOSCredentials("AKIDXXXXXXXX", "1A2Z3YYYYYYYYYY");
-        // 2 设置bucket的区域, COS地域的简称请参照 https://www.qcloud.com/document/product/436/6224
-        ClientConfig clientConfig = new ClientConfig(new Region("ap-beijing-1"));
-        // 3 生成cos客户端
-        COSClient cosclient = new COSClient(cred, clientConfig);
-        // bucket名需包含appid
-        String bucketName = "mybucket-12500000000";
-
-        ExecutorService threadPool = Executors.newFixedThreadPool(4);
-        // 传入一个threadpool, 若不传入线程池, 默认TransferManager中会生成一个单线程的线程池。
-        TransferManager transferManager = new TransferManager(cosclient, threadPool);
+        TransferManager transferManager = createTransferManager();
 
         // 设置要下载的对象的前缀（相当于cos上的一个目录），如果设置成 ""，则下载整个 bucket。
         String cos_path = "/prefix";
@@ -269,23 +229,11 @@ private static void downloadDirectory() {
         }
 
         transferManager.shutdownNow();
-        cosclient.shutdown();
     }
 
     // 将文件下载到本地
     private static void downLoadFile() {
-        // 1 初始化用户身份信息(secretId, secretKey)
-        COSCredentials cred = new BasicCOSCredentials("AKIDXXXXXXXX", "1A2Z3YYYYYYYYYY");
-        // 2 设置bucket的区域, COS地域的简称请参照 https://www.qcloud.com/document/product/436/6224
-        ClientConfig clientConfig = new ClientConfig(new Region("ap-beijing-1"));
-        // 3 生成cos客户端
-        COSClient cosclient = new COSClient(cred, clientConfig);
-        // bucket名需包含appid
-        String bucketName = "mybucket-12500000000";
-
-        ExecutorService threadPool = Executors.newFixedThreadPool(32);
-        // 传入一个threadpool, 若不传入线程池, 默认TransferManager中会生成一个单线程的线程池。
-        TransferManager transferManager = new TransferManager(cosclient, threadPool);
+        TransferManager transferManager = createTransferManager();
 
         String key = "aaa/bbb.txt";
         File downloadFile = new File("src/test/resources/download.txt");
@@ -303,23 +251,11 @@ private static void downLoadFile() {
         }
 
         transferManager.shutdownNow();
-        cosclient.shutdown();
     }
 
     // 将文件下载到本地(中途暂停并继续开始)
     private static void pauseDownloadFileAndResume() {
-        // 1 初始化用户身份信息(secretId, secretKey)
-        COSCredentials cred = new BasicCOSCredentials("AKIDXXXXXXXX", "1A2Z3YYYYYYYYYY");
-        // 2 设置bucket的区域, COS地域的简称请参照 https://www.qcloud.com/document/product/436/6224
-        ClientConfig clientConfig = new ClientConfig(new Region("ap-beijing-1"));
-        // 3 生成cos客户端
-        COSClient cosclient = new COSClient(cred, clientConfig);
-        // bucket名需包含appid
-        String bucketName = "mybucket-12500000000";
-
-        ExecutorService threadPool = Executors.newFixedThreadPool(32);
-        // 传入一个threadpool, 若不传入线程池, 默认TransferManager中会生成一个单线程的线程池。
-        TransferManager transferManager = new TransferManager(cosclient, threadPool);
+        TransferManager transferManager = createTransferManager();
 
         String key = "aaa/bbb.txt";
         File downloadFile = new File("src/test/resources/download.txt");
@@ -340,23 +276,20 @@ private static void pauseDownloadFileAndResume() {
             e.printStackTrace();
         }
 
-
         transferManager.shutdownNow();
-        cosclient.shutdown();
     }
 
 
     // copy接口支持根据文件大小自动选择copy或者分块copy
     // 以下代码展示跨园区拷贝, 即将一个园区的文件拷贝到另一个园区
     private static void copyFileForDiffRegion() {
         // 1 初始化用户身份信息(secretId, secretKey)
-        COSCredentials cred = new BasicCOSCredentials("AKIDXXXXXXXX", "1A2Z3YYYYYYYYYY");
+        COSCredentials cred = new BasicCOSCredentials(secretId, secretKey);
         // 2 设置bucket的区域, COS地域的简称请参照 https://www.qcloud.com/document/product/436/6224
-        ClientConfig clientConfig = new ClientConfig(new Region("ap-beijing-1"));
+        ClientConfig clientConfig = new ClientConfig(new Region(cosRegion));
         // 3 生成cos客户端
         COSClient cosclient = new COSClient(cred, clientConfig);
 
-
         ExecutorService threadPool = Executors.newFixedThreadPool(32);
         // 传入一个threadpool, 若不传入线程池, 默认TransferManager中会生成一个单线程的线程池。
         TransferManager transferManager = new TransferManager(cosclient, threadPool);
@@ -395,17 +328,7 @@ private static void copyFileForDiffRegion() {
     // copy接口支持根据文件大小自动选择copy或者分块copy
     // 以下代码展示同园区拷贝, 即将同园区的文件拷贝到另一个园区
     private static void copyFileForSameRegion() {
-        // 1 初始化用户身份信息(secretId, secretKey)
-        COSCredentials cred = new BasicCOSCredentials("AKIDXXXXXXXX", "1A2Z3YYYYYYYYYY");
-        // 2 设置bucket的区域, COS地域的简称请参照 https://www.qcloud.com/document/product/436/6224
-        ClientConfig clientConfig = new ClientConfig(new Region("ap-beijing-1"));
-        // 3 生成cos客户端
-        COSClient cosclient = new COSClient(cred, clientConfig);
-
-
-        ExecutorService threadPool = Executors.newFixedThreadPool(32);
-        // 传入一个threadpool, 若不传入线程池, 默认TransferManager中会生成一个单线程的线程池。
-        TransferManager transferManager = new TransferManager(cosclient, threadPool);
+        TransferManager transferManager = createTransferManager();
         TransferManagerConfiguration transferManagerConfiguration = new TransferManagerConfiguration();
         transferManagerConfiguration.setMultipartCopyThreshold(20*1024*1024);
         transferManager.setConfiguration(transferManagerConfiguration);
@@ -437,14 +360,13 @@ private static void copyFileForSameRegion() {
         }
 
         transferManager.shutdownNow();
-        cosclient.shutdown();
     }
 
     private static void copyFileSetMetadata() {
         // 1 初始化用户身份信息(secretId, secretKey)
-        COSCredentials cred = new BasicCOSCredentials("AKIDXXXXXXXX", "1A2Z3YYYYYYYYYY");
+        COSCredentials cred = new BasicCOSCredentials(secretId, secretKey);
         // 2 设置bucket的区域, COS地域的简称请参照 https://www.qcloud.com/document/product/436/6224
-        ClientConfig clientConfig = new ClientConfig(new Region("ap-guangzhou"));
+        ClientConfig clientConfig = new ClientConfig(new Region(cosRegion));
         // 3 生成cos客户端
         COSClient cosclient = new COSClient(cred, clientConfig);
 
@@ -497,18 +419,9 @@ private static void copyFileSetMetadata() {
     }
 
     private static void resumableDownloadFile() {
-        // 1 初始化用户身份信息(secretId, secretKey)
-        COSCredentials cred = new BasicCOSCredentials("AKIDXXXXXXXX", "1A2Z3YYYYYYYYYY");
-        // 2 设置bucket的区域, COS地域的简称请参照 https://www.qcloud.com/document/product/436/6224
-        ClientConfig clientConfig = new ClientConfig(new Region("ap-shanghai"));
-        // 3 生成cos客户端
-        COSClient cosclient = new COSClient(cred, clientConfig);
+        TransferManager transferManager = createTransferManager();
 
-        ExecutorService threadPool = Executors.newFixedThreadPool(5);
-        // 传入一个threadpool, 若不传入线程池, 默认TransferManager中会生成一个单线程的线程池。
-        TransferManager transferManager = new TransferManager(cosclient, threadPool);
-
-        GetObjectRequest getObj = new GetObjectRequest("mybucket-1000000000", "/path/to/key");
+        GetObjectRequest getObj = new GetObjectRequest(bucketName, "/path/to/key");
 
         File dstFile = new File("dstFile");
         Download download = transferManager.download(getObj, dstFile, true);
@@ -524,6 +437,5 @@ private static void resumableDownloadFile() {
         }
 
         transferManager.shutdownNow();
-        cosclient.shutdown();
     }
 }
diff --git a/src/main/java/com/qcloud/cos/http/DefaultCosHttpClient.java b/src/main/java/com/qcloud/cos/http/DefaultCosHttpClient.java
@@ -509,6 +509,7 @@ public <X, Y extends CosServiceRequest> X exeute(CosHttpRequest<Y> request,
                 }
                 if (retryIndex != 0) {
                     long delay = backoffStrategy.computeDelayBeforeNextRetry(retryIndex);
+                    request.addHeader("x-cos-sdk-retry", "true");
                     Thread.sleep(delay);
                 }
                 HttpContext context = HttpClientContext.create();
diff --git a/src/main/java/com/qcloud/cos/model/GetObjectRequest.java b/src/main/java/com/qcloud/cos/model/GetObjectRequest.java
@@ -329,6 +329,20 @@ public void setRange(long start, long end) {
         range = new long[] {start, end};
     }
 
+    public void setRangeStart(long start) {
+        if (start < 0) {
+            throw new IllegalArgumentException("The range start should be greater than or equal to 0");
+        }
+        range = new long[] {start, -1L};
+    }
+
+    public void setRangeEnd(long end) {
+        if (end < 0) {
+            throw new IllegalArgumentException("The range end should be greater than or equal to 0");
+        }
+        range = new long[] {-1L, end};
+    }
+
     /**
      * <p>
      * Sets the optional inclusive byte range within the desired object that will be downloaded by
diff --git a/src/main/java/com/qcloud/cos/transfer/TransferManager.java b/src/main/java/com/qcloud/cos/transfer/TransferManager.java
@@ -721,7 +721,7 @@ private Download doDownload(final GetObjectRequest getObjectRequest, final File
         long lastByte;
 
         long[] range = getObjectRequest.getRange();
-        if (range != null && range.length == 2) {
+        if (range != null && range.length == 2 && range[0] != -1 && range[1] != -1) {
             startingByte = range[0];
             lastByte = range[1];
         } else {
@@ -734,6 +734,13 @@ private Download doDownload(final GetObjectRequest getObjectRequest, final File
             final ObjectMetadata objectMetadata = cos.getObjectMetadata(getObjectMetadataRequest);
 
             lastByte = objectMetadata.getContentLength() - 1;
+            if (range != null && range.length == 2) {
+                if (range[0] == -1) {
+                    startingByte = lastByte - range[1] + 1;
+                } else {
+                    startingByte = range[0];
+                }
+            }
         }
         final long origStartingByte = startingByte;
         // We still pass the unfiltered listener chain into DownloadImpl

Original file line number	Diff line number	Diff line change
`@@ -509,6 +509,7 @@ public <X, Y extends CosServiceRequest> X exeute(CosHttpRequest<Y> request,`
`509`	`509`	`}`
`510`	`510`	`if (retryIndex != 0) {`
`511`	`511`	`long delay = backoffStrategy.computeDelayBeforeNextRetry(retryIndex);`
	`512`	`+ request.addHeader("x-cos-sdk-retry", "true");`
`512`	`513`	`Thread.sleep(delay);`
`513`	`514`	`}`
`514`	`515`	`HttpContext context = HttpClientContext.create();`