Refactor client-side checksum validation

azure/src/main/java/ch/cyberduck/core/azure/AzureUploadFeature.java

@@ -29,7 +29,6 @@ public class AzureUploadFeature extends DefaultUploadFeature<Void> {
     private final AzureSession session;
 
     public AzureUploadFeature(final AzureSession session) {
-        super(session);
         this.session = session;
     }
 

backblaze/src/main/java/ch/cyberduck/core/b2/B2LargeUploadService.java

@@ -43,7 +43,6 @@
 import org.apache.logging.log4j.Logger;
 
 import java.io.IOException;
-import java.security.MessageDigest;
 import java.text.MessageFormat;
 import java.util.ArrayList;
 import java.util.Comparator;
@@ -61,7 +60,7 @@
 import static ch.cyberduck.core.b2.B2MetadataFeature.X_BZ_INFO_SRC_CREATION_DATE_MILLIS;
 import static ch.cyberduck.core.b2.B2MetadataFeature.X_BZ_INFO_SRC_LAST_MODIFIED_MILLIS;
 
-public class B2LargeUploadService extends HttpUploadFeature<BaseB2Response, MessageDigest> {
+public class B2LargeUploadService extends HttpUploadFeature<BaseB2Response> {
     private static final Logger log = LogManager.getLogger(B2LargeUploadService.class);
 
     /**
@@ -229,7 +228,7 @@ public B2UploadPartResponse call() throws BackgroundException {
                 status.setChecksum(write.checksum(file, status).compute(local.getInputStream(), status));
                 status.setSegment(true);
                 status.setPart(partNumber);
-                return (B2UploadPartResponse) B2LargeUploadService.this.upload(write, file, local, throttle, counter, status, overall, status, callback);
+                return (B2UploadPartResponse) B2LargeUploadService.this.transfer(write, file, local, throttle, counter, status, overall, status, callback);
             }
         }, overall, counter));
     }

backblaze/src/main/java/ch/cyberduck/core/b2/B2SingleUploadService.java

@@ -15,57 +15,10 @@
  * GNU General Public License for more details.
  */
 
-import ch.cyberduck.core.Path;
-import ch.cyberduck.core.exception.BackgroundException;
 import ch.cyberduck.core.http.HttpUploadFeature;
-import ch.cyberduck.core.io.Checksum;
-import ch.cyberduck.core.preferences.HostPreferencesFactory;
 
-import org.apache.commons.lang3.StringUtils;
-
-import java.io.IOException;
-import java.io.InputStream;
-import java.security.DigestInputStream;
-import java.security.MessageDigest;
-import java.security.NoSuchAlgorithmException;
-
-import synapticloop.b2.response.B2FileResponse;
 import synapticloop.b2.response.BaseB2Response;
 
-public class B2SingleUploadService extends HttpUploadFeature<BaseB2Response, MessageDigest> {
-
-    private final B2Session session;
-
-    public B2SingleUploadService(final B2Session session) {
-        this.session = session;
-    }
-
-    @Override
-    protected InputStream decorate(final InputStream in, final MessageDigest digest) throws IOException {
-        if(null == digest) {
-            return super.decorate(in, null);
-        }
-        else {
-            return new DigestInputStream(in, digest);
-        }
-    }
-
-    @Override
-    protected MessageDigest digest() throws IOException {
-        MessageDigest digest = null;
-        if(HostPreferencesFactory.get(session.getHost()).getBoolean("b2.upload.checksum.verify")) {
-            try {
-                digest = MessageDigest.getInstance("SHA1");
-            }
-            catch(NoSuchAlgorithmException e) {
-                throw new IOException(e.getMessage(), e);
-            }
-        }
-        return digest;
-    }
+public class B2SingleUploadService extends HttpUploadFeature<BaseB2Response> {
 
-    @Override
-    protected void post(final Path file, final MessageDigest digest, final BaseB2Response response) throws BackgroundException {
-        this.verify(file, digest, Checksum.parse(StringUtils.removeStart(((B2FileResponse) response).getContentSha1(), "unverified:")));
-    }
 }

backblaze/src/main/java/ch/cyberduck/core/b2/B2ThresholdUploadService.java

@@ -64,7 +64,7 @@ public BaseB2Response upload(final Write<BaseB2Response> write, final Path file,
             return new B2LargeUploadService(session, fileid).upload(write, file, local, throttle, progress, streamListener, status, callback);
         }
         else {
-            return new B2SingleUploadService(session).upload(write, file, local, throttle, progress, streamListener, status, callback);
+            return new B2SingleUploadService().upload(write, file, local, throttle, progress, streamListener, status, callback);
         }
     }
 

backblaze/src/test/java/ch/cyberduck/core/b2/B2LargeUploadServiceTest.java

@@ -118,11 +118,11 @@ public void testAppendNoPartCompleted() throws Exception {
         final B2VersionIdProvider fileid = new B2VersionIdProvider(session);
         final B2LargeUploadService service = new B2LargeUploadService(session, fileid, 5 * 1000L * 1000L, 1) {
             @Override
-            public BaseB2Response upload(final Write<BaseB2Response> write, final Path file, final Local local, final BandwidthThrottle throttle, final StreamListener listener, final TransferStatus status, final StreamCancelation cancel, final StreamProgress progress, final ConnectionCallback callback) throws BackgroundException {
+            public BaseB2Response transfer(final Write<BaseB2Response> write, final Path file, final Local local, final BandwidthThrottle throttle, final StreamListener listener, final TransferStatus status, final StreamCancelation cancel, final StreamProgress progress, final ConnectionCallback callback) throws BackgroundException {
                 if(!interrupt.get()) {
                     throw new ConnectionTimeoutException("Test");
                 }
-                return super.upload(write, file, local, throttle, listener, status, cancel, progress, callback);
+                return super.transfer(write, file, local, throttle, listener, status, cancel, progress, callback);
             }
         };
         try {
@@ -171,13 +171,13 @@ public void testAppendSecondPart() throws Exception {
         final B2VersionIdProvider fileid = new B2VersionIdProvider(session);
         final B2LargeUploadService feature = new B2LargeUploadService(session, fileid, 5 * 1000L * 1000L, 1) {
             @Override
-            public BaseB2Response upload(final Write<BaseB2Response> write, final Path file, final Local local, final BandwidthThrottle throttle, final StreamListener listener, final TransferStatus status, final StreamCancelation cancel, final StreamProgress progress, final ConnectionCallback callback) throws BackgroundException {
+            public BaseB2Response transfer(final Write<BaseB2Response> write, final Path file, final Local local, final BandwidthThrottle throttle, final StreamListener listener, final TransferStatus status, final StreamCancelation cancel, final StreamProgress progress, final ConnectionCallback callback) throws BackgroundException {
                 if(!interrupt.get()) {
                     if(status.getOffset() >= 5L * 1000L * 1000L) {
                         throw new ConnectionTimeoutException("Test");
                     }
                 }
-                return super.upload(write, file, local, throttle, listener, status, cancel, progress, callback);
+                return super.transfer(write, file, local, throttle, listener, status, cancel, progress, callback);
             }
         };
         final BytecountStreamListener count = new BytecountStreamListener();

backblaze/src/test/java/ch/cyberduck/core/b2/B2ReadFeatureTest.java

@@ -128,7 +128,7 @@ public void testReadRange() throws Exception {
         assertNotNull(out);
         IOUtils.write(content, out);
         out.close();
-        final BaseB2Response reply = new B2SingleUploadService(session).upload(
+        final BaseB2Response reply = new B2SingleUploadService().upload(
                 new B2WriteFeature(session, fileid), test, local, new BandwidthThrottle(BandwidthThrottle.UNLIMITED), new DisabledProgressListener(), new DisabledStreamListener(),
                 new TransferStatus().setLength(content.length),
                 new DisabledConnectionCallback());
@@ -160,7 +160,7 @@ public void testReadRangeUnknownLength() throws Exception {
         assertNotNull(out);
         IOUtils.write(content, out);
         out.close();
-        new B2SingleUploadService(session).upload(
+        new B2SingleUploadService().upload(
                 new B2WriteFeature(session, fileid), test, local, new BandwidthThrottle(BandwidthThrottle.UNLIMITED), new DisabledProgressListener(), new DisabledStreamListener(),
                 new TransferStatus().setLength(content.length),
                 new DisabledConnectionCallback());

backblaze/src/test/java/ch/cyberduck/core/b2/B2SingleUploadServiceTest.java

@@ -61,7 +61,7 @@ public void testUpload() throws Exception {
         final Checksum checksum = new SHA1ChecksumCompute().compute(new ByteArrayInputStream(content), new TransferStatus());
         status.setChecksum(checksum);
         final B2VersionIdProvider fileid = new B2VersionIdProvider(session);
-        final B2SingleUploadService upload = new B2SingleUploadService(session);
+        final B2SingleUploadService upload = new B2SingleUploadService();
         upload.upload(new B2WriteFeature(session, fileid), test, local, new BandwidthThrottle(BandwidthThrottle.UNLIMITED), new DisabledProgressListener(), new DisabledStreamListener(),
                 status, new DisabledConnectionCallback());
         assertEquals(checksum, new B2AttributesFinderFeature(session, fileid).find(test).getChecksum());

backblaze/src/test/java/ch/cyberduck/core/worker/B2ConcurrentTransferWorkerTest.java

@@ -56,7 +56,6 @@
 import java.io.InputStream;
 import java.io.OutputStream;
 import java.net.SocketTimeoutException;
-import java.security.MessageDigest;
 import java.util.Collections;
 import java.util.EnumSet;
 import java.util.Random;
@@ -113,7 +112,7 @@ public Session create() {
                             final B2LargeUploadService upload = new B2LargeUploadService(this, new B2VersionIdProvider(this)
                             ) {
                                 @Override
-                                protected InputStream decorate(final InputStream in, final MessageDigest digest) {
+                                protected InputStream decorate(final InputStream in, final TransferStatus status) {
                                     if(failed.get()) {
                                         // Second attempt successful
                                         return in;

backblaze/src/test/java/ch/cyberduck/core/worker/B2SingleTransferWorkerTest.java

@@ -48,6 +48,7 @@
 import ch.cyberduck.core.transfer.TransferItem;
 import ch.cyberduck.core.transfer.TransferOptions;
 import ch.cyberduck.core.transfer.TransferSpeedometer;
+import ch.cyberduck.core.transfer.TransferStatus;
 import ch.cyberduck.core.transfer.UploadTransfer;
 import ch.cyberduck.test.IntegrationTest;
 import ch.cyberduck.test.VaultTest;
@@ -61,7 +62,6 @@
 import java.io.InputStream;
 import java.io.OutputStream;
 import java.net.SocketTimeoutException;
-import java.security.MessageDigest;
 import java.util.Collections;
 import java.util.EnumSet;
 import java.util.Random;
@@ -110,7 +110,7 @@ public String getProperty(final String key) {
             final B2LargeUploadService upload = new B2LargeUploadService(this, new B2VersionIdProvider(this)
             ) {
                 @Override
-                protected InputStream decorate(final InputStream in, final MessageDigest digest) {
+                protected InputStream decorate(final InputStream in, final TransferStatus status) {
                     if(failed.get()) {
                         // Second attempt successful
                         return in;

box/src/main/java/ch/cyberduck/core/box/BoxLargeUploadService.java

@@ -44,7 +44,6 @@
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 
-import java.security.MessageDigest;
 import java.text.MessageFormat;
 import java.util.ArrayList;
 import java.util.HashMap;
@@ -54,7 +53,7 @@
 import java.util.concurrent.Future;
 import java.util.stream.Collectors;
 
-public class BoxLargeUploadService extends HttpUploadFeature<File, MessageDigest> {
+public class BoxLargeUploadService extends HttpUploadFeature<File> {
     private static final Logger log = LogManager.getLogger(BoxLargeUploadService.class);
 
     public static final String UPLOAD_SESSION_ID = "uploadSessionId";
@@ -137,7 +136,7 @@ public Part call() throws BackgroundException {
                 parameters.put(UPLOAD_SESSION_ID, uploadSessionId);
                 parameters.put(OVERALL_LENGTH, String.valueOf(overall.getLength()));
                 status.setParameters(parameters);
-                final File response = BoxLargeUploadService.this.upload(
+                final File response = BoxLargeUploadService.this.transfer(
                         write, file, local, throttle, listener, status, overall, status, callback);
                 log.info("Received response {} for part {}", response, partNumber);
                 return new Part(response, status);

box/src/main/java/ch/cyberduck/core/box/BoxSmallUploadService.java

@@ -18,7 +18,5 @@
 import ch.cyberduck.core.box.io.swagger.client.model.File;
 import ch.cyberduck.core.http.HttpUploadFeature;
 
-import java.security.MessageDigest;
-
-public class BoxSmallUploadService extends HttpUploadFeature<File, MessageDigest> {
+public class BoxSmallUploadService extends HttpUploadFeature<File> {
 }

brick/src/main/java/ch/cyberduck/core/brick/BrickUploadFeature.java

@@ -47,14 +47,13 @@
 import org.apache.logging.log4j.Logger;
 import org.joda.time.DateTime;
 
-import java.security.MessageDigest;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Optional;
 import java.util.concurrent.Future;
 import java.util.stream.Collectors;
 
-public class BrickUploadFeature extends HttpUploadFeature<FileEntity, MessageDigest> {
+public class BrickUploadFeature extends HttpUploadFeature<FileEntity> {
     private static final Logger log = LogManager.getLogger(BrickUploadFeature.class);
 
     /**
@@ -168,7 +167,7 @@ public TransferStatus call() throws BackgroundException {
                 status.setUrl(url);
                 status.setPart(partNumber);
                 status.setHeader(overall.getHeader());
-                BrickUploadFeature.super.upload(
+                BrickUploadFeature.super.transfer(
                         write, file, local, throttle, listener, status, overall, status, callback);
                 log.info("Received response for part number {}", partNumber);
                 return status;

brick/src/main/java/ch/cyberduck/core/brick/BrickWriteFeature.java

@@ -88,24 +88,6 @@ public FileEntity call(final HttpEntity entity) throws BackgroundException {
                             case HttpStatus.SC_OK:
                                 log.info("Received response {} for part number {}", response, status.getPart());
                                 // Upload complete
-                                if(response.containsHeader("ETag")) {
-                                    if(file.getType().contains(Path.Type.encrypted)) {
-                                        log.warn("Skip checksum verification for {} with client side encryption enabled", file);
-                                    }
-                                    else {
-                                        if(HashAlgorithm.md5.equals(status.getChecksum().algorithm)) {
-                                            final Checksum etag = Checksum.parse(StringUtils.remove(response.getFirstHeader("ETag").getValue(), '"'));
-                                            if(!status.getChecksum().equals(etag)) {
-                                                throw new ChecksumException(MessageFormat.format(LocaleFactory.localizedString("Upload {0} failed", "Error"), file.getName()),
-                                                        MessageFormat.format("Mismatch between {0} hash {1} of uploaded data and ETag {2} returned by the server",
-                                                                etag.algorithm.toString(), status.getChecksum().hash, etag.hash));
-                                            }
-                                        }
-                                    }
-                                }
-                                else {
-                                    log.debug("No ETag header in response available");
-                                }
                                 return null;
                             default:
                                 EntityUtils.updateEntity(response, new BufferedHttpEntity(response.getEntity()));

brick/src/test/java/ch/cyberduck/core/brick/BrickReadFeatureTest.java

@@ -106,7 +106,7 @@ public void testReadRange() throws Exception {
         upload.setExists(true);
         new BrickUploadFeature(session).upload(
                 new BrickWriteFeature(session), test, local, new BandwidthThrottle(BandwidthThrottle.UNLIMITED), new DisabledProgressListener(), new DisabledStreamListener(), upload,
-            new DisabledConnectionCallback());
+                new DisabledConnectionCallback());
         final TransferStatus status = new TransferStatus();
         status.setLength(content.length);
         status.setAppend(true);
@@ -138,7 +138,7 @@ public void testReadRangeUnknownLength() throws Exception {
         upload.setExists(true);
         new BrickUploadFeature(session).upload(
                 new BrickWriteFeature(session), test, local, new BandwidthThrottle(BandwidthThrottle.UNLIMITED), new DisabledProgressListener(), new DisabledStreamListener(), upload,
-            new DisabledConnectionCallback());
+                new DisabledConnectionCallback());
         final TransferStatus status = new TransferStatus();
         status.setLength(-1L);
         status.setAppend(true);

core/src/main/java/ch/cyberduck/core/Session.java

@@ -291,7 +291,7 @@ public <T> T getFeature(final Class<T> type, final T feature) {
     @SuppressWarnings("unchecked")
     public <T> T _getFeature(final Class<T> type) {
         if(type == Upload.class) {
-            return (T) new DefaultUploadFeature<>(this);
+            return (T) new DefaultUploadFeature<>();
         }
         if(type == Download.class) {
             return (T) new DefaultDownloadFeature(this);