Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Set progress when concatenating files. #16440

Draft
wants to merge 5 commits into
base: master
Choose a base branch
from
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -19,8 +19,10 @@
import ch.cyberduck.core.ConnectionCallback;
import ch.cyberduck.core.DefaultIOExceptionMappingService;
import ch.cyberduck.core.Local;
import ch.cyberduck.core.LocaleFactory;
import ch.cyberduck.core.Path;
import ch.cyberduck.core.PathContainerService;
import ch.cyberduck.core.ProgressListener;
import ch.cyberduck.core.concurrency.Interruptibles;
import ch.cyberduck.core.exception.BackgroundException;
import ch.cyberduck.core.features.Upload;
Expand All @@ -42,6 +44,7 @@

import java.io.IOException;
import java.security.MessageDigest;
import java.text.MessageFormat;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.HashMap;
Expand Down Expand Up @@ -94,7 +97,7 @@ public B2LargeUploadService(final B2Session session, final B2VersionIdProvider f

@Override
public BaseB2Response upload(final Path file, final Local local, final BandwidthThrottle throttle,
final StreamListener listener, final TransferStatus status, final ConnectionCallback callback) throws BackgroundException {
final ProgressListener progress, final StreamListener streamListener, final TransferStatus status, final ConnectionCallback callback) throws BackgroundException {
final long partSize;
if(file.getType().contains(Path.Type.encrypted)) {
// For uploads to vault part size must be a multiple of 32 * 1024. Recommended partsize from B2 API may not meet that requirement.
Expand All @@ -103,12 +106,12 @@ public BaseB2Response upload(final Path file, final Local local, final Bandwidth
else {
partSize = this.partSize;
}
return this.upload(file, local, throttle, listener, status, callback,
return this.upload(file, local, throttle, progress, streamListener, status, callback,
partSize < status.getLength() ? partSize : PreferencesFactory.get().getLong("b2.upload.largeobject.size.minimum"));
}

public BaseB2Response upload(final Path file, final Local local,
final BandwidthThrottle throttle, final StreamListener listener, final TransferStatus status,
final BandwidthThrottle throttle, final ProgressListener progress, final StreamListener streamListener, final TransferStatus status,
final ConnectionCallback callback, final Long partSize) throws BackgroundException {
final ThreadPool pool = ThreadPoolFactory.get("largeupload", concurrency);
try {
Expand Down Expand Up @@ -175,7 +178,7 @@ public BaseB2Response upload(final Path file, final Local local,
if(!skip) {
final long length = Math.min(Math.max((size / B2LargeUploadService.MAXIMUM_UPLOAD_PARTS), partSize), remaining);
// Submit to queue
parts.add(this.submit(pool, file, local, throttle, listener, status, fileId, partNumber, offset, length, callback));
parts.add(this.submit(pool, file, local, throttle, streamListener, status, fileId, partNumber, offset, length, callback));
if(log.isDebugEnabled()) {
log.debug(String.format("Part %s submitted with size %d and offset %d", partNumber, length, offset));
}
Expand All @@ -194,6 +197,8 @@ public int compare(final B2UploadPartResponse o1, final B2UploadPartResponse o2)
for(B2UploadPartResponse part : completed) {
checksums.add(part.getContentSha1());
}
progress.message(MessageFormat.format(LocaleFactory.localizedString("Finalize {0}", "Status"),
file.getName()));
final B2FinishLargeFileResponse response = session.getClient().finishLargeFileUpload(fileId, checksums.toArray(new String[checksums.size()]));
if(log.isInfoEnabled()) {
log.info(String.format("Finished large file upload %s with %d parts", file, completed.size()));
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
import ch.cyberduck.core.ConnectionCallback;
import ch.cyberduck.core.Local;
import ch.cyberduck.core.Path;
import ch.cyberduck.core.ProgressListener;
import ch.cyberduck.core.exception.BackgroundException;
import ch.cyberduck.core.features.Upload;
import ch.cyberduck.core.features.Write;
Expand Down Expand Up @@ -59,13 +60,13 @@ public Write.Append append(final Path file, final TransferStatus status) throws
}

@Override
public BaseB2Response upload(final Path file, final Local local, final BandwidthThrottle throttle, final StreamListener listener,
public BaseB2Response upload(final Path file, final Local local, final BandwidthThrottle throttle, final ProgressListener progress, final StreamListener streamListener,
final TransferStatus status, final ConnectionCallback callback) throws BackgroundException {
if(this.threshold(status)) {
return new B2LargeUploadService(session, fileid, writer).upload(file, local, throttle, listener, status, callback);
return new B2LargeUploadService(session, fileid, writer).upload(file, local, throttle, progress, streamListener, status, callback);
}
else {
return new B2SingleUploadService(session, writer).upload(file, local, throttle, listener, status, callback);
return new B2SingleUploadService(session, writer).upload(file, local, throttle, progress, streamListener, status, callback);
}
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
import ch.cyberduck.core.ConnectionCallback;
import ch.cyberduck.core.DisabledConnectionCallback;
import ch.cyberduck.core.DisabledLoginCallback;
import ch.cyberduck.core.DisabledProgressListener;
import ch.cyberduck.core.Local;
import ch.cyberduck.core.Path;
import ch.cyberduck.core.PathAttributes;
Expand Down Expand Up @@ -82,7 +83,7 @@ public void testUpload() throws Exception {
final B2LargeUploadService upload = new B2LargeUploadService(session, fileid,
new B2WriteFeature(session, fileid), 5 * 1000L * 1000L, 5);

upload.upload(test, local, new BandwidthThrottle(BandwidthThrottle.UNLIMITED), new DisabledStreamListener(),
upload.upload(test, local, new BandwidthThrottle(BandwidthThrottle.UNLIMITED), new DisabledProgressListener(), new DisabledStreamListener(),
status, new DisabledConnectionCallback());
final PathAttributes attr = new B2AttributesFinderFeature(session, fileid).find(test);
assertNotEquals(Checksum.NONE, attr.getChecksum());
Expand Down Expand Up @@ -127,7 +128,7 @@ public BaseB2Response upload(final Path file, final Local local, final Bandwidth
}
};
try {
service.upload(test, new Local(System.getProperty("java.io.tmpdir"), name), new BandwidthThrottle(BandwidthThrottle.UNLIMITED), count, status, new DisabledLoginCallback());
service.upload(test, new Local(System.getProperty("java.io.tmpdir"), name), new BandwidthThrottle(BandwidthThrottle.UNLIMITED), new DisabledProgressListener(), count, status, new DisabledLoginCallback());
}
catch(BackgroundException e) {
// Expected
Expand All @@ -142,7 +143,7 @@ public BaseB2Response upload(final Path file, final Local local, final Bandwidth
assertEquals(0L, resume.offset, 0L);
final TransferStatus append = new TransferStatus().append(true).withLength(content.length);
service.upload(test, local,
new BandwidthThrottle(BandwidthThrottle.UNLIMITED), new DisabledStreamListener(), append,
new BandwidthThrottle(BandwidthThrottle.UNLIMITED), new DisabledProgressListener(), new DisabledStreamListener(), append,
new DisabledLoginCallback());
assertEquals(content.length, append.getResponse().getSize());
assertTrue(new B2FindFeature(session, fileid).find(test));
Expand Down Expand Up @@ -184,7 +185,7 @@ public BaseB2Response upload(final Path file, final Local local, final Bandwidth
};
final BytecountStreamListener count = new BytecountStreamListener();
try {
feature.upload(test, new Local(System.getProperty("java.io.tmpdir"), name), new BandwidthThrottle(BandwidthThrottle.UNLIMITED), count, status, new DisabledLoginCallback());
feature.upload(test, new Local(System.getProperty("java.io.tmpdir"), name), new BandwidthThrottle(BandwidthThrottle.UNLIMITED), new DisabledProgressListener(), count, status, new DisabledLoginCallback());
}
catch(BackgroundException e) {
// Expected
Expand All @@ -202,7 +203,7 @@ public BaseB2Response upload(final Path file, final Local local, final Bandwidth
assertEquals(5 * 1000L * 1000L, new B2AttributesFinderFeature(session, fileid).find(upload).getSize(), 0L);
final TransferStatus append = new TransferStatus().append(true).withLength(2L * 1000L * 1000L).withOffset(5 * 1000L * 1000L);
feature.upload(test, local,
new BandwidthThrottle(BandwidthThrottle.UNLIMITED), count, append,
new BandwidthThrottle(BandwidthThrottle.UNLIMITED), new DisabledProgressListener(), count, append,
new DisabledLoginCallback());
assertEquals(6 * 1000L * 1000L, count.getSent());
assertTrue(append.isComplete());
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@
import ch.cyberduck.core.DisabledConnectionCallback;
import ch.cyberduck.core.DisabledListProgressListener;
import ch.cyberduck.core.DisabledLoginCallback;
import ch.cyberduck.core.DisabledProgressListener;
import ch.cyberduck.core.Local;
import ch.cyberduck.core.Path;
import ch.cyberduck.core.exception.NotfoundException;
Expand Down Expand Up @@ -127,7 +128,7 @@ public void testReadRange() throws Exception {
IOUtils.write(content, out);
out.close();
final BaseB2Response reply = new B2SingleUploadService(session, new B2WriteFeature(session, fileid)).upload(
test, local, new BandwidthThrottle(BandwidthThrottle.UNLIMITED), new DisabledStreamListener(),
test, local, new BandwidthThrottle(BandwidthThrottle.UNLIMITED), new DisabledProgressListener(), new DisabledStreamListener(),
new TransferStatus().withLength(content.length),
new DisabledConnectionCallback());
final TransferStatus status = new TransferStatus();
Expand Down Expand Up @@ -159,7 +160,7 @@ public void testReadRangeUnknownLength() throws Exception {
IOUtils.write(content, out);
out.close();
new B2SingleUploadService(session, new B2WriteFeature(session, fileid)).upload(
test, local, new BandwidthThrottle(BandwidthThrottle.UNLIMITED), new DisabledStreamListener(),
test, local, new BandwidthThrottle(BandwidthThrottle.UNLIMITED), new DisabledProgressListener(), new DisabledStreamListener(),
new TransferStatus().withLength(content.length),
new DisabledConnectionCallback());
final TransferStatus status = new TransferStatus();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
import ch.cyberduck.core.AlphanumericRandomStringService;
import ch.cyberduck.core.DisabledConnectionCallback;
import ch.cyberduck.core.DisabledLoginCallback;
import ch.cyberduck.core.DisabledProgressListener;
import ch.cyberduck.core.Local;
import ch.cyberduck.core.Path;
import ch.cyberduck.core.features.Delete;
Expand Down Expand Up @@ -61,7 +62,7 @@ public void testUpload() throws Exception {
status.setChecksum(checksum);
final B2VersionIdProvider fileid = new B2VersionIdProvider(session);
final B2SingleUploadService upload = new B2SingleUploadService(session, new B2WriteFeature(session, fileid));
upload.upload(test, local, new BandwidthThrottle(BandwidthThrottle.UNLIMITED), new DisabledStreamListener(),
upload.upload(test, local, new BandwidthThrottle(BandwidthThrottle.UNLIMITED), new DisabledProgressListener(), new DisabledStreamListener(),
status, new DisabledConnectionCallback());
assertEquals(checksum, new B2AttributesFinderFeature(session, fileid).find(test).getChecksum());
status.validate();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@
import ch.cyberduck.core.DisabledLoginCallback;
import ch.cyberduck.core.DisabledPasswordCallback;
import ch.cyberduck.core.DisabledPasswordStore;
import ch.cyberduck.core.DisabledProgressListener;
import ch.cyberduck.core.Local;
import ch.cyberduck.core.Path;
import ch.cyberduck.core.b2.AbstractB2Test;
Expand Down Expand Up @@ -90,7 +91,7 @@ public void testWrite() throws Exception {
writeStatus.setLength(content.length);
final Path test = new Path(vault, new AlphanumericRandomStringService().random(), EnumSet.of(Path.Type.file));
final BytecountStreamListener counter = new BytecountStreamListener();
service.upload(test, local, new BandwidthThrottle(BandwidthThrottle.UNLIMITED), counter, writeStatus, new DisabledConnectionCallback());
service.upload(test, local, new BandwidthThrottle(BandwidthThrottle.UNLIMITED), new DisabledProgressListener(), counter, writeStatus, new DisabledConnectionCallback());
assertEquals(content.length, counter.getSent());
assertTrue(writeStatus.isComplete());
assertTrue(cryptomator.getFeature(session, Find.class, new B2FindFeature(session, fileid)).find(test));
Expand Down Expand Up @@ -126,7 +127,7 @@ public void testUploadWithBulk() throws Exception {
final Local local = new Local(System.getProperty("java.io.tmpdir"), UUID.randomUUID().toString());
IOUtils.write(content, local.getOutputStream(false));
final BytecountStreamListener counter = new BytecountStreamListener();
service.upload(test, local, new BandwidthThrottle(BandwidthThrottle.UNLIMITED), counter, writeStatus, new DisabledConnectionCallback());
service.upload(test, local, new BandwidthThrottle(BandwidthThrottle.UNLIMITED), new DisabledProgressListener(), counter, writeStatus, new DisabledConnectionCallback());
assertEquals(content.length, counter.getSent());
assertTrue(writeStatus.isComplete());
assertTrue(cryptomator.getFeature(session, Find.class, new B2FindFeature(session, fileid)).find(test));
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,9 @@
import ch.cyberduck.core.BytecountStreamListener;
import ch.cyberduck.core.ConnectionCallback;
import ch.cyberduck.core.Local;
import ch.cyberduck.core.LocaleFactory;
import ch.cyberduck.core.Path;
import ch.cyberduck.core.ProgressListener;
import ch.cyberduck.core.box.io.swagger.client.model.File;
import ch.cyberduck.core.box.io.swagger.client.model.Files;
import ch.cyberduck.core.box.io.swagger.client.model.UploadPart;
Expand All @@ -44,6 +46,7 @@
import org.apache.logging.log4j.Logger;

import java.security.MessageDigest;
import java.text.MessageFormat;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
Expand Down Expand Up @@ -78,7 +81,7 @@ public BoxLargeUploadService(final BoxSession session, final BoxFileidProvider f
}

@Override
public File upload(final Path file, final Local local, final BandwidthThrottle throttle, final StreamListener listener,
public File upload(final Path file, final Local local, final BandwidthThrottle throttle, final ProgressListener progress, final StreamListener streamListener,
final TransferStatus status, final ConnectionCallback callback) throws BackgroundException {
final ThreadPool pool = ThreadPoolFactory.get("multipart", concurrency);
try {
Expand All @@ -92,13 +95,15 @@ public File upload(final Path file, final Local local, final BandwidthThrottle t
final UploadSession uploadSession = helper.createUploadSession(status, file);
for(int partNumber = 1; remaining > 0; partNumber++) {
final long length = Math.min(uploadSession.getPartSize(), remaining);
parts.add(this.submit(pool, file, local, throttle, listener, status,
parts.add(this.submit(pool, file, local, throttle, streamListener, status,
uploadSession.getId(), partNumber, offset, length, callback));
remaining -= length;
offset += length;
}
// Checksums for uploaded segments
final List<Part> chunks = Interruptibles.awaitAll(parts);
progress.message(MessageFormat.format(LocaleFactory.localizedString("Finalize {0}", "Status"),
file.getName()));
final Files files = helper.commitUploadSession(file, uploadSession.getId(), status,
chunks.stream().map(f -> new UploadPart().sha1(f.part.getSha1())
.size(f.status.getLength()).offset(f.status.getOffset()).partId(f.part.getId())).collect(Collectors.toList()));
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
import ch.cyberduck.core.ConnectionCallback;
import ch.cyberduck.core.Local;
import ch.cyberduck.core.Path;
import ch.cyberduck.core.ProgressListener;
import ch.cyberduck.core.box.io.swagger.client.model.File;
import ch.cyberduck.core.exception.BackgroundException;
import ch.cyberduck.core.features.Upload;
Expand Down Expand Up @@ -45,15 +46,15 @@ public BoxThresholdUploadService(final BoxSession session, final BoxFileidProvid
}

@Override
public File upload(final Path file, final Local local, final BandwidthThrottle throttle, final StreamListener listener,
public File upload(final Path file, final Local local, final BandwidthThrottle throttle, final ProgressListener progress, final StreamListener streamListener,
final TransferStatus status, final ConnectionCallback callback) throws BackgroundException {
if(this.threshold(status.getLength())) {
if(Vault.DISABLED == registry.find(session, file)) {
return new BoxLargeUploadService(session, fileid, new BoxChunkedWriteFeature(session, fileid)).upload(file, local, throttle, listener, status, callback);
return new BoxLargeUploadService(session, fileid, new BoxChunkedWriteFeature(session, fileid)).upload(file, local, throttle, progress, streamListener, status, callback);
}
// Cannot comply with chunk size requirement from server
}
return new BoxSmallUploadService(session, fileid, writer).upload(file, local, throttle, listener, status, callback);
return new BoxSmallUploadService(session, fileid, writer).upload(file, local, throttle, progress, streamListener, status, callback);
}

@Override
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
import ch.cyberduck.core.BytecountStreamListener;
import ch.cyberduck.core.DisabledConnectionCallback;
import ch.cyberduck.core.DisabledLoginCallback;
import ch.cyberduck.core.DisabledProgressListener;
import ch.cyberduck.core.Local;
import ch.cyberduck.core.Path;
import ch.cyberduck.core.box.io.swagger.client.model.File;
Expand Down Expand Up @@ -57,7 +58,7 @@ public void testUploadLargeFileInChunks() throws Exception {
status.setChecksum(new SHA1ChecksumCompute().compute(local.getInputStream(), new TransferStatus()));
status.setLength(content.length);
final BytecountStreamListener count = new BytecountStreamListener();
final File response = s.upload(file, local, new BandwidthThrottle(BandwidthThrottle.UNLIMITED), count, status, new DisabledConnectionCallback());
final File response = s.upload(file, local, new BandwidthThrottle(BandwidthThrottle.UNLIMITED), new DisabledProgressListener(), count, status, new DisabledConnectionCallback());
assertTrue(status.isComplete());
assertNotNull(response.getSha1());
assertEquals(content.length, count.getSent());
Expand Down
Loading