Skip to content

Commit

Permalink
Merge pull request #14963 from iterate-ch/bugfix/MD-19684
Browse files Browse the repository at this point in the history 
Do not cache file id when versioned object listings are disabled.
  • Loading branch information
@ylangisc
ylangisc authored Aug 4, 2023
2 parents cf4dd1e + 7809568 commit 742299a
Show file tree
Hide file tree
Showing 5 changed files with 139 additions and 29 deletions.
16 changes: 14 additions & 2 deletions backblaze/src/main/java/ch/cyberduck/core/b2/B2AttributesFinderFeature.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,12 +22,14 @@
import ch.cyberduck.core.Path;
import ch.cyberduck.core.PathAttributes;
import ch.cyberduck.core.PathContainerService;
import ch.cyberduck.core.VersioningConfiguration;
import ch.cyberduck.core.exception.BackgroundException;
import ch.cyberduck.core.exception.NotfoundException;
import ch.cyberduck.core.features.AttributesAdapter;
import ch.cyberduck.core.features.AttributesFinder;
import ch.cyberduck.core.features.Write;
import ch.cyberduck.core.io.Checksum;
import ch.cyberduck.core.preferences.HostPreferences;
import ch.cyberduck.core.transfer.TransferStatus;

import org.apache.commons.lang3.StringUtils;
Expand Down Expand Up @@ -56,10 +58,16 @@ public class B2AttributesFinderFeature implements AttributesFinder, AttributesAd

private final B2Session session;
private final B2VersionIdProvider fileid;
private final VersioningConfiguration versioning;

public B2AttributesFinderFeature(final B2Session session, final B2VersionIdProvider fileid) {
this(session, fileid, new VersioningConfiguration(new HostPreferences(session.getHost()).getBoolean("b2.listing.versioning.enable")));
}

public B2AttributesFinderFeature(final B2Session session, final B2VersionIdProvider fileid, final VersioningConfiguration versioning) {
this.session = session;
this.fileid = fileid;
this.versioning = versioning;
}

@Override
Expand Down Expand Up @@ -155,7 +163,9 @@ protected PathAttributes toAttributes(final B2FileInfoResponse response) {
if(!response.getFileInfo().isEmpty()) {
attributes.setMetadata(new HashMap<>(response.getFileInfo()));
}
attributes.setVersionId(response.getFileId());
if(versioning.isEnabled()) {
attributes.setVersionId(response.getFileId());
}
final long timestamp = response.getUploadTimestamp();
if(response.getFileInfo().containsKey(X_BZ_INFO_SRC_LAST_MODIFIED_MILLIS)) {
final String value = response.getFileInfo().get(X_BZ_INFO_SRC_LAST_MODIFIED_MILLIS);
Expand Down Expand Up @@ -197,7 +207,9 @@ protected PathAttributes toAttributes(final B2FileResponse response) {
if(!response.getFileInfo().isEmpty()) {
attributes.setMetadata(new HashMap<>(response.getFileInfo()));
}
attributes.setVersionId(response.getFileId());
if(versioning.isEnabled()) {
attributes.setVersionId(response.getFileId());
}
final long timestamp = response.getUploadTimestamp();
if(response.getFileInfo().containsKey(X_BZ_INFO_SRC_LAST_MODIFIED_MILLIS)) {
final String value = response.getFileInfo().get(X_BZ_INFO_SRC_LAST_MODIFIED_MILLIS);
Expand Down
20 changes: 11 additions & 9 deletions backblaze/src/main/java/ch/cyberduck/core/b2/B2ObjectListService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -126,8 +126,8 @@ private String createPrefix(final Path directory) {
}

private Marker parse(final Path directory, final AttributedList<Path> objects,
final B2ListFilesResponse response, final Map<String, Long> revisions) throws BackgroundException {
final B2AttributesFinderFeature attr = new B2AttributesFinderFeature(session, fileid);
final B2ListFilesResponse response, final Map<String, Long> revisions) {
final B2AttributesFinderFeature attr = new B2AttributesFinderFeature(session, fileid, versioning);
for(B2FileInfoResponse info : response.getFiles()) {
if(StringUtils.equals(PathNormalizer.name(info.getFileName()), B2PathContainerService.PLACEHOLDER)) {
continue;
Expand All @@ -147,14 +147,16 @@ private Marker parse(final Path directory, final AttributedList<Path> objects,
continue;
}
final PathAttributes attributes = attr.toAttributes(info);
long revision = 0;
if(revisions.containsKey(info.getFileName())) {
// Later version already found
attributes.setDuplicate(true);
revision = revisions.get(info.getFileName()) + 1L;
attributes.setRevision(revision);
if(versioning.isEnabled()) {
long revision = 0;
if(revisions.containsKey(info.getFileName())) {
// Later version already found
attributes.setDuplicate(true);
revision = revisions.get(info.getFileName()) + 1L;
attributes.setRevision(revision);
}
revisions.put(info.getFileName(), revision);
}
revisions.put(info.getFileName(), revision);
final Path f = new Path(directory.isDirectory() ? directory : directory.getParent(), PathNormalizer.name(info.getFileName()),
info.getAction() == Action.start ? EnumSet.of(Path.Type.file, Path.Type.upload) : EnumSet.of(Path.Type.file), attributes);
fileid.cache(f, info.getFileId());
Expand Down
47 changes: 33 additions & 14 deletions backblaze/src/main/java/ch/cyberduck/core/b2/B2VersionIdProvider.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,9 +19,11 @@
import ch.cyberduck.core.DefaultIOExceptionMappingService;
import ch.cyberduck.core.Path;
import ch.cyberduck.core.PathContainerService;
import ch.cyberduck.core.VersioningConfiguration;
import ch.cyberduck.core.exception.BackgroundException;
import ch.cyberduck.core.exception.NotfoundException;
import ch.cyberduck.core.features.VersionIdProvider;
import ch.cyberduck.core.preferences.HostPreferences;

import org.apache.commons.lang3.StringUtils;
import org.apache.logging.log4j.LogManager;
Expand All @@ -39,35 +41,44 @@ public class B2VersionIdProvider extends CachingVersionIdProvider implements Ver

private final PathContainerService containerService = new B2PathContainerService();
private final B2Session session;
private final VersioningConfiguration versioning;

public B2VersionIdProvider(final B2Session session) {
this(session, new VersioningConfiguration(new HostPreferences(session.getHost()).getBoolean("b2.listing.versioning.enable")));
}

public B2VersionIdProvider(final B2Session session, final VersioningConfiguration versioning) {
super(session.getCaseSensitivity());
this.session = session;
this.versioning = versioning;
}

@Override
public String getVersionId(final Path file) throws BackgroundException {
if(StringUtils.isNotBlank(file.attributes().getVersionId())) {
if(log.isDebugEnabled()) {
log.debug(String.format("Return version %s from attributes for file %s", file.attributes().getVersionId(), file));
}
return file.attributes().getVersionId();
}
final String cached = super.getVersionId(file);
if(cached != null) {
if(log.isDebugEnabled()) {
log.debug(String.format("Return cached versionid %s for file %s", cached, file));
}
return cached;
}
try {
if(containerService.isContainer(file)) {
if(StringUtils.isNotBlank(file.attributes().getFileId())) {
return file.attributes().getFileId();
}
final B2BucketResponse info = session.getClient().listBucket(file.getName());
if(null == info) {
throw new NotfoundException(file.getAbsolute());
}
// Cache in file attributes
return this.cache(file, info.getBucketId());
return info.getBucketId();
}
if(StringUtils.isNotBlank(file.attributes().getVersionId())) {
if(log.isDebugEnabled()) {
log.debug(String.format("Return version %s from attributes for file %s", file.attributes().getVersionId(), file));
}
return file.attributes().getVersionId();
}
final String cached = super.getVersionId(file);
if(cached != null) {
if(log.isDebugEnabled()) {
log.debug(String.format("Return cached versionid %s for file %s", cached, file));
}
return cached;
}
// Files that have been hidden will not be returned
final B2ListFilesResponse response = session.getClient().listFileNames(
Expand All @@ -87,4 +98,12 @@ public String getVersionId(final Path file) throws BackgroundException {
throw new DefaultIOExceptionMappingService().map(e);
}
}

@Override
public String cache(final Path file, final String id) {
if(versioning.isEnabled()) {
return super.cache(file, id);
}
return null;
}
}
10 changes: 7 additions & 3 deletions backblaze/src/main/java/ch/cyberduck/core/b2/B2WriteFeature.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -53,16 +53,20 @@ public class B2WriteFeature extends AbstractHttpWriteFeature<BaseB2Response> imp
private static final Logger log = LogManager.getLogger(B2WriteFeature.class);

private final PathContainerService containerService
= new B2PathContainerService();
= new B2PathContainerService();

private final B2Session session;
private final B2VersionIdProvider fileid;

private final ThreadLocal<B2GetUploadUrlResponse> urls
= new ThreadLocal<>();
= new ThreadLocal<>();

public B2WriteFeature(final B2Session session, final B2VersionIdProvider fileid) {
super(new B2AttributesFinderFeature(session, fileid));
this(session, fileid, new B2AttributesFinderFeature(session, fileid));
}

public B2WriteFeature(final B2Session session, final B2VersionIdProvider fileid, final B2AttributesFinderFeature attributes) {
super(attributes);
this.session = session;
this.fileid = fileid;
}
Expand Down
75 changes: 74 additions & 1 deletion backblaze/src/test/java/ch/cyberduck/core/b2/B2ObjectListServiceTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -174,7 +174,8 @@ public void testListChunking() throws Exception {
@Test
public void testListRevisions() throws Exception {
final B2VersionIdProvider fileid = new B2VersionIdProvider(session);
final Path bucket = new B2DirectoryFeature(session, fileid).mkdir(new Path(String.format("test-%s", new AsciiRandomStringService().random()), EnumSet.of(Path.Type.directory, Path.Type.volume)), new TransferStatus());
final Path bucket = new B2DirectoryFeature(session, fileid).mkdir(new Path(
String.format("test-%s", new AsciiRandomStringService().random()), EnumSet.of(Path.Type.directory, Path.Type.volume)), new TransferStatus());
final String name = new AsciiRandomStringService().random();
final Path file = new Path(bucket, name, EnumSet.of(Path.Type.file));
{
Expand All @@ -191,6 +192,7 @@ public void testListRevisions() throws Exception {
final AttributedList<Path> list = new B2ObjectListService(session, fileid).list(bucket, new DisabledListProgressListener());
assertTrue(list.contains(file));
assertNull(list.find(new SimplePathPredicate(file)).attributes().getRevision());
assertNotNull(list.find(new SimplePathPredicate(file)).attributes().getVersionId());
assertEquals(content.length, list.find(new SimplePathPredicate(file)).attributes().getSize());
assertEquals(bucket, list.find(new SimplePathPredicate(file)).getParent());
}
Expand All @@ -210,6 +212,7 @@ public void testListRevisions() throws Exception {
assertTrue(list.contains(file));
assertEquals(bucket, list.get(file).getParent());
assertNull(list.get(file).attributes().getRevision());
assertNotNull(list.find(new SimplePathPredicate(file)).attributes().getVersionId());
assertEquals(Long.valueOf(1L), list.find(path -> path.attributes().isDuplicate()).attributes().getRevision());
}
// Add hide marker
Expand Down Expand Up @@ -243,6 +246,76 @@ public void testListRevisions() throws Exception {
new B2DeleteFeature(session, fileid).delete(Arrays.asList(other, bucket), new DisabledLoginCallback(), new Delete.DisabledCallback());
}

@Test
public void testListRevisionsNoVersioning() throws Exception {
final B2VersionIdProvider fileid = new B2VersionIdProvider(session, VersioningConfiguration.empty());
final Path bucket = new B2DirectoryFeature(session, fileid,
new B2WriteFeature(session, fileid, new B2AttributesFinderFeature(session, fileid,
VersioningConfiguration.empty()))).mkdir(new Path(
String.format("test-%s", new AsciiRandomStringService().random()), EnumSet.of(Path.Type.directory, Path.Type.volume)), new TransferStatus());
final String name = new AsciiRandomStringService().random();
final Path file = new Path(bucket, name, EnumSet.of(Path.Type.file));
{
final byte[] content = RandomUtils.nextBytes(1);
final TransferStatus status = new TransferStatus();
status.setLength(content.length);
status.setChecksum(new SHA1ChecksumCompute().compute(new ByteArrayInputStream(content), status));
final HttpResponseOutputStream<BaseB2Response> out = new B2WriteFeature(session, fileid, new B2AttributesFinderFeature(session, fileid,
VersioningConfiguration.empty())).write(file, status, new DisabledConnectionCallback());
IOUtils.write(content, out);
out.close();
final B2FileResponse response = (B2FileResponse) out.getStatus();
assertNotNull(response.getFileId());
assertNull(file.attributes().getVersionId());
final AttributedList<Path> list = new B2ObjectListService(session, fileid, 10,
VersioningConfiguration.empty()).list(bucket, new DisabledListProgressListener());
assertTrue(list.contains(file));
assertNull(list.find(new SimplePathPredicate(file)).attributes().getRevision());
assertNull(list.find(new SimplePathPredicate(file)).attributes().getVersionId());
assertEquals(content.length, list.find(new SimplePathPredicate(file)).attributes().getSize());
assertEquals(bucket, list.find(new SimplePathPredicate(file)).getParent());
}
// Replace
{
final byte[] content = RandomUtils.nextBytes(1);
final TransferStatus status = new TransferStatus();
status.setLength(content.length);
status.setChecksum(new SHA1ChecksumCompute().compute(new ByteArrayInputStream(content), status));
final HttpResponseOutputStream<BaseB2Response> out = new B2WriteFeature(session, fileid, new B2AttributesFinderFeature(session, fileid,
VersioningConfiguration.empty())).write(file, status, new DisabledConnectionCallback());
IOUtils.write(content, out);
out.close();
final B2FileResponse response = (B2FileResponse) out.getStatus();
assertNotNull(response.getFileId());
assertNull(file.attributes().getVersionId());
final AttributedList<Path> list = new B2ObjectListService(session, fileid, 10,
VersioningConfiguration.empty()).list(bucket, new DisabledListProgressListener());
assertEquals(1, list.size());
assertTrue(list.contains(file));
assertEquals(bucket, list.get(file).getParent());
assertNull(list.get(file).attributes().getRevision());
assertNull(list.find(new SimplePathPredicate(file)).attributes().getVersionId());
}
// Add hide marker
new B2DeleteFeature(session, fileid).delete(Collections.singletonList(file), new DisabledLoginCallback(), new Delete.DisabledCallback());
assertTrue(new B2ObjectListService(session, fileid, 1, VersioningConfiguration.empty()).list(bucket, new DisabledListProgressListener()).isEmpty());
assertFalse(new B2FindFeature(session, fileid).find(file));
assertFalse(new DefaultFindFeature(session).find(file));
try {
new B2AttributesFinderFeature(session, fileid).find(file);
fail();
}
catch(NotfoundException e) {
//
}
final AttributedList<Path> list = new B2ObjectListService(session, fileid).list(bucket, new DisabledListProgressListener());
assertEquals(list, new B2VersioningFeature(session, fileid).list(file, new DisabledListProgressListener()));
for(Path f : list) {
new B2DeleteFeature(session, fileid).delete(Collections.singletonList(f), new DisabledLoginCallback(), new Delete.DisabledCallback());
}
new B2DeleteFeature(session, fileid).delete(Collections.singletonList(bucket), new DisabledLoginCallback(), new Delete.DisabledCallback());
}

@Test
public void testListFolder() throws Exception {
final B2VersionIdProvider fileid = new B2VersionIdProvider(session);
Expand Down

0 comments on commit 742299a

Please sign in to comment.