From be6394360e3c15c72e03c0a9f0f7f5d1ea56de2e Mon Sep 17 00:00:00 2001 From: Tobias Bischoff Date: Tue, 6 Nov 2018 16:16:54 +0100 Subject: [PATCH 1/4] Implements versioned files for sirius biz Enables us to version e.g. javascript written jobs - Fixes: SE-4561 --- .../sirius/biz/storage/VersionedFile.java | 98 +++++++++ .../sirius/biz/storage/VersionedFiles.java | 200 ++++++++++++++++++ src/main/resources/biz_de.properties | 2 + src/main/resources/component-biz.conf | 10 + .../biz/storage/VersionedFilesSpec.groovy | 73 +++++++ src/test/resources/test.conf | 3 + 6 files changed, 386 insertions(+) create mode 100644 src/main/java/sirius/biz/storage/VersionedFile.java create mode 100644 src/main/java/sirius/biz/storage/VersionedFiles.java create mode 100644 src/test/java/sirius/biz/storage/VersionedFilesSpec.groovy diff --git a/src/main/java/sirius/biz/storage/VersionedFile.java b/src/main/java/sirius/biz/storage/VersionedFile.java new file mode 100644 index 000000000..b7109270f --- /dev/null +++ b/src/main/java/sirius/biz/storage/VersionedFile.java @@ -0,0 +1,98 @@ +package sirius.biz.storage; + +import sirius.biz.tenants.SQLTenantAware; +import sirius.db.mixing.Mapping; +import sirius.db.mixing.annotations.Length; +import sirius.db.mixing.annotations.Lob; +import sirius.db.mixing.annotations.NullAllowed; +import sirius.db.mixing.annotations.Trim; +import sirius.kernel.commons.Explain; + +import java.time.LocalDateTime; + +/** + * Entity holding meta information about a versioned file. + */ +@Explain("No need to override the equals method. Mapping and property can have the same name, this is sirius standard.") +@SuppressWarnings({"squid:S2160", "squid:MaximumInheritanceDepth", "squid:S1845"}) +public class VersionedFile extends SQLTenantAware { + + /** + * Path of the versioned file. + *

+ * This string is used to identify the file which is versioned. + */ + public static final Mapping UNIQUE_IDENTIFIER = Mapping.named("uniqueIdentifier"); + @Length(255) + @Trim + private String uniqueIdentifier; + + /** + * The comment explaining what was changed with this change. + */ + public static final Mapping COMMENT = Mapping.named("comment"); + @NullAllowed + @Lob + @Trim + private String comment; + + /** + * The timestamp marking the version of the file. + */ + public static final Mapping TIMESTAMP = Mapping.named("timestamp"); + private LocalDateTime timestamp; + + /** + * The file holding the versioned code. + */ + public static final Mapping STORED_FILE = Mapping.named("storedFile"); + @NullAllowed + private final StoredObjectRef storedFile = new StoredObjectRef(VersionedFiles.VERSIONED_FILES, false); + + /** + * Contains meta information about this versioned file. + *

+ * We can store for e.g. if this file was automatically created or if this is the default version of a template. + */ + public static final Mapping ADDITIONAL_INFORMATION = Mapping.named("additionalInformation"); + @Length(255) + @NullAllowed + @Trim + private String additionalInformation; + + public String getUniqueIdentifier() { + return uniqueIdentifier; + } + + public void setUniqueIdentifier(String uniqueIdentifier) { + this.uniqueIdentifier = uniqueIdentifier; + } + + public String getComment() { + return comment; + } + + public void setComment(String comment) { + this.comment = comment; + } + + public StoredObjectRef getStoredFile() { + return storedFile; + } + + public LocalDateTime getTimestamp() { + return timestamp; + } + + public void setTimestamp(LocalDateTime timestamp) { + this.timestamp = timestamp; + } + + public String getAdditionalInformation() { + return additionalInformation; + } + + public void setAdditionalInformation(String additionalInformation) { + this.additionalInformation = additionalInformation; + } +} diff --git a/src/main/java/sirius/biz/storage/VersionedFiles.java b/src/main/java/sirius/biz/storage/VersionedFiles.java new file mode 100644 index 000000000..b44078075 --- /dev/null +++ b/src/main/java/sirius/biz/storage/VersionedFiles.java @@ -0,0 +1,200 @@ +package sirius.biz.storage; + +import sirius.biz.tenants.Tenant; +import sirius.db.jdbc.OMA; +import sirius.kernel.di.std.ConfigValue; +import sirius.kernel.di.std.Part; +import sirius.kernel.di.std.Register; +import sirius.kernel.health.Exceptions; +import sirius.kernel.nls.NLS; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.io.OutputStream; +import java.time.LocalDateTime; +import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.stream.Collectors; + +/** + * Small helper class to access and create {@link VersionedFile versioned files}. + */ +@Register(classes = VersionedFiles.class) +public class VersionedFiles { + + @Part + private static OMA oma; + + @Part + private static Storage storage; + + /** + * Name of the used bucked. + */ + public static final String VERSIONED_FILES = "versioned-files"; + + @ConfigValue("storage.buckets.versioned-files.maxNumberOfVersions") + private int maxNumberOfVersions = 0; + + /** + * Retrieves all versions of a versioned file. + * + * @param tenant the owning tenant + * @param uniqueIdentifier the identifier of the versioned file + * @return {@link List} list of versioned files (sorted: most current first) + */ + public List getVersions(Tenant tenant, String uniqueIdentifier) { + return oma.select(VersionedFile.class) + .eq(VersionedFile.TENANT, tenant) + .eq(VersionedFile.UNIQUE_IDENTIFIER, uniqueIdentifier) + .orderDesc(VersionedFile.TIMESTAMP) + .queryList(); + } + + /** + * Checks if there are any versions for the given path. + * + * @param tenant the owning tenant + * @param uniqueIdentifier the identifier of the versioned file + * @return boolean true if there are any versions, false otherwise + */ + public boolean hasVersions(Tenant tenant, String uniqueIdentifier) { + return oma.select(VersionedFile.class) + .eq(VersionedFile.TENANT, tenant) + .eq(VersionedFile.UNIQUE_IDENTIFIER, uniqueIdentifier) + .exists(); + } + + /** + * Deletes all versions of a versioned file. + * + * @param tenant the owning tenant + * @param uniqueIdentifier the identifier of the versioned file + */ + public void deleteVersions(Tenant tenant, String uniqueIdentifier) { + oma.select(VersionedFile.class) + .eq(VersionedFile.TENANT, tenant) + .eq(VersionedFile.UNIQUE_IDENTIFIER, uniqueIdentifier) + .delete(); + } + + /** + * Retrieves the {@link VersionedFile} for the given id. + * + * @param tenant the owning tenant + * @param versionId the database id of the versioned file entity + * @return the found {@link VersionedFile} + */ + public VersionedFile getFile(Tenant tenant, String versionId) { + return oma.select(VersionedFile.class) + .eq(VersionedFile.ID, versionId) + .eq(VersionedFile.TENANT, tenant) + .first() + .orElseThrow(() -> Exceptions.createHandled().withNLSKey("VersionedFiles.noVersion").handle()); + } + + /** + * Gets the content of the {@link VersionedFile}. + * + * @param file the {@link VersionedFile} holding the content + * @return {@link List} each string holding one line of the file content + */ + public List getContent(VersionedFile file) { + try (InputStream data = storage.getData(file.getStoredFile().getObject())) { + return new BufferedReader(new InputStreamReader(data)).lines().collect(Collectors.toList()); + } catch (IOException e) { + throw Exceptions.handle(Storage.LOG, e); + } + } + + /** + * Creates a new {@link VersionedFile}. + * + * @param tenant the owning tenant + * @param uniqueIdentifier the identifier of the versioned file + * @param content the content of the current version + * @param comment the comment explaining the changes of the current version + * @return {@link VersionedFile} the versioned file + */ + public VersionedFile createVersion(Tenant tenant, String uniqueIdentifier, String content, String comment) { + VersionedFile file = new VersionedFile(); + file.setComment(comment); + file.setUniqueIdentifier(uniqueIdentifier); + file.setTimestamp(LocalDateTime.now()); + file.getTenant().setValue(tenant); + file.getStoredFile().setObject(generateNewFile(file, uniqueIdentifier, content)); + + oma.update(file); + + deleteOldVersions(tenant, uniqueIdentifier); + + return file; + } + + /** + * Generates a new {@link StoredObject} holding the code to be versioned. + *

+ * Will check if the path already exists to avoid overwriting existing versions. + * + * @param file the {@link VersionedFile} associated with the versioned code + * @param uniqueIdentifier the identifier of the versioned file + * @param code the code to be versioned + * @return {@link StoredObject} the generated file + */ + private StoredObject generateNewFile(VersionedFile file, String uniqueIdentifier, String code) { + String fullPath = uniqueIdentifier + file.getTimestamp(); + + if (storage.findByPath(file.getTenant().getValue(), VERSIONED_FILES, fullPath).isPresent()) { + throw Exceptions.createHandled() + .withNLSKey("VersionedFiles.versionExistsConflict") + .set("date", NLS.toUserString(file.getTimestamp())) + .set("path", uniqueIdentifier) + .handle(); + } + + StoredObject object = storage.createTemporaryObject(file.getTenant().getValue(), + VERSIONED_FILES, + file.getStoredFile().getReference(), + fullPath); + + try (OutputStream out = storage.updateFile(object)) { + out.write(code.getBytes()); + } catch (IOException e) { + throw Exceptions.handle(Storage.LOG, e); + } + + return object; + } + + /** + * Deletes old versions of a {@link VersionedFile} identified by an unique identifier. + * + * @param tenant the owning tenant + * @param uniqueIdentifier the unique identifier of a versioned file + */ + public void deleteOldVersions(Tenant tenant, String uniqueIdentifier) { + if (maxNumberOfVersions == 0) { + return; + } + + AtomicInteger filesToSkip = new AtomicInteger(maxNumberOfVersions); + + oma.select(VersionedFile.class) + .eq(VersionedFile.TENANT, tenant) + .eq(VersionedFile.UNIQUE_IDENTIFIER, uniqueIdentifier) + .orderDesc(VersionedFile.TIMESTAMP) + .iterateAll(file -> { + if (filesToSkip.getAndDecrement() > 0) { + return; + } + + if (file.getStoredFile().isFilled()) { + storage.delete(file.getStoredFile().getObject()); + } + + oma.delete(file); + }); + } +} diff --git a/src/main/resources/biz_de.properties b/src/main/resources/biz_de.properties index 77112ba7e..0ff49b0da 100644 --- a/src/main/resources/biz_de.properties +++ b/src/main/resources/biz_de.properties @@ -275,6 +275,8 @@ UserAccountController.forgotPassword.reason = Die "Passwort-Vergessen" Funktion UserAccountController.logout = Abmelden UserAccountController.noUserFoundForEmail = Für die angegebene eMail-Adresse wurde kein Benutzer gefunden. UserAccountController.tooManyUsersFoundForEmail = Die angegebene eMail-Adresse ist nicht eindeutig. +VersionedFiles.noVersion = Keine Version gefunden +VersionedFiles.versionExistsConflict = Die Version ${date} für den Pfad ${uniqueIdentifier} existiert bereits und kann deswegen nicht verwendet werden. VirtualObject.download = Herunterladen VirtualObject.fileSize = Größe VirtualObject.physicalKey = Physikalischer Schlüssel diff --git a/src/main/resources/component-biz.conf b/src/main/resources/component-biz.conf index cbe664618..c1bb44b7e 100644 --- a/src/main/resources/component-biz.conf +++ b/src/main/resources/component-biz.conf @@ -295,6 +295,16 @@ storage { deleteFilesAfterDays = 30 } + # Defines storage for versioned files + versioned-files { + canCreate = false + canEdit = false + canDelete = false + + # number of versions kept from one versioned file + # setting this number to 0 will keep all versions + maxNumberOfVersions = 0 + } } } diff --git a/src/test/java/sirius/biz/storage/VersionedFilesSpec.groovy b/src/test/java/sirius/biz/storage/VersionedFilesSpec.groovy new file mode 100644 index 000000000..02eaac573 --- /dev/null +++ b/src/test/java/sirius/biz/storage/VersionedFilesSpec.groovy @@ -0,0 +1,73 @@ +package sirius.biz.storage + +import sirius.biz.tenants.Tenant +import sirius.biz.tenants.TenantsHelper +import sirius.db.jdbc.OMA +import sirius.kernel.BaseSpecification +import sirius.kernel.Sirius +import sirius.kernel.di.std.Part +import sirius.web.security.UserContext + +import java.time.Duration + +/** + * Provides tests for {@link VersionedFiles}. + */ +class VersionedFilesSpec extends BaseSpecification { + @Part + private static OMA oma + + @Part + private static VersionedFiles versionedFiles + + private Tenant tenant + + + def setupSpec() { + oma.getReadyFuture().await(Duration.ofSeconds(60)) + } + + def setup() { + TenantsHelper.installTestTenant() + tenant = UserContext.getCurrentUser().as(Tenant.class) + } + + def "has version and has no version"() { + given: + String identifier = "versioned-file-hasVersion" + assert !versionedFiles.hasVersions(tenant, identifier) + when: + versionedFiles.createVersion(tenant, identifier, "test content", "test comment") + then: + versionedFiles.hasVersions(tenant, identifier) + !versionedFiles.hasVersions(tenant, "versioned-file-hasNoVersion") + } + + def "createVersion"() { + given: + String identifier = "created-version" + when: + versionedFiles.createVersion(tenant, identifier, "test content", "test comment") + then: + versionedFiles.hasVersions(tenant, identifier) + and: + VersionedFile file = versionedFiles.getVersions(tenant, identifier).get(0) + "test content" == versionedFiles.getContent(file).get(0) + and: + file.getComment() == "test comment" + } + + def "delete older versions"() { + given: + String identifier = "created-version" + int maxNumberOfVersions = Sirius.getSettings().get("storage.buckets.versioned-files.maxNumberOfVersions").asInt(1) + when: + for (int i = 0; i < maxNumberOfVersions * 2; i++) { + versionedFiles.createVersion(tenant, identifier, "test content", "test comment " + i) + } + then: + versionedFiles.hasVersions(tenant, identifier) + and: + versionedFiles.getVersions(tenant, identifier).size() == maxNumberOfVersions + } +} diff --git a/src/test/resources/test.conf b/src/test/resources/test.conf index d6863faa7..f1f8ab0e4 100644 --- a/src/test/resources/test.conf +++ b/src/test/resources/test.conf @@ -102,3 +102,6 @@ async { } } } + + +storage.buckets.versioned-files.maxNumberOfVersions = 2 From de7693a19dfd8dac64de306fda5b66059b467a21 Mon Sep 17 00:00:00 2001 From: Tobias Bischoff Date: Wed, 7 Nov 2018 09:41:20 +0100 Subject: [PATCH 2/4] Removes unnecessary annotations --- src/main/java/sirius/biz/storage/VersionedFile.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/main/java/sirius/biz/storage/VersionedFile.java b/src/main/java/sirius/biz/storage/VersionedFile.java index b7109270f..7d5a4e04b 100644 --- a/src/main/java/sirius/biz/storage/VersionedFile.java +++ b/src/main/java/sirius/biz/storage/VersionedFile.java @@ -6,15 +6,12 @@ import sirius.db.mixing.annotations.Lob; import sirius.db.mixing.annotations.NullAllowed; import sirius.db.mixing.annotations.Trim; -import sirius.kernel.commons.Explain; import java.time.LocalDateTime; /** * Entity holding meta information about a versioned file. */ -@Explain("No need to override the equals method. Mapping and property can have the same name, this is sirius standard.") -@SuppressWarnings({"squid:S2160", "squid:MaximumInheritanceDepth", "squid:S1845"}) public class VersionedFile extends SQLTenantAware { /** From d2a763e76d2b1d95421bc55b9161da628ffe283f Mon Sep 17 00:00:00 2001 From: Tobias Bischoff Date: Thu, 8 Nov 2018 14:43:48 +0100 Subject: [PATCH 3/4] Requires the storage framework --- src/main/java/sirius/biz/storage/VersionedFiles.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/sirius/biz/storage/VersionedFiles.java b/src/main/java/sirius/biz/storage/VersionedFiles.java index b44078075..d24a2c956 100644 --- a/src/main/java/sirius/biz/storage/VersionedFiles.java +++ b/src/main/java/sirius/biz/storage/VersionedFiles.java @@ -21,7 +21,7 @@ /** * Small helper class to access and create {@link VersionedFile versioned files}. */ -@Register(classes = VersionedFiles.class) +@Register(classes = VersionedFiles.class, framework = Storage.FRAMEWORK_STORAGE) public class VersionedFiles { @Part From b22b8f76fdcd4ff1a997123f886507ebcd0ef60d Mon Sep 17 00:00:00 2001 From: Tobias Bischoff Date: Mon, 12 Nov 2018 16:53:15 +0100 Subject: [PATCH 4/4] Adds the framework restriction to the entity this prevents the table from being created if the framwork is not active --- .../sirius/biz/storage/{ => versions}/VersionedFile.java | 6 +++++- .../sirius/biz/storage/{ => versions}/VersionedFiles.java | 4 +++- src/test/java/sirius/biz/storage/VersionedFilesSpec.groovy | 4 +++- 3 files changed, 11 insertions(+), 3 deletions(-) rename src/main/java/sirius/biz/storage/{ => versions}/VersionedFile.java (92%) rename src/main/java/sirius/biz/storage/{ => versions}/VersionedFiles.java (98%) diff --git a/src/main/java/sirius/biz/storage/VersionedFile.java b/src/main/java/sirius/biz/storage/versions/VersionedFile.java similarity index 92% rename from src/main/java/sirius/biz/storage/VersionedFile.java rename to src/main/java/sirius/biz/storage/versions/VersionedFile.java index 7d5a4e04b..aa2fea6be 100644 --- a/src/main/java/sirius/biz/storage/VersionedFile.java +++ b/src/main/java/sirius/biz/storage/versions/VersionedFile.java @@ -1,17 +1,21 @@ -package sirius.biz.storage; +package sirius.biz.storage.versions; +import sirius.biz.storage.Storage; +import sirius.biz.storage.StoredObjectRef; import sirius.biz.tenants.SQLTenantAware; import sirius.db.mixing.Mapping; import sirius.db.mixing.annotations.Length; import sirius.db.mixing.annotations.Lob; import sirius.db.mixing.annotations.NullAllowed; import sirius.db.mixing.annotations.Trim; +import sirius.kernel.di.std.Register; import java.time.LocalDateTime; /** * Entity holding meta information about a versioned file. */ +@Register(framework = Storage.FRAMEWORK_STORAGE) public class VersionedFile extends SQLTenantAware { /** diff --git a/src/main/java/sirius/biz/storage/VersionedFiles.java b/src/main/java/sirius/biz/storage/versions/VersionedFiles.java similarity index 98% rename from src/main/java/sirius/biz/storage/VersionedFiles.java rename to src/main/java/sirius/biz/storage/versions/VersionedFiles.java index d24a2c956..852b04933 100644 --- a/src/main/java/sirius/biz/storage/VersionedFiles.java +++ b/src/main/java/sirius/biz/storage/versions/VersionedFiles.java @@ -1,5 +1,7 @@ -package sirius.biz.storage; +package sirius.biz.storage.versions; +import sirius.biz.storage.Storage; +import sirius.biz.storage.StoredObject; import sirius.biz.tenants.Tenant; import sirius.db.jdbc.OMA; import sirius.kernel.di.std.ConfigValue; diff --git a/src/test/java/sirius/biz/storage/VersionedFilesSpec.groovy b/src/test/java/sirius/biz/storage/VersionedFilesSpec.groovy index 02eaac573..beab76581 100644 --- a/src/test/java/sirius/biz/storage/VersionedFilesSpec.groovy +++ b/src/test/java/sirius/biz/storage/VersionedFilesSpec.groovy @@ -1,5 +1,7 @@ package sirius.biz.storage +import sirius.biz.storage.versions.VersionedFile +import sirius.biz.storage.versions.VersionedFiles import sirius.biz.tenants.Tenant import sirius.biz.tenants.TenantsHelper import sirius.db.jdbc.OMA @@ -11,7 +13,7 @@ import sirius.web.security.UserContext import java.time.Duration /** - * Provides tests for {@link VersionedFiles}. + * Provides tests for {@link sirius.biz.storage.versions.VersionedFiles}. */ class VersionedFilesSpec extends BaseSpecification { @Part