diff --git a/cli/src/main/java/com/devonfw/tools/ide/io/HttpErrorResponse.java b/cli/src/main/java/com/devonfw/tools/ide/io/HttpErrorResponse.java new file mode 100644 index 000000000..c82372f9e --- /dev/null +++ b/cli/src/main/java/com/devonfw/tools/ide/io/HttpErrorResponse.java @@ -0,0 +1,75 @@ +package com.devonfw.tools.ide.io; + +import java.net.URI; +import java.net.http.HttpClient.Version; +import java.net.http.HttpHeaders; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.util.Collections; +import java.util.Optional; +import javax.net.ssl.SSLSession; + +/** + * Implementation of {@link HttpResponse} in case of an {@link Throwable error} to prevent sub-sequent {@link NullPointerException}s. + */ +public class HttpErrorResponse implements HttpResponse { + + private final Throwable error; + + private final HttpRequest request; + + private final URI uri; + private static final HttpHeaders NO_HEADERS = HttpHeaders.of(Collections.emptyMap(), (x, y) -> true); + + /** + * @param error the {@link Throwable} that was preventing the HTTP request for {@link #body()}. + * @param request the {@link HttpRequest} for {@link #request()}. + * @param uri the {@link URI} for {@link #uri()}. + */ + public HttpErrorResponse(Throwable error, HttpRequest request, URI uri) { + super(); + this.error = error; + this.request = request; + this.uri = uri; + } + + @Override + public int statusCode() { + return -1; + } + + @Override + public HttpRequest request() { + return this.request; + } + + @Override + public Optional> previousResponse() { + return Optional.empty(); + } + + @Override + public HttpHeaders headers() { + return NO_HEADERS; + } + + @Override + public Throwable body() { + return this.error; + } + + @Override + public Optional sslSession() { + return Optional.empty(); + } + + @Override + public URI uri() { + return this.uri; + } + + @Override + public Version version() { + return Version.HTTP_2; + } +} diff --git a/cli/src/main/java/com/devonfw/tools/ide/tool/eclipse/EclipseJavaUrlUpdater.java b/cli/src/main/java/com/devonfw/tools/ide/tool/eclipse/EclipseJavaUrlUpdater.java index 78aa2ba71..07ec659a7 100644 --- a/cli/src/main/java/com/devonfw/tools/ide/tool/eclipse/EclipseJavaUrlUpdater.java +++ b/cli/src/main/java/com/devonfw/tools/ide/tool/eclipse/EclipseJavaUrlUpdater.java @@ -6,12 +6,7 @@ public class EclipseJavaUrlUpdater extends EclipseUrlUpdater { @Override - protected String getEdition() { - - return "eclipse"; - } - - @Override + // getEdition() must still return "eclipse" protected String getEclipseEdition() { return "java"; diff --git a/cli/src/main/java/com/devonfw/tools/ide/tool/eclipse/EclipseUrlUpdater.java b/cli/src/main/java/com/devonfw/tools/ide/tool/eclipse/EclipseUrlUpdater.java index 38157c7c0..ccefb8079 100644 --- a/cli/src/main/java/com/devonfw/tools/ide/tool/eclipse/EclipseUrlUpdater.java +++ b/cli/src/main/java/com/devonfw/tools/ide/tool/eclipse/EclipseUrlUpdater.java @@ -13,9 +13,9 @@ */ public abstract class EclipseUrlUpdater extends WebsiteUrlUpdater { - private static final String[] MIRRORS = { "https://ftp.snt.utwente.nl/pub/software/eclipse/technology/epp/downloads", - "https://ftp.osuosl.org/pub/eclipse/technology/epp/downloads", - "https://archive.eclipse.org/technology/epp/downloads" }; + private static final String[] MIRRORS = { + "https://archive.eclipse.org/technology/epp/downloads", + "https://ftp.osuosl.org/pub/eclipse/technology/epp/downloads" }; @Override protected String getTool() { @@ -24,7 +24,7 @@ protected String getTool() { } /** - * @return the eclipse edition name. + * @return the eclipse edition name. May be different from {@link #getEdition()} allowing a different edition name (e.g. eclipse) for IDEasy. */ protected String getEclipseEdition() { @@ -79,7 +79,6 @@ protected String getVersionUrl() { @Override protected Pattern getVersionPattern() { - // return Pattern.compile("\\d{4}-\\d{2}(\\s\\w{2})?"); return Pattern.compile("\\d{4}-\\d{2}"); } diff --git a/cli/src/main/java/com/devonfw/tools/ide/url/UpdateInitiator.java b/cli/src/main/java/com/devonfw/tools/ide/url/UpdateInitiator.java index 92e1f503b..d163267c5 100644 --- a/cli/src/main/java/com/devonfw/tools/ide/url/UpdateInitiator.java +++ b/cli/src/main/java/com/devonfw/tools/ide/url/UpdateInitiator.java @@ -32,6 +32,7 @@ public static void main(String[] args) { String pathToRepo = args[0]; Instant expirationTime = null; + String selectedTool = null; if (args.length < 2) { logger.warn("Timeout was not set, setting timeout to infinite instead."); @@ -44,6 +45,9 @@ public static void main(String[] args) { logger.error("Error: Provided timeout format is not valid.", e); System.exit(1); } + if (args.length > 2) { + selectedTool = args[2]; + } } Path repoPath = Path.of(pathToRepo); @@ -56,7 +60,11 @@ public static void main(String[] args) { UrlFinalReport urlFinalReport = new UrlFinalReport(); UpdateManager updateManager = new UpdateManager(repoPath, urlFinalReport, expirationTime); - updateManager.updateAll(); + if (selectedTool == null) { + updateManager.updateAll(); + } else { + updateManager.update(selectedTool); + } logger.info(urlFinalReport.toString()); } diff --git a/cli/src/main/java/com/devonfw/tools/ide/url/model/AbstractUrlArtifactWithParent.java b/cli/src/main/java/com/devonfw/tools/ide/url/model/AbstractUrlArtifactWithParent.java index 08c18fd48..43d824bff 100644 --- a/cli/src/main/java/com/devonfw/tools/ide/url/model/AbstractUrlArtifactWithParent.java +++ b/cli/src/main/java/com/devonfw/tools/ide/url/model/AbstractUrlArtifactWithParent.java @@ -31,4 +31,8 @@ public P getParent() { return this.parent; } + @Override + public void delete() { + getParent().deleteChild(getName()); + } } diff --git a/cli/src/main/java/com/devonfw/tools/ide/url/model/UrlArtifactWithParent.java b/cli/src/main/java/com/devonfw/tools/ide/url/model/UrlArtifactWithParent.java index cd3b85064..a6851d841 100644 --- a/cli/src/main/java/com/devonfw/tools/ide/url/model/UrlArtifactWithParent.java +++ b/cli/src/main/java/com/devonfw/tools/ide/url/model/UrlArtifactWithParent.java @@ -13,4 +13,12 @@ public interface UrlArtifactWithParent

> extends UrlArtifa * @return the parent {@link UrlFolder} owning this artifact as child. */ P getParent(); + + + /** + * Physically deletes this artifact with all its potential children from the disc. Will also remove it from its {@link #getParent() parent}. + */ + default void delete() { + getParent().deleteChild(getName()); + } } diff --git a/cli/src/main/java/com/devonfw/tools/ide/url/model/file/AbstractUrlFile.java b/cli/src/main/java/com/devonfw/tools/ide/url/model/file/AbstractUrlFile.java index 07e971b5e..5f03c67ae 100644 --- a/cli/src/main/java/com/devonfw/tools/ide/url/model/file/AbstractUrlFile.java +++ b/cli/src/main/java/com/devonfw/tools/ide/url/model/file/AbstractUrlFile.java @@ -2,6 +2,8 @@ import java.nio.file.Files; +import org.jline.utils.Log; + import com.devonfw.tools.ide.url.model.AbstractUrlArtifactWithParent; import com.devonfw.tools.ide.url.model.folder.AbstractUrlFolder; import com.devonfw.tools.ide.url.model.folder.UrlFolder; @@ -54,6 +56,7 @@ public void load(boolean recursive) { public void save() { if (this.modified) { + Log.debug("Saving {}", getPath()); doSave(); this.modified = false; } diff --git a/cli/src/main/java/com/devonfw/tools/ide/url/model/file/UrlStatusFile.java b/cli/src/main/java/com/devonfw/tools/ide/url/model/file/UrlStatusFile.java index a628deef8..c7bbf9a29 100644 --- a/cli/src/main/java/com/devonfw/tools/ide/url/model/file/UrlStatusFile.java +++ b/cli/src/main/java/com/devonfw/tools/ide/url/model/file/UrlStatusFile.java @@ -4,7 +4,6 @@ import java.io.BufferedWriter; import java.nio.file.Files; import java.nio.file.Path; -import java.nio.file.StandardOpenOption; import com.devonfw.tools.ide.json.JsonMapping; import com.devonfw.tools.ide.url.model.file.json.StatusJson; @@ -23,7 +22,7 @@ public class UrlStatusFile extends AbstractUrlFile { private static final ObjectMapper MAPPER = JsonMapping.create(); - private StatusJson statusJson = new StatusJson(); + private StatusJson statusJson; /** * The constructor. @@ -33,6 +32,7 @@ public class UrlStatusFile extends AbstractUrlFile { public UrlStatusFile(UrlVersion parent) { super(parent, STATUS_JSON); + this.statusJson = new StatusJson(); } /** @@ -70,11 +70,23 @@ protected void doLoad() { @Override protected void doSave() { - try (BufferedWriter writer = Files.newBufferedWriter(getPath(), StandardOpenOption.CREATE)) { + try (BufferedWriter writer = Files.newBufferedWriter(getPath())) { MAPPER.writeValue(writer, this.statusJson); } catch (Exception e) { throw new IllegalStateException("Failed to save file " + getPath(), e); } } + /** + * Performs a cleanup and removes all unused entries. + * + * @see StatusJson#cleanup() + */ + public void cleanup() { + + boolean changed = this.statusJson.cleanup(); + if (changed) { + this.modified = true; + } + } } diff --git a/cli/src/main/java/com/devonfw/tools/ide/url/model/file/json/StatusJson.java b/cli/src/main/java/com/devonfw/tools/ide/url/model/file/json/StatusJson.java index b878292c5..49ba1bca0 100644 --- a/cli/src/main/java/com/devonfw/tools/ide/url/model/file/json/StatusJson.java +++ b/cli/src/main/java/com/devonfw/tools/ide/url/model/file/json/StatusJson.java @@ -1,7 +1,9 @@ package com.devonfw.tools.ide.url.model.file.json; +import java.util.Iterator; import java.util.LinkedHashMap; import java.util.Map; +import java.util.Map.Entry; import com.devonfw.tools.ide.url.model.file.UrlStatusFile; @@ -27,7 +29,7 @@ public StatusJson() { /** * @return {@code true} if this file has been created manually and the containing version folder shall be ignored by the automatic update process, - * {@code false} otherwise. + * {@code false} otherwise. */ public boolean isManual() { @@ -58,13 +60,70 @@ public void setUrls(Map urlStatuses) { this.urls = urlStatuses; } + /** + * @param url the URL to get the {@link UrlStatus} for. + * @return the existing {@link UrlStatus} for the given URL or a {@code null} if not found. + */ + public UrlStatus getStatus(String url) { + + return getStatus(url, false); + } + /** * @param url the URL to get or create the {@link UrlStatus} for. * @return the existing {@link UrlStatus} for the given URL or a new {@link UrlStatus} associated with the given URL. */ public UrlStatus getOrCreateUrlStatus(String url) { + return getStatus(url, true); + } + + /** + * @param url the URL to get or create the {@link UrlStatus} for. + * @param create {@code true} for {@link #getOrCreateUrlStatus(String)} and {@code false} for {@link #getStatus(String)}. + * @return the existing {@link UrlStatus} for the given URL or {@code null} or created status according to {@code create} flag. + */ + public UrlStatus getStatus(String url, boolean create) { + + UrlStatus urlStatus; + Integer key = computeKey(url); + if (create) { + urlStatus = this.urls.computeIfAbsent(key, hash -> new UrlStatus()); + } else { + urlStatus = this.urls.get(key); + } + if (urlStatus != null) { + urlStatus.markStillUsed(); + } + return urlStatus; + } + + static Integer computeKey(String url) { Integer key = Integer.valueOf(url.hashCode()); - return this.urls.computeIfAbsent(key, hash -> new UrlStatus()); + return key; + } + + public void remove(String url) { + + this.urls.remove(computeKey(url)); + } + + /** + * Performs a cleanup and removes all unused entries. + * + * @return {@code true} if something changed during cleanup, {@code false} otherwise. + */ + public boolean cleanup() { + + boolean changed = false; + Iterator> entryIterator = this.urls.entrySet().iterator(); + while (entryIterator.hasNext()) { + Entry entry = entryIterator.next(); + if (!entry.getValue().checkStillUsed()) { + entryIterator.remove(); + changed = true; + } + } + return changed; } } diff --git a/cli/src/main/java/com/devonfw/tools/ide/url/model/file/json/UrlStatus.java b/cli/src/main/java/com/devonfw/tools/ide/url/model/file/json/UrlStatus.java index ca530650d..1f9c8fecd 100644 --- a/cli/src/main/java/com/devonfw/tools/ide/url/model/file/json/UrlStatus.java +++ b/cli/src/main/java/com/devonfw/tools/ide/url/model/file/json/UrlStatus.java @@ -9,6 +9,8 @@ public class UrlStatus { private UrlStatusState error; + private transient boolean stillUsed; + /** * The constructor. */ @@ -48,4 +50,32 @@ public void setError(UrlStatusState error) { this.error = error; } + + /** + * @return {@code true} if entirely empty, {@code false} otherwise. + */ + public boolean checkEmpty() { + + return (this.error == null) && (this.success == null); + } + + /** + * ATTENTION: This is not a standard getter (isStillUsed()) since otherwise Jackson will write it to JSON. + * + * @return {@code true} if still {@link StatusJson#getOrCreateUrlStatus(String) used}, {@code false} otherwise (if the entire version has been processed, it + * will be removed via {@link StatusJson#cleanup()}. + */ + public boolean checkStillUsed() { + + return this.stillUsed; + } + + /** + * Sets {@link #checkStillUsed()} to {@code true}. + */ + void markStillUsed() { + + this.stillUsed = true; + } + } diff --git a/cli/src/main/java/com/devonfw/tools/ide/url/model/file/json/UrlStatusState.java b/cli/src/main/java/com/devonfw/tools/ide/url/model/file/json/UrlStatusState.java index 7c56b1ca9..161f0c2df 100644 --- a/cli/src/main/java/com/devonfw/tools/ide/url/model/file/json/UrlStatusState.java +++ b/cli/src/main/java/com/devonfw/tools/ide/url/model/file/json/UrlStatusState.java @@ -21,7 +21,17 @@ public final class UrlStatusState { */ public UrlStatusState() { - this.timestamp = Instant.now(); + this(Instant.now()); + } + + /** + * The constructor. + * + * @param timestamp the {@link #getTimestamp() timestamp}. + */ + public UrlStatusState(Instant timestamp) { + + this.timestamp = timestamp; } /** @@ -78,4 +88,4 @@ public String toString() { return getClass().getSimpleName() + "@" + this.timestamp + ((this.message == null || this.message.isEmpty()) ? "" : ":" + this.message); } -} \ No newline at end of file +} diff --git a/cli/src/main/java/com/devonfw/tools/ide/url/model/folder/AbstractUrlFolder.java b/cli/src/main/java/com/devonfw/tools/ide/url/model/folder/AbstractUrlFolder.java index 6bf7f7af6..2daa4f1c8 100644 --- a/cli/src/main/java/com/devonfw/tools/ide/url/model/folder/AbstractUrlFolder.java +++ b/cli/src/main/java/com/devonfw/tools/ide/url/model/folder/AbstractUrlFolder.java @@ -6,10 +6,14 @@ import java.util.Collection; import java.util.Collections; import java.util.HashMap; +import java.util.Iterator; import java.util.Map; import java.util.Set; import java.util.stream.Stream; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + import com.devonfw.tools.ide.url.model.AbstractUrlArtifact; import com.devonfw.tools.ide.url.model.UrlArtifactWithParent; @@ -21,6 +25,8 @@ public abstract class AbstractUrlFolder> extends AbstractUrlArtifact implements UrlFolder { + private static final Logger LOG = LoggerFactory.getLogger(AbstractUrlFolder.class); + private final Map childMap; private final Set childNames; @@ -61,6 +67,45 @@ public C getOrCreateChild(String name) { return this.childMap.computeIfAbsent(name, p -> newChild(name)); } + @Override + public void deleteChild(String name) { + + C child = this.childMap.remove(name); + if (child != null) { + delete(child.getPath()); + } + } + + private static void delete(Path path) { + + LOG.debug("Deleting {}", path); + if (Files.exists(path)) { + try { + deleteRecursive(path); + } catch (IOException e) { + throw new RuntimeException("Failed to delete " + path); + } + } else { + LOG.warn("Could not delete file {} because it does not exist.", path); + } + } + + private static void deleteRecursive(Path path) throws IOException { + + if (Files.isDirectory(path)) { + try (Stream childStream = Files.list(path)) { + Iterator iterator = childStream.iterator(); + while (iterator.hasNext()) { + Path child = iterator.next(); + deleteRecursive(child); + } + } + } + LOG.trace("Deleting {}", path); + Files.delete(path); + + } + @Override public Collection getChildren() { @@ -72,7 +117,7 @@ public Collection getChildren() { * @param name the plain filename (excluding any path). * @param folder - {@code true} in case of a folder, {@code false} otherwise (plain data file). * @return {@code true} if the existing file from the file-system should be {@link #getOrCreateChild(String) created as child}, {@code false} otherwise - * (ignore the file). + * (ignore the file). */ protected boolean isAllowedChild(String name, boolean folder) { @@ -128,7 +173,7 @@ private void loadChild(Path childPath, boolean recursive) { public void save() { for (C child : this.childMap.values()) { - ((AbstractUrlArtifact) child).save(); + child.save(); } } } diff --git a/cli/src/main/java/com/devonfw/tools/ide/url/model/folder/UrlFolder.java b/cli/src/main/java/com/devonfw/tools/ide/url/model/folder/UrlFolder.java index c9f781b89..8bf5ff65c 100644 --- a/cli/src/main/java/com/devonfw/tools/ide/url/model/folder/UrlFolder.java +++ b/cli/src/main/java/com/devonfw/tools/ide/url/model/folder/UrlFolder.java @@ -34,4 +34,11 @@ public interface UrlFolder> extends UrlArtifa * @return the {@link Collection} of all children of this folder. */ Collection getChildren(); + + /** + * Physically deletes the {@link #getChild(String) child} with the given name from the disc and removes it from this {@link AbstractUrlFolder}. + * + * @param name the {@link #getName() name} of the child to delete. + */ + void deleteChild(String name); } diff --git a/cli/src/main/java/com/devonfw/tools/ide/url/model/report/UrlUpdaterReport.java b/cli/src/main/java/com/devonfw/tools/ide/url/model/report/UrlUpdaterReport.java index 34499c8cb..e86d2c22b 100644 --- a/cli/src/main/java/com/devonfw/tools/ide/url/model/report/UrlUpdaterReport.java +++ b/cli/src/main/java/com/devonfw/tools/ide/url/model/report/UrlUpdaterReport.java @@ -1,5 +1,7 @@ package com.devonfw.tools.ide.url.model.report; +import java.util.Objects; + /** * An instance of this class represent the result of updating a tool with specific url updater. It counts the number of successful and failed versions and * verifications. @@ -30,6 +32,25 @@ public UrlUpdaterReport(String tool, String edition) { this.edition = edition; } + /** + * The constructor. + * + * @param tool the name of the tool {@link #getTool() tool name} + * @param edition the name of edition of the tool {@link #getEdition()} can be the same as the tool name if no editions exist. + * @param addSuccess see {@link #getAddVersionSuccess()}. + * @param addFailure see {@link #getAddVersionFailure()}. + * @param verificationSuccess see {@link #getVerificationSuccess()}. + * @param verificationFailure see {@link #getVerificationFailure()}. + */ + public UrlUpdaterReport(String tool, String edition, int addSuccess, int addFailure, int verificationSuccess, int verificationFailure) { + + this(tool, edition); + this.addVersionSuccess = addSuccess; + this.addVersionFailure = addFailure; + this.verificationSuccess = verificationSuccess; + this.verificationFailure = verificationFailure; + } + public String getTool() { return tool; @@ -120,4 +141,31 @@ public double getErrorRateVerificiations() { return 0; } } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } else if (o == null || getClass() != o.getClass()) { + return false; + } + UrlUpdaterReport that = (UrlUpdaterReport) o; + return this.addVersionSuccess == that.addVersionSuccess && this.addVersionFailure == that.addVersionFailure + && this.verificationSuccess == that.verificationSuccess + && this.verificationFailure == that.verificationFailure && Objects.equals(this.tool, that.tool) && Objects.equals(this.edition, that.edition); + } + + @Override + public int hashCode() { + return Objects.hash(this.tool, this.edition, this.addVersionSuccess, this.addVersionFailure, this.verificationSuccess, this.verificationFailure); + } + + @Override + public String toString() { + return this.tool + '/' + this.edition + ':' + + "addVersionSuccess=" + addVersionSuccess + + ", addVersionFailure=" + addVersionFailure + + ", verificationSuccess=" + verificationSuccess + + ", verificationFailure=" + verificationFailure; + } } diff --git a/cli/src/main/java/com/devonfw/tools/ide/url/updater/AbstractUrlUpdater.java b/cli/src/main/java/com/devonfw/tools/ide/url/updater/AbstractUrlUpdater.java index 68c6bea19..eba14898f 100644 --- a/cli/src/main/java/com/devonfw/tools/ide/url/updater/AbstractUrlUpdater.java +++ b/cli/src/main/java/com/devonfw/tools/ide/url/updater/AbstractUrlUpdater.java @@ -7,11 +7,14 @@ import java.net.http.HttpClient.Redirect; import java.net.http.HttpRequest; import java.net.http.HttpResponse; +import java.nio.file.Path; import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; import java.time.Duration; import java.time.Instant; +import java.util.ArrayList; import java.util.Collection; +import java.util.HashSet; import java.util.List; import java.util.Locale; import java.util.Objects; @@ -20,6 +23,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import com.devonfw.tools.ide.io.HttpErrorResponse; import com.devonfw.tools.ide.os.OperatingSystem; import com.devonfw.tools.ide.os.SystemArchitecture; import com.devonfw.tools.ide.url.model.file.UrlChecksum; @@ -43,7 +47,9 @@ */ public abstract class AbstractUrlUpdater extends AbstractProcessorWithTimeout implements UrlUpdater { - private static final Duration TWO_DAYS = Duration.ofDays(2); + private static final Duration VERSION_RECHECK_DELAY = Duration.ofDays(3); + + private static final Duration DAYS_UNTIL_DELETION_OF_BROKEN_URL = Duration.ofDays(14); /** {@link OperatingSystem#WINDOWS}. */ protected static final OperatingSystem WINDOWS = OperatingSystem.WINDOWS; @@ -72,6 +78,13 @@ public abstract class AbstractUrlUpdater extends AbstractProcessorWithTimeout im private static final Logger logger = LoggerFactory.getLogger(AbstractUrlUpdater.class); + /** + * The constructor. + */ + public AbstractUrlUpdater() { + super(); + } + /** * @return the name of the {@link UrlTool tool} handled by this updater. */ @@ -160,7 +173,7 @@ protected HttpResponse doGetResponseAsStream(String url) { * * @param urlVersion the {@link UrlVersion} with the {@link UrlVersion#getName() version-number} to process. * @param downloadUrl the URL of the download for the tool. - * @return true if the version was successfully updated, false otherwise. + * @return {@code true} if the version was successfully added, {@code false} otherwise. */ protected boolean doAddVersion(UrlVersion urlVersion, String downloadUrl) { @@ -173,7 +186,7 @@ protected boolean doAddVersion(UrlVersion urlVersion, String downloadUrl) { * @param edition the edition of the tool. * @param urlVersion the {@link UrlVersion} with the {@link UrlVersion#getName() version-number} to process. * @param downloadUrl the URL of the download for the tool. - * @return true if the version was successfully updated, false otherwise. + * @return {@code true} if the version was successfully added, {@code false} otherwise. */ protected boolean doAddVersion(String edition, UrlVersion urlVersion, String downloadUrl) { @@ -187,7 +200,7 @@ protected boolean doAddVersion(String edition, UrlVersion urlVersion, String dow * @param urlVersion the {@link UrlVersion} with the {@link UrlVersion#getName() version-number} to process. * @param downloadUrl the URL of the download for the tool. * @param os the {@link OperatingSystem} for the tool (can be null). - * @return true if the version was successfully updated, false otherwise. + * @return {@code true} if the version was successfully added, {@code false} otherwise. */ protected boolean doAddVersion(UrlVersion urlVersion, String downloadUrl, OperatingSystem os) { @@ -201,7 +214,7 @@ protected boolean doAddVersion(UrlVersion urlVersion, String downloadUrl, Operat * @param urlVersion the {@link UrlVersion} with the {@link UrlVersion#getName() version-number} to process. * @param downloadUrl the URL of the download for the tool. * @param os the {@link OperatingSystem} for the tool (can be null). - * @return true if the version was successfully updated, false otherwise. + * @return {@code true} if the version was successfully added, {@code false} otherwise. */ protected boolean doAddVersion(String edition, UrlVersion urlVersion, String downloadUrl, OperatingSystem os) { @@ -215,7 +228,7 @@ protected boolean doAddVersion(String edition, UrlVersion urlVersion, String dow * @param downloadUrl the URL of the download for the tool. * @param os the {@link OperatingSystem} for the tool (can be null). * @param architecture the optional {@link SystemArchitecture}. - * @return true if the version was successfully updated, false otherwise. + * @return {@code true} if the version was successfully added, {@code false} otherwise. */ protected boolean doAddVersion(UrlVersion urlVersion, String downloadUrl, OperatingSystem os, SystemArchitecture architecture) { @@ -231,7 +244,7 @@ protected boolean doAddVersion(UrlVersion urlVersion, String downloadUrl, Operat * @param downloadUrl the URL of the download for the tool. * @param os the {@link OperatingSystem} for the tool (can be null). * @param architecture the optional {@link SystemArchitecture}. - * @return true if the version was successfully updated, false otherwise. + * @return {@code true} if the version was successfully added, {@code false} otherwise. */ protected boolean doAddVersion(String edition, UrlVersion urlVersion, String downloadUrl, OperatingSystem os, SystemArchitecture architecture) { @@ -247,8 +260,8 @@ protected boolean doAddVersion(String edition, UrlVersion urlVersion, String dow * @param url the URL of the download for the tool. * @param os the optional {@link OperatingSystem}. * @param architecture the optional {@link SystemArchitecture}. - * @param checksum String of the checksum to utilize - * @return {@code true} if the version was successfully updated, {@code false} otherwise. + * @param checksum the existing checksum (e.g. from JSON metadata) or the empty {@link String} if not available and computation needed. + * @return {@code true} if the version was successfully added, {@code false} otherwise. */ protected boolean doAddVersion(UrlVersion urlVersion, String url, OperatingSystem os, SystemArchitecture architecture, String checksum) { return doAddVersion(getEdition(), urlVersion, url, os, architecture, checksum); @@ -262,8 +275,8 @@ protected boolean doAddVersion(UrlVersion urlVersion, String url, OperatingSyste * @param url the URL of the download for the tool. * @param os the optional {@link OperatingSystem}. * @param architecture the optional {@link SystemArchitecture}. - * @param checksum String of the checksum to utilize - * @return {@code true} if the version was successfully updated, {@code false} otherwise. + * @param checksum the existing checksum (e.g. from JSON metadata) or the empty {@link String} if not available and computation needed. + * @return {@code true} if the version was successfully added, {@code false} otherwise. */ protected boolean doAddVersion(String edition, UrlVersion urlVersion, String url, OperatingSystem os, SystemArchitecture architecture, String checksum) { @@ -284,8 +297,7 @@ protected boolean doAddVersion(String edition, UrlVersion urlVersion, String url } url = url.replace("${edition}", edition); - return checkDownloadUrl(edition, url, urlVersion, os, architecture, checksum); - + return doAddVersionUrlIfNewAndValid(edition, url, urlVersion, os, architecture, checksum); } /** @@ -303,28 +315,23 @@ protected boolean isSuccess(HttpResponse response) { /** * Checks if the download file checksum is still valid * - * @param url String of the URL to check - * @param urlVersion the {@link UrlVersion} with the {@link UrlVersion#getName() version-number} to process. - * @param os the {@link OperatingSystem} - * @param architecture the {@link SystemArchitecture} - * @param checksum String of the new checksum to check - * @param tool String of the tool - * @param version String of the version + * @param checksum the newly computed checksum. + * @param urlChecksum the {@link UrlChecksum} to compare. + * @param toolWithEdition the tool/edition information for logging. + * @param version the tool version. + * @param url the URL the checksum belongs to. * @return {@code true} if update of checksum was successful, {@code false} otherwise. */ - private static boolean isChecksumStillValid(String url, UrlVersion urlVersion, OperatingSystem os, - SystemArchitecture architecture, String checksum, String tool, String version) { + private static boolean isChecksumStillValid(String checksum, UrlChecksum urlChecksum, + String toolWithEdition, String version, String url) { - UrlDownloadFile urlDownloadFile = urlVersion.getOrCreateUrls(os, architecture); - UrlChecksum urlChecksum = urlVersion.getOrCreateChecksum(urlDownloadFile.getName()); - String oldChecksum = urlChecksum.getChecksum(); + String existingChecksum = urlChecksum.getChecksum(); - if ((oldChecksum != null) && !Objects.equals(oldChecksum, checksum)) { - logger.error("For tool {} and version {} the mirror URL {} points to a different checksum {} but expected {}.", - tool, version, url, checksum, oldChecksum); + if ((existingChecksum != null) && !existingChecksum.equals(checksum)) { + logger.error("For tool {} and version {} the download URL {} results in checksum {} but expected {}.", + toolWithEdition, version, url, checksum, existingChecksum); return false; } else { - urlDownloadFile.addUrl(url); urlChecksum.setChecksum(checksum); } return true; @@ -333,19 +340,19 @@ private static boolean isChecksumStillValid(String url, UrlVersion urlVersion, O /** * Checks if the content type is valid (not of type text) * - * @param url String of the url to check - * @param tool String of the tool name - * @param version String of the version + * @param url the URL to check. + * @param toolWithEdition the tool/edition information for logging. + * @param version the tool version. * @param response the {@link HttpResponse}. * @return {@code true} if the content type is not of type text, {@code false} otherwise. */ - private boolean isValidDownload(String url, String tool, String version, HttpResponse response) { + private boolean isValidDownload(String url, String toolWithEdition, String version, HttpResponse response) { if (isSuccess(response)) { String contentType = response.headers().firstValue("content-type").orElse("undefined"); boolean isValidContentType = isValidContentType(contentType); if (!isValidContentType) { - logger.error("For tool {} and version {} the download has an invalid content type {} for URL {}", tool, version, + logger.error("For toolWithEdition {} and version {} the download has an invalid content type {} for URL {}", toolWithEdition, version, contentType, url); return false; } @@ -377,43 +384,57 @@ protected boolean isValidContentType(String contentType) { * @param urlVersion the {@link UrlVersion} where to store the collected information like status and checksum. * @param os the {@link OperatingSystem} * @param architecture the {@link SystemArchitecture} + * @param checksum the existing checksum (e.g. from JSON metadata) or the empty {@link String} if not available and computation needed. * @return {@code true} if the download was checked successfully, {@code false} otherwise. */ - private boolean checkDownloadUrl(String edition, String url, UrlVersion urlVersion, OperatingSystem os, + private boolean doAddVersionUrlIfNewAndValid(String edition, String url, UrlVersion urlVersion, OperatingSystem os, SystemArchitecture architecture, String checksum) { + UrlDownloadFile urlDownloadFile = urlVersion.getUrls(os, architecture); + if (urlDownloadFile != null) { + if (urlDownloadFile.getUrls().contains(url)) { + logger.debug("Skipping add of already existing URL {}", url); + return false; + } + } HttpResponse response = doCheckDownloadViaHeadRequest(url); int statusCode = response.statusCode(); - String tool = getToolWithEdition(edition); + String toolWithEdition = getToolWithEdition(edition); String version = urlVersion.getName(); - boolean success = isValidDownload(url, tool, version, response); + boolean success = isValidDownload(url, toolWithEdition, version, response); - // Checks if checksum for URL is already existing - UrlDownloadFile urlDownloadFile = urlVersion.getUrls(os, architecture); - if (urlDownloadFile != null) { - UrlChecksum urlChecksum = urlVersion.getChecksum(urlDownloadFile.getName()); - if (urlChecksum != null) { - logger.warn("Checksum is already existing for: {}, skipping.", url); - doUpdateStatusJson(success, statusCode, edition, urlVersion, url, true); - return true; - } - } + boolean update = false; if (success) { + UrlChecksum urlChecksum = null; + if (urlDownloadFile != null) { + urlChecksum = urlVersion.getChecksum(urlDownloadFile.getName()); + if (urlChecksum != null) { + logger.warn("Checksum is already existing for: {}, skipping.", url); + update = true; + } + } if (checksum == null || checksum.isEmpty()) { String contentType = response.headers().firstValue("content-type").orElse("undefined"); checksum = doGenerateChecksum(doGetResponseAsStream(url), url, edition, version, contentType); } - - success = isChecksumStillValid(url, urlVersion, os, architecture, checksum, tool, version); + // we only use getOrCreate here to avoid creating empty file if doGenerateChecksum fails + if (urlChecksum == null) { + if (urlDownloadFile == null) { + urlDownloadFile = urlVersion.getOrCreateUrls(os, architecture); + } + urlChecksum = urlVersion.getOrCreateChecksum(urlDownloadFile.getName()); + } + success = isChecksumStillValid(checksum, urlChecksum, toolWithEdition, version, url); + if (success) { + urlDownloadFile.addUrl(url); + } } - if (success) { - urlVersion.save(); - } + doUpdateStatusJson(success, statusCode, edition, urlVersion, url, urlDownloadFile, update); - doUpdateStatusJson(success, statusCode, edition, urlVersion, url, false); + urlVersion.save(); return success; } @@ -428,6 +449,7 @@ private boolean checkDownloadUrl(String edition, String url, UrlVersion urlVersi private String doGenerateChecksum(HttpResponse response, String url, String edition, String version, String contentType) { + logger.info("Computing checksum for download with URL {}", url); try (InputStream inputStream = response.body()) { MessageDigest md = MessageDigest.getInstance(UrlChecksum.HASH_ALGORITHM); @@ -458,18 +480,21 @@ private String doGenerateChecksum(HttpResponse response, String url * Checks if a download URL works and if the file is available for download. * * @param url the URL to check. - * @return a URLRequestResult object representing the success or failure of the URL check. + * @return the {@link HttpResponse} to the HEAD request. */ protected HttpResponse doCheckDownloadViaHeadRequest(String url) { + URI uri = null; + HttpRequest request = null; try { - HttpRequest request = HttpRequest.newBuilder().uri(URI.create(url)) + uri = URI.create(url); + request = HttpRequest.newBuilder().uri(uri) .method("HEAD", HttpRequest.BodyPublishers.noBody()).timeout(Duration.ofSeconds(5)).build(); return this.client.send(request, HttpResponse.BodyHandlers.ofString()); } catch (Exception e) { logger.error("Failed to perform HEAD request of URL {}", url, e); - return null; + return new HttpErrorResponse(e, request, uri); } } @@ -480,29 +505,36 @@ protected HttpResponse doCheckDownloadViaHeadRequest(String url) { * @param statusCode the HTTP status code of the response. * @param urlVersion the {@link UrlVersion} instance to create or refresh the status JSON file for. * @param url the checked download URL. + * @param downloadFile the {@link UrlDownloadFile} where the given {@code url} comes from. * @param update - {@code true} in case the URL was updated (verification), {@code false} otherwise (version/URL initially added). */ @SuppressWarnings("null") // Eclipse is too stupid to check this - private void doUpdateStatusJson(boolean success, int statusCode, String edition, UrlVersion urlVersion, String url, boolean update) { + private void doUpdateStatusJson(boolean success, int statusCode, String edition, UrlVersion urlVersion, String url, UrlDownloadFile downloadFile, + boolean update) { - UrlStatusFile urlStatusFile = null; + UrlStatusFile urlStatusFile = urlVersion.getStatus(); + boolean forceCreation = (success || update); + if ((urlStatusFile == null) && forceCreation) { + urlStatusFile = urlVersion.getOrCreateStatus(); + } StatusJson statusJson = null; UrlStatus status = null; UrlStatusState errorStatus = null; Instant errorTimestamp = null; UrlStatusState successStatus = null; Instant successTimestamp = null; - if (success || update) { - urlStatusFile = urlVersion.getOrCreateStatus(); + if (urlStatusFile != null) { statusJson = urlStatusFile.getStatusJson(); - status = statusJson.getOrCreateUrlStatus(url); - errorStatus = status.getError(); - if (errorStatus != null) { - errorTimestamp = errorStatus.getTimestamp(); - } - successStatus = status.getSuccess(); - if (successStatus != null) { - successTimestamp = successStatus.getTimestamp(); + status = statusJson.getStatus(url, forceCreation); + if (status != null) { + errorStatus = status.getError(); + if (errorStatus != null) { + errorTimestamp = errorStatus.getTimestamp(); + } + successStatus = status.getSuccess(); + if (successStatus != null) { + successTimestamp = successStatus.getTimestamp(); + } } } Integer code = Integer.valueOf(statusCode); @@ -511,7 +543,7 @@ private void doUpdateStatusJson(boolean success, int statusCode, String edition, boolean modified = false; if (success) { - boolean setSuccess = !update; + boolean setSuccess = !update || (successStatus == null); if (errorStatus != null) { // we avoid git diff overhead by only updating success timestamp if last check was an error @@ -535,8 +567,26 @@ private void doUpdateStatusJson(boolean success, int statusCode, String edition, logger.warn("For tool {} and version {} the error status-code changed from {} to {} for URL {}.", tool, version, code, errorStatus.getCode(), url); modified = true; - } - if (!modified) { + } else if (isErrorCodeForAutomaticUrlRemoval(code)) { + boolean urlBroken; + Instant ts = successTimestamp; + if (ts == null) { + ts = errorTimestamp; + } + if (ts == null) { + urlBroken = true; // this code should never be reached, but if it does something is very wrong + } else { + Duration errorDuration = Duration.between(ts, Instant.now()); + // if more than this number of days the download URL is broken, we delete it + urlBroken = isGreaterThan(errorDuration, DAYS_UNTIL_DELETION_OF_BROKEN_URL); + if (!urlBroken) { + logger.info("Leaving broken URL since error could be temporary - error duration {} for URL {}", errorDuration, url); + } + } + if (urlBroken) { + removeUrl(url, downloadFile, tool, version, code, urlStatusFile, status); + } + } else { // we avoid git diff overhead by only updating error timestamp if last check was a success if (DateTimeUtil.isAfter(successTimestamp, errorTimestamp)) { modified = true; @@ -551,11 +601,54 @@ private void doUpdateStatusJson(boolean success, int statusCode, String edition, } logger.warn("For tool {} and version {} the download verification failed with status code {} for URL {}.", tool, version, code, url); + getUrlUpdaterReport().incrementVerificationFailure(); } if (modified) { urlStatusFile.setStatusJson(statusJson); // hack to set modified (better solution welcome) } + if (status != null) { + assert !status.checkEmpty() : "Invalid status!"; + } + } + + private static void removeUrl(String url, UrlDownloadFile downloadFile, String tool, String version, Integer code, UrlStatusFile urlStatusFile, + UrlStatus status) { + logger.warn("For tool {} and version {} the the URL {} is broken (status code {}) for a long time and will be removed.", tool, + version, code, url); + downloadFile.removeUrl(url); + if (downloadFile.getUrls().isEmpty()) { + Path downloadPath = downloadFile.getPath(); + logger.warn("For tool {} and version {} all URLs have been removed so the download file {} will be removed.", tool, + version, downloadPath); + downloadFile.delete(); + UrlChecksum urlChecksum = downloadFile.getParent().getChecksum(downloadFile.getName()); + if (urlChecksum == null) { + logger.warn("Was missing checksum file for {}", downloadFile.getPath()); + } else { + urlChecksum.delete(); + } + } else { + downloadFile.save(); + } + StatusJson statusJson = urlStatusFile.getStatusJson(); + statusJson.remove(url); + if (statusJson.getUrls().isEmpty()) { + urlStatusFile.delete(); + } else { + urlStatusFile.setStatusJson(statusJson); + } + } + + private static boolean isErrorCodeForAutomaticUrlRemoval(Integer code) { + return Integer.valueOf(404).equals(code); + } + + private boolean isGreaterThan(Duration errorDuration, Duration daysUntilDeletionOfBrokenUrl) { + + int delta = errorDuration.compareTo(daysUntilDeletionOfBrokenUrl); + // delta: 1 = greater, 0 = equal, -1 = less + return (delta > 0); } /** @@ -573,7 +666,7 @@ protected Set getUrlFilenames() { /** * Checks if we are dependent on OS URL file names, can be overridden to disable OS dependency * - * @return true if we want to check for missing OS URL file names, false if not + * @return {@code true} if we want to check for missing OS URL file names, {@code false} if not. */ protected boolean isOsDependent() { @@ -584,7 +677,7 @@ protected boolean isOsDependent() { * Checks if an OS URL file name was missing in {@link UrlVersion} * * @param urlVersion the {@link UrlVersion} to check - * @return true if an OS type was missing, false if not + * @return {@code true} if an OS type was missing, {@code false} if not. */ public boolean isMissingOs(UrlVersion urlVersion) { @@ -642,7 +735,8 @@ public void update(UrlRepository urlRepository) { */ protected void updateExistingVersions(UrlEdition edition) { - Set existingVersions = edition.getChildNames(); + // since Java collections do not support modification while iterating, we need to create a copy + String[] existingVersions = edition.getChildNames().toArray(i -> new String[i]); for (String version : existingVersions) { UrlVersion urlVersion = edition.getChild(version); if (urlVersion != null) { @@ -653,7 +747,12 @@ protected void updateExistingVersions(UrlEdition edition) { version); } else { updateExistingVersion(edition.getName(), version, urlVersion, statusJson, urlStatusFile); - urlVersion.save(); + if (urlVersion.getChildren().isEmpty()) { + logger.warn("Finally deleting broken or disappeared version {}", urlVersion.getPath()); + urlVersion.delete(); + } else { + urlVersion.save(); + } } } } @@ -662,22 +761,26 @@ protected void updateExistingVersions(UrlEdition edition) { private void updateExistingVersion(String edition, String version, UrlVersion urlVersion, StatusJson statusJson, UrlStatusFile urlStatusFile) { - boolean modified = false; String toolWithEdition = getToolWithEdition(edition); Instant now = Instant.now(); - for (UrlFile child : urlVersion.getChildren()) { - if (child instanceof UrlDownloadFile) { - Set urls = ((UrlDownloadFile) child).getUrls(); + // since Java collections do not support modification while iterating, we need to create a copy + Collection> urlFiles = new ArrayList<>(urlVersion.getChildren()); + for (UrlFile child : urlFiles) { + if (child instanceof UrlDownloadFile urlDownloadFile) { + // since Java collections do not support modification while iterating, we need to create a copy + Set urls = new HashSet<>(urlDownloadFile.getUrls()); for (String url : urls) { - if (shouldVerifyDownloadUrl(version, statusJson, toolWithEdition, now)) { + if (shouldVerifyDownloadUrl(url, statusJson, toolWithEdition, now)) { HttpResponse response = doCheckDownloadViaHeadRequest(url); - doUpdateStatusJson(isSuccess(response), response.statusCode(), edition, urlVersion, url, true); - modified = true; + doUpdateStatusJson(isSuccess(response), response.statusCode(), edition, urlVersion, url, urlDownloadFile, true); } } } } - if (modified) { + urlStatusFile.cleanup(); + if (statusJson.getUrls().isEmpty()) { + urlStatusFile.delete(); + } else { urlStatusFile.save(); } } @@ -688,10 +791,12 @@ private boolean shouldVerifyDownloadUrl(String url, StatusJson statusJson, Strin UrlStatusState success = urlStatus.getSuccess(); if (success != null) { Instant timestamp = success.getTimestamp(); - Integer delta = DateTimeUtil.compareDuration(timestamp, now, TWO_DAYS); - if ((delta != null) && (delta.intValue() <= 0)) { - logger.debug("For tool {} the URL {} has already been checked recently on {}", toolWithEdition, url, timestamp); - return false; + if (timestamp != null) { + Integer delta = DateTimeUtil.compareDuration(timestamp, now, VERSION_RECHECK_DELAY); + if ((delta != null) && (delta.intValue() <= 0)) { + logger.debug("For tool {} the URL {} has already been checked recently on {}", toolWithEdition, url, timestamp); + return false; + } } } return true; @@ -741,12 +846,9 @@ protected String getVersionPrefixToRemove() { */ protected final boolean addVersion(String version, Collection versions) { - String mappedVersion = mapVersion(version); - if ((mappedVersion == null) || mappedVersion.isBlank()) { - logger.debug("Filtered version {}", version); + String mappedVersion = getMappedVersion(version); + if (mappedVersion == null) { return false; - } else if (!version.equals(mappedVersion)) { - logger.debug("Mapped version {} to {}", version, mappedVersion); } boolean added = versions.add(mappedVersion); if (!added) { @@ -755,6 +857,24 @@ protected final boolean addVersion(String version, Collection versions) return added; } + /** + * Higher level method for {@link #mapVersion(String)} with additional logging. + * + * @param version the version to {@link #mapVersion(String) map}. + * @return the mapped version or {@code null} if the version was filtered and shall be ignored. + * @see #mapVersion(String) + */ + private String getMappedVersion(String version) { + String mappedVersion = mapVersion(version); + if ((mappedVersion == null) || mappedVersion.isBlank()) { + logger.debug("Filtered version {}", version); + return null; + } else if (!version.equals(mappedVersion)) { + logger.debug("Mapped version {} to {}", version, mappedVersion); + } + return mappedVersion; + } + /** * Updates the version of a given URL version. * diff --git a/cli/src/main/java/com/devonfw/tools/ide/url/updater/UpdateManager.java b/cli/src/main/java/com/devonfw/tools/ide/url/updater/UpdateManager.java index 0404d250c..be8954257 100644 --- a/cli/src/main/java/com/devonfw/tools/ide/url/updater/UpdateManager.java +++ b/cli/src/main/java/com/devonfw/tools/ide/url/updater/UpdateManager.java @@ -94,14 +94,36 @@ public void updateAll() { if (isTimeoutExpired()) { break; } - try { - updater.setExpirationTime(getExpirationTime()); - updater.setUrlFinalReport(this.urlFinalReport); - updater.update(this.urlRepository); - } catch (Exception e) { - logger.error("Failed to update {}", updater.getToolWithEdition(), e); + update(updater); + } + } + + /** + * Update only a single tool. Mainly used in local development only to test updater only for a tool where changes have been made. + * + * @param tool the name of the tool to update. + */ + public void update(String tool) { + + for (AbstractUrlUpdater updater : this.updaters) { + if (updater.getTool().equals(tool)) { + update(updater); } } } + private void update(AbstractUrlUpdater updater) { + try { + updater.setExpirationTime(getExpirationTime()); + updater.setUrlFinalReport(this.urlFinalReport); + String updaterName = updater.getClass().getSimpleName(); + String toolName = updater.getTool(); + logger.debug("Starting {} for tool {}", updaterName, toolName); + updater.update(this.urlRepository); + logger.debug("Ended {} for tool {}", updaterName, updater.getTool()); + } catch (Exception e) { + logger.error("Failed to update {}", updater.getToolWithEdition(), e); + } + } + } diff --git a/cli/src/main/java/com/devonfw/tools/ide/util/DateTimeUtil.java b/cli/src/main/java/com/devonfw/tools/ide/util/DateTimeUtil.java index 148d39005..0fd836d9a 100644 --- a/cli/src/main/java/com/devonfw/tools/ide/util/DateTimeUtil.java +++ b/cli/src/main/java/com/devonfw/tools/ide/util/DateTimeUtil.java @@ -29,11 +29,10 @@ private DateTimeUtil() { */ public static boolean isAfter(Instant start, Instant end) { - Integer delta = compareDuration(start, end, Duration.ZERO); - if (delta == null) { + if ((start == null) || (end == null)) { return false; } - return delta.intValue() < 0; + return start.isAfter(end); } /** @@ -43,11 +42,10 @@ public static boolean isAfter(Instant start, Instant end) { */ public static boolean isBefore(Instant start, Instant end) { - Integer delta = compareDuration(start, end, Duration.ZERO); - if (delta == null) { + if ((start == null) || (end == null)) { return false; } - return delta.intValue() > 0; + return start.isBefore(end); } /** @@ -55,7 +53,7 @@ public static boolean isBefore(Instant start, Instant end) { * @param end the end {@link Instant}. * @param duration the {@link Duration} to compare to. * @return {@code 0} if the {@link Duration} from {@code start} to {@code end} is equal to the given {@link Duration}, negative value if less, positive value - * is greater and {@code null} if one of the given values was {@code null}. + * is greater and {@code null} if one of the given values was {@code null}. */ public static Integer compareDuration(Instant start, Instant end, Duration duration) { diff --git a/cli/src/main/resources/logback.xml b/cli/src/main/resources/logback.xml index a80f349a0..f0b844725 100644 --- a/cli/src/main/resources/logback.xml +++ b/cli/src/main/resources/logback.xml @@ -8,8 +8,9 @@ + - \ No newline at end of file + diff --git a/cli/src/test/java/com/devonfw/tools/ide/tool/UrlUpdaterMock.java b/cli/src/test/java/com/devonfw/tools/ide/tool/UrlUpdaterMock.java index 14d212e84..9911aca7c 100644 --- a/cli/src/test/java/com/devonfw/tools/ide/tool/UrlUpdaterMock.java +++ b/cli/src/test/java/com/devonfw/tools/ide/tool/UrlUpdaterMock.java @@ -4,7 +4,6 @@ import java.util.HashSet; import java.util.Set; -import com.devonfw.tools.ide.url.model.folder.UrlRepository; import com.devonfw.tools.ide.url.model.folder.UrlVersion; import com.devonfw.tools.ide.url.updater.AbstractUrlUpdater; import com.devonfw.tools.ide.url.updater.UrlUpdater; @@ -35,11 +34,6 @@ protected String getTool() { return "mocked"; } - @Override - public void update(UrlRepository urlRepository) { - super.update(urlRepository); - } - @Override protected Set getVersions() { return versions; diff --git a/cli/src/test/java/com/devonfw/tools/ide/tool/UrlUpdaterTest.java b/cli/src/test/java/com/devonfw/tools/ide/tool/UrlUpdaterTest.java index a428c8c69..b312a98a5 100644 --- a/cli/src/test/java/com/devonfw/tools/ide/tool/UrlUpdaterTest.java +++ b/cli/src/test/java/com/devonfw/tools/ide/tool/UrlUpdaterTest.java @@ -8,14 +8,25 @@ import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; +import java.time.Duration; import java.time.Instant; +import java.time.temporal.ChronoUnit; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; +import com.devonfw.tools.ide.os.OperatingSystem; +import com.devonfw.tools.ide.os.SystemArchitecture; +import com.devonfw.tools.ide.url.model.file.UrlChecksum; +import com.devonfw.tools.ide.url.model.file.UrlDownloadFile; +import com.devonfw.tools.ide.url.model.file.UrlStatusFile; import com.devonfw.tools.ide.url.model.file.json.StatusJson; import com.devonfw.tools.ide.url.model.file.json.UrlStatus; +import com.devonfw.tools.ide.url.model.file.json.UrlStatusState; +import com.devonfw.tools.ide.url.model.folder.UrlEdition; import com.devonfw.tools.ide.url.model.folder.UrlRepository; +import com.devonfw.tools.ide.url.model.folder.UrlTool; +import com.devonfw.tools.ide.url.model.folder.UrlVersion; import com.github.tomakehurst.wiremock.junit5.WireMockRuntimeInfo; import com.github.tomakehurst.wiremock.junit5.WireMockTest; @@ -93,9 +104,7 @@ public void testUrlUpdaterIsNotUpdatingWhenStatusManualIsTrue(@TempDir Path temp } /** - * Tests if the timestamps of the status.json get updated properly. Creates an initial status.json with a success timestamp. Updates the status.json with an - * error timestamp and compares it with the success timestamp. Updates the status.json with a final success timestamp and compares it with the error - * timestamp. + * Tests if the timestamps of the status.json get updated properly if a first error is detected. *

* See: #1343 for reference. * @@ -103,69 +112,145 @@ public void testUrlUpdaterIsNotUpdatingWhenStatusManualIsTrue(@TempDir Path temp * @param wmRuntimeInfo wireMock server on a random port */ @Test - public void testUrlUpdaterStatusJsonRefreshBugStillExisting(@TempDir Path tempDir, WireMockRuntimeInfo wmRuntimeInfo) { + public void testStatusJsonUpdateOnFirstError(@TempDir Path tempDir, WireMockRuntimeInfo wmRuntimeInfo) { - stubFor(any(urlMatching("/os/.*")).willReturn(aResponse().withStatus(200).withBody("aBody"))); - - UrlRepository urlRepository = UrlRepository.load(tempDir); - UrlUpdaterMockSingle updater = new UrlUpdaterMockSingle(wmRuntimeInfo); - - String statusUrl = wmRuntimeInfo.getHttpBaseUrl() + "/os/windows_x64_url.tgz"; + // arrange String toolName = "mocked"; String editionName = "mocked"; String versionName = "1.0"; + String url = wmRuntimeInfo.getHttpBaseUrl() + "/os/windows_x64_url.tgz"; + Instant now = Instant.now(); + Instant lastMonth = now.minus(31, ChronoUnit.DAYS); + UrlRepository urlRepository = UrlRepository.load(tempDir); + UrlTool urlTool = urlRepository.getOrCreateChild(toolName); + UrlEdition urlEdition = urlTool.getOrCreateChild(editionName); + UrlVersion urlVersion = urlEdition.getOrCreateChild(versionName); + // we create the structure of our tool version and URL to simulate it was valid last moth + UrlStatusFile statusFile = urlVersion.getOrCreateStatus(); + UrlStatus status = statusFile.getStatusJson().getOrCreateUrlStatus(url); + UrlStatusState successState = new UrlStatusState(lastMonth); // ensure that we trigger a recheck of the URL + status.setSuccess(successState); + UrlDownloadFile urlDownloadFile = urlVersion.getOrCreateUrls(OperatingSystem.WINDOWS, SystemArchitecture.X64); + urlDownloadFile.addUrl(url); + UrlChecksum urlChecksum = urlVersion.getOrCreateChecksum(urlDownloadFile.getName()); + urlChecksum.setChecksum("1234567890"); + urlVersion.save(); + UrlUpdaterMockSingle updater = new UrlUpdaterMockSingle(wmRuntimeInfo); + // now we want to simulate that the url got broken (404) and the updater is properly handling this + stubFor(any(urlMatching("/os/.*")).willReturn(aResponse().withStatus(404))); - // when + // act updater.update(urlRepository); - Path versionsPath = tempDir.resolve(toolName).resolve(editionName).resolve(versionName); - - // then - assertThat(versionsPath.resolve("status.json")).exists(); - + // assert StatusJson statusJson = retrieveStatusJson(urlRepository, toolName, editionName, versionName); + status = statusJson.getStatus(url); + successState = status.getSuccess(); + assertThat(successState).isNotNull(); + assertThat(successState.getTimestamp()).isEqualTo(lastMonth); + UrlStatusState errorState = status.getError(); + assertThat(errorState).isNotNull(); + assertThat(errorState.getCode()).isEqualTo(404); + assertThat(Duration.between(errorState.getTimestamp(), now)).isLessThan(Duration.ofSeconds(5)); + } - UrlStatus urlStatus = statusJson.getOrCreateUrlStatus(statusUrl); - - Instant successTimestamp = urlStatus.getSuccess().getTimestamp(); - - assertThat(successTimestamp).isNotNull(); - - stubFor(any(urlMatching("/os/.*")).willReturn(aResponse().withStatus(404))); - - // re-initialize UrlRepository for error timestamp - UrlRepository urlRepositoryWithError = UrlRepository.load(tempDir); - updater.update(urlRepositoryWithError); - - statusJson = retrieveStatusJson(urlRepositoryWithError, toolName, editionName, versionName); - - urlStatus = statusJson.getOrCreateUrlStatus(statusUrl); - successTimestamp = urlStatus.getSuccess().getTimestamp(); - Instant errorTimestamp = urlStatus.getError().getTimestamp(); - Integer errorCode = urlStatus.getError().getCode(); - - assertThat(errorCode).isEqualTo(404); - assertThat(errorTimestamp).isAfter(successTimestamp); + /** + * Tests if the timestamps of the status.json get updated properly on success after an error. + *

+ * See: #1343 for reference. + * + * @param tempDir Temporary directory + * @param wmRuntimeInfo wireMock server on a random port + */ + @Test + public void testSuccessStateUpdatedAfterError(@TempDir Path tempDir, WireMockRuntimeInfo wmRuntimeInfo) { + // arrange + String toolName = "mocked"; + String editionName = "mocked"; + String versionName = "1.0"; + String url = wmRuntimeInfo.getHttpBaseUrl() + "/os/windows_x64_url.tgz"; + Instant now = Instant.now(); + Instant lastMonth = now.minus(31, ChronoUnit.DAYS); + Instant lastSuccess = lastMonth.minus(1, ChronoUnit.DAYS); + UrlRepository urlRepository = UrlRepository.load(tempDir); + UrlTool urlTool = urlRepository.getOrCreateChild(toolName); + UrlEdition urlEdition = urlTool.getOrCreateChild(editionName); + UrlVersion urlVersion = urlEdition.getOrCreateChild(versionName); + // we create the structure of our tool version and URL to simulate it was valid last moth + UrlStatusFile statusFile = urlVersion.getOrCreateStatus(); + UrlStatus status = statusFile.getStatusJson().getOrCreateUrlStatus(url); + UrlStatusState successState = new UrlStatusState(lastSuccess); + status.setSuccess(successState); + UrlStatusState errorState = new UrlStatusState(lastMonth); + errorState.setCode(404); + status.setError(errorState); + UrlDownloadFile urlDownloadFile = urlVersion.getOrCreateUrls(OperatingSystem.WINDOWS, SystemArchitecture.X64); + urlDownloadFile.addUrl(url); + UrlChecksum urlChecksum = urlVersion.getOrCreateChecksum(urlDownloadFile.getName()); + urlChecksum.setChecksum("1234567890"); + urlVersion.save(); + UrlUpdaterMockSingle updater = new UrlUpdaterMockSingle(wmRuntimeInfo); + // now we want to simulate that the broken url is working again stubFor(any(urlMatching("/os/.*")).willReturn(aResponse().withStatus(200).withHeader("Content-Type", "text/plain"))); - // re-initialize UrlRepository for error timestamp - UrlRepository urlRepositoryWithSuccess = UrlRepository.load(tempDir); - updater.update(urlRepositoryWithSuccess); - - assertThat(versionsPath.resolve("status.json")).exists(); + // act + updater.update(urlRepository); - statusJson = retrieveStatusJson(urlRepositoryWithSuccess, toolName, editionName, versionName); + // assert + status = retrieveStatusJson(urlRepository, toolName, editionName, versionName).getStatus(url); + successState = status.getSuccess(); + assertThat(successState).isNotNull(); + assertThat(Duration.between(successState.getTimestamp(), now)).isLessThan(Duration.ofSeconds(5)); + errorState = status.getError(); + assertThat(errorState).isNotNull(); + assertThat(errorState.getCode()).isEqualTo(404); + assertThat(errorState.getTimestamp()).isEqualTo(lastMonth); + } - urlStatus = statusJson.getOrCreateUrlStatus(statusUrl); + /** + * Tests if the the tool version gets entirely removed if all versions are broken for a long time. + * + * @param tempDir Temporary directory + * @param wmRuntimeInfo wireMock server on a random port + */ + @Test + public void testVersionRemovedIfErrorPersists(@TempDir Path tempDir, WireMockRuntimeInfo wmRuntimeInfo) { - successTimestamp = urlStatus.getSuccess().getTimestamp(); - errorTimestamp = urlStatus.getError().getTimestamp(); - errorCode = urlStatus.getError().getCode(); + // arrange + String toolName = "mocked"; + String editionName = "mocked"; + String versionName = "1.0"; + String url = wmRuntimeInfo.getHttpBaseUrl() + "/os/windows_x64_url.tgz"; + Instant now = Instant.now(); + Instant lastMonth = now.minus(31, ChronoUnit.DAYS); + Instant lastSuccess = lastMonth.minus(1, ChronoUnit.DAYS); + UrlRepository urlRepository = UrlRepository.load(tempDir); + UrlTool urlTool = urlRepository.getOrCreateChild(toolName); + UrlEdition urlEdition = urlTool.getOrCreateChild(editionName); + UrlVersion urlVersion = urlEdition.getOrCreateChild(versionName); + // we create the structure of our tool version and URL to simulate it was valid last moth + UrlStatusFile statusFile = urlVersion.getOrCreateStatus(); + UrlStatus status = statusFile.getStatusJson().getOrCreateUrlStatus(url); + UrlStatusState successState = new UrlStatusState(lastSuccess); + status.setSuccess(successState); + UrlStatusState errorState = new UrlStatusState(lastMonth); + errorState.setCode(404); + status.setError(errorState); + UrlDownloadFile urlDownloadFile = urlVersion.getOrCreateUrls(OperatingSystem.WINDOWS, SystemArchitecture.X64); + urlDownloadFile.addUrl(url); + UrlChecksum urlChecksum = urlVersion.getOrCreateChecksum(urlDownloadFile.getName()); + urlChecksum.setChecksum("1234567890"); + urlVersion.save(); + UrlUpdaterMockSingle updater = new UrlUpdaterMockSingle(wmRuntimeInfo); + // now we want to simulate that the url got broken (404) and the updater is properly handling this + stubFor(any(urlMatching("/os/.*")).willReturn(aResponse().withStatus(404))); - assertThat(errorCode).isEqualTo(200); - assertThat(errorTimestamp).isAfter(successTimestamp); + // act + updater.update(urlRepository); + // assert + assertThat(urlVersion.getPath()).doesNotExist(); } /** diff --git a/cli/src/test/java/com/devonfw/tools/ide/url/model/UrlStatusFileTest.java b/cli/src/test/java/com/devonfw/tools/ide/url/model/file/UrlStatusFileTest.java similarity index 94% rename from cli/src/test/java/com/devonfw/tools/ide/url/model/UrlStatusFileTest.java rename to cli/src/test/java/com/devonfw/tools/ide/url/model/file/UrlStatusFileTest.java index 527cbd651..168860e2a 100644 --- a/cli/src/test/java/com/devonfw/tools/ide/url/model/UrlStatusFileTest.java +++ b/cli/src/test/java/com/devonfw/tools/ide/url/model/file/UrlStatusFileTest.java @@ -1,4 +1,4 @@ -package com.devonfw.tools.ide.url.model; +package com.devonfw.tools.ide.url.model.file; import java.nio.file.Path; import java.time.Instant; @@ -6,7 +6,6 @@ import org.assertj.core.api.Assertions; import org.junit.jupiter.api.Test; -import com.devonfw.tools.ide.url.model.file.UrlStatusFile; import com.devonfw.tools.ide.url.model.file.json.StatusJson; import com.devonfw.tools.ide.url.model.file.json.UrlStatus; import com.devonfw.tools.ide.url.model.file.json.UrlStatusState; diff --git a/cli/src/test/java/com/devonfw/tools/ide/url/model/file/json/StatusJsonTest.java b/cli/src/test/java/com/devonfw/tools/ide/url/model/file/json/StatusJsonTest.java new file mode 100644 index 000000000..6216b143d --- /dev/null +++ b/cli/src/test/java/com/devonfw/tools/ide/url/model/file/json/StatusJsonTest.java @@ -0,0 +1,33 @@ +package com.devonfw.tools.ide.url.model.file.json; + +import java.util.HashSet; +import java.util.Set; + +import org.assertj.core.api.Assertions; +import org.junit.jupiter.api.Test; + +/** + * Test of {@link StatusJson}. + */ +public class StatusJsonTest extends Assertions { + + @Test + public void testHashCollision() { + + // arrange + Set urls = Set.of("https://archive.eclipse.org/technology/epp/downloads/release/2022-09/R/eclipse-cpp-2022-09-R-linux-gtk-aarch64.tar.gz", + "https://archive.eclipse.org/technology/epp/downloads/release/2022-09/R/eclipse-cpp-2022-09-R-linux-gtk-x86_64.tar.gz", + "https://archive.eclipse.org/technology/epp/downloads/release/2022-09/R/eclipse-cpp-2022-09-R-macosx-cocoa-aarch64.tar.gz", + "https://archive.eclipse.org/technology/epp/downloads/release/2022-09/R/eclipse-cpp-2022-09-R-macosx-cocoa-x86_64.tar.gz", + "https://archive.eclipse.org/technology/epp/downloads/release/2022-09/R/eclipse-cpp-2022-09-R-win32-x86_64.zip"); + Set hashes = new HashSet<>(urls.size()); + + // act + for (String url : urls) { + boolean added = hashes.add(StatusJson.computeKey(url)); + // assert + assertThat(added).as("hash of %s is unique").isTrue(); + } + } + +} diff --git a/cli/src/test/java/com/devonfw/tools/ide/url/report/UrlUpdaterReportTest.java b/cli/src/test/java/com/devonfw/tools/ide/url/report/UrlUpdaterReportTest.java index 6080c60b9..7ac5e32a8 100644 --- a/cli/src/test/java/com/devonfw/tools/ide/url/report/UrlUpdaterReportTest.java +++ b/cli/src/test/java/com/devonfw/tools/ide/url/report/UrlUpdaterReportTest.java @@ -9,6 +9,7 @@ import java.nio.file.Files; import java.nio.file.Path; +import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; @@ -26,136 +27,220 @@ @WireMockTest public class UrlUpdaterReportTest extends AbstractUrlUpdaterTest { + private UrlRepository urlRepository; + private UrlUpdaterMock updater; + /** - * Function to test the equality of two {@link UrlUpdaterReport} instances + * Configure {@link UrlRepository} and {@link UrlUpdaterMock} before each test * - * @param report1 first report - * @param report2 second report - * @return true if equal otherwise false + * @param tempDir temporary directory to use + * @param wmRuntimeInfo wireMock server on a random port */ - public boolean equalsReport(UrlUpdaterReport report1, UrlUpdaterReport report2) { + @BeforeEach + public void setup(@TempDir Path tempDir, WireMockRuntimeInfo wmRuntimeInfo) { - return (report1.getAddVersionSuccess() == report2.getAddVersionSuccess()) && (report1.getAddVersionFailure() == report2.getAddVersionFailure()) - && (report1.getVerificationSuccess() == report2.getVerificationSuccess()) && (report1.getVerificationFailure() == report2.getVerificationFailure()); + urlRepository = UrlRepository.load(tempDir); + updater = new UrlUpdaterMock(wmRuntimeInfo); } /** - * Tests for {@link UrlUpdaterReport} if information of updaters is collected correctly - * - * @param tempDir Temporary directory - * @param wmRuntimeInfo wireMock server on a random port - * @throws IOException test fails + * Test report on the first (initial) run when all URLs are successful */ @Test - public void testReportWithoutErrorsAndEmptyRepo(@TempDir Path tempDir, WireMockRuntimeInfo wmRuntimeInfo) throws IOException { + public void testReportOnInitialRunWithAllUrlsSuccessful() { - UrlRepository urlRepository = UrlRepository.load(tempDir); - UrlUpdaterMock updater = new UrlUpdaterMock(wmRuntimeInfo); + // assign + stubSuccessfulUrlRequest(); + // 3 versions x 4 urls --> 3 additions and 12 verifications + UrlUpdaterReport expectedReport = createReport(3, 0, 12, 0); UrlFinalReport urlFinalReport = new UrlFinalReport(); updater.setUrlFinalReport(urlFinalReport); - // assign - stubFor(any(urlMatching("/os.*")).willReturn(aResponse().withStatus(200).withBody("aBody"))); - UrlUpdaterTestReport expectedReport = new UrlUpdaterTestReport("mocked", "mocked", 3, 0, 12, 0); // act updater.update(urlRepository); - // assert - assertThat(equalsReport(urlFinalReport.getUrlUpdaterReports().get(0), expectedReport)).isEqualTo(true); + // assert + assertThat(urlFinalReport.getUrlUpdaterReports()).contains(expectedReport); } /** - * Test with existing versions and not empty repo and some failing urls + * Test report on the first (initial) run when all URLs are failing */ @Test - public void testReportWithExistVersionsAndFailedDownloads(@TempDir Path tempDir, WireMockRuntimeInfo wmRuntimeInfo) throws IOException { + public void testReportOnInitialRunWithAllUrlsFailing() { - UrlRepository urlRepository = UrlRepository.load(tempDir); - UrlUpdaterMock updater = new UrlUpdaterMock(wmRuntimeInfo); + // assign + stubFailedUrlRequest(); + UrlUpdaterReport expectedReport = createReport(3, 0, 0, 12); UrlFinalReport urlFinalReport = new UrlFinalReport(); updater.setUrlFinalReport(urlFinalReport); - Path versionsPath = tempDir.resolve("mocked").resolve("mocked").resolve("1.0"); - // pre configuration - stubFor(any(urlMatching("/os.*")).willReturn(aResponse().withStatus(200).withBody("aBody"))); - updater.update(urlRepository); - // assign - stubFor(any(urlMatching("/os/mac.*")).willReturn(aResponse().withStatus(400).withBody("aBody"))); - UrlUpdaterTestReport expectedReport = new UrlUpdaterTestReport("mocked", "mocked", 0, 0, 6, 6); // act updater.update(urlRepository); + // assert - assertThat(equalsReport(urlFinalReport.getUrlUpdaterReports().get(1), expectedReport)).isEqualTo(true); + assertThat(urlFinalReport.getUrlUpdaterReports()).contains(expectedReport); } /** - * Test one os version url file is removed in already existing repository + * Test report on first (initial) run when urls for mac_x64 and mac_arm64 are failing */ @Test - public void testReportAfterVersionForOsRemoved(@TempDir Path tempDir, WireMockRuntimeInfo wmRuntimeInfo) throws IOException { + public void testReportOnInitialRunWithFailedUrlsForMac() { - UrlRepository urlRepository = UrlRepository.load(tempDir); - UrlUpdaterMock updater = new UrlUpdaterMock(wmRuntimeInfo); + // assign + stubSuccessfulUrlRequest(); + stubFailedUrlRequest("/os/mac.*"); + updater.update(urlRepository); // init successful update + UrlUpdaterReport expectedReport = createReport(3, 0, 0, 6); UrlFinalReport urlFinalReport = new UrlFinalReport(); updater.setUrlFinalReport(urlFinalReport); - Path versionsPath = tempDir.resolve("mocked").resolve("mocked").resolve("1.0"); - // pre configuration - stubFor(any(urlMatching("/os.*")).willReturn(aResponse().withStatus(200).withBody("aBody"))); - updater.update(urlRepository); - stubFor(any(urlMatching("/os/mac.*")).willReturn(aResponse().withStatus(400).withBody("aBody"))); + // act updater.update(urlRepository); + + // assert + assertThat(urlFinalReport.getUrlUpdaterReports()).contains(expectedReport); + } + + /** + * Test report on second run with existing versions already verified within the timeframe + */ + @Test + public void testReportOnSecondRunWithExistVersionsAlreadyVerifiedInTime() { + // assign - stubFor(any(urlMatching("/os/mac.*")).willReturn(aResponse().withStatus(400).withBody("aBody"))); - UrlUpdaterReport expectedReport = new UrlUpdaterTestReport("mocked", "mocked", 1, 0, 7, 8); + stubSuccessfulUrlRequest(); + updater.update(urlRepository); // init successful update + UrlUpdaterReport expectedReport = createReport(0, 0, 0, 0); + UrlFinalReport urlFinalReport = new UrlFinalReport(); + updater.setUrlFinalReport(urlFinalReport); + // act - Files.deleteIfExists(versionsPath.resolve("windows_x64.urls")); - Files.deleteIfExists(versionsPath.resolve("windows_x64.urls.sha256")); - UrlRepository urlRepositoryWithError = UrlRepository.load(tempDir); - updater.update(urlRepositoryWithError); + updater.update(urlRepository); + // assert - assertThat(equalsReport(urlFinalReport.getUrlUpdaterReports().get(2), expectedReport)).isEqualTo(true); + assertThat(urlFinalReport.getUrlUpdaterReports()).contains(expectedReport); } /** - * Test after one version is completely removed in already existing repository and download urls response is reversed + * Test report on second run when an url is removed from one version */ @Test - public void testReportAfterVersionsForRemovedAndReversedUrlResponses(@TempDir Path tempDir, WireMockRuntimeInfo wmRuntimeInfo) throws IOException { - UrlRepository urlRepository = UrlRepository.load(tempDir); - UrlUpdaterMock updater = new UrlUpdaterMock(wmRuntimeInfo); + public void testReportOnSecondRunAfterOneVersionIsRemoved() throws IOException { + + // assign + stubSuccessfulUrlRequest(); + updater.update(urlRepository); // init successful update + UrlUpdaterReport expectedReport = createReport(1, 0, 1, 0); + Path urlPath = urlRepository.getPath().resolve("mocked").resolve("mocked").resolve("1.0"); + Files.deleteIfExists(urlPath.resolve("windows_x64.urls")); + Files.deleteIfExists(urlPath.resolve("windows_x64.urls.sha256")); + urlRepository = UrlRepository.load(urlRepository.getPath()); UrlFinalReport urlFinalReport = new UrlFinalReport(); updater.setUrlFinalReport(urlFinalReport); - Path versionsPath = tempDir.resolve("mocked").resolve("mocked").resolve("1.0"); - // pre configuration - stubFor(any(urlMatching("/os.*")).willReturn(aResponse().withStatus(200).withBody("aBody"))); - updater.update(urlRepository); - stubFor(any(urlMatching("/os/mac.*")).willReturn(aResponse().withStatus(400).withBody("aBody"))); - updater.update(urlRepository); - stubFor(any(urlMatching("/os/mac.*")).willReturn(aResponse().withStatus(400).withBody("aBody"))); + // act updater.update(urlRepository); + + // assert + assertThat(urlFinalReport.getUrlUpdaterReports()).contains(expectedReport); + } + + /** + * Test report total additions and verifications operations + */ + @Test + public void testReportTotalAdditionsAndVerificationsOperations() { + // assign - stubFor(any(urlMatching("/os.*")).willReturn(aResponse().withStatus(400).withBody("aBody"))); - stubFor(any(urlMatching("/os/mac.*")).willReturn(aResponse().withStatus(200).withBody("aBody"))); - UrlUpdaterReport expectedReport = new UrlUpdaterTestReport("mocked", "mocked", 1, 0, 6, 6); + int addVersionSuccess = 5; + int addVersionFailure = 0; + int addVerificationSuccess = 10; + int addVerificationFailure = 10; + UrlUpdaterReport report = createReport(addVersionSuccess, addVersionFailure, addVerificationSuccess, addVerificationFailure); + + // assert + assertThat(report.getTotalAdditions()).isEqualTo(report.getAddVersionSuccess() + report.getAddVersionFailure()); + assertThat(report.getTotalVerificitations()).isEqualTo(report.getVerificationSuccess() + report.getVerificationFailure()); + } + + /** + * Test report increment operations for additions and verifications + */ + @Test + public void testReportIncrementOperations() { + + // assign + int addVersionSuccess = 5; + int addVersionFailure = 0; + int addVerificationSuccess = 10; + int addVerificationFailure = 10; + UrlUpdaterReport report = createReport(addVersionSuccess, addVersionFailure, addVerificationSuccess, addVerificationFailure); + // act - Files.deleteIfExists(versionsPath.resolve("windows_x64.urls")); - Files.deleteIfExists(versionsPath.resolve("windows_x64.urls.sha256")); - Files.deleteIfExists(versionsPath.resolve("mac_x64.urls")); - Files.deleteIfExists(versionsPath.resolve("mac_x64.urls.sha256")); - Files.deleteIfExists(versionsPath.resolve("mac_arm64.urls")); - Files.deleteIfExists(versionsPath.resolve("mac_arm64.urls.sha256")); - Files.deleteIfExists(versionsPath.resolve("linux_x64.urls")); - Files.deleteIfExists(versionsPath.resolve("linux_x64.urls.sha256")); - Files.deleteIfExists(versionsPath.resolve("status.json")); - Files.deleteIfExists(versionsPath); - UrlRepository urlRepositoryWithError = UrlRepository.load(tempDir); - updater.update(urlRepositoryWithError); + report.incrementAddVersionSuccess(); + report.incrementAddVersionFailure(); + report.incrementVerificationSuccess(); + report.incrementVerificationFailure(); + // assert - assertThat(equalsReport(urlFinalReport.getUrlUpdaterReports().get(3), expectedReport)).isEqualTo(true); + assertThat(report.getAddVersionSuccess()).isEqualTo(addVersionSuccess + 1); + assertThat(report.getAddVersionFailure()).isEqualTo(addVersionFailure + 1); + assertThat(report.getVerificationSuccess()).isEqualTo(addVerificationSuccess + 1); + assertThat(report.getVerificationFailure()).isEqualTo(addVerificationFailure + 1); + } + + /** + * Test report error rate operations for additions and verifications + */ + @Test + public void testReportErrorRateOperations() { + + // assign + int addVersionSuccess = 20; + int addVersionFailureNull = 0; + int addVerificationSuccessNull = 0; + int addVerificationFailure = 10; + int addVersionFailureIncremented = 5; // for testing without null + int addVerificationSuccessIncremented = 10; // for testing without null + UrlUpdaterReport reportWithNull = createReport(addVersionSuccess, addVersionFailureNull, addVerificationSuccessNull, addVerificationFailure); + UrlUpdaterReport reportWithoutNull = createReport(addVersionSuccess, addVersionFailureIncremented, addVerificationSuccessIncremented, + addVerificationFailure); + // act + double errorRateWithNullAdd = reportWithNull.getErrorRateAdditions(); + double errorRateWithNullVer = reportWithNull.getErrorRateVerificiations(); + double errorRateWithoutNullAdd = reportWithoutNull.getErrorRateAdditions(); + double errorRateWithoutNullVer = reportWithoutNull.getErrorRateVerificiations(); + + // assert (failures / total) * 100 + assertThat(errorRateWithNullAdd).isEqualTo(0.00); + assertThat(errorRateWithNullVer).isEqualTo(0.00); + assertThat(errorRateWithoutNullAdd).isEqualTo(20.0); + assertThat(errorRateWithoutNullVer).isEqualTo(50.0); } -} + // some utils + private void stubSuccessfulUrlRequest() { + + stubFor(any(urlMatching("/os.*")).willReturn(aResponse().withStatus(200).withBody("aBody"))); + } + + private void stubFailedUrlRequest(String urlPattern) { + + stubFor(any(urlMatching(urlPattern)).willReturn(aResponse().withStatus(400).withBody("aBody"))); + } + + private void stubFailedUrlRequest() { + + stubFor(any(urlMatching("/os.*")).willReturn(aResponse().withStatus(400).withBody("aBody"))); + } + + private UrlUpdaterReport createReport(int addSucc, int addFail, int verSucc, int verFail) { + + return new UrlUpdaterReport("mocked", "mocked", addSucc, addFail, verSucc, verFail); + } + +} diff --git a/cli/src/test/java/com/devonfw/tools/ide/url/report/UrlUpdaterTestReport.java b/cli/src/test/java/com/devonfw/tools/ide/url/report/UrlUpdaterTestReport.java deleted file mode 100644 index ac336666d..000000000 --- a/cli/src/test/java/com/devonfw/tools/ide/url/report/UrlUpdaterTestReport.java +++ /dev/null @@ -1,20 +0,0 @@ -package com.devonfw.tools.ide.url.report; - -import com.devonfw.tools.ide.url.model.report.UrlUpdaterReport; - -/** - * Test class for {@link UrlUpdaterReport} to create an instance with specific attributes - */ -public class UrlUpdaterTestReport extends UrlUpdaterReport { - - public UrlUpdaterTestReport(String tool, String edition, int addSuccess, int addFailure, int verificationSuccess, int verificationFailure) { - - super(tool, edition); - this.addVersionSuccess = addSuccess; - this.addVersionFailure = addFailure; - this.verificationSuccess = verificationSuccess; - this.verificationFailure = verificationFailure; - } - - -} diff --git a/documentation/DoD.adoc b/documentation/DoD.adoc index 4a29de515..69dce4b52 100644 --- a/documentation/DoD.adoc +++ b/documentation/DoD.adoc @@ -21,11 +21,11 @@ Otherwise if a check failed (red cross) you need to click the `Details` link, re ** [ ] The build and all automated tests succeeded. If failed and you clicked on `Details` add read the logs to find the error. ** [ ] The contributors license agreement (CLA) is signed by all contributors of the PR. -** [ ] Git-Gardian did not report any security issue +** [ ] Git-Guardian did not report any security issue * [ ] The feature branch of the PR is up-to-date with the `main` branch. If you see `This branch is out-of-date with the base branch` in the PR click the `Update branch` button to fix (or manually merge with the `main` from upstream locally and push your changes). In case you see `This branch has conflicts that must be resolved` instead, you need to resolve conflicts. -Very simple conficts may be resolved in the browser on github. +Very simple conflicts may be resolved in the browser on github. But as a general recommendation you should resolve the conflicts locally with proper merge tool support and rerun tests before you push the merged changes. * [ ] You followed all link:coding-conventions.adoc[coding conventions] * [ ] You have already added the issue implemented by your PR in https://github.com/devonfw/ide/blob/master/CHANGELOG.adoc[CHANGELOG.adoc] to the next open release (see milestones or https://github.com/devonfw/IDEasy/blob/main/.mvn/maven.config[maven.config]). @@ -41,11 +41,11 @@ There are very few tools as exception to this rule like `Docker` that extend `Gl ** [ ] The tool can be configured locally inside `$IDE_HOME/conf` and not from a global location (e.g. in `$HOME`). Note: If a tool reads configuration files from the users home directory this is not given as two IDEasy projects using the same tool then would read the same config so one installation would influence the other. ** [ ] The help page displays information about the commandlet and its properties (CLI parameters) explaining how to use it properly. -There are no warnings logged in the help output (like `Cound not find key 'cmd-gcviewer' in ResourceBundle nls.Ide.properties`). +There are no warnings logged in the help output (like `Could not find key 'cmd-gcviewer' in ResourceBundle nls.Ide.properties`). Therefore add proper help texts for all supported languages https://github.com/devonfw/IDEasy/tree/main/cli/src/main/resources/nls[here]. -** [ ] The new tool is added to the table of tools in https://github.com/devonfw/ide/blob/master/documentation/LICENSE.asciidoc#license[LICENSE.asciidoc] with its according licesne. +** [ ] The new tool is added to the table of tools in https://github.com/devonfw/ide/blob/master/documentation/LICENSE.asciidoc#license[LICENSE.asciidoc] with its according license. If that license is not yet included, the full license text needs to be added. ** [ ] The new commandlet installs potential dependencies automatically (e.g. `getCommandlet(«DependentTool».class).install()` in overridden `install` method). ** [ ] The variables `«TOOL»_VERSION` and `«TOOL»_EDITION` are honored by your commandlet so if present that edition and version will be downloaded and installed (happens by default but important if you implement custom installation logic). -** [ ] The new commandlet is tested on all plattforms it is availible for. +** [ ] The new commandlet is tested on all platforms it is available for. Assuming you are using Windows, testing for Linux can be done with WSL or Virtual Box and for MacOS we have a virtual cloud instance. diff --git a/documentation/coding-conventions.adoc b/documentation/coding-conventions.adoc index d50a4e5ca..31fde0a68 100644 --- a/documentation/coding-conventions.adoc +++ b/documentation/coding-conventions.adoc @@ -14,7 +14,7 @@ We follow these additional naming rules: * Always use short but speaking names (for types, methods, fields, parameters, variables, constants, etc.). * Avoid using existing type names from JDK (from `java.lang.*`, `java.util.*`, etc.) - so e.g. never name your own Java type `List`, `Error`, etc. * Strictly avoid special characters in technical names (for files, types, fields, methods, properties, variables, database tables, columns, constraints, etc.). -In other words only use Latin alpahnumeric ASCII characters with the common allowed technical separators for the accordign context (e.g. underscore) for technical names (even excluding whitespaces). +In other words only use Latin alphanumeric ASCII characters with the common allowed technical separators for the according context (e.g. underscore) for technical names (even excluding whitespaces). * For package segments and type names prefer singular forms (`CustomerEntity` instead of [line-through]`CustomersEntity`). Only use plural forms when there is no singular or it is really semantically required (e.g. for a container that contains multiple of such objects). * Avoid having duplicate type names. @@ -116,10 +116,10 @@ In IDEasy for commandlets, etc. we do not need to define `LOG` and can simply us |*Level*|*Type*|*Meaning* |`error`|Standard|Only for real errors that should raise the end-users attention. If an error is logged something went wrong and action needs to be taken and usually the operation failed. |`warning`|Standard|For warnings when something is not correct and the end-user should have a look. E.g. if something is misconfigured. Unlike error the process can continue and may hopefully success. -|`interaction`|Proprietary|For interaction with the end-user. Typically for questions the end-user needs to answer (use dedidcated `question` or `askForInput` methods of `context`). +|`interaction`|Proprietary|For interaction with the end-user. Typically for questions the end-user needs to answer (use dedicated `question` or `askForInput` methods of `context`). |`step`|Proprietary|For steps of advanced processing. Allows to divide some processing into logical steps (use `newStep` method of `context`). This increases the user-experience as the end-user sees the progress and can get a report of these steps and see how long they took and if they succeeded or not. -|`debug`|Standard|Only used for debugging. Disabled by default to avoid "spammming" the end-user. Can be enabled with `-d` or `--debug` option to get more details and analyze what happens in detail. -|`trace`|Standard|Only used for very fine grained details. Disabled by default to avoid "spammming" the end-user. Can be enabled with `-t` or `--trace` option to get even more details if debug is not enough. +|`debug`|Standard|Only used for debugging. Disabled by default to avoid "spamming" the end-user. Can be enabled with `-d` or `--debug` option to get more details and analyze what happens in detail. +|`trace`|Standard|Only used for very fine grained details. Disabled by default to avoid "spamming" the end-user. Can be enabled with `-t` or `--trace` option to get even more details if debug is not enough. |======================= The Log-Levels with type `Proprietary` only exist in `IdeLogger` for allowing different syntax coloring for these specific use-cases. @@ -180,7 +180,7 @@ try { doSomething(); } catch (Exception e) { // fine - throw new IllegalStateExeception("Something failed", e); + throw new IllegalStateException("Something failed", e); } ---- @@ -331,7 +331,7 @@ Do refactorings with care and follow these best-practices: Otherwise your diff may show that a file has been deleted somewhere and another file has been added but you cannot see that this file was moved/renamed and what changed inside the file. * do not change Java signatures like in a text editor but use refactoring capabilities of your IDE. So e.g. when changing a method name, adding or removing a parameter, always use refactoring as otherwise you easily break references (and JavaDoc references will not give you compile errors so you break things without noticing). -* when adding paramaters to methods, please always consider to keep the existing signature and just create a new variant of the method with an additional parameter. +* when adding parameters to methods, please always consider to keep the existing signature and just create a new variant of the method with an additional parameter. Lets assume we have this method: @@ -416,10 +416,10 @@ if (foo == null) { ---- Please note that the term `Exception` is used for something exceptional. -Further creating an instance of an `Exception` or `Throable` in Java is expensive as the entire Strack has to be collected and copied into arrays, etc. causing significant overhead. +Further creating an instance of an `Exception` or `Throwable` in Java is expensive as the entire stack has to be collected and copied into arrays, etc. causing significant overhead. This should always be avoided in situations we can easily avoid with a simple `if` check. -== Consider extractig local variable for multiple method calls +== Consider extracting local variable for multiple method calls Calling the same method (cascades) multiple times is redundant and reduces readability and performance: @@ -484,7 +484,7 @@ Instead this applies to things like `IdeContext` and all its related child-objec Such classes shall never be modified after initialization. Methods called at runtime (after initialization) do not assign fields (member variables of your class) or mutate the object stored in a field. This allows your component or bean to be stateless and thread-safe. -Therefore it can be initialized as a singleton so only one instance is created and shared accross all threads of the application. +Therefore it can be initialized as a singleton so only one instance is created and shared across all threads of the application. Ideally all fields are declared `final` otherwise be careful not to change them dynamically (except for lazy-initializations). Here is an example: @@ -495,7 +495,7 @@ public class GitHelperImpl implements GitHelper { // bad private boolean force; - @Overide + @Override public void gitPullOrClone(boolean force, Path target, String gitUrl) { this.force = force; if (Files.isDirectory(target.resolve(".git"))) { @@ -512,7 +512,7 @@ public class GitHelperImpl implements GitHelper { ---- As you can see in the `bad` code fields of the class are assigned at runtime. -Since IDEasy is not implementing a concurremt multi-user application this is not really critical. +Since IDEasy is not implementing a concurrent multi-user application this is not really critical. However, it is best-practice to avoid this pattern and generally follow thread-safe programming as best-practice: [source,java] @@ -520,7 +520,7 @@ However, it is best-practice to avoid this pattern and generally follow thread-s public class GitHelperImpl implements GitHelper { // fine - @Overide + @Override public void gitPullOrClone(boolean force, Path target, String gitUrl) { if (Files.isDirectory(target.resolve(".git"))) { gitPull(force, target); @@ -616,7 +616,7 @@ if (condition()) { //System.err.println("that"); ---- -== Field Initializion +== Field Initialization Non-static fields should never been initialized outside of the constructor call. @@ -683,7 +683,7 @@ John Doe Hi John Doe ``` -One could assume this since during the constructor call the overridden `computeMessage` method is invoked that assignes the `name` variable to `John Doe`. +One could assume this since during the constructor call the overridden `computeMessage` method is invoked that assigns the `name` variable to `John Doe`. However, the output of this program is actually this: