Skip to content

Commit

Permalink
Added Mac Os aarch64 support for helm and terraform.
Browse files Browse the repository at this point in the history 
Refactored UpdateManager
Fixxed problem with HelmCrawler, which caused no versions to be found.
Fixxed MavenCrawler not creating status.json files.
Rancher now is part of the folder docker.
Added support For Kotlin due to ticket devonfw#1039
  • Loading branch information
@Amueller36
Amueller36 committed Feb 21, 2023
1 parent 970e34a commit 17f26de
Show file tree
Hide file tree
Showing 8 changed files with 82 additions and 119 deletions.
2 changes: 1 addition & 1 deletion url-updater/src/main/java/com/devonfw/tools/ide/url/updater/AbstractCrawler.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -187,7 +187,7 @@ public void update(UrlRepository urlRepository) {
}
}

public void updateExistingVersions(UrlEdition edition) {
protected void updateExistingVersions(UrlEdition edition) {
List<String> existingVersions = edition.getListOfAllChildren();
edition.getListOfAllChildren().stream().filter(existingVersions::contains).forEach(version -> {
UrlVersion urlVersion = edition.getChild(version);
Expand Down
8 changes: 1 addition & 7 deletions url-updater/src/main/java/com/devonfw/tools/ide/url/updater/MavenCrawler.java
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
package com.devonfw.tools.ide.url.updater;

import com.devonfw.tools.ide.url.updater.mavenapiclasses.Metadata;
import com.devonfw.tools.ide.url.folderhandling.UrlDownloadFile;
import com.devonfw.tools.ide.url.folderhandling.UrlVersion;
import com.fasterxml.jackson.dataformat.xml.XmlMapper;

Expand Down Expand Up @@ -33,12 +32,7 @@ protected String getExtension(){
protected void updateVersion(UrlVersion urlVersion) {
String version = urlVersion.getName();
String url = mavenBaseRepoUrl + version + "/" + getArtifcatId() + "-" + version + getExtension();
Result resultOfHttpRequest= doCheckIfDownloadUrlWorks(url);
if(resultOfHttpRequest.isSuccess()){
UrlDownloadFile urlDownloadFile = urlVersion.getOrCreateUrls();
urlDownloadFile.addUrl(url);
}
urlVersion.save();
doUpdateVersion(urlVersion, url);
}


Expand Down
144 changes: 37 additions & 107 deletions url-updater/src/main/java/com/devonfw/tools/ide/url/updater/UpdateManager.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
import com.devonfw.tools.ide.url.updater.intellij.IntelliJUltimateEditionCrawler;
import com.devonfw.tools.ide.url.updater.java.JavaCrawler;
import com.devonfw.tools.ide.url.updater.jenkins.JenkinsCrawler;
import com.devonfw.tools.ide.url.updater.kotlin.KotlinCrawler;
import com.devonfw.tools.ide.url.updater.lazydocker.LazyDockerCrawler;
import com.devonfw.tools.ide.url.updater.maven.MavenCrawler;
import com.devonfw.tools.ide.url.updater.nodejs.NodeJsCrawler;
Expand All @@ -30,6 +31,8 @@
import com.devonfw.tools.ide.url.folderhandling.UrlRepository;

import java.nio.file.Path;
import java.util.Arrays;
import java.util.List;

public class UpdateManager {
private final UrlRepository urlRepository;
Expand All @@ -38,117 +41,44 @@ public UpdateManager(Path pathToRepository) {
this.urlRepository = UrlRepository.load(pathToRepository);
}

private final AWSCrawler awsCrawler = new AWSCrawler();
private final AzureCrawler azureCrawler = new AzureCrawler();
private final CobigenCrawler cobigenCrawler = new CobigenCrawler();
private final DotNetCrawler dotNetCrawler = new DotNetCrawler();
private final DockerCrawler dockerCrawler = new DockerCrawler();
private final EclipseCppCrawler eclipseCppCrawler = new EclipseCppCrawler();
private final List<AbstractCrawler> crawlers = Arrays.asList(
new AWSCrawler(),
new AzureCrawler(),
new CobigenCrawler(),
new DotNetCrawler(),
new DockerCrawler(),
new EclipseCppCrawler(),
new EclipseJavaCrawler(),
new GCViewerCrawler(),
new GHCrawler(),
new GraalVMCrawler(),
new GradleCrawler(),
new HelmCrawler(),
new IntelliJUltimateEditionCrawler(),
new IntelliJCommunityEditionCrawler(),
new JavaCrawler(),
new JenkinsCrawler(),
new KotlinCrawler(),
new LazyDockerCrawler(),
new MavenCrawler(),
new NodeJsCrawler(),
new NpmCrawler(),
new OcCrawler(),
new PipCrawler(),
new PythonCrawler(),
new QuarkusCrawler(),
new RancherCrawler(),
new SonarqubeCrawler(),
new TerraformCrawler(),
new VSCodeCrawler()
);

private final EclipseJavaCrawler eclipseJavaCrawler = new EclipseJavaCrawler();

private final GCViewerCrawler gcViewerCrawler = new GCViewerCrawler();

private final GHCrawler ghCrawler = new GHCrawler();

private final GraalVMCrawler graalvmCrawler = new GraalVMCrawler();

private final GradleCrawler gradleCrawler = new GradleCrawler();

private final HelmCrawler helmCrawler = new HelmCrawler();

private final IntelliJUltimateEditionCrawler intellijUltimateCrawler = new IntelliJUltimateEditionCrawler();

private final IntelliJCommunityEditionCrawler intellijCommunityEditionCrawler = new IntelliJCommunityEditionCrawler();

private final JavaCrawler javaCrawler = new JavaCrawler();

private final JenkinsCrawler jenkinsCrawler = new JenkinsCrawler();

private final LazyDockerCrawler lazyDockerCrawler = new LazyDockerCrawler();

private final MavenCrawler mavenCrawler = new MavenCrawler();

private final NodeJsCrawler nodeJSCrawler = new NodeJsCrawler();

private final NpmCrawler npmCrawler = new NpmCrawler();

private final OcCrawler ocCrawler = new OcCrawler();

private final PipCrawler pipCrawler = new PipCrawler();

private final PythonCrawler pythonCrawler = new PythonCrawler();

private final QuarkusCrawler quarkusCrawler = new QuarkusCrawler();

private final RancherCrawler rancherCrawler = new RancherCrawler();

private final SonarqubeCrawler sonarqubeCrawler = new SonarqubeCrawler();

private final TerraformCrawler terraformCrawler = new TerraformCrawler();

private final VSCodeCrawler vsCodeCrawler = new VSCodeCrawler();

public void updateAll() {
awsCrawler.update(urlRepository);
azureCrawler.update(urlRepository);
cobigenCrawler.update(urlRepository);
dotNetCrawler.update(urlRepository);
dockerCrawler.update(urlRepository);
eclipseCppCrawler.update(urlRepository);
eclipseJavaCrawler.update(urlRepository);
gcViewerCrawler.update(urlRepository);
ghCrawler.update(urlRepository);
graalvmCrawler.update(urlRepository);
gradleCrawler.update(urlRepository);
helmCrawler.update(urlRepository);
javaCrawler.update(urlRepository);
intellijUltimateCrawler.update(urlRepository);
intellijCommunityEditionCrawler.update(urlRepository);
jenkinsCrawler.update(urlRepository);
lazyDockerCrawler.update(urlRepository);
mavenCrawler.update(urlRepository);
nodeJSCrawler.update(urlRepository);
npmCrawler.update(urlRepository);
ocCrawler.update(urlRepository);
pipCrawler.update(urlRepository);
pythonCrawler.update(urlRepository);
quarkusCrawler.update(urlRepository);
rancherCrawler.update(urlRepository);
sonarqubeCrawler.update(urlRepository);
terraformCrawler.update(urlRepository);
vsCodeCrawler.update(urlRepository);
for (AbstractCrawler crawler : crawlers) {
crawler.update(urlRepository);
}
}

// public void updateOnlyExistingVersions() {
// awsCrawler.updateExistingVersions(urlRepository);
// azureCrawler.updateExistingVersions(urlRepository);
// cobigenCrawler.updateExistingVersions(urlRepository);
// dotNetCrawler.updateExistingVersions(urlRepository);
// dockerCrawler.updateExistingVersions(urlRepository);
// eclipseCppCrawler.updateExistingVersions(urlRepository);
// eclipseJavaCrawler.updateExistingVersions(urlRepository);
// gcViewerCrawler.updateExistingVersions(urlRepository);
// ghCrawler.updateExistingVersions(urlRepository);
// graalvmCrawler.updateExistingVersions(urlRepository);
// gradleCrawler.updateExistingVersions(urlRepository);
// helmCrawler.updateExistingVersions(urlRepository);
// javaCrawler.updateExistingVersions(urlRepository);
// intellijUltimateCrawler.updateExistingVersions(urlRepository);
// intellijCommunityEditionCrawler.updateExistingVersions(urlRepository);
// jenkinsCrawler.updateExistingVersions(urlRepository);
// lazyDockerCrawler.updateExistingVersions(urlRepository);
// mavenCrawler.updateExistingVersions(urlRepository);
// nodeJSCrawler.updateExistingVersions(urlRepository);
// npmCrawler.updateExistingVersions(urlRepository);
// ocCrawler.updateExistingVersions(urlRepository);
// pipCrawler.updateExistingVersions(urlRepository);
// pythonCrawler.updateExistingVersions(urlRepository);
// quarkusCrawler.updateExistingVersions(urlRepository);
// rancherCrawler.updateExistingVersions(urlRepository);
// sonarqubeCrawler.updateExistingVersions(urlRepository);
// terraformCrawler.updateExistingVersions(urlRepository);
// vsCodeCrawler.updateExistingVersions(urlRepository);
// }

}
7 changes: 4 additions & 3 deletions url-updater/src/main/java/com/devonfw/tools/ide/url/updater/helm/HelmCrawler.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,9 +13,10 @@ protected String getToolName() {

@Override
protected void updateVersion(UrlVersion urlVersion) {
doUpdateVersion(urlVersion,"https://get.helm.sh/helm-${version}-windows-amd64.zip", OSType.WINDOWS);
doUpdateVersion(urlVersion,"https://get.helm.sh/helm-${version}-linux-amd64.tar.gz", OSType.LINUX);
doUpdateVersion(urlVersion,"https://get.helm.sh/helm-${version}-darwin-amd64.tar.gz", OSType.MAC);
doUpdateVersion(urlVersion,"https://get.helm.sh/helm-v${version}-windows-amd64.zip", OSType.WINDOWS);
doUpdateVersion(urlVersion,"https://get.helm.sh/helm-v${version}-linux-amd64.tar.gz", OSType.LINUX);
doUpdateVersion(urlVersion,"https://get.helm.sh/helm-v${version}-darwin-amd64.tar.gz", OSType.MAC);
doUpdateVersion(urlVersion,"https://get.helm.sh/helm-v${version}-darwin-arm64.tar.gz", OSType.MAC, "arm64");
}

@Override
Expand Down
2 changes: 1 addition & 1 deletion url-updater/src/main/java/com/devonfw/tools/ide/url/updater/intellij/IntelliJCrawler.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,8 +41,8 @@ protected Set<String> getVersions() {
try {
driver.get(url);
String html = driver.getPageSource();
Thread.sleep(5000);
driver.quit();
System.out.println(html);
return doGetRegexMatchesAsList(html);
} catch (Exception e) {
throw new IllegalStateException("Failed to retrieve response body from url: " + url, e);
Expand Down
32 changes: 32 additions & 0 deletions url-updater/src/main/java/com/devonfw/tools/ide/url/updater/kotlin/KotlinCrawler.java
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,32 @@
package com.devonfw.tools.ide.url.updater.kotlin;

import com.devonfw.tools.ide.url.folderhandling.UrlVersion;
import com.devonfw.tools.ide.url.updater.OSType;
import com.devonfw.tools.ide.url.updater.WebsiteCrawler;

import java.util.regex.Pattern;

public class KotlinCrawler extends WebsiteCrawler {
@Override
protected String getToolName() {
return "kotlin";
}

@Override
protected void updateVersion(UrlVersion urlVersion) {
doUpdateVersion(urlVersion, "https://github.com/JetBrains/kotlin/releases/download/v${version}/kotlin-native-windows-x86_64-${version}.zip", OSType.WINDOWS,"x64");
doUpdateVersion(urlVersion, "https://github.com/JetBrains/kotlin/releases/download/v${version}/kotlin-native-linux-x86_64-${version}.tar.gz", OSType.LINUX,"x64");
doUpdateVersion(urlVersion, "https://github.com/JetBrains/kotlin/releases/download/v${version}/kotlin-native-macos-x86_64-${version}.tar.gz", OSType.MAC,"x64");
doUpdateVersion(urlVersion, "https://github.com/JetBrains/kotlin/releases/download/v${version}/kotlin-native-macos-aarch64-${version}.tar.gz", OSType.MAC,"arm64");
}

@Override
protected Pattern getVersionPattern() {
return Pattern.compile("[0-9]+\\.[0-9]+\\.[0-9]+");
}

@Override
protected String getVersionUrl() {
return "https://api.github.com/repos/JetBrains/kotlin/releases";
}
}
5 changes: 5 additions & 0 deletions url-updater/src/main/java/com/devonfw/tools/ide/url/updater/rancher/RancherCrawler.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,11 @@
public class RancherCrawler extends GithubCrawler {
@Override
protected String getToolName() {
return "docker";
}

@Override
protected String getEdition() {
return "rancher-desktop";
}

Expand Down
1 change: 1 addition & 0 deletions url-updater/src/main/java/com/devonfw/tools/ide/url/updater/terraform/TerraformCrawler.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ protected void updateVersion(UrlVersion urlVersion) {
doUpdateVersion(urlVersion, "https://releases.hashicorp.com/terraform/${version}/terraform_${version}_windows_amd64.zip", OSType.WINDOWS);
doUpdateVersion(urlVersion, "https://releases.hashicorp.com/terraform/${version}/terraform_${version}_linux_amd64.zip", OSType.LINUX);
doUpdateVersion(urlVersion, "https://releases.hashicorp.com/terraform/${version}/terraform_${version}_darwin_amd64.zip", OSType.MAC);
doUpdateVersion(urlVersion, "https://releases.hashicorp.com/terraform/${version}/terraform_${version}_darwin_arm64.zip", OSType.MAC, "arm64");
}

@Override
Expand Down

0 comments on commit 17f26de

Please sign in to comment.