ORCH-487 Add support for downloading artifacts without repox

README.md

@@ -71,6 +71,9 @@ Aliases can be used to define the versions of SonarQube and plugins to be instal
 
 The alias `LTS` is no more supported for SonarQube since Orchestrator 3.17. It should be replaced by `LATEST_RELEASE[6.7]`.
 
+Please note that since Orchestrator 4.7, if the default value of `orchestrator.artifactory.url` (https://repox.jfrog.io/repox) is _not_ used, 
+the `DEV` and `DOGFOOD` aliases will not work.
+
 ## Local Cache
 
 The artifacts downloaded from Artifactory (SonarQube, plugins) are copied to the local directory `~/.sonar/orchestrator/cache`.
@@ -103,9 +106,11 @@ The test environment is configured in the file `~/.sonar/orchestrator/orchestrat
     # Default is ~/.m2/repository
     #maven.localRepository=/path/to/maven/repository
 
-    # Instance of Artifactory. Default is SonarSource's instance.
+    # Instance of Artifactory. Default is SonarSource's instance (https://repox.jfrog.io/repox).
     # This property can be replaced by the environment variable ARTIFACTORY_URL.
-    #orchestrator.artifactory.url=https://repox.jfrog.io/repox
+    # To use maven central instead, use https://repo1.maven.org/maven2
+    # To use a custom instance, use your own URL that points to a Maven repository.
+    #orchestrator.artifactory.url=https://repo1.maven.org/maven2
 
 The path to configuration file can be overridden with the system property `orchestrator.configUrl`
 or the environment variable `ORCHESTRATOR_CONFIG_URL`.

sonar-orchestrator/pom.xml

@@ -12,7 +12,7 @@
   <properties>
     <maven.test.redirectTestOutputToFile>true</maven.test.redirectTestOutputToFile>
     <okhttp.version>4.5.0</okhttp.version>
-
+    <jupiter.version>5.9.1</jupiter.version>
     <jacoco.version>0.8.10</jacoco.version>
   </properties>
 
@@ -52,6 +52,11 @@
       <artifactId>minimal-json</artifactId>
       <version>0.9.5</version>
     </dependency>
+    <dependency>
+      <groupId>com.fasterxml.jackson.dataformat</groupId>
+      <artifactId>jackson-dataformat-xml</artifactId>
+      <version>2.13.2</version>
+    </dependency>
 
     <!-- Apache HTTP Client is kept for deprecated methods Server#getWsClient()
     and Server#getAdminWsClient() -->
@@ -154,7 +159,12 @@
       <version>${okhttp.version}</version>
       <scope>test</scope>
     </dependency>
-
+    <dependency>
+      <groupId>org.junit.jupiter</groupId>
+      <artifactId>junit-jupiter-params</artifactId>
+      <version>${jupiter.version}</version>
+      <scope>test</scope>
+    </dependency>
   </dependencies>
 
   <build>

sonar-orchestrator/src/main/java/com/sonar/orchestrator/config/Configuration.java

@@ -19,7 +19,7 @@
  */
 package com.sonar.orchestrator.config;
 
-import com.sonar.orchestrator.locator.ArtifactoryImpl;
+import com.sonar.orchestrator.locator.ArtifactoryFactory;
 import com.sonar.orchestrator.locator.FileLocation;
 import com.sonar.orchestrator.locator.Locators;
 import java.io.File;
@@ -56,7 +56,7 @@ public class Configuration {
   private Configuration(File homeDir, Map<String, String> props) {
     this.props = Collections.unmodifiableMap(new HashMap<>(props));
     this.fileSystem = new FileSystem(homeDir, this);
-    this.locators = new Locators(this.fileSystem, ArtifactoryImpl.create(this));
+    this.locators = new Locators(this.fileSystem, ArtifactoryFactory.createArtifactory(this));
   }
 
   public FileSystem fileSystem() {

sonar-orchestrator/src/main/java/com/sonar/orchestrator/locator/Artifactory.java

@@ -19,13 +19,138 @@
  */
 package com.sonar.orchestrator.locator;
 
+import com.eclipsesource.json.Json;
+import com.eclipsesource.json.JsonArray;
+import com.sonar.orchestrator.http.HttpCall;
+import com.sonar.orchestrator.http.HttpClientFactory;
+import com.sonar.orchestrator.http.HttpException;
 import java.io.File;
+import java.io.IOException;
 import java.util.Optional;
+import java.util.stream.Collectors;
+import java.util.stream.StreamSupport;
+import javax.annotation.Nullable;
+import okhttp3.HttpUrl;
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.lang.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
-public interface Artifactory {
-  boolean downloadToFile(MavenLocation location, File toFile);
+import static com.sonar.orchestrator.util.OrchestratorUtils.isEmpty;
+import static java.net.HttpURLConnection.HTTP_FORBIDDEN;
+import static java.net.HttpURLConnection.HTTP_NOT_FOUND;
+import static java.net.HttpURLConnection.HTTP_UNAUTHORIZED;
 
-  Optional<File> downloadToDir(MavenLocation location, File toDir);
+public abstract class Artifactory {
+
+  private static final Logger LOG = LoggerFactory.getLogger(Artifactory.class);
+
+  protected final File tempDir;
+  protected final String baseUrl;
+  @Nullable
+  protected final String apiKey;
+  @Nullable
+  protected final String accessToken;
+
+  protected Artifactory(File tempDir, String baseUrl, @Nullable String accessToken, @Nullable String apiKey) {
+    this.tempDir = tempDir;
+    this.baseUrl = baseUrl;
+    this.apiKey = apiKey;
+    this.accessToken = accessToken;
+  }
+
+  protected boolean moveFile(File tempFile, File toFile) {
+    try {
+      FileUtils.deleteQuietly(toFile);
+      FileUtils.moveFile(tempFile, toFile);
+      return true;
+    } catch (IOException e) {
+      throw new IllegalStateException("Fail to move file " + toFile, e);
+    }
+  }
+
+  protected Optional<File> downloadToDir(MavenLocation location, File toDir, @Nullable String repository) {
+    HttpUrl.Builder urlBuilder = HttpUrl.parse(baseUrl).newBuilder();
+    if (!StringUtils.isEmpty(repository)) {
+      urlBuilder.addPathSegment(repository);
+    }
+    HttpUrl url = urlBuilder.addEncodedPathSegments(StringUtils.replace(location.getGroupId(), ".", "/"))
+      .addPathSegment(location.getArtifactId())
+      .addPathSegment(location.getVersion())
+      .addPathSegment(location.getFilename())
+      .build();
+
+    HttpCall call = newArtifactoryCall(url);
+    try {
+      LOG.info("Downloading {}", url);
+      File toFile = call.downloadToDirectory(toDir);
+      LOG.info("Found {} at {}", location, url);
+      return Optional.of(toFile);
+    } catch (HttpException e) {
+      if (e.getCode() != HTTP_NOT_FOUND && e.getCode() != HTTP_UNAUTHORIZED && e.getCode() != HTTP_FORBIDDEN) {
+        throw new IllegalStateException("Failed to request " + url, e);
+      } else {
+        String errorMessage;
+        try {
+          JsonArray errors = Json.parse(e.getBody()).asObject().get("errors").asArray();
+          errorMessage = StreamSupport.stream(errors.spliterator(), false)
+            .map(item -> item.asObject().get("message").asString())
+            .collect(Collectors.joining(", "));
+        } catch (Exception ignored) {
+          errorMessage = "--- Failed to parse response body -- ";
+        }
+        LOG.warn("Could not download artifact from repository '{}': {} - {}",
+          repository,
+          e.getCode(),
+          errorMessage
+        );
+      }
+    }
+    return Optional.empty();
+  }
+
+  protected HttpCall newArtifactoryCall(HttpUrl url) {
+    HttpCall call = HttpClientFactory.create().newCall(url);
+    if (!isEmpty(accessToken)) {
+      call.setHeader("Authorization", "Bearer " + accessToken);
+    } else if (!isEmpty(apiKey)) {
+      call.setHeader("X-JFrog-Art-Api", apiKey);
+    }
+    return call;
+  }
+
+  /**
+   * Examples:
+   * "LATEST_RELEASE" -> ""
+   * "LATEST_RELEASE[7]" -> "7"
+   * "LATEST_RELEASE[7.1]" -> "7.1"
+   * "LATEST_RELEASE[7.1.2]" -> "7.1.2"
+   */
+  protected static String extractVersionFromAlias(String s) {
+    int start = s.indexOf('[');
+    int end = s.indexOf(']');
+    if (start >= 0 && end > start) {
+      return s.substring(start + 1, end);
+    }
+    return "";
+  }
+
+  protected String getBaseUrl() {
+    return baseUrl;
+  }
+
+  protected String getApiKey() {
+    return apiKey;
+  }
+
+  protected String getAccessToken() {
+    return accessToken;
+  }
+
+  public abstract Optional<String> resolveVersion(MavenLocation location);
+
+  public abstract boolean downloadToFile(MavenLocation location, File toFile);
+
+  public abstract Optional<File> downloadToDir(MavenLocation location, File toDir);
 
-  Optional<String> resolveVersion(MavenLocation location);
 }

sonar-orchestrator/src/main/java/com/sonar/orchestrator/locator/ArtifactoryFactory.java

@@ -0,0 +1,55 @@
+/*
+ * Orchestrator
+ * Copyright (C) 2011-2024 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+package com.sonar.orchestrator.locator;
+
+import com.sonar.orchestrator.config.Configuration;
+import java.io.File;
+
+import static com.sonar.orchestrator.util.OrchestratorUtils.defaultIfEmpty;
+
+public class ArtifactoryFactory {
+
+  private static final String DEFAULT_ARTIFACTORY_URL = "https://repox.jfrog.io/repox";
+
+  /**
+   * Two types of Artifactory are supported: Maven and Default.
+   *
+   * <p>
+   * Default repox artifactory is used if orchestrator.artifactory.url is empty or specifically set to <a href="https://repox.jfrog.io/repox">repox</a>.
+   * Otherwise, we assume the URL points to a maven repository.
+   * </p>
+   */
+  public static Artifactory createArtifactory(Configuration configuration) {
+    File downloadTempDir = new File(configuration.fileSystem().workspace(), "temp-downloads");
+    String baseUrl = defaultIfEmpty(configuration.getStringByKeys("orchestrator.artifactory.url", "ARTIFACTORY_URL"), DEFAULT_ARTIFACTORY_URL);
+
+    if (baseUrl.startsWith(DEFAULT_ARTIFACTORY_URL)) {
+      String accessToken = configuration.getStringByKeys("orchestrator.artifactory.accessToken", "ARTIFACTORY_ACCESS_TOKEN");
+      String apiKey = configuration.getStringByKeys("orchestrator.artifactory.apiKey", "ARTIFACTORY_API_KEY");
+      return new DefaultArtifactory(downloadTempDir, baseUrl, accessToken, apiKey);
+    } else {
+      return new MavenArtifactory(downloadTempDir, baseUrl);
+    }
+  }
+
+  private ArtifactoryFactory() {
+  }
+
+}