Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Add support for new repository target location type #2844

Merged
merged 1 commit into from
Sep 21, 2023
Merged

Add support for new repository target location type #2844

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
180 changes: 101 additions & 79 deletions ...en-plugin/src/main/java/org/eclipse/tycho/p2maven/transport/TychoRepositoryTransport.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,8 @@
import java.io.OutputStream;
import java.net.URI;
import java.net.URLConnection;
import java.nio.file.Files;
import java.nio.file.Path;
import java.text.NumberFormat;
import java.util.Map;
import java.util.concurrent.Executor;
Expand All @@ -44,25 +46,27 @@

@Component(role = org.eclipse.equinox.internal.p2.repository.Transport.class, hint = "tycho")
public class TychoRepositoryTransport extends org.eclipse.equinox.internal.p2.repository.Transport
implements IAgentServiceFactory {
implements IAgentServiceFactory {

private static final int MAX_DOWNLOAD_THREADS = Integer.getInteger("tycho.p2.transport.max-download-threads", 4);

private static final boolean DEBUG_REQUESTS = Boolean.getBoolean("tycho.p2.transport.debug");

private static final Executor DOWNLOAD_EXECUTOR = Executors.newFixedThreadPool(MAX_DOWNLOAD_THREADS, new ThreadFactory() {

private AtomicInteger cnt = new AtomicInteger();
@Override
public Thread newThread(Runnable r) {
Thread thread = new Thread(r);
thread.setName("Tycho-Download-Thread-" + cnt.getAndIncrement());
thread.setDaemon(true);
return thread;
}
});
private static final Executor DOWNLOAD_EXECUTOR = Executors.newFixedThreadPool(MAX_DOWNLOAD_THREADS,
new ThreadFactory() {

private AtomicInteger cnt = new AtomicInteger();

private NumberFormat numberFormat = NumberFormat.getNumberInstance();
@Override
public Thread newThread(Runnable r) {
Thread thread = new Thread(r);
thread.setName("Tycho-Download-Thread-" + cnt.getAndIncrement());
thread.setDaemon(true);
return thread;
}
});

private NumberFormat numberFormat = NumberFormat.getNumberInstance();

@Requirement
Logger logger;
Expand All @@ -73,24 +77,24 @@ public Thread newThread(Runnable r) {
@Requirement(role = TransportProtocolHandler.class)
Map<String, TransportProtocolHandler> transportProtocolHandlers;

private LongAdder requests = new LongAdder();
private LongAdder indexRequests = new LongAdder();
private LongAdder requests = new LongAdder();
private LongAdder indexRequests = new LongAdder();

public TychoRepositoryTransport() {
numberFormat.setMaximumFractionDigits(2);
}

@Override
public IStatus download(URI toDownload, OutputStream target, long startPos, IProgressMonitor monitor) {
if (startPos > 0) {
return new Status(IStatus.ERROR, TychoRepositoryTransport.class.getName(),
"range downloads are not implemented");
}
return download(toDownload, target, monitor);
}

@Override
public IStatus download(URI toDownload, OutputStream target, IProgressMonitor monitor) {
numberFormat.setMaximumFractionDigits(2);
}

@Override
public IStatus download(URI toDownload, OutputStream target, long startPos, IProgressMonitor monitor) {
if (startPos > 0) {
return new Status(IStatus.ERROR, TychoRepositoryTransport.class.getName(),
"range downloads are not implemented");
}
return download(toDownload, target, monitor);
}

@Override
public IStatus download(URI toDownload, OutputStream target, IProgressMonitor monitor) {
String id = "p2"; // TODO we might compute the id from the IRepositoryIdManager based on the URI?
if (cacheConfig.isInteractive()) {
logger.info("Downloading from " + id + ": " + toDownload);
Expand All @@ -101,41 +105,40 @@ public IStatus download(URI toDownload, OutputStream target, IProgressMonitor mo
stream(toDownload, monitor).transferTo(statusOutputStream);
DownloadStatus downloadStatus = statusOutputStream.getStatus();
if (cacheConfig.isInteractive()) {
logger.info(
"Downloaded from " + id + ": " + toDownload + " ("
+ FileUtils.byteCountToDisplaySize(downloadStatus.getFileSize())
+ " at " + FileUtils.byteCountToDisplaySize(downloadStatus.getTransferRate()) + "/s)");
logger.info("Downloaded from " + id + ": " + toDownload + " ("
+ FileUtils.byteCountToDisplaySize(downloadStatus.getFileSize()) + " at "
+ FileUtils.byteCountToDisplaySize(downloadStatus.getTransferRate()) + "/s)");
}
return reportStatus(downloadStatus, target);
} catch (AuthenticationFailedException e) {
return new Status(IStatus.ERROR, TychoRepositoryTransport.class.getName(),
"authentication failed for " + toDownload, e);
} catch (IOException e) {
return reportStatus(new Status(IStatus.ERROR, TychoRepositoryTransport.class.getName(),
"download from " + toDownload + " failed", e), target);
} catch (CoreException e) {
return reportStatus(e.getStatus(), target);
}
}
} catch (AuthenticationFailedException e) {
return new Status(IStatus.ERROR, TychoRepositoryTransport.class.getName(),
"authentication failed for " + toDownload, e);
} catch (IOException e) {
return reportStatus(new Status(IStatus.ERROR, TychoRepositoryTransport.class.getName(),
"download from " + toDownload + " failed", e), target);
} catch (CoreException e) {
return reportStatus(e.getStatus(), target);
}
}

private IStatus reportStatus(IStatus status, OutputStream target) {
if (target instanceof IStateful stateful) {
stateful.setStatus(status);
}
return status;
}
if (target instanceof IStateful stateful) {
stateful.setStatus(status);
}
return status;
}

@Override
@Override
public InputStream stream(URI toDownload, IProgressMonitor monitor)
throws FileNotFoundException, CoreException, AuthenticationFailedException {
if (DEBUG_REQUESTS) {
logger.debug("Request stream for " + toDownload);
logger.debug("Request stream for " + toDownload);
}
requests.increment();
if (toDownload.toASCIIString().endsWith("p2.index")) {
indexRequests.increment();
}
try {
requests.increment();
if (toDownload.toASCIIString().endsWith("p2.index")) {
indexRequests.increment();
}
try {
TransportProtocolHandler handler = getHandler(toDownload);
if (handler != null) {
File cachedFile = handler.getFile(toDownload);
Expand All @@ -146,28 +149,28 @@ public InputStream stream(URI toDownload, IProgressMonitor monitor)
return new FileInputStream(cachedFile);
}
}
return toDownload.toURL().openStream();
} catch (FileNotFoundException e) {
return toDownload.toURL().openStream();
} catch (FileNotFoundException e) {
if (DEBUG_REQUESTS) {
logger.debug(" --> not found!");
}
throw e;
} catch (IOException e) {
logger.debug(" --> not found!");
}
throw e;
} catch (IOException e) {
if (e instanceof AuthenticationFailedException afe) {
throw afe;
}
if (DEBUG_REQUESTS) {
logger.debug(" --> generic error: " + e);
}
throw new CoreException(new Status(IStatus.ERROR, TychoRepositoryTransport.class.getName(),
"download from " + toDownload + " failed", e));
} finally {
logger.debug(" --> generic error: " + e);
}
throw new CoreException(new Status(IStatus.ERROR, TychoRepositoryTransport.class.getName(),
"download from " + toDownload + " failed", e));
} finally {
if (DEBUG_REQUESTS) {
logger.debug("Total number of requests: " + requests.longValue() + " (" + indexRequests.longValue()
+ " for p2.index)");
}
}
}
logger.debug("Total number of requests: " + requests.longValue() + " (" + indexRequests.longValue()
+ " for p2.index)");
}
}
}

TransportProtocolHandler getHandler(URI uri) {
String scheme = uri.getScheme();
Expand All @@ -181,9 +184,9 @@ TransportProtocolHandler getHandler(URI uri) {
return null;
}

@Override
public long getLastModified(URI toDownload, IProgressMonitor monitor)
throws CoreException, FileNotFoundException, AuthenticationFailedException {
@Override
public long getLastModified(URI toDownload, IProgressMonitor monitor)
throws CoreException, FileNotFoundException, AuthenticationFailedException {
try {
TransportProtocolHandler handler = getHandler(toDownload);
if (handler != null) {
Expand All @@ -199,14 +202,33 @@ public long getLastModified(URI toDownload, IProgressMonitor monitor)
throw new CoreException(new Status(IStatus.ERROR, TychoRepositoryTransport.class.getName(),
"download from " + toDownload + " failed", e));
}
}
@Override
public Object createService(IProvisioningAgent agent) {
return this;
}
}

@Override
public Object createService(IProvisioningAgent agent) {
return this;
}

public static Executor getDownloadExecutor() {
return DOWNLOAD_EXECUTOR;
}

public File downloadToFile(URI uri) throws IOException {
TransportProtocolHandler handler = getHandler(uri);
if (handler != null) {
File file = handler.getFile(uri);
if (file != null) {
return file;
}
}
Path tempFile = Files.createTempFile("tycho", ".tmp");
tempFile.toFile().deleteOnExit();
try {
Files.copy(stream(uri, null), tempFile);
return tempFile.toFile();
} catch (CoreException e) {
throw new IOException(e);
}
}

}
132 changes: 132 additions & 0 deletions tycho-core/src/main/java/org/eclipse/tycho/p2resolver/RepositoryLocationContent.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,132 @@
/*******************************************************************************
* Copyright (c) 2023 Christoph Läubrich and others.
*
* This program and the accompanying materials
* are made available under the terms of the Eclipse Public License 2.0
* which accompanies this distribution, and is available at
* https://www.eclipse.org/legal/epl-2.0/
*
* SPDX-License-Identifier: EPL-2.0
*
* Contributors:
* Christoph Läubrich - initial API and implementation
*******************************************************************************/
package org.eclipse.tycho.p2resolver;

import java.io.File;
import java.io.InputStream;
import java.net.URI;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.function.Predicate;

import org.apache.commons.io.FilenameUtils;
import org.eclipse.equinox.internal.p2.publisher.eclipse.FeatureParser;
import org.eclipse.equinox.internal.p2.repository.Transport;
import org.eclipse.equinox.p2.core.IProvisioningAgent;
import org.eclipse.equinox.p2.metadata.IArtifactKey;
import org.eclipse.equinox.p2.metadata.IInstallableUnit;
import org.eclipse.equinox.p2.publisher.IPublisherInfo;
import org.eclipse.equinox.p2.publisher.PublisherInfo;
import org.eclipse.equinox.p2.publisher.eclipse.BundlesAction;
import org.eclipse.equinox.p2.publisher.eclipse.Feature;
import org.eclipse.equinox.p2.repository.artifact.IArtifactDescriptor;
import org.eclipse.equinox.p2.repository.artifact.IArtifactRepository;
import org.eclipse.equinox.p2.repository.metadata.IMetadataRepository;
import org.eclipse.osgi.service.resolver.BundleDescription;
import org.eclipse.tycho.core.resolver.target.FileArtifactRepository;
import org.eclipse.tycho.core.resolver.target.SupplierMetadataRepository;
import org.eclipse.tycho.core.shared.MavenLogger;
import org.eclipse.tycho.p2.resolver.BundlePublisher;
import org.eclipse.tycho.p2.resolver.FeaturePublisher;
import org.eclipse.tycho.p2maven.transport.TychoRepositoryTransport;
import org.eclipse.tycho.targetplatform.TargetDefinitionContent;
import org.eclipse.tycho.targetplatform.TargetDefinitionResolutionException;
import org.osgi.resource.Capability;
import org.osgi.resource.Requirement;

import aQute.bnd.osgi.repository.ResourcesRepository;
import aQute.bnd.osgi.repository.XMLResourceParser;
import aQute.bnd.osgi.resource.ResourceUtils;
import aQute.bnd.osgi.resource.ResourceUtils.ContentCapability;

public class RepositoryLocationContent implements TargetDefinitionContent {
private final Map<IArtifactDescriptor, IInstallableUnit> repositoryContent = new HashMap<>();
private SupplierMetadataRepository metadataRepository;
private FileArtifactRepository artifactRepository;

public RepositoryLocationContent(URI uri, Collection<Requirement> requirements, IProvisioningAgent agent,
MavenLogger logger) throws TargetDefinitionResolutionException {
TychoRepositoryTransport tychoTransport = (TychoRepositoryTransport) agent.getService(Transport.class);

metadataRepository = new SupplierMetadataRepository(agent, () -> repositoryContent.values().iterator());
metadataRepository.setLocation(uri);
metadataRepository.setName(String.valueOf(uri));
artifactRepository = new FileArtifactRepository(agent, () -> repositoryContent.keySet().stream()
.filter(Predicate.not(FeaturePublisher::isMetadataOnly)).iterator());
artifactRepository.setName(String.valueOf(uri));
artifactRepository.setLocation(uri);
List<Feature> features = new ArrayList<>();
ResourcesRepository repository;
try (InputStream stream = tychoTransport.stream(uri, null)) {
repository = new ResourcesRepository(XMLResourceParser.getResources(stream, uri));
} catch (Exception e) {
throw new TargetDefinitionResolutionException("Can't load the repository from URI " + uri, e);
}
Map<Requirement, Collection<Capability>> providers = repository.findProviders(requirements);
//TODO once we have changed Tycho to use resources this can be optimized to not download all selected content here ...
List<ContentCapability> contentCapabilities = providers.values().stream().flatMap(Collection::stream)
.map(Capability::getResource).distinct().map(ResourceUtils::getContentCapability)
.filter(Objects::nonNull).toList();
for (ContentCapability content : contentCapabilities) {
URI url = content.url();
logger.info("Loading " + url + "...");
try {
File file = tychoTransport.downloadToFile(url);
if (!"jar".equalsIgnoreCase(FilenameUtils.getExtension(file.getName()))) {
logger.info("Skip non-jar artifact (" + file + ")");
continue;
}
Feature feature = new FeatureParser().parse(file);
if (feature != null) {
feature.setLocation(file.getAbsolutePath());
features.add(feature);
continue;
}
BundleDescription bundleDescription = BundlesAction.createBundleDescription(file);
if (bundleDescription == null || bundleDescription.getSymbolicName() == null) {
continue;
}
publish(bundleDescription, file);
} catch (Exception e) {
throw new TargetDefinitionResolutionException("Can't fetch resource from " + url, e);
}
}
FeaturePublisher.publishFeatures(features, repositoryContent::put, logger);
}

private void publish(BundleDescription bundleDescription, File bundleLocation) {
IArtifactKey key = BundlesAction.createBundleArtifactKey(bundleDescription.getSymbolicName(),
bundleDescription.getVersion().toString());
IArtifactDescriptor descriptor = FileArtifactRepository.forFile(bundleLocation, key);
PublisherInfo publisherInfo = new PublisherInfo();
publisherInfo.setArtifactOptions(IPublisherInfo.A_INDEX);
IInstallableUnit iu = BundlePublisher.publishBundle(bundleDescription, descriptor, publisherInfo);
repositoryContent.put(descriptor, iu);
}

@Override
public IMetadataRepository getMetadataRepository() {
return metadataRepository;
}

@Override
public IArtifactRepository getArtifactRepository() {
return artifactRepository;
}

}
Loading
Loading