diff --git a/p2-maven-plugin/src/main/java/org/eclipse/tycho/p2maven/transport/SharedHttpCacheStorage.java b/p2-maven-plugin/src/main/java/org/eclipse/tycho/p2maven/transport/SharedHttpCacheStorage.java index 8621450136..ca3e8cc54d 100644 --- a/p2-maven-plugin/src/main/java/org/eclipse/tycho/p2maven/transport/SharedHttpCacheStorage.java +++ b/p2-maven-plugin/src/main/java/org/eclipse/tycho/p2maven/transport/SharedHttpCacheStorage.java @@ -33,6 +33,7 @@ import java.util.concurrent.TimeUnit; import java.util.function.Function; import java.util.stream.Collectors; +import java.util.zip.GZIPInputStream; import org.apache.commons.io.FileUtils; import org.codehaus.plexus.component.annotations.Component; @@ -45,112 +46,108 @@ public class SharedHttpCacheStorage implements HttpCache { private static final int MAX_CACHE_LINES = Integer.getInteger("tycho.p2.transport.max-cache-lines", 1000); /** - * Assumes the following minimum caching period for remote files in minutes - */ - //TODO can we sync this with the time where maven updates snapshots? - public static final long MIN_CACHE_PERIOD = Long.getLong("tycho.p2.transport.min-cache-minutes", - TimeUnit.HOURS.toMinutes(1)); - private static final String LAST_MODIFIED_HEADER = "Last-Modified"; - private static final String EXPIRES_HEADER = "Expires"; - private static final String CACHE_CONTROL_HEADER = "Cache-Control"; - private static final String MAX_AGE_DIRECTIVE = "max-age"; - private static final String MUST_REVALIDATE_DIRECTIVE = "must-revalidate"; + * Assumes the following minimum caching period for remote files in minutes + */ + // TODO can we sync this with the time where maven updates snapshots? + public static final long MIN_CACHE_PERIOD = Long.getLong("tycho.p2.transport.min-cache-minutes", + TimeUnit.HOURS.toMinutes(1)); + private static final String LAST_MODIFIED_HEADER = "Last-Modified"; + private static final String EXPIRES_HEADER = "Expires"; + private static final String CACHE_CONTROL_HEADER = "Cache-Control"; + private static final String MAX_AGE_DIRECTIVE = "max-age"; + private static final String MUST_REVALIDATE_DIRECTIVE = "must-revalidate"; - private static final String ETAG_HEADER = "ETag"; + private static final String ETAG_HEADER = "ETag"; - private static final int MAX_IN_MEMORY = 1000; + private static final int MAX_IN_MEMORY = 1000; @Requirement TransportCacheConfig cacheConfig; - private final Map entryCache; - + private final Map entryCache; public SharedHttpCacheStorage() { entryCache = new LinkedHashMap<>(MAX_CACHE_LINES, 0.75f, true) { - private static final long serialVersionUID = 1L; + private static final long serialVersionUID = 1L; - @Override - protected boolean removeEldestEntry(final Map.Entry eldest) { - return (size() > MAX_IN_MEMORY); - } + @Override + protected boolean removeEldestEntry(final Map.Entry eldest) { + return (size() > MAX_IN_MEMORY); + } - }; - } + }; + } - /** - * Fetches the cache entry for this URI - * - * @param uri - * @return - * @throws FileNotFoundException - * if the URI is know to be not found - */ + /** + * Fetches the cache entry for this URI + * + * @param uri + * @return + * @throws FileNotFoundException if the URI is know to be not found + */ @Override public CacheEntry getCacheEntry(URI uri, Logger logger) throws FileNotFoundException { URI normalized = uri.normalize(); CacheLine cacheLine = getCacheLine(normalized); if (!cacheConfig.isUpdate()) { // if not updates are forced ... - int code = cacheLine.getResponseCode(); - if (code == HttpURLConnection.HTTP_NOT_FOUND) { + int code = cacheLine.getResponseCode(); + if (code == HttpURLConnection.HTTP_NOT_FOUND) { throw new FileNotFoundException(normalized.toASCIIString()); - } - if (code == HttpURLConnection.HTTP_MOVED_PERM) { + } + if (code == HttpURLConnection.HTTP_MOVED_PERM) { return getCacheEntry(cacheLine.getRedirect(normalized), logger); - } - } - return new CacheEntry() { + } + } + return new CacheEntry() { - @Override - public long getLastModified(HttpTransportFactory transportFactory) - throws IOException { + @Override + public long getLastModified(HttpTransportFactory transportFactory) throws IOException { if (cacheConfig.isOffline()) { return cacheLine.getLastModified(normalized, transportFactory, - SharedHttpCacheStorage::mavenIsOffline, logger); - } - try { + SharedHttpCacheStorage::mavenIsOffline, logger); + } + try { return cacheLine.fetchLastModified(normalized, transportFactory, logger); - } catch (FileNotFoundException | AuthenticationFailedException e) { - //for not found and failed authentication we can't do anything useful - throw e; - } catch (IOException e) { + } catch (FileNotFoundException | AuthenticationFailedException e) { + // for not found and failed authentication we can't do anything useful + throw e; + } catch (IOException e) { if (!cacheConfig.isUpdate() && cacheLine.getResponseCode() > 0) { - //if we have something cached, use that ... + // if we have something cached, use that ... logger.warn("Request to " + normalized + " failed, trying cache instead"); return cacheLine.getLastModified(normalized, transportFactory, nil -> e, logger); - } - throw e; - } - } - - @Override - public File getCacheFile(HttpTransportFactory transportFactory) - throws IOException { + } + throw e; + } + } + + @Override + public File getCacheFile(HttpTransportFactory transportFactory) throws IOException { if (cacheConfig.isOffline()) { - return cacheLine.getFile(normalized, transportFactory, - SharedHttpCacheStorage::mavenIsOffline, logger); - } - try { + return cacheLine.getFile(normalized, transportFactory, SharedHttpCacheStorage::mavenIsOffline, + logger); + } + try { return cacheLine.fetchFile(normalized, transportFactory, logger); - } catch (FileNotFoundException | AuthenticationFailedException e) { - //for not found and failed authentication we can't do anything useful - throw e; - } catch (IOException e) { + } catch (FileNotFoundException | AuthenticationFailedException e) { + // for not found and failed authentication we can't do anything useful + throw e; + } catch (IOException e) { if (!cacheConfig.isUpdate() && cacheLine.getResponseCode() > 0) { - //if we have something cached, use that ... + // if we have something cached, use that ... logger.warn("Request to " + normalized + " failed, trying cache instead"); return cacheLine.getFile(normalized, transportFactory, nil -> e, logger); - } - throw e; - } - } + } + throw e; + } + } - }; - } + }; + } - private synchronized CacheLine getCacheLine(URI uri) { + private synchronized CacheLine getCacheLine(URI uri) { String cleanPath = uri.normalize().toASCIIString().replace(':', '/').replace('?', '/').replace('&', '/') .replace('*', '/').replaceAll("/+", "/"); if (cleanPath.endsWith("/")) { @@ -159,96 +156,101 @@ private synchronized CacheLine getCacheLine(URI uri) { cleanPath += ".idx"; } File file = new File(cacheConfig.getCacheLocation(), cleanPath); - File location; - try { - location = file.getCanonicalFile(); - } catch (IOException e) { - location = file.getAbsoluteFile(); - } - return entryCache.computeIfAbsent(location, CacheLine::new); - - } - - private final class CacheLine { - - private static final String RESPONSE_CODE = "HTTP_RESPONSE_CODE"; - private static final String LAST_UPDATED = "FILE-LAST_UPDATED"; - private static final String STATUS_LINE = "HTTP_STATUS_LINE"; - private final File file; - private final File headerFile; - private Properties header; - private final DateFormat httpDateFormat = new SimpleDateFormat("EEE, dd MMM yyyy HH:mm:ss z", Locale.US); - - public CacheLine(File file) { - this.file = file; - this.headerFile = new File(file.getParent(), file.getName() + ".headers"); - httpDateFormat.setTimeZone(TimeZone.getTimeZone("GMT")); - } + File location; + try { + location = file.getCanonicalFile(); + } catch (IOException e) { + location = file.getAbsoluteFile(); + } + return entryCache.computeIfAbsent(location, CacheLine::new); + + } + + private final class CacheLine { + + private static final String ENCODING_IDENTITY = "identity"; + private static final String HEADER_ACCEPT_ENCODING = "Accept-Encoding"; + private static final String HEADER_CONTENT_ENCODING = "Content-Encoding"; + private static final String ENCODING_GZIP = "gzip"; + private static final String RESPONSE_CODE = "HTTP_RESPONSE_CODE"; + private static final String LAST_UPDATED = "FILE-LAST_UPDATED"; + private static final String STATUS_LINE = "HTTP_STATUS_LINE"; + private final File file; + private final File headerFile; + private Properties header; + private final DateFormat httpDateFormat = new SimpleDateFormat("EEE, dd MMM yyyy HH:mm:ss z", Locale.US); + + public CacheLine(File file) { + this.file = file; + this.headerFile = new File(file.getParent(), file.getName() + ".headers"); + httpDateFormat.setTimeZone(TimeZone.getTimeZone("GMT")); + } public synchronized long fetchLastModified(URI uri, HttpTransportFactory transportFactory, Logger logger) throws IOException { - //TODO its very likely that the file is downloaded here if it has changed... so probably just download it right now? + // TODO its very likely that the file is downloaded here if it has changed... so + // probably just download it right now? HttpTransport transport = transportFactory.createTransport(uri); try (Response response = transport.head()) { int code = response.statusCode(); - if (isAuthFailure(code)) { - throw new AuthenticationFailedException(); //FIXME why is there no constructor to give a cause? - } - if (isNotFound(code)) { + if (isAuthFailure(code)) { + throw new AuthenticationFailedException(); // FIXME why is there no constructor to give a cause? + } + if (isNotFound(code)) { updateHeader(response, code); - throw new FileNotFoundException(uri.toString()); - } - if (isRedirected(code)) { + throw new FileNotFoundException(uri.toString()); + } + if (isRedirected(code)) { updateHeader(response, code); return SharedHttpCacheStorage.this.getCacheEntry(uri, logger).getLastModified(transportFactory); - } + } return response.getLastModified(); - } - } + } + } public synchronized long getLastModified(URI uri, HttpTransportFactory transportFactory, - Function notAviableExceptionSupplier, - Logger logger) throws IOException { - int code = getResponseCode(); - if (code > 0) { - if (isAuthFailure(code)) { - throw new AuthenticationFailedException(); //FIXME why is there no constructor to give a cause? - } - if (isNotFound(code)) { - throw new FileNotFoundException(uri.toString()); - } - if (isRedirected(code)) { + Function notAviableExceptionSupplier, Logger logger) throws IOException { + int code = getResponseCode(); + if (code > 0) { + if (isAuthFailure(code)) { + throw new AuthenticationFailedException(); // FIXME why is there no constructor to give a cause? + } + if (isNotFound(code)) { + throw new FileNotFoundException(uri.toString()); + } + if (isRedirected(code)) { return SharedHttpCacheStorage.this.getCacheEntry(uri, logger).getLastModified(transportFactory); - } - Properties offlineHeader = getHeader(); - Date lastModified = pareHttpDate(offlineHeader.getProperty(LAST_MODIFIED_HEADER.toLowerCase())); - if (lastModified != null) { - return lastModified.getTime(); - } - return -1; - } else { - throw notAviableExceptionSupplier.apply(uri); - } - } + } + Properties offlineHeader = getHeader(); + Date lastModified = pareHttpDate(offlineHeader.getProperty(LAST_MODIFIED_HEADER.toLowerCase())); + if (lastModified != null) { + return lastModified.getTime(); + } + return -1; + } else { + throw notAviableExceptionSupplier.apply(uri); + } + } public synchronized File fetchFile(URI uri, HttpTransportFactory transportFactory, Logger logger) throws IOException { - boolean exits = file.isFile(); - if (exits && !mustValidate()) { - return file; - } + boolean exits = file.isFile(); + if (exits && !mustValidate()) { + return file; + } HttpTransport transport = transportFactory.createTransport(uri); - Properties lastHeader = getHeader(); - if (exits) { - if (lastHeader.containsKey(ETAG_HEADER.toLowerCase())) { + Properties lastHeader = getHeader(); + if (exits) { + if (lastHeader.containsKey(ETAG_HEADER.toLowerCase())) { transport.setHeader("If-None-Match", lastHeader.getProperty(ETAG_HEADER.toLowerCase())); - } - if (lastHeader.contains(LAST_MODIFIED_HEADER.toLowerCase())) { + } + if (lastHeader.contains(LAST_MODIFIED_HEADER.toLowerCase())) { transport.setHeader("If-Modified-Since", - lastHeader.getProperty(LAST_MODIFIED_HEADER.toLowerCase())); - } - } + lastHeader.getProperty(LAST_MODIFIED_HEADER.toLowerCase())); + } + } + transport.setHeader(HEADER_ACCEPT_ENCODING, ENCODING_GZIP); try (Response response = transport.get()) { int code = response.statusCode(); if (exits && code == HttpURLConnection.HTTP_NOT_MODIFIED) { @@ -269,7 +271,14 @@ public synchronized File fetchFile(URI uri, HttpTransportFactory transportFactor response.checkResponseCode(); File tempFile = File.createTempFile("download", ".tmp", file.getParentFile()); try (InputStream inputStream = response.body(); FileOutputStream os = new FileOutputStream(tempFile)) { - inputStream.transferTo(os); + String encoding = response.getHeader(HEADER_CONTENT_ENCODING); + if (ENCODING_GZIP.equals(encoding)) { + new GZIPInputStream(inputStream).transferTo(os); + } else if (encoding == null || encoding.isEmpty() || ENCODING_IDENTITY.equals(encoding)) { + inputStream.transferTo(os); + } else { + throw new IOException("Unknown content encoding: " + encoding); + } } catch (IOException e) { tempFile.delete(); throw e; @@ -277,170 +286,174 @@ public synchronized File fetchFile(URI uri, HttpTransportFactory transportFactor FileUtils.moveFile(tempFile, file); } return file; - } + } public synchronized File getFile(URI uri, HttpTransportFactory transportFactory, - Function notAviableExceptionSupplier, - Logger logger) throws IOException { - int code = getResponseCode(); - if (code > 0) { - if (isAuthFailure(code)) { - throw new AuthenticationFailedException(); //FIXME why is there no constructor to give a cause? - } - if (isNotFound(code)) { - throw new FileNotFoundException(uri.toString()); - } - if (isRedirected(code)) { - return SharedHttpCacheStorage.this.getCacheEntry(getRedirect(uri), logger) + Function notAviableExceptionSupplier, Logger logger) throws IOException { + int code = getResponseCode(); + if (code > 0) { + if (isAuthFailure(code)) { + throw new AuthenticationFailedException(); // FIXME why is there no constructor to give a cause? + } + if (isNotFound(code)) { + throw new FileNotFoundException(uri.toString()); + } + if (isRedirected(code)) { + return SharedHttpCacheStorage.this.getCacheEntry(getRedirect(uri), logger) .getCacheFile(transportFactory); - } - if (file.isFile()) { - return file; - } - } - throw notAviableExceptionSupplier.apply(uri); - } - - private boolean mustValidate() { + } + if (file.isFile()) { + return file; + } + } + throw notAviableExceptionSupplier.apply(uri); + } + + private boolean mustValidate() { if (cacheConfig.isUpdate()) { - //user enforced validation - return true; - } - String[] cacheControls = getCacheControl(); - for (String directive : cacheControls) { - if (MUST_REVALIDATE_DIRECTIVE.equals(directive)) { - //server enforced validation - return true; - } - } - Properties properties = getHeader(); - long lastUpdated = parseLong(properties.getProperty(LAST_UPDATED)); - if (lastUpdated + TimeUnit.MINUTES.toMillis(MIN_CACHE_PERIOD) > System.currentTimeMillis()) { - return false; - } - //Cache-Control header with "max-age" directive takes precedence over Expires Header. - for (String directive : cacheControls) { - if (directive.toLowerCase().startsWith(MAX_AGE_DIRECTIVE)) { - long maxAge = parseLong(directive.substring(MAX_AGE_DIRECTIVE.length() + 1)); - if (maxAge <= 0) { - return true; - } - return (lastUpdated + TimeUnit.SECONDS.toMillis(maxAge)) < System.currentTimeMillis(); - } - } - Date expiresDate = pareHttpDate(properties.getProperty(EXPIRES_HEADER.toLowerCase())); - if (expiresDate != null) { - return expiresDate.after(new Date()); - } - return true; - } - - protected long parseLong(String value) { - if (value != null) { - try { - return Long.parseLong(value); - } catch (NumberFormatException e) { - //ignore... - } - } - return 0; - } - - private String[] getCacheControl() { - String property = getHeader().getProperty(CACHE_CONTROL_HEADER); - if (property != null) { - return property.split(",\\s*"); - } - return new String[0]; - } - - protected boolean isAuthFailure(int code) { - return code == HttpURLConnection.HTTP_PROXY_AUTH || code == HttpURLConnection.HTTP_UNAUTHORIZED; - } - - protected void updateHeader(Response response, int code) - throws IOException, FileNotFoundException { - header = new Properties(); - header.setProperty(RESPONSE_CODE, String.valueOf(code)); - header.setProperty(LAST_UPDATED, String.valueOf(System.currentTimeMillis())); + // user enforced validation + return true; + } + String[] cacheControls = getCacheControl(); + for (String directive : cacheControls) { + if (MUST_REVALIDATE_DIRECTIVE.equals(directive)) { + // server enforced validation + return true; + } + } + Properties properties = getHeader(); + long lastUpdated = parseLong(properties.getProperty(LAST_UPDATED)); + if (lastUpdated + TimeUnit.MINUTES.toMillis(MIN_CACHE_PERIOD) > System.currentTimeMillis()) { + return false; + } + // Cache-Control header with "max-age" directive takes precedence over Expires + // Header. + for (String directive : cacheControls) { + if (directive.toLowerCase().startsWith(MAX_AGE_DIRECTIVE)) { + long maxAge = parseLong(directive.substring(MAX_AGE_DIRECTIVE.length() + 1)); + if (maxAge <= 0) { + return true; + } + return (lastUpdated + TimeUnit.SECONDS.toMillis(maxAge)) < System.currentTimeMillis(); + } + } + Date expiresDate = pareHttpDate(properties.getProperty(EXPIRES_HEADER.toLowerCase())); + if (expiresDate != null) { + return expiresDate.after(new Date()); + } + return true; + } + + protected long parseLong(String value) { + if (value != null) { + try { + return Long.parseLong(value); + } catch (NumberFormatException e) { + // ignore... + } + } + return 0; + } + + private String[] getCacheControl() { + String property = getHeader().getProperty(CACHE_CONTROL_HEADER); + if (property != null) { + return property.split(",\\s*"); + } + return new String[0]; + } + + protected boolean isAuthFailure(int code) { + return code == HttpURLConnection.HTTP_PROXY_AUTH || code == HttpURLConnection.HTTP_UNAUTHORIZED; + } + + protected void updateHeader(Response response, int code) throws IOException, FileNotFoundException { + header = new Properties(); + header.setProperty(RESPONSE_CODE, String.valueOf(code)); + header.setProperty(LAST_UPDATED, String.valueOf(System.currentTimeMillis())); Map> headerFields = response.headers(); - for (var entry : headerFields.entrySet()) { - String key = entry.getKey(); - if (key == null) { - key = STATUS_LINE; - } - key = key.toLowerCase(); + for (var entry : headerFields.entrySet()) { + String key = entry.getKey(); + if (key == null) { + key = STATUS_LINE; + } + key = key.toLowerCase(); if (MavenAuthenticator.AUTHORIZATION_HEADER.equalsIgnoreCase(key) || MavenAuthenticator.PROXY_AUTHORIZATION_HEADER.equalsIgnoreCase(key)) { - //Don't store sensitive information here... - continue; - } - if (key.toLowerCase().startsWith("x-")) { - //don't store non default header... - continue; - } - List value = entry.getValue(); - if (value.size() == 1) { - header.put(key, value.get(0)); - } else { - header.put(key, value.stream().collect(Collectors.joining(","))); - } - } - FileUtils.forceMkdir(file.getParentFile()); - try (FileOutputStream out = new FileOutputStream(headerFile)) { - //we store the header here, this might be a 404 response or (permanent) redirect we probably need to work with later on - header.store(out, null); - } - } - - private synchronized Date pareHttpDate(String input) { - if (input != null) { - try { - return httpDateFormat.parse(input); - } catch (ParseException e) { - //can't use it then.. - } - } - return null; - } - - public int getResponseCode() { - return Integer.parseInt(getHeader().getProperty(RESPONSE_CODE, "-1")); - } - - public URI getRedirect(URI base) throws FileNotFoundException { - String location = getHeader().getProperty("location"); - if (location == null) { - throw new FileNotFoundException(base.toASCIIString()); - } - return base.resolve(location); - } - - public Properties getHeader() { - if (header == null) { - header = new Properties(); - if (headerFile.isFile()) { - try { - header.load(new FileInputStream(headerFile)); - } catch (IOException e) { - //can't use the headers then... - } - } - } - return header; - } - } - - private static boolean isRedirected(int code) { - return code == HttpURLConnection.HTTP_MOVED_PERM || code == HttpURLConnection.HTTP_MOVED_TEMP; - } - - private static boolean isNotFound(int code) { - return code == HttpURLConnection.HTTP_NOT_FOUND; - } - - private static IOException mavenIsOffline(URI uri) { - return new IOException("maven is currently in offline mode requested URL " + uri + " does not exist locally!"); - } + // Don't store sensitive information here... + continue; + } + if (key.toLowerCase().startsWith("x-")) { + // don't store non default header... + continue; + } + if (HEADER_CONTENT_ENCODING.equals(key)) { + // we already decode the content before... + continue; + } + List value = entry.getValue(); + if (value.size() == 1) { + header.put(key, value.get(0)); + } else { + header.put(key, value.stream().collect(Collectors.joining(","))); + } + } + FileUtils.forceMkdir(file.getParentFile()); + try (FileOutputStream out = new FileOutputStream(headerFile)) { + // we store the header here, this might be a 404 response or (permanent) + // redirect we probably need to work with later on + header.store(out, null); + } + } + + private synchronized Date pareHttpDate(String input) { + if (input != null) { + try { + return httpDateFormat.parse(input); + } catch (ParseException e) { + // can't use it then.. + } + } + return null; + } + + public int getResponseCode() { + return Integer.parseInt(getHeader().getProperty(RESPONSE_CODE, "-1")); + } + + public URI getRedirect(URI base) throws FileNotFoundException { + String location = getHeader().getProperty("location"); + if (location == null) { + throw new FileNotFoundException(base.toASCIIString()); + } + return base.resolve(location); + } + + public Properties getHeader() { + if (header == null) { + header = new Properties(); + if (headerFile.isFile()) { + try { + header.load(new FileInputStream(headerFile)); + } catch (IOException e) { + // can't use the headers then... + } + } + } + return header; + } + } + + private static boolean isRedirected(int code) { + return code == HttpURLConnection.HTTP_MOVED_PERM || code == HttpURLConnection.HTTP_MOVED_TEMP; + } + + private static boolean isNotFound(int code) { + return code == HttpURLConnection.HTTP_NOT_FOUND; + } + + private static IOException mavenIsOffline(URI uri) { + return new IOException("maven is currently in offline mode requested URL " + uri + " does not exist locally!"); + } }