From 2bba79211e2254e26df83463402ff9720d904714 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20L=C3=A4ubrich?= Date: Mon, 2 Oct 2023 10:28:00 +0200 Subject: [PATCH] Enable content encoding Actually one would expect that p2 site content is always compressed already (e.g. jar or xz) but there are some known cases where this is not the case and plain xml is used. To account for this cases we enable transport compression in such a case --- .../transport/SharedHttpCacheStorage.java | 613 +++++++++--------- 1 file changed, 313 insertions(+), 300 deletions(-) diff --git a/p2-maven-plugin/src/main/java/org/eclipse/tycho/p2maven/transport/SharedHttpCacheStorage.java b/p2-maven-plugin/src/main/java/org/eclipse/tycho/p2maven/transport/SharedHttpCacheStorage.java index 8621450136..ca3e8cc54d 100644 --- a/p2-maven-plugin/src/main/java/org/eclipse/tycho/p2maven/transport/SharedHttpCacheStorage.java +++ b/p2-maven-plugin/src/main/java/org/eclipse/tycho/p2maven/transport/SharedHttpCacheStorage.java @@ -33,6 +33,7 @@ import java.util.concurrent.TimeUnit; import java.util.function.Function; import java.util.stream.Collectors; +import java.util.zip.GZIPInputStream; import org.apache.commons.io.FileUtils; import org.codehaus.plexus.component.annotations.Component; @@ -45,112 +46,108 @@ public class SharedHttpCacheStorage implements HttpCache { private static final int MAX_CACHE_LINES = Integer.getInteger("tycho.p2.transport.max-cache-lines", 1000); /** - * Assumes the following minimum caching period for remote files in minutes - */ - //TODO can we sync this with the time where maven updates snapshots? - public static final long MIN_CACHE_PERIOD = Long.getLong("tycho.p2.transport.min-cache-minutes", - TimeUnit.HOURS.toMinutes(1)); - private static final String LAST_MODIFIED_HEADER = "Last-Modified"; - private static final String EXPIRES_HEADER = "Expires"; - private static final String CACHE_CONTROL_HEADER = "Cache-Control"; - private static final String MAX_AGE_DIRECTIVE = "max-age"; - private static final String MUST_REVALIDATE_DIRECTIVE = "must-revalidate"; + * Assumes the following minimum caching period for remote files in minutes + */ + // TODO can we sync this with the time where maven updates snapshots? + public static final long MIN_CACHE_PERIOD = Long.getLong("tycho.p2.transport.min-cache-minutes", + TimeUnit.HOURS.toMinutes(1)); + private static final String LAST_MODIFIED_HEADER = "Last-Modified"; + private static final String EXPIRES_HEADER = "Expires"; + private static final String CACHE_CONTROL_HEADER = "Cache-Control"; + private static final String MAX_AGE_DIRECTIVE = "max-age"; + private static final String MUST_REVALIDATE_DIRECTIVE = "must-revalidate"; - private static final String ETAG_HEADER = "ETag"; + private static final String ETAG_HEADER = "ETag"; - private static final int MAX_IN_MEMORY = 1000; + private static final int MAX_IN_MEMORY = 1000; @Requirement TransportCacheConfig cacheConfig; - private final Map entryCache; - + private final Map entryCache; public SharedHttpCacheStorage() { entryCache = new LinkedHashMap<>(MAX_CACHE_LINES, 0.75f, true) { - private static final long serialVersionUID = 1L; + private static final long serialVersionUID = 1L; - @Override - protected boolean removeEldestEntry(final Map.Entry eldest) { - return (size() > MAX_IN_MEMORY); - } + @Override + protected boolean removeEldestEntry(final Map.Entry eldest) { + return (size() > MAX_IN_MEMORY); + } - }; - } + }; + } - /** - * Fetches the cache entry for this URI - * - * @param uri - * @return - * @throws FileNotFoundException - * if the URI is know to be not found - */ + /** + * Fetches the cache entry for this URI + * + * @param uri + * @return + * @throws FileNotFoundException if the URI is know to be not found + */ @Override public CacheEntry getCacheEntry(URI uri, Logger logger) throws FileNotFoundException { URI normalized = uri.normalize(); CacheLine cacheLine = getCacheLine(normalized); if (!cacheConfig.isUpdate()) { // if not updates are forced ... - int code = cacheLine.getResponseCode(); - if (code == HttpURLConnection.HTTP_NOT_FOUND) { + int code = cacheLine.getResponseCode(); + if (code == HttpURLConnection.HTTP_NOT_FOUND) { throw new FileNotFoundException(normalized.toASCIIString()); - } - if (code == HttpURLConnection.HTTP_MOVED_PERM) { + } + if (code == HttpURLConnection.HTTP_MOVED_PERM) { return getCacheEntry(cacheLine.getRedirect(normalized), logger); - } - } - return new CacheEntry() { + } + } + return new CacheEntry() { - @Override - public long getLastModified(HttpTransportFactory transportFactory) - throws IOException { + @Override + public long getLastModified(HttpTransportFactory transportFactory) throws IOException { if (cacheConfig.isOffline()) { return cacheLine.getLastModified(normalized, transportFactory, - SharedHttpCacheStorage::mavenIsOffline, logger); - } - try { + SharedHttpCacheStorage::mavenIsOffline, logger); + } + try { return cacheLine.fetchLastModified(normalized, transportFactory, logger); - } catch (FileNotFoundException | AuthenticationFailedException e) { - //for not found and failed authentication we can't do anything useful - throw e; - } catch (IOException e) { + } catch (FileNotFoundException | AuthenticationFailedException e) { + // for not found and failed authentication we can't do anything useful + throw e; + } catch (IOException e) { if (!cacheConfig.isUpdate() && cacheLine.getResponseCode() > 0) { - //if we have something cached, use that ... + // if we have something cached, use that ... logger.warn("Request to " + normalized + " failed, trying cache instead"); return cacheLine.getLastModified(normalized, transportFactory, nil -> e, logger); - } - throw e; - } - } - - @Override - public File getCacheFile(HttpTransportFactory transportFactory) - throws IOException { + } + throw e; + } + } + + @Override + public File getCacheFile(HttpTransportFactory transportFactory) throws IOException { if (cacheConfig.isOffline()) { - return cacheLine.getFile(normalized, transportFactory, - SharedHttpCacheStorage::mavenIsOffline, logger); - } - try { + return cacheLine.getFile(normalized, transportFactory, SharedHttpCacheStorage::mavenIsOffline, + logger); + } + try { return cacheLine.fetchFile(normalized, transportFactory, logger); - } catch (FileNotFoundException | AuthenticationFailedException e) { - //for not found and failed authentication we can't do anything useful - throw e; - } catch (IOException e) { + } catch (FileNotFoundException | AuthenticationFailedException e) { + // for not found and failed authentication we can't do anything useful + throw e; + } catch (IOException e) { if (!cacheConfig.isUpdate() && cacheLine.getResponseCode() > 0) { - //if we have something cached, use that ... + // if we have something cached, use that ... logger.warn("Request to " + normalized + " failed, trying cache instead"); return cacheLine.getFile(normalized, transportFactory, nil -> e, logger); - } - throw e; - } - } + } + throw e; + } + } - }; - } + }; + } - private synchronized CacheLine getCacheLine(URI uri) { + private synchronized CacheLine getCacheLine(URI uri) { String cleanPath = uri.normalize().toASCIIString().replace(':', '/').replace('?', '/').replace('&', '/') .replace('*', '/').replaceAll("/+", "/"); if (cleanPath.endsWith("/")) { @@ -159,96 +156,101 @@ private synchronized CacheLine getCacheLine(URI uri) { cleanPath += ".idx"; } File file = new File(cacheConfig.getCacheLocation(), cleanPath); - File location; - try { - location = file.getCanonicalFile(); - } catch (IOException e) { - location = file.getAbsoluteFile(); - } - return entryCache.computeIfAbsent(location, CacheLine::new); - - } - - private final class CacheLine { - - private static final String RESPONSE_CODE = "HTTP_RESPONSE_CODE"; - private static final String LAST_UPDATED = "FILE-LAST_UPDATED"; - private static final String STATUS_LINE = "HTTP_STATUS_LINE"; - private final File file; - private final File headerFile; - private Properties header; - private final DateFormat httpDateFormat = new SimpleDateFormat("EEE, dd MMM yyyy HH:mm:ss z", Locale.US); - - public CacheLine(File file) { - this.file = file; - this.headerFile = new File(file.getParent(), file.getName() + ".headers"); - httpDateFormat.setTimeZone(TimeZone.getTimeZone("GMT")); - } + File location; + try { + location = file.getCanonicalFile(); + } catch (IOException e) { + location = file.getAbsoluteFile(); + } + return entryCache.computeIfAbsent(location, CacheLine::new); + + } + + private final class CacheLine { + + private static final String ENCODING_IDENTITY = "identity"; + private static final String HEADER_ACCEPT_ENCODING = "Accept-Encoding"; + private static final String HEADER_CONTENT_ENCODING = "Content-Encoding"; + private static final String ENCODING_GZIP = "gzip"; + private static final String RESPONSE_CODE = "HTTP_RESPONSE_CODE"; + private static final String LAST_UPDATED = "FILE-LAST_UPDATED"; + private static final String STATUS_LINE = "HTTP_STATUS_LINE"; + private final File file; + private final File headerFile; + private Properties header; + private final DateFormat httpDateFormat = new SimpleDateFormat("EEE, dd MMM yyyy HH:mm:ss z", Locale.US); + + public CacheLine(File file) { + this.file = file; + this.headerFile = new File(file.getParent(), file.getName() + ".headers"); + httpDateFormat.setTimeZone(TimeZone.getTimeZone("GMT")); + } public synchronized long fetchLastModified(URI uri, HttpTransportFactory transportFactory, Logger logger) throws IOException { - //TODO its very likely that the file is downloaded here if it has changed... so probably just download it right now? + // TODO its very likely that the file is downloaded here if it has changed... so + // probably just download it right now? HttpTransport transport = transportFactory.createTransport(uri); try (Response response = transport.head()) { int code = response.statusCode(); - if (isAuthFailure(code)) { - throw new AuthenticationFailedException(); //FIXME why is there no constructor to give a cause? - } - if (isNotFound(code)) { + if (isAuthFailure(code)) { + throw new AuthenticationFailedException(); // FIXME why is there no constructor to give a cause? + } + if (isNotFound(code)) { updateHeader(response, code); - throw new FileNotFoundException(uri.toString()); - } - if (isRedirected(code)) { + throw new FileNotFoundException(uri.toString()); + } + if (isRedirected(code)) { updateHeader(response, code); return SharedHttpCacheStorage.this.getCacheEntry(uri, logger).getLastModified(transportFactory); - } + } return response.getLastModified(); - } - } + } + } public synchronized long getLastModified(URI uri, HttpTransportFactory transportFactory, - Function notAviableExceptionSupplier, - Logger logger) throws IOException { - int code = getResponseCode(); - if (code > 0) { - if (isAuthFailure(code)) { - throw new AuthenticationFailedException(); //FIXME why is there no constructor to give a cause? - } - if (isNotFound(code)) { - throw new FileNotFoundException(uri.toString()); - } - if (isRedirected(code)) { + Function notAviableExceptionSupplier, Logger logger) throws IOException { + int code = getResponseCode(); + if (code > 0) { + if (isAuthFailure(code)) { + throw new AuthenticationFailedException(); // FIXME why is there no constructor to give a cause? + } + if (isNotFound(code)) { + throw new FileNotFoundException(uri.toString()); + } + if (isRedirected(code)) { return SharedHttpCacheStorage.this.getCacheEntry(uri, logger).getLastModified(transportFactory); - } - Properties offlineHeader = getHeader(); - Date lastModified = pareHttpDate(offlineHeader.getProperty(LAST_MODIFIED_HEADER.toLowerCase())); - if (lastModified != null) { - return lastModified.getTime(); - } - return -1; - } else { - throw notAviableExceptionSupplier.apply(uri); - } - } + } + Properties offlineHeader = getHeader(); + Date lastModified = pareHttpDate(offlineHeader.getProperty(LAST_MODIFIED_HEADER.toLowerCase())); + if (lastModified != null) { + return lastModified.getTime(); + } + return -1; + } else { + throw notAviableExceptionSupplier.apply(uri); + } + } public synchronized File fetchFile(URI uri, HttpTransportFactory transportFactory, Logger logger) throws IOException { - boolean exits = file.isFile(); - if (exits && !mustValidate()) { - return file; - } + boolean exits = file.isFile(); + if (exits && !mustValidate()) { + return file; + } HttpTransport transport = transportFactory.createTransport(uri); - Properties lastHeader = getHeader(); - if (exits) { - if (lastHeader.containsKey(ETAG_HEADER.toLowerCase())) { + Properties lastHeader = getHeader(); + if (exits) { + if (lastHeader.containsKey(ETAG_HEADER.toLowerCase())) { transport.setHeader("If-None-Match", lastHeader.getProperty(ETAG_HEADER.toLowerCase())); - } - if (lastHeader.contains(LAST_MODIFIED_HEADER.toLowerCase())) { + } + if (lastHeader.contains(LAST_MODIFIED_HEADER.toLowerCase())) { transport.setHeader("If-Modified-Since", - lastHeader.getProperty(LAST_MODIFIED_HEADER.toLowerCase())); - } - } + lastHeader.getProperty(LAST_MODIFIED_HEADER.toLowerCase())); + } + } + transport.setHeader(HEADER_ACCEPT_ENCODING, ENCODING_GZIP); try (Response response = transport.get()) { int code = response.statusCode(); if (exits && code == HttpURLConnection.HTTP_NOT_MODIFIED) { @@ -269,7 +271,14 @@ public synchronized File fetchFile(URI uri, HttpTransportFactory transportFactor response.checkResponseCode(); File tempFile = File.createTempFile("download", ".tmp", file.getParentFile()); try (InputStream inputStream = response.body(); FileOutputStream os = new FileOutputStream(tempFile)) { - inputStream.transferTo(os); + String encoding = response.getHeader(HEADER_CONTENT_ENCODING); + if (ENCODING_GZIP.equals(encoding)) { + new GZIPInputStream(inputStream).transferTo(os); + } else if (encoding == null || encoding.isEmpty() || ENCODING_IDENTITY.equals(encoding)) { + inputStream.transferTo(os); + } else { + throw new IOException("Unknown content encoding: " + encoding); + } } catch (IOException e) { tempFile.delete(); throw e; @@ -277,170 +286,174 @@ public synchronized File fetchFile(URI uri, HttpTransportFactory transportFactor FileUtils.moveFile(tempFile, file); } return file; - } + } public synchronized File getFile(URI uri, HttpTransportFactory transportFactory, - Function notAviableExceptionSupplier, - Logger logger) throws IOException { - int code = getResponseCode(); - if (code > 0) { - if (isAuthFailure(code)) { - throw new AuthenticationFailedException(); //FIXME why is there no constructor to give a cause? - } - if (isNotFound(code)) { - throw new FileNotFoundException(uri.toString()); - } - if (isRedirected(code)) { - return SharedHttpCacheStorage.this.getCacheEntry(getRedirect(uri), logger) + Function notAviableExceptionSupplier, Logger logger) throws IOException { + int code = getResponseCode(); + if (code > 0) { + if (isAuthFailure(code)) { + throw new AuthenticationFailedException(); // FIXME why is there no constructor to give a cause? + } + if (isNotFound(code)) { + throw new FileNotFoundException(uri.toString()); + } + if (isRedirected(code)) { + return SharedHttpCacheStorage.this.getCacheEntry(getRedirect(uri), logger) .getCacheFile(transportFactory); - } - if (file.isFile()) { - return file; - } - } - throw notAviableExceptionSupplier.apply(uri); - } - - private boolean mustValidate() { + } + if (file.isFile()) { + return file; + } + } + throw notAviableExceptionSupplier.apply(uri); + } + + private boolean mustValidate() { if (cacheConfig.isUpdate()) { - //user enforced validation - return true; - } - String[] cacheControls = getCacheControl(); - for (String directive : cacheControls) { - if (MUST_REVALIDATE_DIRECTIVE.equals(directive)) { - //server enforced validation - return true; - } - } - Properties properties = getHeader(); - long lastUpdated = parseLong(properties.getProperty(LAST_UPDATED)); - if (lastUpdated + TimeUnit.MINUTES.toMillis(MIN_CACHE_PERIOD) > System.currentTimeMillis()) { - return false; - } - //Cache-Control header with "max-age" directive takes precedence over Expires Header. - for (String directive : cacheControls) { - if (directive.toLowerCase().startsWith(MAX_AGE_DIRECTIVE)) { - long maxAge = parseLong(directive.substring(MAX_AGE_DIRECTIVE.length() + 1)); - if (maxAge <= 0) { - return true; - } - return (lastUpdated + TimeUnit.SECONDS.toMillis(maxAge)) < System.currentTimeMillis(); - } - } - Date expiresDate = pareHttpDate(properties.getProperty(EXPIRES_HEADER.toLowerCase())); - if (expiresDate != null) { - return expiresDate.after(new Date()); - } - return true; - } - - protected long parseLong(String value) { - if (value != null) { - try { - return Long.parseLong(value); - } catch (NumberFormatException e) { - //ignore... - } - } - return 0; - } - - private String[] getCacheControl() { - String property = getHeader().getProperty(CACHE_CONTROL_HEADER); - if (property != null) { - return property.split(",\\s*"); - } - return new String[0]; - } - - protected boolean isAuthFailure(int code) { - return code == HttpURLConnection.HTTP_PROXY_AUTH || code == HttpURLConnection.HTTP_UNAUTHORIZED; - } - - protected void updateHeader(Response response, int code) - throws IOException, FileNotFoundException { - header = new Properties(); - header.setProperty(RESPONSE_CODE, String.valueOf(code)); - header.setProperty(LAST_UPDATED, String.valueOf(System.currentTimeMillis())); + // user enforced validation + return true; + } + String[] cacheControls = getCacheControl(); + for (String directive : cacheControls) { + if (MUST_REVALIDATE_DIRECTIVE.equals(directive)) { + // server enforced validation + return true; + } + } + Properties properties = getHeader(); + long lastUpdated = parseLong(properties.getProperty(LAST_UPDATED)); + if (lastUpdated + TimeUnit.MINUTES.toMillis(MIN_CACHE_PERIOD) > System.currentTimeMillis()) { + return false; + } + // Cache-Control header with "max-age" directive takes precedence over Expires + // Header. + for (String directive : cacheControls) { + if (directive.toLowerCase().startsWith(MAX_AGE_DIRECTIVE)) { + long maxAge = parseLong(directive.substring(MAX_AGE_DIRECTIVE.length() + 1)); + if (maxAge <= 0) { + return true; + } + return (lastUpdated + TimeUnit.SECONDS.toMillis(maxAge)) < System.currentTimeMillis(); + } + } + Date expiresDate = pareHttpDate(properties.getProperty(EXPIRES_HEADER.toLowerCase())); + if (expiresDate != null) { + return expiresDate.after(new Date()); + } + return true; + } + + protected long parseLong(String value) { + if (value != null) { + try { + return Long.parseLong(value); + } catch (NumberFormatException e) { + // ignore... + } + } + return 0; + } + + private String[] getCacheControl() { + String property = getHeader().getProperty(CACHE_CONTROL_HEADER); + if (property != null) { + return property.split(",\\s*"); + } + return new String[0]; + } + + protected boolean isAuthFailure(int code) { + return code == HttpURLConnection.HTTP_PROXY_AUTH || code == HttpURLConnection.HTTP_UNAUTHORIZED; + } + + protected void updateHeader(Response response, int code) throws IOException, FileNotFoundException { + header = new Properties(); + header.setProperty(RESPONSE_CODE, String.valueOf(code)); + header.setProperty(LAST_UPDATED, String.valueOf(System.currentTimeMillis())); Map> headerFields = response.headers(); - for (var entry : headerFields.entrySet()) { - String key = entry.getKey(); - if (key == null) { - key = STATUS_LINE; - } - key = key.toLowerCase(); + for (var entry : headerFields.entrySet()) { + String key = entry.getKey(); + if (key == null) { + key = STATUS_LINE; + } + key = key.toLowerCase(); if (MavenAuthenticator.AUTHORIZATION_HEADER.equalsIgnoreCase(key) || MavenAuthenticator.PROXY_AUTHORIZATION_HEADER.equalsIgnoreCase(key)) { - //Don't store sensitive information here... - continue; - } - if (key.toLowerCase().startsWith("x-")) { - //don't store non default header... - continue; - } - List value = entry.getValue(); - if (value.size() == 1) { - header.put(key, value.get(0)); - } else { - header.put(key, value.stream().collect(Collectors.joining(","))); - } - } - FileUtils.forceMkdir(file.getParentFile()); - try (FileOutputStream out = new FileOutputStream(headerFile)) { - //we store the header here, this might be a 404 response or (permanent) redirect we probably need to work with later on - header.store(out, null); - } - } - - private synchronized Date pareHttpDate(String input) { - if (input != null) { - try { - return httpDateFormat.parse(input); - } catch (ParseException e) { - //can't use it then.. - } - } - return null; - } - - public int getResponseCode() { - return Integer.parseInt(getHeader().getProperty(RESPONSE_CODE, "-1")); - } - - public URI getRedirect(URI base) throws FileNotFoundException { - String location = getHeader().getProperty("location"); - if (location == null) { - throw new FileNotFoundException(base.toASCIIString()); - } - return base.resolve(location); - } - - public Properties getHeader() { - if (header == null) { - header = new Properties(); - if (headerFile.isFile()) { - try { - header.load(new FileInputStream(headerFile)); - } catch (IOException e) { - //can't use the headers then... - } - } - } - return header; - } - } - - private static boolean isRedirected(int code) { - return code == HttpURLConnection.HTTP_MOVED_PERM || code == HttpURLConnection.HTTP_MOVED_TEMP; - } - - private static boolean isNotFound(int code) { - return code == HttpURLConnection.HTTP_NOT_FOUND; - } - - private static IOException mavenIsOffline(URI uri) { - return new IOException("maven is currently in offline mode requested URL " + uri + " does not exist locally!"); - } + // Don't store sensitive information here... + continue; + } + if (key.toLowerCase().startsWith("x-")) { + // don't store non default header... + continue; + } + if (HEADER_CONTENT_ENCODING.equals(key)) { + // we already decode the content before... + continue; + } + List value = entry.getValue(); + if (value.size() == 1) { + header.put(key, value.get(0)); + } else { + header.put(key, value.stream().collect(Collectors.joining(","))); + } + } + FileUtils.forceMkdir(file.getParentFile()); + try (FileOutputStream out = new FileOutputStream(headerFile)) { + // we store the header here, this might be a 404 response or (permanent) + // redirect we probably need to work with later on + header.store(out, null); + } + } + + private synchronized Date pareHttpDate(String input) { + if (input != null) { + try { + return httpDateFormat.parse(input); + } catch (ParseException e) { + // can't use it then.. + } + } + return null; + } + + public int getResponseCode() { + return Integer.parseInt(getHeader().getProperty(RESPONSE_CODE, "-1")); + } + + public URI getRedirect(URI base) throws FileNotFoundException { + String location = getHeader().getProperty("location"); + if (location == null) { + throw new FileNotFoundException(base.toASCIIString()); + } + return base.resolve(location); + } + + public Properties getHeader() { + if (header == null) { + header = new Properties(); + if (headerFile.isFile()) { + try { + header.load(new FileInputStream(headerFile)); + } catch (IOException e) { + // can't use the headers then... + } + } + } + return header; + } + } + + private static boolean isRedirected(int code) { + return code == HttpURLConnection.HTTP_MOVED_PERM || code == HttpURLConnection.HTTP_MOVED_TEMP; + } + + private static boolean isNotFound(int code) { + return code == HttpURLConnection.HTTP_NOT_FOUND; + } + + private static IOException mavenIsOffline(URI uri) { + return new IOException("maven is currently in offline mode requested URL " + uri + " does not exist locally!"); + } }