diff --git a/docs/reference-manual/native-image/BuildOutput.md b/docs/reference-manual/native-image/BuildOutput.md new file mode 100644 index 000000000000..e2713ebde012 --- /dev/null +++ b/docs/reference-manual/native-image/BuildOutput.md @@ -0,0 +1,42 @@ +--- +layout: docs +toc_group: native-image +link_title: Build Output +permalink: /reference-manual/native-image/BuildOutput/ +--- +# Native Image Build Output + +## Glossary + +### User-provided features + +### Classes registered for reflection + +### Reachable classes and methods + +### Runtime compiled methods + +### Code Area + +### Image Heap + +### Peak RSS + +### CPU load + + +## Build Stages + +### Initializing + +### Performing analysis + +### Building universe + +### Parsing methods + +### Inlining methods + +### Compiling methods + +### Creating image diff --git a/sdk/src/org.graalvm.nativeimage/src/org/graalvm/nativeimage/impl/RuntimeReflectionSupport.java b/sdk/src/org.graalvm.nativeimage/src/org/graalvm/nativeimage/impl/RuntimeReflectionSupport.java index c1df478a92b6..a6a3ffa773ce 100644 --- a/sdk/src/org.graalvm.nativeimage/src/org/graalvm/nativeimage/impl/RuntimeReflectionSupport.java +++ b/sdk/src/org.graalvm.nativeimage/src/org/graalvm/nativeimage/impl/RuntimeReflectionSupport.java @@ -52,4 +52,6 @@ public interface RuntimeReflectionSupport extends ReflectionRegistry { * from reflection queries. */ Set getHidingMethods(); + + int getReflectionClassesCount(); } diff --git a/substratevm/CHANGELOG.md b/substratevm/CHANGELOG.md index 8d4fbd15c2ff..7540b0b54b48 100644 --- a/substratevm/CHANGELOG.md +++ b/substratevm/CHANGELOG.md @@ -11,4 +11,4 @@ This changelog summarizes major changes to GraalVM Native Image. * (GR-32403) Use more compressed encoding for stack frame metadata. * (GR-35152) Add -H:DisableURLProtocols to allow specifying URL protocols that must never be included in the image. * (GR-35085) Custom prologue/epilogue/handleException customizations of @CEntryPoint must be annotated with @Uninterruptible. The entry points synthetic methods are now implicilty annotated with @Uninterruptible too. - +* (GR-33602) Enable new user-friendly build output mode. The old output can be restored with `-H:-BuildOutputUseNewStyle`. Run `native-image --expert-options-all | grep "BuildOutput` to see all options for the new output. diff --git a/substratevm/mx.substratevm/mx_substratevm.py b/substratevm/mx.substratevm/mx_substratevm.py index 87cb3321b28f..27b9de816729 100644 --- a/substratevm/mx.substratevm/mx_substratevm.py +++ b/substratevm/mx.substratevm/mx_substratevm.py @@ -780,6 +780,7 @@ def native_image_context_run(func, func_args=None, config=None, build_if_missing 'substratevm:NATIVE_IMAGE_BASE', ], support_distributions=['substratevm:SVM_GRAALVM_SUPPORT'], + support_libraries_distributions=['substratevm:SVM_GRAALVM_LIBRARIES_SUPPORT'], stability="earlyadopter", jlink=False, )) diff --git a/substratevm/mx.substratevm/suite.py b/substratevm/mx.substratevm/suite.py index b67ec406fa82..d363fa320d97 100644 --- a/substratevm/mx.substratevm/suite.py +++ b/substratevm/mx.substratevm/suite.py @@ -471,6 +471,26 @@ }, }, + "com.oracle.svm.native.reporterchelper": { + "subDir": "src", + "native": "shared_lib", + "deliverable": "reporterchelper", + "platformDependent": True, + "use_jdk_headers": True, + "os_arch": { + "windows": { + "": { + "cflags": ["-Wall"] + } + }, + "": { + "": { + "cflags": ["-Wall", "-Werror"], + }, + }, + }, + }, + "com.oracle.svm.native.darwin": { "subDir": "src", "native": "static_lib", @@ -1450,7 +1470,14 @@ "com.oracle.graal.pointsto.infrastructure", "com.oracle.graal.pointsto.flow.context.object", ], + "requires": [ + "java.management", + "jdk.management", + ], "requiresConcealed" : { + "java.management": [ + "sun.management", + ], "jdk.internal.vm.ci" : [ "jdk.vm.ci.meta", "jdk.vm.ci.common", @@ -1525,6 +1552,16 @@ }, }, + "SVM_GRAALVM_LIBRARIES_SUPPORT" : { + "native" : True, + "platformDependent" : True, + "description" : "SubstrateVM support libraries for the GraalVM", + "layout" : { + "svm/builder/lib/" : ["dependency:com.oracle.svm.native.reporterchelper"], + }, + "maven" : False, + }, + "SVM_NFI_GRAALVM_SUPPORT" : { "native" : True, "platformDependent" : True, diff --git a/substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/reports/CallTreePrinter.java b/substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/reports/CallTreePrinter.java index 57d883526efe..e47bda91fef3 100644 --- a/substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/reports/CallTreePrinter.java +++ b/substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/reports/CallTreePrinter.java @@ -360,7 +360,7 @@ private static void printCsvFiles(Map methodToNode, private static void toCsvFile(String description, String reportsPath, String prefix, String reportName, Consumer reporter) { final String name = prefix + "_" + reportName; - final String csvFile = ReportUtils.report(description, reportsPath, name, "csv", reporter); + final Path csvFile = ReportUtils.report(description, reportsPath, name, "csv", reporter); final Path csvLink = Paths.get(reportsPath).resolve(prefix + ".csv"); if (Files.exists(csvLink, LinkOption.NOFOLLOW_LINKS)) { @@ -372,7 +372,7 @@ private static void toCsvFile(String description, String reportsPath, String pre } try { - Files.createSymbolicLink(csvLink, Paths.get(csvFile)); + Files.createSymbolicLink(csvLink, csvFile); } catch (IOException e) { throw new RuntimeException(e); } diff --git a/substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/reports/ReportUtils.java b/substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/reports/ReportUtils.java index 63e4a8218dfa..62e80c71ff2f 100644 --- a/substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/reports/ReportUtils.java +++ b/substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/reports/ReportUtils.java @@ -64,6 +64,10 @@ public class ReportUtils { static final Comparator positionMethodComparator = Comparator.comparing(pos -> pos.getMethod().format("%H.%n(%p)")); static final Comparator positionComparator = positionMethodComparator.thenComparing(pos -> pos.getBCI()); + public static Path report(String description, String path, String name, String extension, Consumer reporter) { + return report(description, path, name, extension, reporter, true); + } + /** * Print a report in the format: path/name_timeStamp.extension. The path is relative to the * working directory. @@ -75,18 +79,21 @@ public class ReportUtils { * @param extension the extension of the report * @param reporter a consumer that writes to a PrintWriter */ - public static String report(String description, String path, String name, String extension, Consumer reporter) { + public static Path report(String description, String path, String name, String extension, Consumer reporter, boolean enablePrint) { String fileName = timeStampedFileName(name, extension); Path reportDir = Paths.get(path); - reportImpl(description, reportDir, fileName, reporter); - return fileName; + return reportImpl(enablePrint, description, reportDir, fileName, reporter); } public static String timeStampedFileName(String name, String extension) { + String fileName = name + "_" + getTimeStampString(); + return extension.isEmpty() ? fileName : fileName + "." + extension; + } + + public static String getTimeStampString() { DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyyMMdd_HHmmss"); String timeStamp = LocalDateTime.now().format(formatter); - String fileName = name + "_" + timeStamp; - return extension.isEmpty() ? fileName : fileName + "." + extension; + return timeStamp; } public static File reportFile(String path, String name, String extension) { @@ -101,6 +108,10 @@ public static File reportFile(String path, String name, String extension) { } } + public static void report(String description, Path file, Consumer reporter) { + report(description, file, reporter, true); + } + /** * Print a report in the file given by {@code file} parameter. If the {@code file} is relative * it's resolved to the working directory. @@ -109,17 +120,18 @@ public static File reportFile(String path, String name, String extension) { * @param file the path (relative to the working directory if the argument represents a relative * path) to file to store a report into. * @param reporter a consumer that writes to a PrintWriter + * @param enablePrint of a notice to stdout */ - public static void report(String description, Path file, Consumer reporter) { + public static Path report(String description, Path file, Consumer reporter, boolean enablePrint) { Path folder = file.getParent(); Path fileName = file.getFileName(); if (folder == null || fileName == null) { throw new IllegalArgumentException("File parameter must be a file, got: " + file); } - reportImpl(description, folder, fileName.toString(), reporter); + return reportImpl(enablePrint, description, folder, fileName.toString(), reporter); } - private static void reportImpl(String description, Path folder, String fileName, Consumer reporter) { + private static Path reportImpl(boolean enablePrint, String description, Path folder, String fileName, Consumer reporter) { try { Path reportDir = Files.createDirectories(folder); Path file = reportDir.resolve(fileName); @@ -127,11 +139,13 @@ private static void reportImpl(String description, Path folder, String fileName, try (FileWriter fw = new FileWriter(Files.createFile(file).toFile())) { try (PrintWriter writer = new PrintWriter(fw)) { - System.out.println("# Printing " + description + " to: " + file); + if (enablePrint) { + System.out.println("# Printing " + description + " to: " + file); + } reporter.accept(writer); } } - + return file; } catch (IOException e) { throw JVMCIError.shouldNotReachHere(e); } diff --git a/substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/util/Timer.java b/substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/util/Timer.java index 7e53d1eb5cdb..a7aadaf94b1a 100644 --- a/substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/util/Timer.java +++ b/substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/util/Timer.java @@ -27,6 +27,7 @@ import org.graalvm.compiler.serviceprovider.GraalServices; public class Timer { + private static boolean disablePrinting = false; private String prefix; @@ -57,6 +58,10 @@ public Timer(String prefix, String name, boolean autoPrint) { this.autoPrint = autoPrint; } + public static void disablePrinting() { + disablePrinting = true; + } + /** * Registers the prefix to be used when {@linkplain Timer#print(long) printing} a timer. This * allows the output of interlaced native image executions to be disambiguated. @@ -79,6 +84,9 @@ public void stop() { } private void print(long time) { + if (disablePrinting) { + return; + } final String concurrentPrefix; if (prefix != null) { // Add the PID to further disambiguate concurrent builds of images with the same name diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateOptions.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateOptions.java index 52d5485db676..fe491e0b62ec 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateOptions.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateOptions.java @@ -273,6 +273,30 @@ public Boolean getValue(OptionValues values) { @Option(help = "Alignment of AOT and JIT compiled code in bytes.")// public static final HostedOptionKey CodeAlignment = new HostedOptionKey<>(16); + /* + * Build output options. + */ + @Option(help = "Use new build output style", type = OptionType.User)// + public static final HostedOptionKey BuildOutputUseNewStyle = new HostedOptionKey<>(true); + + @Option(help = "Prefix build output with ':'", type = OptionType.User)// + public static final HostedOptionKey BuildOutputPrefix = new HostedOptionKey<>(false); + + @Option(help = "Colorize build output", type = OptionType.User)// + public static final HostedOptionKey BuildOutputColorful = new HostedOptionKey<>(true); + + @Option(help = "Show links in build output", type = OptionType.User)// + public static final HostedOptionKey BuildOutputLinks = new HostedOptionKey<>(true); + + @Option(help = "Report progress in build output", type = OptionType.User)// + public static final HostedOptionKey BuildOutputProgress = new HostedOptionKey<>(true); + + @Option(help = "Show code and heap breakdowns as part of the build output", type = OptionType.User)// + public static final HostedOptionKey BuildOutputBreakdowns = new HostedOptionKey<>(true); + + @Option(help = "Print GC warnings as part of build output", type = OptionType.User)// + public static final HostedOptionKey BuildOutputGCWarnings = new HostedOptionKey<>(true); + /* * Object and array allocation options. */ diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/reflect/MethodMetadataDecoder.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/reflect/MethodMetadataDecoder.java index 4393e14505dc..7aad8c2adcc1 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/reflect/MethodMetadataDecoder.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/reflect/MethodMetadataDecoder.java @@ -37,6 +37,8 @@ public interface MethodMetadataDecoder { MethodDescriptor[] getAllReachableMethods(); + long getMetadataByteLength(); + class MethodDescriptor { private final Class declaringClass; private final String name; diff --git a/substratevm/src/com.oracle.svm.graal/src/com/oracle/svm/graal/hosted/GraalFeature.java b/substratevm/src/com.oracle.svm.graal/src/com/oracle/svm/graal/hosted/GraalFeature.java index 7da590487b41..1978e12dde90 100644 --- a/substratevm/src/com.oracle.svm.graal/src/com/oracle/svm/graal/hosted/GraalFeature.java +++ b/substratevm/src/com.oracle.svm.graal/src/com/oracle/svm/graal/hosted/GraalFeature.java @@ -132,6 +132,7 @@ import com.oracle.svm.hosted.phases.StrengthenStampsPhase; import com.oracle.svm.hosted.phases.SubstrateClassInitializationPlugin; import com.oracle.svm.hosted.phases.SubstrateGraphBuilderPhase; +import com.oracle.svm.hosted.reporting.ProgressReporter; import com.oracle.truffle.api.CompilerDirectives.TruffleBoundary; import jdk.vm.ci.meta.DeoptimizationAction; @@ -647,7 +648,8 @@ public void beforeCompilation(BeforeCompilationAccess c) { if (Options.PrintRuntimeCompileMethods.getValue()) { printCallTree(); } - System.out.println(methods.size() + " method(s) included for runtime compilation"); + + ProgressReporter.singleton().printRuntimeCompileMethods(methods.size(), config.getMethods().size()); if (Options.PrintStaticTruffleBoundaries.getValue()) { printStaticTruffleBoundaries(); @@ -720,6 +722,7 @@ public void beforeCompilation(BeforeCompilationAccess c) { } } + ProgressReporter.singleton().setGraphEncodingByteLength(graphEncoder.getEncoding().length); GraalSupport.setGraphEncoding(graphEncoder.getEncoding(), graphEncoder.getObjects(), graphEncoder.getNodeClasses()); objectReplacer.updateDataDuringAnalysis((AnalysisMetaAccess) hMetaAccess.getWrapped()); diff --git a/substratevm/src/com.oracle.svm.hosted.jdk11/src/com/oracle/svm/hosted/StringAccess.java b/substratevm/src/com.oracle.svm.hosted.jdk11/src/com/oracle/svm/hosted/StringAccess.java new file mode 100644 index 000000000000..44cfaabca31c --- /dev/null +++ b/substratevm/src/com.oracle.svm.hosted.jdk11/src/com/oracle/svm/hosted/StringAccess.java @@ -0,0 +1,46 @@ +/* + * Copyright (c) 2021, 2021, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package com.oracle.svm.hosted; + +import java.lang.invoke.MethodHandles; +import java.lang.invoke.VarHandle; + +/* TODO: Added as a temporary workaround to provide compatibility with JDK8 (GR-35238). */ +public class StringAccess { + + private static final VarHandle STRING_VALUE; + static { + try { + MethodHandles.Lookup privateLookup = MethodHandles.privateLookupIn(String.class, MethodHandles.lookup()); + STRING_VALUE = privateLookup.unreflectVarHandle(String.class.getDeclaredField("value")); + } catch (ReflectiveOperationException e) { + throw new ExceptionInInitializerError(e); + } + } + + public static int getInternalByteArrayLength(String string) { + return ((byte[]) STRING_VALUE.get(string)).length; + } +} diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/FeatureHandler.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/FeatureHandler.java index 5fc9604f6995..82b097808998 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/FeatureHandler.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/FeatureHandler.java @@ -40,6 +40,7 @@ import org.graalvm.nativeimage.hosted.Feature; import com.oracle.graal.pointsto.reports.ReportUtils; +import com.oracle.svm.core.ClassLoaderSupport; import com.oracle.svm.core.SubstrateOptions; import com.oracle.svm.core.annotate.AutomaticFeature; import com.oracle.svm.core.graal.GraalFeature; @@ -194,4 +195,13 @@ private void registerFeature(Class baseFeatureClass, Function, Class featureInstances.add(feature); } + + public List getUserFeatureNames() { + ClassLoaderSupport classLoaderSupport = ImageSingletons.lookup(ClassLoaderSupport.class); + return featureInstances.stream() + .filter(f -> classLoaderSupport.isNativeImageClassLoader(f.getClass().getClassLoader())) + .map(f -> f.getClass().getName()) + .sorted() + .collect(Collectors.toList()); + } } diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/NativeImageGenerator.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/NativeImageGenerator.java index ec20775d4a12..ffa43252d57f 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/NativeImageGenerator.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/NativeImageGenerator.java @@ -30,9 +30,7 @@ import static org.graalvm.compiler.hotspot.JVMCIVersionCheck.JVMCI8_RELEASES_URL; import static org.graalvm.compiler.replacements.StandardGraphBuilderPlugins.registerInvocationPlugins; -import java.io.File; import java.io.IOException; -import java.io.PrintWriter; import java.lang.ref.Reference; import java.lang.reflect.Method; import java.lang.reflect.Modifier; @@ -56,7 +54,6 @@ import java.util.concurrent.ForkJoinPool; import java.util.concurrent.atomic.AtomicBoolean; import java.util.function.BooleanSupplier; -import java.util.function.Consumer; import java.util.stream.Collectors; import org.graalvm.collections.EconomicSet; @@ -147,7 +144,6 @@ import com.oracle.graal.pointsto.meta.AnalysisUniverse; import com.oracle.graal.pointsto.meta.HostedProviders; import com.oracle.graal.pointsto.reports.AnalysisReporter; -import com.oracle.graal.pointsto.reports.ReportUtils; import com.oracle.graal.pointsto.typestate.TypeState; import com.oracle.graal.pointsto.util.AnalysisError; import com.oracle.graal.pointsto.util.GraalAccess; @@ -271,6 +267,8 @@ import com.oracle.svm.hosted.phases.SubstrateClassInitializationPlugin; import com.oracle.svm.hosted.phases.VerifyDeoptFrameStatesLIRPhase; import com.oracle.svm.hosted.phases.VerifyNoGuardsPhase; +import com.oracle.svm.hosted.reporting.ProgressReporter; +import com.oracle.svm.hosted.reporting.ProgressReporter.ReporterClosable; import com.oracle.svm.hosted.snippets.SubstrateGraphBuilderPlugins; import com.oracle.svm.hosted.substitute.AnnotationSubstitutionProcessor; import com.oracle.svm.hosted.substitute.DeletedFieldsPlugin; @@ -297,6 +295,7 @@ public class NativeImageGenerator { protected final FeatureHandler featureHandler; protected final ImageClassLoader loader; protected final HostedOptionProvider optionProvider; + private final ProgressReporter reporter; private DeadlockWatchdog watchdog; private AnalysisUniverse aUniverse; @@ -308,13 +307,14 @@ public class NativeImageGenerator { private Pair mainEntryPoint; - final Map> buildArtifacts = new EnumMap<>(ArtifactType.class); + private final Map> buildArtifacts = new EnumMap<>(ArtifactType.class); - public NativeImageGenerator(ImageClassLoader loader, HostedOptionProvider optionProvider, Pair mainEntryPoint) { + public NativeImageGenerator(ImageClassLoader loader, HostedOptionProvider optionProvider, Pair mainEntryPoint, ProgressReporter reporter) { this.loader = loader; this.mainEntryPoint = mainEntryPoint; this.featureHandler = new FeatureHandler(); this.optionProvider = optionProvider; + this.reporter = reporter; /* * Substrate VM parses all graphs, including snippets, early. We do not support bytecode * parsing at run time. @@ -457,7 +457,7 @@ public static SubstrateTargetDescription createTarget(Platform platform) { * Executes the image build. Only one image can be built with this generator. */ public void run(Map entryPoints, - JavaMainSupport javaMainSupport, String imageName, + JavaMainSupport javaMainSupport, String imageName, Timer classlistTimer, NativeImageKind k, SubstitutionProcessor harnessSubstitutions, ForkJoinPool compilationExecutor, ForkJoinPool analysisExecutor, @@ -482,13 +482,14 @@ public void run(Map entryPoints, ImageSingletonsSupportImpl.HostedManagement.install(new ImageSingletonsSupportImpl.HostedManagement()); + ImageSingletons.add(ProgressReporter.class, reporter); ImageSingletons.add(BuildArtifacts.class, (type, artifact) -> buildArtifacts.computeIfAbsent(type, t -> new ArrayList<>()).add(artifact)); ImageSingletons.add(HostedOptionValues.class, new HostedOptionValues(optionProvider.getHostedValues())); ImageSingletons.add(RuntimeOptionValues.class, new RuntimeOptionValues(optionProvider.getRuntimeValues(), allOptionNames)); watchdog = new DeadlockWatchdog(); try (TemporaryBuildDirectoryProviderImpl tempDirectoryProvider = new TemporaryBuildDirectoryProviderImpl()) { ImageSingletons.add(TemporaryBuildDirectoryProvider.class, tempDirectoryProvider); - doRun(entryPoints, javaMainSupport, imageName, k, harnessSubstitutions, compilationExecutor, analysisExecutor); + doRun(entryPoints, javaMainSupport, imageName, classlistTimer, k, harnessSubstitutions, compilationExecutor, analysisExecutor); } finally { watchdog.close(); } @@ -518,7 +519,7 @@ protected static void clearSystemPropertiesForImage() { @SuppressWarnings("try") private void doRun(Map entryPoints, - JavaMainSupport javaMainSupport, String imageName, NativeImageKind k, + JavaMainSupport javaMainSupport, String imageName, Timer classlistTimer, NativeImageKind k, SubstitutionProcessor harnessSubstitutions, ForkJoinPool compilationExecutor, ForkJoinPool analysisExecutor) { List hostedEntryPoints = new ArrayList<>(); @@ -527,7 +528,8 @@ private void doRun(Map entryPoints, SnippetReflectionProvider originalSnippetReflection = GraalAccess.getOriginalSnippetReflection(); try (DebugContext debug = new Builder(options, new GraalDebugHandlersFactory(originalSnippetReflection)).build(); DebugCloseable featureCleanup = () -> featureHandler.forEachFeature(Feature::cleanup)) { - setupNativeImage(imageName, options, entryPoints, javaMainSupport, harnessSubstitutions, analysisExecutor, originalSnippetReflection, debug); + setupNativeImage(imageName, classlistTimer, options, entryPoints, javaMainSupport, harnessSubstitutions, analysisExecutor, originalSnippetReflection, debug); + reporter.printFeatures(featureHandler.getUserFeatureNames()); boolean returnAfterAnalysis = runPointsToAnalysis(imageName, options, debug); if (returnAfterAnalysis) { @@ -537,7 +539,7 @@ private void doRun(Map entryPoints, NativeImageHeap heap; HostedMetaAccess hMetaAccess; SharedRuntimeConfigurationBuilder runtime; - try (StopTimer t = new Timer(imageName, "universe").start()) { + try (ReporterClosable c = reporter.printUniverse(new Timer(imageName, "universe"))) { hUniverse = new HostedUniverse(bb); hMetaAccess = new HostedMetaAccess(hUniverse, bb.getMetaAccess()); @@ -615,10 +617,11 @@ private void doRun(Map entryPoints, featureHandler.forEachFeature(feature -> feature.afterCompilation(config)); } CodeCacheProvider codeCacheProvider = runtime.getRuntimeConfig().getBackendForNormalMethod().getProviders().getCodeCache(); + reporter.printCreationStart(); + Timer imageTimer = new Timer(imageName, "image"); try (Indent indent = debug.logAndIndent("create native image")) { try (DebugContext.Scope buildScope = debug.scope("CreateImage", codeCacheProvider)) { - try (StopTimer t = new Timer(imageName, "image").start()) { - + try (StopTimer t = imageTimer.start()) { // Start building the model of the native image heap. heap.addInitialObjects(); // Then build the model of the code cache, which can @@ -640,6 +643,7 @@ private void doRun(Map entryPoints, */ codeCache.printCompilationResults(); } + } } catch (Throwable e) { throw VMError.shouldNotReachHere(e); @@ -650,7 +654,8 @@ private void doRun(Map entryPoints, runtime.getRuntimeConfig(), aUniverse, hUniverse, optionProvider, hMetaAccess, debug); featureHandler.forEachFeature(feature -> feature.beforeImageWrite(beforeConfig)); - try (StopTimer t = new Timer(imageName, "write").start()) { + Timer writeTimer = new Timer(imageName, "write"); + try (StopTimer t = writeTimer.start()) { /* * This will write the debug info too -- i.e. we may be writing more than one file, * if the debug info is in a separate file. We need to push writing the file to the @@ -666,22 +671,12 @@ private void doRun(Map entryPoints, AfterImageWriteAccessImpl afterConfig = new AfterImageWriteAccessImpl(featureHandler, loader, hUniverse, inv, tmpDir, image.getImageKind(), debug); featureHandler.forEachFeature(feature -> feature.afterImageWrite(afterConfig)); } - } - } - - void reportBuildArtifacts(String imageName) { - Path buildDir = generatedFiles(HostedOptionValues.singleton()); - Consumer writerConsumer = writer -> buildArtifacts.forEach((artifactType, paths) -> { - writer.println("[" + artifactType + "]"); - if (artifactType == BuildArtifacts.ArtifactType.JDK_LIB_SHIM) { - writer.println("# Note that shim JDK libraries depend on this"); - writer.println("# particular native image (including its name)"); - writer.println("# and therefore cannot be used with others."); + reporter.printCreationEnd(imageTimer, writeTimer, image.getImageSize(), bb.getUniverse(), heap.getObjectCount(), image.getImageHeapSize(), codeCache.getCodeCacheSize(), + codeCache.getCompilations().size()); + if (SubstrateOptions.BuildOutputBreakdowns.getValue()) { + ProgressReporter.singleton().printBreakdowns(compileQueue.getCompilationTasks(), image.getHeap().getObjects()); } - paths.stream().map(Path::toAbsolutePath).map(buildDir::relativize).forEach(writer::println); - writer.println(); - }); - ReportUtils.report("build artifacts", buildDir.resolve(imageName + ".build_artifacts.txt"), writerConsumer); + } } @SuppressWarnings("try") @@ -693,7 +688,7 @@ private boolean runPointsToAnalysis(String imageName, OptionValues options, Debu bb.getHostVM().getClassInitializationSupport().setConfigurationSealed(true); } - try (StopTimer t = bb.getAnalysisTimer().start()) { + try (ReporterClosable c = ProgressReporter.singleton().printAnalysis(bb)) { DuringAnalysisAccessImpl config = new DuringAnalysisAccessImpl(featureHandler, loader, bb, nativeLibraries, debug); try { bb.runAnalysis(debug, (universe) -> { @@ -763,10 +758,12 @@ private boolean verifyAssignableTypes(String imageName) { } @SuppressWarnings("try") - private void setupNativeImage(String imageName, OptionValues options, Map entryPoints, JavaMainSupport javaMainSupport, SubstitutionProcessor harnessSubstitutions, + private void setupNativeImage(String imageName, Timer classlistTimer, OptionValues options, Map entryPoints, JavaMainSupport javaMainSupport, + SubstitutionProcessor harnessSubstitutions, ForkJoinPool analysisExecutor, SnippetReflectionProvider originalSnippetReflection, DebugContext debug) { try (Indent ignored = debug.logAndIndent("setup native-image builder")) { - try (StopTimer ignored1 = new Timer(imageName, "setup").start()) { + Timer timer = new Timer(imageName, "setup"); + try (StopTimer ignored1 = timer.start()) { SubstrateTargetDescription target = createTarget(loader.platform); ImageSingletons.add(Platform.class, loader.platform); ImageSingletons.add(SubstrateTargetDescription.class, target); @@ -848,6 +845,8 @@ private void setupNativeImage(String imageName, OptionValues options, Map CEntryPointCallStubSupport.singleton().registerStubForMethod(method, () -> entryPointData)); } + + ProgressReporter.singleton().printInitializeEnd(classlistTimer, timer, nativeLibraries.getLibraries()); } } @@ -1561,6 +1560,10 @@ public BigBang getBigbang() { return bb; } + public Map> getBuildArtifacts() { + return buildArtifacts; + } + private void printTypes() { for (HostedType type : hUniverse.getTypes()) { System.out.format("%8d %s ", type.getTypeID(), type.toJavaName(true)); @@ -1667,19 +1670,6 @@ private static void printMethod(HostedMethod method, int vtableIndex) { System.out.println("]"); } - void printImageBuildStatistics(String imageName) { - Consumer reporter = ImageSingletons.lookup(ImageBuildStatistics.class).getReporter(); - String description = "image build statistics"; - if (ImageBuildStatistics.Options.ImageBuildStatisticsFile.hasBeenSet(bb.getOptions())) { - final File file = new File(ImageBuildStatistics.Options.ImageBuildStatisticsFile.getValue(bb.getOptions())); - ReportUtils.report(description, file.getAbsoluteFile().toPath(), reporter); - } else { - String name = "image_build_statistics_" + ReportUtils.extractImageName(imageName); - String path = SubstrateOptions.Path.getValue() + File.separatorChar + "reports"; - ReportUtils.report(description, path, name, "json", reporter); - } - } - private static String slotsToString(short[] slots) { if (slots == null) { return "null"; diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/NativeImageGeneratorRunner.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/NativeImageGeneratorRunner.java index 82cddbbde9a2..50ecd696c98f 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/NativeImageGeneratorRunner.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/NativeImageGeneratorRunner.java @@ -72,8 +72,8 @@ import com.oracle.svm.hosted.code.CEntryPointData; import com.oracle.svm.hosted.image.AbstractImage.NativeImageKind; import com.oracle.svm.hosted.option.HostedOptionParser; +import com.oracle.svm.hosted.reporting.ProgressReporter; import com.oracle.svm.util.ClassUtil; -import com.oracle.svm.util.ImageBuildStatistics; import com.oracle.svm.util.ModuleSupport; import com.oracle.svm.util.ReflectionUtil; import com.oracle.svm.util.ReflectionUtil.ReflectionUtilError; @@ -298,6 +298,9 @@ private int buildImage(ImageClassLoader classLoader) { ForkJoinPool analysisExecutor = null; ForkJoinPool compilationExecutor = null; + + ProgressReporter reporter = new ProgressReporter(parsedHostedOptions); + try (StopTimer ignored = totalTimer.start()) { Timer classlistTimer = new Timer("classlist", false); try (StopTimer ignored1 = classlistTimer.start()) { @@ -310,6 +313,7 @@ private int buildImage(ImageClassLoader classLoader) { if (imageName.length() == 0) { throw UserError.abort("No output file name specified. Use '%s'.", SubstrateOptionsParser.commandArgument(SubstrateOptions.Name, "")); } + reporter.printStart(imageName); totalTimer.setPrefix(imageName); classlistTimer.setPrefix(imageName); @@ -416,8 +420,8 @@ private int buildImage(ImageClassLoader classLoader) { int maxConcurrentThreads = NativeImageOptions.getMaximumNumberOfConcurrentThreads(parsedHostedOptions); analysisExecutor = NativeImagePointsToAnalysis.createExecutor(debug, NativeImageOptions.getMaximumNumberOfAnalysisThreads(parsedHostedOptions)); compilationExecutor = NativeImagePointsToAnalysis.createExecutor(debug, maxConcurrentThreads); - generator = new NativeImageGenerator(classLoader, optionParser, mainEntryPointData); - generator.run(entryPoints, javaMainSupport, imageName, imageKind, SubstitutionProcessor.IDENTITY, + generator = new NativeImageGenerator(classLoader, optionParser, mainEntryPointData, reporter); + generator.run(entryPoints, javaMainSupport, imageName, classlistTimer, imageKind, SubstitutionProcessor.IDENTITY, compilationExecutor, analysisExecutor, optionParser.getRuntimeOptionNames()); } catch (InterruptImageBuilding e) { if (analysisExecutor != null) { @@ -468,10 +472,7 @@ private int buildImage(ImageClassLoader classLoader) { } finally { totalTimer.print(); if (imageName != null && generator != null) { - if (ImageBuildStatistics.Options.CollectImageBuildStatistics.getValue(parsedHostedOptions)) { - generator.printImageBuildStatistics(imageName); - } - generator.reportBuildArtifacts(imageName); + reporter.printEpilog(generator, imageName, totalTimer, parsedHostedOptions); } NativeImageGenerator.clearSystemPropertiesForImage(); ImageSingletonsSupportImpl.HostedManagement.clear(); diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/StringAccess.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/StringAccess.java new file mode 100644 index 000000000000..630404f47099 --- /dev/null +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/StringAccess.java @@ -0,0 +1,32 @@ +/* + * Copyright (c) 2021, 2021, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package com.oracle.svm.hosted; + +/* TODO: Added as a temporary workaround to provide compatibility with JDK8 (GR-35238). */ +public class StringAccess { + public static int getInternalByteArrayLength(@SuppressWarnings("unused") String string) { + return 0; // not supported + } +} diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/code/CompileQueue.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/code/CompileQueue.java index fd2f2e77364b..39377b767663 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/code/CompileQueue.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/code/CompileQueue.java @@ -122,7 +122,6 @@ import com.oracle.graal.pointsto.util.CompletionExecutor; import com.oracle.graal.pointsto.util.CompletionExecutor.DebugContextRunnable; import com.oracle.graal.pointsto.util.Timer; -import com.oracle.graal.pointsto.util.Timer.StopTimer; import com.oracle.svm.core.SubstrateOptions; import com.oracle.svm.core.annotate.AlwaysInlineAllCallees; import com.oracle.svm.core.annotate.AlwaysInlineSelectCallees; @@ -158,6 +157,8 @@ import com.oracle.svm.hosted.phases.ImageBuildStatisticsCounterPhase; import com.oracle.svm.hosted.phases.ImplicitAssertionsPhase; import com.oracle.svm.hosted.phases.StrengthenStampsPhase; +import com.oracle.svm.hosted.reporting.ProgressReporter; +import com.oracle.svm.hosted.reporting.ProgressReporter.ReporterClosable; import com.oracle.svm.hosted.substitute.DeletedMethod; import com.oracle.svm.util.ImageBuildStatistics; @@ -361,9 +362,10 @@ protected void callForReplacements(DebugContext debug, @SuppressWarnings("hiding @SuppressWarnings("try") public void finish(DebugContext debug) { + ProgressReporter reporter = ProgressReporter.singleton(); try { String imageName = universe.getBigBang().getHostVM().getImageName(); - try (StopTimer t = new Timer(imageName, "(parse)").start()) { + try (ReporterClosable ac = reporter.printParsing(new Timer(imageName, "(parse)"))) { parseAll(); } // Checking @Uninterruptible annotations does not take long enough to justify a timer. @@ -382,14 +384,16 @@ public void finish(DebugContext debug) { } if (SubstrateOptions.AOTInline.getValue() && SubstrateOptions.AOTTrivialInline.getValue()) { - try (StopTimer ignored = new Timer(imageName, "(inline)").start()) { + try (ReporterClosable ac = reporter.printInlining(new Timer(imageName, "(inline)"))) { inlineTrivialMethods(debug); } + } else { + reporter.printInliningSkipped(); } assert suitesNotCreated(); createSuites(); - try (StopTimer t = new Timer(imageName, "(compile)").start()) { + try (ReporterClosable ac = reporter.printCompiling(new Timer(imageName, "(compile)"))) { compileAll(); } } catch (InterruptedException ie) { @@ -577,6 +581,7 @@ protected void inlineTrivialMethods(DebugContext debug) throws InterruptedExcept int round = 0; do { + ProgressReporter.singleton().printStageProgress(); inliningProgress = false; round++; try (Indent ignored = debug.logAndIndent("==== Trivial Inlining round %d\n", round)) { diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/image/AbstractImage.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/image/AbstractImage.java index 1fbc17cc60cd..3745dddc4217 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/image/AbstractImage.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/image/AbstractImage.java @@ -159,6 +159,8 @@ public NativeImageHeap getHeap() { return heap; } + public abstract long getImageHeapSize(); + public abstract ObjectFile getOrCreateDebugObjectFile(); public boolean requiresCustomDebugRelocation() { diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/image/NativeImage.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/image/NativeImage.java index dfc5e46b487b..7f2057e9cebe 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/image/NativeImage.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/image/NativeImage.java @@ -114,6 +114,7 @@ import com.oracle.svm.hosted.meta.HostedMetaAccess; import com.oracle.svm.hosted.meta.HostedMethod; import com.oracle.svm.hosted.meta.HostedUniverse; +import com.oracle.svm.hosted.reporting.ProgressReporter; import com.oracle.svm.util.ReflectionUtil; import com.oracle.svm.util.ReflectionUtil.ReflectionUtilError; @@ -133,6 +134,8 @@ public abstract class NativeImage extends AbstractImage { private final int wordSize; private final Set uniqueEntryPoints = new HashSet<>(); + private long imageHeapSize = -1; + // The sections of the native image. private Section textSection; private Section roDataSection; @@ -413,6 +416,8 @@ public void build(String imageName, DebugContext debug) { // after this point, the layout is final and must not be changed anymore assert !hasDuplicatedObjects(heap.getObjects()) : "heap.getObjects() must not contain any duplicates"; + imageHeapSize = heapLayout.getImageHeapSize(); + // Text section (code) final int textSectionSize = codeCache.getCodeCacheSize(); final RelocatableBuffer textBuffer = new RelocatableBuffer(textSectionSize, objectFile.getByteOrder()); @@ -454,17 +459,19 @@ public void build(String imageName, DebugContext debug) { * If we constructed debug info give the object file a chance to install it */ if (SubstrateOptions.GenerateDebugInfo.getValue(HostedOptionValues.singleton()) > 0) { - try (Timer.StopTimer t = new Timer(imageName, "dbginfo").start()) { + Timer timer = new Timer(imageName, "dbginfo"); + try (Timer.StopTimer t = timer.start()) { ImageSingletons.add(SourceManager.class, new SourceManager()); DebugInfoProvider provider = new NativeImageDebugInfoProvider(debug, codeCache, heap); objectFile.installDebugInfo(provider); } + ProgressReporter.singleton().setDebugInfoTimer(timer); } // - Write the heap to its own section. // Dynamic linkers/loaders generally don't ensure any alignment to more than page // boundaries, so we take care of this ourselves in CommittedMemoryProvider, if we can. int alignment = objectFile.getPageSize(); - RelocatableBuffer heapSectionBuffer = new RelocatableBuffer(heapLayout.getImageHeapSize(), objectFile.getByteOrder()); + RelocatableBuffer heapSectionBuffer = new RelocatableBuffer(imageHeapSize, objectFile.getByteOrder()); ProgbitsSectionImpl heapSectionImpl = new BasicProgbitsSectionImpl(heapSectionBuffer.getBackingArray()); heapSection = objectFile.newProgbitsSection(SectionName.SVM_HEAP.getFormatDependentName(objectFile.getFormat()), alignment, writable, false, heapSectionImpl); objectFile.createDefinedSymbol(heapSection.getName(), heapSection, 0, 0, false, false); @@ -473,7 +480,7 @@ public void build(String imageName, DebugContext debug) { assert !SubstrateOptions.SpawnIsolates.getValue() || heapSectionBuffer.getByteBuffer().getLong((int) offsetOfARelocatablePointer) == 0L; defineDataSymbol(Isolates.IMAGE_HEAP_BEGIN_SYMBOL_NAME, heapSection, 0); - defineDataSymbol(Isolates.IMAGE_HEAP_END_SYMBOL_NAME, heapSection, heapLayout.getImageHeapSize()); + defineDataSymbol(Isolates.IMAGE_HEAP_END_SYMBOL_NAME, heapSection, imageHeapSize); defineDataSymbol(Isolates.IMAGE_HEAP_RELOCATABLE_BEGIN_SYMBOL_NAME, heapSection, heapLayout.getReadOnlyRelocatableOffset()); defineDataSymbol(Isolates.IMAGE_HEAP_RELOCATABLE_END_SYMBOL_NAME, heapSection, heapLayout.getReadOnlyRelocatableOffset() + heapLayout.getReadOnlyRelocatableSize()); defineDataSymbol(Isolates.IMAGE_HEAP_A_RELOCATABLE_POINTER_SYMBOL_NAME, heapSection, offsetOfARelocatablePointer); @@ -746,6 +753,12 @@ public static String globalSymbolNameForMethod(ResolvedJavaMethod sm) { return mangleName((sm instanceof HostedMethod ? ((HostedMethod) sm).getUniqueShortName() : SubstrateUtil.uniqueShortName(sm))); } + @Override + public long getImageHeapSize() { + assert imageHeapSize > -1 : "imageHeapSize accessed before set"; + return imageHeapSize; + } + @Override public ObjectFile getOrCreateDebugObjectFile() { assert objectFile != null; diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/reporting/ProgressReporter.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/reporting/ProgressReporter.java new file mode 100644 index 000000000000..001e203f8289 --- /dev/null +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/reporting/ProgressReporter.java @@ -0,0 +1,985 @@ +/* + * Copyright (c) 2021, 2021, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package com.oracle.svm.hosted.reporting; + +import java.io.File; +import java.io.PrintWriter; +import java.lang.management.GarbageCollectorMXBean; +import java.lang.management.ManagementFactory; +import java.lang.management.OperatingSystemMXBean; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Comparator; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledFuture; +import java.util.concurrent.TimeUnit; +import java.util.function.Consumer; + +import org.graalvm.compiler.options.OptionValues; +import org.graalvm.compiler.serviceprovider.GraalServices; +import org.graalvm.nativeimage.ImageSingletons; +import org.graalvm.nativeimage.hosted.Feature; +import org.graalvm.nativeimage.impl.RuntimeReflectionSupport; + +import com.oracle.graal.pointsto.BigBang; +import com.oracle.graal.pointsto.meta.AnalysisMethod; +import com.oracle.graal.pointsto.meta.AnalysisUniverse; +import com.oracle.graal.pointsto.reports.ReportUtils; +import com.oracle.graal.pointsto.util.Timer; +import com.oracle.svm.core.BuildArtifacts; +import com.oracle.svm.core.BuildArtifacts.ArtifactType; +import com.oracle.svm.core.SubstrateOptions; +import com.oracle.svm.core.VM; +import com.oracle.svm.core.annotate.AutomaticFeature; +import com.oracle.svm.core.option.HostedOptionValues; +import com.oracle.svm.core.reflect.MethodMetadataDecoder; +import com.oracle.svm.hosted.NativeImageGenerator; +import com.oracle.svm.hosted.StringAccess; +import com.oracle.svm.hosted.code.CompileQueue.CompileTask; +import com.oracle.svm.hosted.image.NativeImageHeap.ObjectInfo; +import com.oracle.svm.util.ImageBuildStatistics; + +public class ProgressReporter { + private static final int CHARACTERS_PER_LINE; + private static final int PROGRESS_BAR_START = 30; + private static final boolean IS_CI = System.console() == null || System.getenv("CI") != null; + + private static final int MAX_NUM_FEATURES = 50; + private static final int MAX_NUM_BREAKDOWN = 10; + private static final String CODE_BREAKDOWN_TITLE = String.format("Top %d packages in code area:", MAX_NUM_BREAKDOWN); + private static final String HEAP_BREAKDOWN_TITLE = String.format("Top %d object types in image heap:", MAX_NUM_BREAKDOWN); + private static final String STAGE_DOCS_URL = "https://github.com/oracle/graal/blob/master/docs/reference-manual/native-image/BuildOutput.md"; + private static final double EXCESSIVE_GC_MIN_THRESHOLD_MILLIS = 15_000; + private static final double EXCESSIVE_GC_RATIO = 0.5; + private static final String BREAKDOWN_BYTE_ARRAY_PREFIX = "byte[] for "; + + private static final double MILLIS_TO_SECONDS = 1000d; + private static final double NANOS_TO_SECONDS = 1000d * 1000d * 1000d; + private static final double BYTES_TO_KiB = 1024d; + private static final double BYTES_TO_MiB = 1024d * 1024d; + private static final double BYTES_TO_GiB = 1024d * 1024d * 1024d; + + private final boolean isEnabled; + private final LinePrinter linePrinter; + private final boolean usePrefix; + private final boolean showLinks; + private final ScheduledExecutorService executor = Executors.newSingleThreadScheduledExecutor(); + + private ScheduledFuture periodicPrintingTask; + private int periodicPrintingTaskPeriodSeconds = 1; + + private int numStageChars = 0; + private long lastGCCheckTimeMillis = System.currentTimeMillis(); + private GCStats lastGCStats = getCurrentGCStats(); + private long graphEncodingByteLength = 0; + private Timer debugInfoTimer; + + private enum BuildStage { + INITIALIZE("Initializing"), + ANALYSIS("Performing analysis"), + UNIVERSE("Building universe"), + PARSING("Parsing methods"), + INLINING("Inlining methods"), + COMPILING("Compiling methods"), + CREATION("Creating image"); + + private static final int NUM_STAGES = values().length; + + private final String message; + + BuildStage(String message) { + this.message = message; + } + } + + static { + CHARACTERS_PER_LINE = IS_CI ? ProgressReporterCHelper.MAX_CHARACTERS_PER_LINE : ProgressReporterCHelper.getTerminalWindowColumnsClamped(); + } + + public static ProgressReporter singleton() { + return ImageSingletons.lookup(ProgressReporter.class); + } + + public ProgressReporter(OptionValues options) { + isEnabled = SubstrateOptions.BuildOutputUseNewStyle.getValue(options); + if (isEnabled) { + Timer.disablePrinting(); + } + if (SubstrateOptions.BuildOutputPrefix.hasBeenSet(options)) { + usePrefix = SubstrateOptions.BuildOutputPrefix.getValue(options); + } else { + usePrefix = IS_CI; + } + boolean enableColors = !IS_CI; + if (SubstrateOptions.BuildOutputColorful.hasBeenSet(options)) { + enableColors = SubstrateOptions.BuildOutputColorful.getValue(options); + } + boolean enableProgress = !IS_CI; + if (SubstrateOptions.BuildOutputProgress.hasBeenSet(options)) { + enableProgress = SubstrateOptions.BuildOutputProgress.getValue(options); + } + if (enableColors) { + linePrinter = new ColorfulLinePrinter(enableProgress); + /* Add a shutdown hook to reset the ANSI mode. */ + try { + Runtime.getRuntime().addShutdownHook(new Thread(ProgressReporter::resetANSIMode)); + } catch (IllegalStateException e) { + /* If the VM is already shutting down, we do not need to register shutdownHook. */ + } + } else { + linePrinter = new ColorlessLinePrinter(enableProgress); + } + showLinks = SubstrateOptions.BuildOutputColorful.getValue(options); + } + + private LinePrinter l() { + assert linePrinter.isEmpty() : "Line printer should be empty before printing a new line"; + return linePrinter; + } + + public void setGraphEncodingByteLength(int value) { + graphEncodingByteLength = value; + } + + public void printStart(String imageName) { + if (usePrefix) { + // Add the PID to further disambiguate concurrent builds of images with the same name + linePrinter.outputPrefix = String.format("[%s:%s] ", imageName, GraalServices.getExecutionID()); + } + l().printHeadlineSeparator(); + l().blueBold().link("GraalVM Native Image", "https://www.graalvm.org/native-image/").reset() + .a(": Generating '").bold().a(imageName).a("'...").reset().flushln(); + l().printHeadlineSeparator(); + printStageStart(BuildStage.INITIALIZE); + } + + public void printInitializeEnd(Timer classlistTimer, Timer setupTimer, Collection libraries) { + printStageEnd(classlistTimer.getTotalTime() + setupTimer.getTotalTime()); + l().a(" Version info: '").a(ImageSingletons.lookup(VM.class).version).a("'").flushln(); + printNativeLibraries(libraries); + } + + private void printNativeLibraries(Collection libraries) { + int numLibraries = libraries.size(); + if (numLibraries > 0) { + if (numLibraries == 1) { + l().a(" 1 native library: ").a(libraries.iterator().next()).flushln(); + } else { + l().a(" ").a(numLibraries).a(" native libraries: ").a(String.join(", ", libraries)).flushln(); + } + } + } + + public void printFeatures(List list) { + int numUserFeatures = list.size(); + if (numUserFeatures > 0) { + l().a(" ").a(numUserFeatures).a(" ").doclink("user-provided feature(s)", "#glossary-user-provided-features").flushln(); + if (numUserFeatures <= MAX_NUM_FEATURES) { + for (String name : list) { + l().a(" - ").a(name).flushln(); + } + } else { + for (int i = 0; i < MAX_NUM_FEATURES; i++) { + l().a(" - ").a(list.get(i)).flushln(); + } + l().a(" ... ").a(numUserFeatures - MAX_NUM_FEATURES).a(" more").flushln(); + } + } + } + + public ReporterClosable printAnalysis(BigBang bb) { + int numReflectionClasses = ImageSingletons.lookup(RuntimeReflectionSupport.class).getReflectionClassesCount(); + l().a(" ").a(numReflectionClasses).a(" ").doclink("classes registered for reflection", "#glossary-reflection-classes").flushln(); + Timer timer = bb.getAnalysisTimer(); + timer.start(); + printStageStart(BuildStage.ANALYSIS); + printProgressStart(); + return new ReporterClosable() { + @Override + public void closeAction() { + timer.stop(); + printProgressEnd(); + int analysisMillis = (int) bb.getAnalysisTimer().getTotalTime(); + printStageEnd(analysisMillis); + if (analysisMillis > 30_000) { + // Adjust period for printing task according to duration of analysis. + periodicPrintingTaskPeriodSeconds = analysisMillis / 30_000; + } + Collection methods = bb.getUniverse().getMethods(); + long reachableMethod = methods.stream().filter(m -> m.isReachable()).count(); + int totalMethods = methods.size(); + l().a(" %,6d (%2.2f%%) of %,6d methods ", reachableMethod, reachableMethod / (double) totalMethods * 100, totalMethods) + .doclink("reachable", "#glossary-reachability").flushln(); + long reachableClasses = bb.getUniverse().getTypes().stream().filter(t -> t.isReachable()).count(); + long totalClasses = bb.getUniverse().getTypes().size(); + l().a(" %,6d (%2.2f%%) of %,6d classes ", reachableClasses, reachableClasses / (double) totalClasses * 100, totalClasses) + .doclink("reachable", "#glossary-reachability").flushln(); + } + }; + + } + + public ReporterClosable printUniverse(Timer timer) { + timer.start(); + printStageStart(BuildStage.UNIVERSE); + return new ReporterClosable() { + @Override + public void closeAction() { + timer.stop(); + printStageEnd(timer); + } + }; + } + + public void printRuntimeCompileMethods(int numRuntimeCompileMethods, int numTotalMethods) { + l().a(" %,6d (%2.2f%%) of %,6d methods included for ", numRuntimeCompileMethods, numRuntimeCompileMethods / (double) numTotalMethods * 100, numTotalMethods) + .doclink("runtime compilation", "#glossary-runtime-methods").flushln(); + } + + public ReporterClosable printParsing(Timer timer) { + timer.start(); + printStageStart(BuildStage.PARSING); + printProgressStart(); + startPeriodicPrinting(); + return new ReporterClosable() { + @Override + public void closeAction() { + timer.stop(); + stopPeriodicPrinting(); + printProgressEnd(); + printStageEnd(timer); + } + }; + } + + public ReporterClosable printInlining(Timer timer) { + timer.start(); + printStageStart(BuildStage.INLINING); + printProgressStart(); + return new ReporterClosable() { + @Override + public void closeAction() { + timer.stop(); + printProgressEnd(); + printStageEnd(timer); + } + }; + } + + public void printInliningSkipped() { + printStageStart(BuildStage.INLINING); + linePrinter.dim().a(" (skipped)").reset().flushln(); + numStageChars = 0; + } + + public ReporterClosable printCompiling(Timer timer) { + timer.start(); + printStageStart(BuildStage.COMPILING); + printProgressStart(); + startPeriodicPrinting(); + return new ReporterClosable() { + @Override + public void closeAction() { + timer.stop(); + stopPeriodicPrinting(); + printProgressEnd(); + printStageEnd(timer); + } + }; + } + + // TODO: merge printCreationStart and printCreationEnd at some point (GR-35238). + public void printCreationStart() { + printStageStart(BuildStage.CREATION); + } + + public void setDebugInfoTimer(Timer timer) { + this.debugInfoTimer = timer; + } + + public void printCreationEnd(Timer creationTimer, Timer writeTimer, int imageSize, AnalysisUniverse universe, int numHeapObjects, long imageHeapSize, int codeCacheSize, + int numCompilations) { + printStageEnd(creationTimer.getTotalTime() + writeTimer.getTotalTime()); + String total = bytesToHuman("%4.2f", imageSize); + l().a("%9s in total (%2.2f%% for ", total, codeCacheSize / (double) imageSize * 100).doclink("code area", "#glossary-code-area") + .a(" and %2.2f%% for ", imageHeapSize / (double) imageSize * 100).doclink("image heap", "#glossary-image-heap").flushln(); + l().a("%9s in code size: %,8d compilation units", bytesToHuman("%4.2f", codeCacheSize), numCompilations).flushln(); + long numInstantiatedClasses = universe.getTypes().stream().filter(t -> t.isInstantiated()).count(); + l().a("%9s in heap size: %,8d classes and %,d objects", bytesToHuman("%4.2f", imageHeapSize), numInstantiatedClasses, numHeapObjects).flushln(); + if (debugInfoTimer != null) { + String debugInfoTime = String.format("%4.3fms", debugInfoTimer.getTotalTime()); + l().dim().a("%9s for generating debug info (%2.2f%% of time to create image)", debugInfoTime, debugInfoTimer.getTotalTime() / creationTimer.getTotalTime() * 100).reset().flushln(); + } + } + + public void printBreakdowns(Collection compilationTasks, Collection heapObjects) { + Map codeBreakdown = calculateCodeBreakdown(compilationTasks); + Map heapBreakdown = calculateHeapBreakdown(heapObjects); + l().printLineSeparator(); + int numCodeBreakdownItems = codeBreakdown.size(); + int numHeapBreakdownItems = heapBreakdown.size(); + Iterator> packagesBySize = codeBreakdown.entrySet().stream() + .sorted(Entry.comparingByValue(Comparator.reverseOrder())).iterator(); + Iterator> typesBySizeInHeap = heapBreakdown.entrySet().stream() + .sorted(Entry.comparingByValue(Comparator.reverseOrder())).iterator(); + + l().yellowBold().a(CODE_BREAKDOWN_TITLE).jumpToMiddle().a(HEAP_BREAKDOWN_TITLE).reset().flushln(); + + List> printedCodeSizeEntries = new ArrayList<>(); + List> printedHeapSizeEntries = new ArrayList<>(); + for (int i = 0; i < MAX_NUM_BREAKDOWN; i++) { + String codeSizePart = ""; + if (packagesBySize.hasNext()) { + Entry e = packagesBySize.next(); + String className = truncateClassName(e.getKey()); + codeSizePart = String.format("%9s %s", bytesToHuman("%4.2f", e.getValue()), className); + printedCodeSizeEntries.add(e); + } + + String heapSizePart = ""; + if (typesBySizeInHeap.hasNext()) { + Entry e = typesBySizeInHeap.next(); + String className = e.getKey(); + // Do not truncate special breakdown items, they can contain links. + if (!className.startsWith(BREAKDOWN_BYTE_ARRAY_PREFIX)) { + className = truncateClassName(className); + } + heapSizePart = String.format("%9s %s", bytesToHuman(e.getValue()), className); + printedHeapSizeEntries.add(e); + } + if (codeSizePart.isEmpty() && heapSizePart.isEmpty()) { + break; + } + l().a(codeSizePart).jumpToMiddle().a(heapSizePart).flushln(); + } + + l().a(" ... ").a(numCodeBreakdownItems - printedHeapSizeEntries.size()).a(" additional packages") + .jumpToMiddle() + .a(" ... ").a(numHeapBreakdownItems - printedCodeSizeEntries.size()).a(" additional object types").flushln(); + + l().dim().a("(use ").link("GraalVM Dashboard", "https://www.graalvm.org/dashboard/?ojr=help%3Btopic%3Dgetting-started.md").a(" to see all)").reset().flushCenteredln(); + } + + private static Map calculateCodeBreakdown(Collection compilationTasks) { + Map classNameToCodeSize = new HashMap<>(); + for (CompileTask task : compilationTasks) { + String className = task.method.format("%H"); + classNameToCodeSize.merge(className, (long) task.result.getTargetCodeSize(), (a, b) -> a + b); + } + return classNameToCodeSize; + } + + private Map calculateHeapBreakdown(Collection heapObjects) { + Map classNameToSize = new HashMap<>(); + long stringByteLength = 0; + for (ObjectInfo o : heapObjects) { + classNameToSize.merge(o.getClazz().toJavaName(true), o.getSize(), (a, b) -> a + b); + Object javaObject = o.getObject(); + if (javaObject instanceof String) { + stringByteLength += StringAccess.getInternalByteArrayLength((String) javaObject); + } + } + + Long byteArraySize = classNameToSize.remove("byte[]"); + if (byteArraySize != null) { + classNameToSize.put(BREAKDOWN_BYTE_ARRAY_PREFIX + "java.lang.String[]", stringByteLength); + long metadataByteLength = ImageSingletons.lookup(MethodMetadataDecoder.class).getMetadataByteLength(); + if (metadataByteLength > 0) { + classNameToSize.put(BREAKDOWN_BYTE_ARRAY_PREFIX + linePrinter.asDocLink("method metadata", "#heapbreakdown-method-metadata"), metadataByteLength); + } + if (graphEncodingByteLength > 0) { + classNameToSize.put(BREAKDOWN_BYTE_ARRAY_PREFIX + linePrinter.asDocLink("graph encodings", "#heapbreakdown-graph-encodings"), graphEncodingByteLength); + } + long remaining = byteArraySize - stringByteLength - metadataByteLength - graphEncodingByteLength; + assert remaining >= 0; + classNameToSize.put(BREAKDOWN_BYTE_ARRAY_PREFIX + linePrinter.asDocLink("assorted data", "#heapbreakdown-assorted-data"), remaining); + } + return classNameToSize; + } + + public void printEpilog(NativeImageGenerator generator, String imageName, Timer totalTimer, OptionValues parsedHostedOptions) { + l().printLineSeparator(); + printStats(millisToSeconds(totalTimer.getTotalTime())); + l().printLineSeparator(); + + l().yellowBold().a("Produced artifacts:").reset().flushln(); + generator.getBuildArtifacts().forEach((artifactType, paths) -> { + for (Path p : paths) { + l().a(" ").a(p.toString()).dim().a(" (").a(artifactType.name()).a(")").reset().flushln(); + } + }); + if (ImageBuildStatistics.Options.CollectImageBuildStatistics.getValue(parsedHostedOptions)) { + l().a(" ").a(reportImageBuildStatistics(imageName, generator.getBigbang()).toString()).flushln(); + } + l().a(" ").a(reportBuildArtifacts(imageName, generator.getBuildArtifacts()).toString()).flushln(); + + l().printHeadlineSeparator(); + + double totalSeconds = millisToSeconds(totalTimer.getTotalTime()); + String timeStats; + if (totalSeconds <= 60) { + timeStats = String.format("%.1fs", totalSeconds); + } else { + timeStats = String.format("%.0fm %.0fs", totalSeconds / 60, totalSeconds % 60); + } + l().a("Finished generating '").bold().a(imageName).reset().a(" in ").a(timeStats).a(".").flushln(); + executor.shutdown(); + } + + private Path reportImageBuildStatistics(String imageName, BigBang bb) { + Consumer statsReporter = ImageSingletons.lookup(ImageBuildStatistics.class).getReporter(); + String description = "image build statistics"; + if (ImageBuildStatistics.Options.ImageBuildStatisticsFile.hasBeenSet(bb.getOptions())) { + final File file = new File(ImageBuildStatistics.Options.ImageBuildStatisticsFile.getValue(bb.getOptions())); + return ReportUtils.report(description, file.getAbsoluteFile().toPath(), statsReporter, !isEnabled); + } else { + String name = "image_build_statistics_" + ReportUtils.extractImageName(imageName); + String path = SubstrateOptions.Path.getValue() + File.separatorChar + "reports"; + return ReportUtils.report(description, path, name, "json", statsReporter, !isEnabled); + } + } + + private Path reportBuildArtifacts(String imageName, Map> buildArtifacts) { + Path buildDir = NativeImageGenerator.generatedFiles(HostedOptionValues.singleton()); + + Consumer writerConsumer = writer -> buildArtifacts.forEach((artifactType, paths) -> { + writer.println("[" + artifactType + "]"); + if (artifactType == BuildArtifacts.ArtifactType.JDK_LIB_SHIM) { + writer.println("# Note that shim JDK libraries depend on this"); + writer.println("# particular native image (including its name)"); + writer.println("# and therefore cannot be used with others."); + } + paths.stream().map(Path::toAbsolutePath).map(buildDir::relativize).forEach(writer::println); + writer.println(); + }); + return ReportUtils.report("build artifacts", buildDir.resolve(imageName + ".build_artifacts.txt"), writerConsumer, !isEnabled); + } + + private void printStats(double totalSeconds) { + GCStats gcStats = getCurrentGCStats(); + LinePrinter l = l().a("%.1fs", millisToSeconds(gcStats.totalTimeMillis)).a(" spent in ").a(gcStats.totalCount).a(" GCs | "); + long peakRSS = ProgressReporterCHelper.getPeakRSS(); + if (peakRSS >= 0) { + l.doclink("Peak RSS", "#glossary-peak-rss").a(": ").a("%.2fGB", bytesToGiB(peakRSS)).a(" | "); + } + OperatingSystemMXBean osMXBean = ManagementFactory.getOperatingSystemMXBean(); + double processCPUTime = nanosToSeconds(((com.sun.management.OperatingSystemMXBean) osMXBean).getProcessCpuTime()); + l.doclink("CPU load", "#glossary-cpu-load").a(": ").a("~%.2f%%", processCPUTime / totalSeconds * 100).flushCenteredln(); + } + + private void printStageStart(BuildStage stage) { + assert numStageChars == 0; + l().appendPrefix().flush(); + linePrinter.blue() + .a("[").a(1 + stage.ordinal()).a("/").a(BuildStage.NUM_STAGES).a("] ").reset().blueBold() + .doclink(stage.message, "#step-" + stage.name().toLowerCase()).a("...").reset(); + numStageChars = linePrinter.getCurrentTextLength(); + linePrinter.flush(); + } + + private void printProgressStart() { + linePrinter.a(stringFilledWith(PROGRESS_BAR_START - numStageChars, " ")).dim().a("["); + numStageChars = PROGRESS_BAR_START + 1; /* +1 for [ */ + linePrinter.flush(); + } + + private void printProgressEnd() { + linePrinter.a("]").reset().flush(); + numStageChars++; // for ] + } + + public void printStageProgress() { + linePrinter.printRaw("*"); + numStageChars++; // for * + } + + private ScheduledFuture startPeriodicPrinting() { + periodicPrintingTask = executor.scheduleAtFixedRate(new Runnable() { + @Override + public void run() { + printStageProgress(); + } + }, 0, periodicPrintingTaskPeriodSeconds, TimeUnit.SECONDS); + return periodicPrintingTask; + } + + private void stopPeriodicPrinting() { + periodicPrintingTask.cancel(false); + } + + private void printStageEnd(Timer timer) { + printStageEnd(timer.getTotalTime()); + } + + private void printStageEnd(double totalTime) { + assert numStageChars >= 0; + String suffix = String.format("(%.1fs @ %.2fGB)", millisToSeconds(totalTime), getUsedMemory()); + String padding = stringFilledWith(Math.max(0, CHARACTERS_PER_LINE - numStageChars - suffix.length()), " "); + linePrinter.a(padding).dim().a(suffix).reset().flushln(false); + numStageChars = 0; + if (SubstrateOptions.BuildOutputGCWarnings.getValue()) { + checkForExcessiveGarbageCollection(); + } + } + + private void checkForExcessiveGarbageCollection() { + long current = System.currentTimeMillis(); + long timeDeltaMillis = current - lastGCCheckTimeMillis; + lastGCCheckTimeMillis = current; + GCStats currentGCStats = getCurrentGCStats(); + long gcTimeDeltaMillis = currentGCStats.totalTimeMillis - lastGCStats.totalTimeMillis; + double ratio = gcTimeDeltaMillis / (double) timeDeltaMillis; + if (gcTimeDeltaMillis > EXCESSIVE_GC_MIN_THRESHOLD_MILLIS && ratio > EXCESSIVE_GC_RATIO) { + l().redBold().a("GC warning").reset() + .a(": %.1fs spent in %d GCs during the last stage, taking up %2.2f%% of the time.", + millisToSeconds(gcTimeDeltaMillis), currentGCStats.totalCount - lastGCStats.totalCount, ratio * 100) + .flushln(); + l().a(" Please ensure more than %.2fGB of memory is available for Native Image", bytesToGiB(ProgressReporterCHelper.getPeakRSS())).flushln(); + l().a(" to reduce GC overhead and improve image build time.").flushln(); + } + lastGCStats = currentGCStats; + } + + private static void resetANSIMode() { + System.out.print(ANSIColors.RESET); + } + + /* + * HELPERS + */ + + private static String stringFilledWith(int size, String fill) { + return new String(new char[size]).replace("\0", fill); + } + + protected static String truncateClassName(String className) { + int classNameLength = className.length(); + int maxLength = CHARACTERS_PER_LINE / 2 - 10; + if (classNameLength <= maxLength) { + return className; + } + StringBuilder sb = new StringBuilder(); + int currentDot = -1; + while (true) { + int nextDot = className.indexOf('.', currentDot + 1); + if (nextDot < 0) { // Not more dots, handle the rest and return. + String rest = className.substring(currentDot + 1); + int sbLength = sb.length(); + int restLength = rest.length(); + if (sbLength + restLength <= maxLength) { + sb.append(rest); + } else { + int remainingSpaceDivBy2 = (maxLength - sbLength) / 2; + sb.append(rest.substring(0, remainingSpaceDivBy2 - 1) + "~" + rest.substring(restLength - remainingSpaceDivBy2, restLength)); + } + break; + } + sb.append(className.charAt(currentDot + 1)).append('.'); + if (sb.length() + (classNameLength - nextDot) <= maxLength) { + // Rest fits maxLength, append and return. + sb.append(className.substring(nextDot + 1)); + break; + } + currentDot = nextDot; + } + return sb.toString(); + } + + private static double getUsedMemory() { + return bytesToGiB(Runtime.getRuntime().totalMemory() - Runtime.getRuntime().freeMemory()); + } + + private static String bytesToHuman(long bytes) { + return bytesToHuman("%.2f", bytes); + } + + private static String bytesToHuman(String format, long bytes) { + if (bytes < BYTES_TO_KiB) { + return String.format(format, (double) bytes) + "B"; + } else if (bytes < BYTES_TO_MiB) { + return String.format(format, bytesToKiB(bytes)) + "KB"; + } else if (bytes < BYTES_TO_GiB) { + return String.format(format, bytesToMiB(bytes)) + "MB"; + } else { + return String.format(format, bytesToGiB(bytes)) + "GB"; + } + } + + private static double bytesToKiB(long bytes) { + return bytes / BYTES_TO_KiB; + } + + private static double bytesToGiB(long bytes) { + return bytes / BYTES_TO_GiB; + } + + private static double bytesToMiB(long bytes) { + return bytes / BYTES_TO_MiB; + } + + private static double millisToSeconds(double millis) { + return millis / MILLIS_TO_SECONDS; + } + + private static double nanosToSeconds(double nanos) { + return nanos / NANOS_TO_SECONDS; + } + + private static GCStats getCurrentGCStats() { + long totalCount = 0; + long totalTime = 0; + for (GarbageCollectorMXBean bean : ManagementFactory.getGarbageCollectorMXBeans()) { + long collectionCount = bean.getCollectionCount(); + if (collectionCount > 0) { + totalCount += collectionCount; + } + long collectionTime = bean.getCollectionTime(); + if (collectionTime > 0) { + totalTime += collectionTime; + } + } + return new GCStats(totalCount, totalTime); + } + + private static class GCStats { + private final long totalCount; + private final long totalTimeMillis; + + GCStats(long totalCount, long totalTime) { + this.totalCount = totalCount; + this.totalTimeMillis = totalTime; + } + } + + @AutomaticFeature + public static class ProgressReporterFeature implements Feature { + private final ProgressReporter reporter = ProgressReporter.singleton(); + + @Override + public void duringAnalysis(DuringAnalysisAccess access) { + reporter.printStageProgress(); + } + } + + public abstract static class ReporterClosable implements AutoCloseable { + @Override + public void close() { + closeAction(); + } + + abstract void closeAction(); + } + + /* + * COLORFUL OUTPUT + */ + + abstract class LinePrinter { + private final List textBuffer = new ArrayList<>(); + private final StringBuilder printBuffer; + protected final String headlineSeparator = stringFilledWith(CHARACTERS_PER_LINE, "="); + protected final String lineSeparator = stringFilledWith(CHARACTERS_PER_LINE, "-"); + private String outputPrefix = ""; + + LinePrinter(boolean enableProgress) { + printBuffer = enableProgress ? null : new StringBuilder(); + } + + protected LinePrinter a(String text) { + if (isEnabled) { + textBuffer.add(text); + } + return this; + } + + protected LinePrinter a(String text, Object... args) { + return a(String.format(text, args)); + } + + protected LinePrinter a(int i) { + return a(String.valueOf(i)); + } + + protected LinePrinter a(long i) { + return a(String.valueOf(i)); + } + + protected LinePrinter appendPrefix() { + return a(outputPrefix); + } + + protected LinePrinter jumpToMiddle() { + int remaining = (CHARACTERS_PER_LINE / 2) - getCurrentTextLength(); + assert remaining >= 0 : "Column text too wide"; + a(stringFilledWith(remaining, " ")); + assert getCurrentTextLength() == CHARACTERS_PER_LINE / 2; + return this; + } + + protected abstract LinePrinter bold(); + + protected abstract LinePrinter blue(); + + protected abstract LinePrinter blueBold(); + + protected abstract LinePrinter redBold(); + + protected abstract LinePrinter yellowBold(); + + protected abstract LinePrinter dim(); + + protected abstract LinePrinter reset(); + + protected abstract LinePrinter link(String text, String url); + + protected abstract LinePrinter doclink(String text, String htmlAnchor); + + protected abstract String asDocLink(String text, String htmlAnchor); + + private void flush() { + if (!isEnabled) { + return; + } + if (printBuffer != null) { + textBuffer.forEach(printBuffer::append); + } else { + textBuffer.forEach(System.out::print); + } + textBuffer.clear(); + } + + private void printRaw(String text) { + if (!isEnabled) { + return; + } + if (printBuffer != null) { + printBuffer.append(text); + } else { + System.out.print(text); + } + } + + private void flushln() { + flushln(true); + } + + private void flushln(boolean useOutputPrefix) { + if (!isEnabled) { + return; + } + if (useOutputPrefix) { + System.out.print(outputPrefix); + } + if (printBuffer != null) { + System.out.print(printBuffer.toString()); + printBuffer.setLength(0); // Clear buffer. + } + textBuffer.forEach(System.out::print); + textBuffer.clear(); + System.out.println(); + } + + private void flushCenteredln() { + if (!isEnabled) { + return; + } + String padding = stringFilledWith((Math.max(0, CHARACTERS_PER_LINE - getCurrentTextLength())) / 2, " "); + textBuffer.add(0, padding); + flushln(); + } + + // Ignores ansi escape sequences + private int getCurrentTextLength() { + int textLength = 0; + for (String text : textBuffer) { + if (!text.startsWith(ANSIColors.ESCAPE)) { + textLength += text.length(); + } + } + return textLength; + } + + private boolean isEmpty() { + return textBuffer.isEmpty(); + } + + private void printHeadlineSeparator() { + dim().a(headlineSeparator).reset().flushln(); + } + + private void printLineSeparator() { + dim().a(lineSeparator).reset().flushln(); + } + } + + private final class ColorfulLinePrinter extends LinePrinter { + + ColorfulLinePrinter(boolean enableProgress) { + super(enableProgress); + } + + @Override + protected LinePrinter bold() { + return a(ANSIColors.BOLD); + } + + @Override + protected LinePrinter blue() { + return a(ANSIColors.BLUE); + } + + @Override + protected LinePrinter blueBold() { + return a(ANSIColors.BLUE_BOLD); + } + + @Override + protected LinePrinter redBold() { + return a(ANSIColors.RED_BOLD); + } + + @Override + protected LinePrinter yellowBold() { + return a(ANSIColors.YELLOW_BOLD); + } + + @Override + protected LinePrinter dim() { + return a(ANSIColors.DIM); + } + + @Override + protected LinePrinter link(String text, String url) { + if (showLinks) { + /* links added as a single item so that jumpToMiddle can ignore url text. */ + a(ANSIColors.LINK_START + url).a(ANSIColors.LINK_TEXT).a(text).a(ANSIColors.LINK_END); + } else { + a(text); + } + return this; + } + + @Override + protected LinePrinter doclink(String text, String htmlAnchor) { + return link(text, STAGE_DOCS_URL + htmlAnchor); + } + + @Override + protected LinePrinter reset() { + return a(ANSIColors.RESET); + } + + @Override + protected String asDocLink(String text, String htmlAnchor) { + if (showLinks) { + return String.format(ANSIColors.LINK_FORMAT, STAGE_DOCS_URL + htmlAnchor, text); + } else { + return text; + } + } + } + + private final class ColorlessLinePrinter extends LinePrinter { + + ColorlessLinePrinter(boolean enableProgress) { + super(enableProgress); + } + + @Override + protected LinePrinter bold() { + return this; + } + + @Override + protected LinePrinter blue() { + return this; + } + + @Override + protected LinePrinter blueBold() { + return this; + } + + @Override + protected LinePrinter redBold() { + return this; + } + + @Override + protected LinePrinter yellowBold() { + return this; + } + + @Override + protected LinePrinter dim() { + return this; + } + + @Override + protected LinePrinter link(String text, String url) { + return this; + } + + @Override + protected LinePrinter doclink(String text, String htmlAnchor) { + return this; + } + + @Override + protected LinePrinter reset() { + return this; + } + + @Override + protected String asDocLink(String text, String htmlAnchor) { + return text; + } + } + + private class ANSIColors { + static final String ESCAPE = "\033"; + static final String RESET = ESCAPE + "[0m"; + static final String BOLD = ESCAPE + "[1m"; + static final String DIM = ESCAPE + "[2m"; + + static final String LINK_START = ESCAPE + "]8;;"; + static final String LINK_TEXT = ESCAPE + "\\"; + static final String LINK_END = LINK_START + LINK_TEXT; + static final String LINK_FORMAT = LINK_START + "%s" + LINK_TEXT + "%s" + LINK_END; + + static final String BLUE = ESCAPE + "[0;34m"; + + static final String RED_BOLD = ESCAPE + "[1;31m"; + static final String YELLOW_BOLD = ESCAPE + "[1;33m"; + static final String BLUE_BOLD = ESCAPE + "[1;34m"; + } +} diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/reporting/ProgressReporterCHelper.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/reporting/ProgressReporterCHelper.java new file mode 100644 index 000000000000..f7df8b06a6a5 --- /dev/null +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/reporting/ProgressReporterCHelper.java @@ -0,0 +1,87 @@ +/* + * Copyright (c) 2021, 2021, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +package com.oracle.svm.hosted.reporting; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; + +import org.graalvm.compiler.serviceprovider.JavaVersionUtil; + +import com.oracle.svm.hosted.image.AbstractImage.NativeImageKind; + +public final class ProgressReporterCHelper { + private static final int DEFAULT_CHARACTERS_PER_LINE = 80; + public static final int MAX_CHARACTERS_PER_LINE = 120; + + static { + loadCHelperLibrary(); + } + + private static void loadCHelperLibrary() { + if (JavaVersionUtil.JAVA_SPEC <= 8) { + return; // TODO: remove as part of JDK8 removal (GR-35238). + } + Path libRSSHelperPath = Paths.get(System.getProperty("java.home"), "lib", "svm", "builder", "lib", "libreporterchelper" + NativeImageKind.SHARED_LIBRARY.getFilenameSuffix()); + if (Files.exists(libRSSHelperPath)) { + System.load(libRSSHelperPath.toString()); + } + } + + private ProgressReporterCHelper() { + } + + public static int getTerminalWindowColumnsClamped() { + return Math.min(Math.max(DEFAULT_CHARACTERS_PER_LINE, ProgressReporterCHelper.getTerminalWindowColumns()), MAX_CHARACTERS_PER_LINE); + } + + /** + * @return get terminal window columns or {@link #DEFAULT_CHARACTERS_PER_LINE} if unavailable. + */ + public static int getTerminalWindowColumns() { + try { + return getTerminalWindowColumns0(); + } catch (UnsatisfiedLinkError e) { + return DEFAULT_CHARACTERS_PER_LINE; + } + } + + /** + * @return peak RSS (in bytes) or -1 if unavailable. + */ + public static long getPeakRSS() { + try { + return getPeakRSS0(); + } catch (UnsatisfiedLinkError e) { + return -1; + } + } + + private static native long getPeakRSS0(); + + private static native int getTerminalWindowColumns0(); + +} diff --git a/substratevm/src/com.oracle.svm.native.reporterchelper/src/getPeakRSS.c b/substratevm/src/com.oracle.svm.native.reporterchelper/src/getPeakRSS.c new file mode 100644 index 000000000000..1d397afb098c --- /dev/null +++ b/substratevm/src/com.oracle.svm.native.reporterchelper/src/getPeakRSS.c @@ -0,0 +1,71 @@ +/* + * Copyright (c) 2021, 2021, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +#include + +#if defined(__linux__) + +#include + +JNIEXPORT jlong JNICALL Java_com_oracle_svm_hosted_reporting_ProgressReporterCHelper_getPeakRSS0(void *env, void * ignored) { + struct rusage rusage; + if (getrusage(RUSAGE_SELF, &rusage) == 0) { + return (size_t)rusage.ru_maxrss * 1024; /* (in kilobytes) */ + } else { + return -1; + } +} + +#elif defined(__APPLE__) + +#include +#include +#include + +JNIEXPORT jlong JNICALL Java_com_oracle_svm_hosted_reporting_ProgressReporterCHelper_getPeakRSS0(void *env, void * ignored) { + struct mach_task_basic_info info; + mach_msg_type_number_t count = MACH_TASK_BASIC_INFO_COUNT; + if (task_info(mach_task_self(), MACH_TASK_BASIC_INFO, (task_info_t)&info, &count) == KERN_SUCCESS) { + return (size_t)info.resident_size_max; + } else { + return -1; + } +} + +#elif defined(_WIN64) + +#include +#include + +JNIEXPORT jlong JNICALL Java_com_oracle_svm_hosted_reporting_ProgressReporterCHelper_getPeakRSS0(void *env, void * ignored) { + PROCESS_MEMORY_COUNTERS memCounter; + if (GetProcessMemoryInfo(GetCurrentProcess(), &memCounter, sizeof memCounter)) { + return (size_t)memCounter.PeakWorkingSetSize; + } else { + return -1; + } +} + +#endif diff --git a/substratevm/src/com.oracle.svm.native.reporterchelper/src/getTerminalWindowColumns.c b/substratevm/src/com.oracle.svm.native.reporterchelper/src/getTerminalWindowColumns.c new file mode 100644 index 000000000000..221a4e3700bf --- /dev/null +++ b/substratevm/src/com.oracle.svm.native.reporterchelper/src/getTerminalWindowColumns.c @@ -0,0 +1,51 @@ +/* + * Copyright (c) 2021, 2021, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +#include + +#if defined(__linux__) || defined(__APPLE__) + +#include +#include +#include +#include + +JNIEXPORT jint JNICALL Java_com_oracle_svm_hosted_reporting_ProgressReporterCHelper_getTerminalWindowColumns0(void *env, void * ignored) { + struct winsize w; + ioctl(STDOUT_FILENO, TIOCGWINSZ, &w); + return w.ws_col; +} + +#elif defined(_WIN64) + +#include + +JNIEXPORT jint JNICALL Java_com_oracle_svm_hosted_reporting_ProgressReporterCHelper_getTerminalWindowColumns0(void *env, void * ignored) { + CONSOLE_SCREEN_BUFFER_INFO c; + GetConsoleScreenBufferInfo(GetStdHandle(STD_OUTPUT_HANDLE), &c); + return c.srWindow.Right - c.srWindow.Left + 1; +} + +#endif diff --git a/substratevm/src/com.oracle.svm.reflect/src/com/oracle/svm/reflect/hosted/ReflectionDataBuilder.java b/substratevm/src/com.oracle.svm.reflect/src/com/oracle/svm/reflect/hosted/ReflectionDataBuilder.java index e7d522fd282e..e6230c0baa2a 100644 --- a/substratevm/src/com.oracle.svm.reflect/src/com/oracle/svm/reflect/hosted/ReflectionDataBuilder.java +++ b/substratevm/src/com.oracle.svm.reflect/src/com/oracle/svm/reflect/hosted/ReflectionDataBuilder.java @@ -701,6 +701,11 @@ public Set getHidingMethods() { return hidingMethods != null ? Collections.unmodifiableSet(hidingMethods) : Collections.emptySet(); } + @Override + public int getReflectionClassesCount() { + return reflectionClasses.size(); + } + static final class ReflectionDataAccessors { private final Method reflectionDataMethod; private final Field declaredFieldsField; diff --git a/substratevm/src/com.oracle.svm.reflect/src/com/oracle/svm/reflect/target/MethodMetadataDecoderImpl.java b/substratevm/src/com.oracle.svm.reflect/src/com/oracle/svm/reflect/target/MethodMetadataDecoderImpl.java index e642eaa5d9a2..66fb20da6bca 100644 --- a/substratevm/src/com.oracle.svm.reflect/src/com/oracle/svm/reflect/target/MethodMetadataDecoderImpl.java +++ b/substratevm/src/com.oracle.svm.reflect/src/com/oracle/svm/reflect/target/MethodMetadataDecoderImpl.java @@ -165,6 +165,12 @@ public MethodDescriptor[] getAllReachableMethods() { return allMethods.toArray(new MethodDescriptor[0]); } + @Override + public long getMetadataByteLength() { + MethodMetadataEncoding encoding = ImageSingletons.lookup(MethodMetadataEncoding.class); + return encoding.getMethodsEncoding().length + encoding.getIndexEncoding().length; + } + private static int getOffset(int typeID) { MethodMetadataEncoding encoding = ImageSingletons.lookup(MethodMetadataEncoding.class); byte[] index = encoding.getIndexEncoding();