[GR-33602] Introduce new build output style for Native Image.

PullRequest: graal/10204

docs/reference-manual/native-image/BuildOutput.md

@@ -0,0 +1,42 @@
+---
+layout: docs
+toc_group: native-image
+link_title: Build Output
+permalink: /reference-manual/native-image/BuildOutput/
+---
+# Native Image Build Output
+
+## Glossary
+
+### <a name="glossary-user-provided-features">User-provided features
+
+### <a name="glossary-reflection-classes">Classes registered for reflection
+
+### <a name="glossary-reachability"/>Reachable classes and methods
+
+### <a name="glossary-runtime-methods"/>Runtime compiled methods
+
+### <a name="glossary-code-area"/>Code Area
+
+### <a name="glossary-image-heap"/>Image Heap
+
+### <a name="glossary-peak-rss"/>Peak RSS
+
+### <a name="glossary-cpu-load"/>CPU load
+
+
+## Build Stages
+
+### <a name="step-initializing"/>Initializing
+
+### <a name="step-analysis"/>Performing analysis
+
+### <a name="step-universe"/>Building universe
+
+### <a name="step-parsing"/>Parsing methods
+
+### <a name="step-inlining"/>Inlining methods
+
+### <a name="step-compiling"/>Compiling methods
+
+### <a name="step-creation"/>Creating image

sdk/src/org.graalvm.nativeimage/src/org/graalvm/nativeimage/impl/RuntimeReflectionSupport.java

@@ -52,4 +52,6 @@ public interface RuntimeReflectionSupport extends ReflectionRegistry {
      * from reflection queries.
      */
     Set<?> getHidingMethods();
+
+    int getReflectionClassesCount();
 }

substratevm/CHANGELOG.md

@@ -11,4 +11,4 @@ This changelog summarizes major changes to GraalVM Native Image.
 * (GR-32403) Use more compressed encoding for stack frame metadata.
 * (GR-35152) Add -H:DisableURLProtocols to allow specifying URL protocols that must never be included in the image.
 * (GR-35085) Custom prologue/epilogue/handleException customizations of @CEntryPoint must be annotated with @Uninterruptible. The entry points synthetic methods are now implicilty annotated with @Uninterruptible too.
-
+* (GR-33602) Enable new user-friendly build output mode. The old output can be restored with `-H:-BuildOutputUseNewStyle`. Run `native-image --expert-options-all | grep "BuildOutput` to see all options for the new output.

substratevm/mx.substratevm/mx_substratevm.py

@@ -780,6 +780,7 @@ def native_image_context_run(func, func_args=None, config=None, build_if_missing
         'substratevm:NATIVE_IMAGE_BASE',
     ],
     support_distributions=['substratevm:SVM_GRAALVM_SUPPORT'],
+    support_libraries_distributions=['substratevm:SVM_GRAALVM_LIBRARIES_SUPPORT'],
     stability="earlyadopter",
     jlink=False,
 ))

substratevm/mx.substratevm/suite.py

@@ -471,6 +471,26 @@
             },
         },
 
+        "com.oracle.svm.native.reporterchelper": {
+            "subDir": "src",
+            "native": "shared_lib",
+            "deliverable": "reporterchelper",
+            "platformDependent": True,
+            "use_jdk_headers": True,
+            "os_arch": {
+                "windows": {
+                    "<others>": {
+                        "cflags": ["-Wall"]
+                    }
+                },
+                "<others>": {
+                    "<others>": {
+                        "cflags": ["-Wall", "-Werror"],
+                    },
+                },
+            },
+        },
+
         "com.oracle.svm.native.darwin": {
             "subDir": "src",
             "native": "static_lib",
@@ -1450,7 +1470,14 @@
                 "com.oracle.graal.pointsto.infrastructure",
                 "com.oracle.graal.pointsto.flow.context.object",
               ],
+              "requires": [
+                "java.management",
+                "jdk.management",
+              ],
               "requiresConcealed" : {
+                "java.management": [
+                  "sun.management",
+                ],
                 "jdk.internal.vm.ci" : [
                   "jdk.vm.ci.meta",
                   "jdk.vm.ci.common",
@@ -1525,6 +1552,16 @@
             },
         },
 
+        "SVM_GRAALVM_LIBRARIES_SUPPORT" : {
+            "native" : True,
+            "platformDependent" : True,
+            "description" : "SubstrateVM support libraries for the GraalVM",
+            "layout" : {
+                "svm/builder/lib/" : ["dependency:com.oracle.svm.native.reporterchelper"],
+            },
+            "maven" : False,
+        },
+
         "SVM_NFI_GRAALVM_SUPPORT" : {
             "native" : True,
             "platformDependent" : True,

substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/reports/CallTreePrinter.java

@@ -360,7 +360,7 @@ private static void printCsvFiles(Map<AnalysisMethod, MethodNode> methodToNode,
 
     private static void toCsvFile(String description, String reportsPath, String prefix, String reportName, Consumer<PrintWriter> reporter) {
         final String name = prefix + "_" + reportName;
-        final String csvFile = ReportUtils.report(description, reportsPath, name, "csv", reporter);
+        final Path csvFile = ReportUtils.report(description, reportsPath, name, "csv", reporter);
         final Path csvLink = Paths.get(reportsPath).resolve(prefix + ".csv");
 
         if (Files.exists(csvLink, LinkOption.NOFOLLOW_LINKS)) {
@@ -372,7 +372,7 @@ private static void toCsvFile(String description, String reportsPath, String pre
         }
 
         try {
-            Files.createSymbolicLink(csvLink, Paths.get(csvFile));
+            Files.createSymbolicLink(csvLink, csvFile);
         } catch (IOException e) {
             throw new RuntimeException(e);
         }

substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/reports/ReportUtils.java

@@ -64,6 +64,10 @@ public class ReportUtils {
     static final Comparator<BytecodePosition> positionMethodComparator = Comparator.comparing(pos -> pos.getMethod().format("%H.%n(%p)"));
     static final Comparator<BytecodePosition> positionComparator = positionMethodComparator.thenComparing(pos -> pos.getBCI());
 
+    public static Path report(String description, String path, String name, String extension, Consumer<PrintWriter> reporter) {
+        return report(description, path, name, extension, reporter, true);
+    }
+
     /**
      * Print a report in the format: path/name_timeStamp.extension. The path is relative to the
      * working directory.
@@ -75,18 +79,21 @@ public class ReportUtils {
      * @param extension the extension of the report
      * @param reporter a consumer that writes to a PrintWriter
      */
-    public static String report(String description, String path, String name, String extension, Consumer<PrintWriter> reporter) {
+    public static Path report(String description, String path, String name, String extension, Consumer<PrintWriter> reporter, boolean enablePrint) {
         String fileName = timeStampedFileName(name, extension);
         Path reportDir = Paths.get(path);
-        reportImpl(description, reportDir, fileName, reporter);
-        return fileName;
+        return reportImpl(enablePrint, description, reportDir, fileName, reporter);
     }
 
     public static String timeStampedFileName(String name, String extension) {
+        String fileName = name + "_" + getTimeStampString();
+        return extension.isEmpty() ? fileName : fileName + "." + extension;
+    }
+
+    public static String getTimeStampString() {
         DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyyMMdd_HHmmss");
         String timeStamp = LocalDateTime.now().format(formatter);
-        String fileName = name + "_" + timeStamp;
-        return extension.isEmpty() ? fileName : fileName + "." + extension;
+        return timeStamp;
     }
 
     public static File reportFile(String path, String name, String extension) {
@@ -101,6 +108,10 @@ public static File reportFile(String path, String name, String extension) {
         }
     }
 
+    public static void report(String description, Path file, Consumer<PrintWriter> reporter) {
+        report(description, file, reporter, true);
+    }
+
     /**
      * Print a report in the file given by {@code file} parameter. If the {@code file} is relative
      * it's resolved to the working directory.
@@ -109,29 +120,32 @@ public static File reportFile(String path, String name, String extension) {
      * @param file the path (relative to the working directory if the argument represents a relative
      *            path) to file to store a report into.
      * @param reporter a consumer that writes to a PrintWriter
+     * @param enablePrint of a notice to stdout
      */
-    public static void report(String description, Path file, Consumer<PrintWriter> reporter) {
+    public static Path report(String description, Path file, Consumer<PrintWriter> reporter, boolean enablePrint) {
         Path folder = file.getParent();
         Path fileName = file.getFileName();
         if (folder == null || fileName == null) {
             throw new IllegalArgumentException("File parameter must be a file, got: " + file);
         }
-        reportImpl(description, folder, fileName.toString(), reporter);
+        return reportImpl(enablePrint, description, folder, fileName.toString(), reporter);
     }
 
-    private static void reportImpl(String description, Path folder, String fileName, Consumer<PrintWriter> reporter) {
+    private static Path reportImpl(boolean enablePrint, String description, Path folder, String fileName, Consumer<PrintWriter> reporter) {
         try {
             Path reportDir = Files.createDirectories(folder);
             Path file = reportDir.resolve(fileName);
             Files.deleteIfExists(file);
 
             try (FileWriter fw = new FileWriter(Files.createFile(file).toFile())) {
                 try (PrintWriter writer = new PrintWriter(fw)) {
-                    System.out.println("# Printing " + description + " to: " + file);
+                    if (enablePrint) {
+                        System.out.println("# Printing " + description + " to: " + file);
+                    }
                     reporter.accept(writer);
                 }
             }
-
+            return file;
         } catch (IOException e) {
             throw JVMCIError.shouldNotReachHere(e);
         }

substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/util/Timer.java

@@ -27,6 +27,7 @@
 import org.graalvm.compiler.serviceprovider.GraalServices;
 
 public class Timer {
+    private static boolean disablePrinting = false;
 
     private String prefix;
 
@@ -57,6 +58,10 @@ public Timer(String prefix, String name, boolean autoPrint) {
         this.autoPrint = autoPrint;
     }
 
+    public static void disablePrinting() {
+        disablePrinting = true;
+    }
+
     /**
      * Registers the prefix to be used when {@linkplain Timer#print(long) printing} a timer. This
      * allows the output of interlaced native image executions to be disambiguated.
@@ -79,6 +84,9 @@ public void stop() {
     }
 
     private void print(long time) {
+        if (disablePrinting) {
+            return;
+        }
         final String concurrentPrefix;
         if (prefix != null) {
             // Add the PID to further disambiguate concurrent builds of images with the same name

substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateOptions.java

@@ -273,6 +273,30 @@ public Boolean getValue(OptionValues values) {
     @Option(help = "Alignment of AOT and JIT compiled code in bytes.")//
     public static final HostedOptionKey<Integer> CodeAlignment = new HostedOptionKey<>(16);
 
+    /*
+     * Build output options.
+     */
+    @Option(help = "Use new build output style", type = OptionType.User)//
+    public static final HostedOptionKey<Boolean> BuildOutputUseNewStyle = new HostedOptionKey<>(true);
+
+    @Option(help = "Prefix build output with '<pid>:<image name>'", type = OptionType.User)//
+    public static final HostedOptionKey<Boolean> BuildOutputPrefix = new HostedOptionKey<>(false);
+
+    @Option(help = "Colorize build output", type = OptionType.User)//
+    public static final HostedOptionKey<Boolean> BuildOutputColorful = new HostedOptionKey<>(true);
+
+    @Option(help = "Show links in build output", type = OptionType.User)//
+    public static final HostedOptionKey<Boolean> BuildOutputLinks = new HostedOptionKey<>(true);
+
+    @Option(help = "Report progress in build output", type = OptionType.User)//
+    public static final HostedOptionKey<Boolean> BuildOutputProgress = new HostedOptionKey<>(true);
+
+    @Option(help = "Show code and heap breakdowns as part of the build output", type = OptionType.User)//
+    public static final HostedOptionKey<Boolean> BuildOutputBreakdowns = new HostedOptionKey<>(true);
+
+    @Option(help = "Print GC warnings as part of build output", type = OptionType.User)//
+    public static final HostedOptionKey<Boolean> BuildOutputGCWarnings = new HostedOptionKey<>(true);
+
     /*
      * Object and array allocation options.
      */

substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/reflect/MethodMetadataDecoder.java

@@ -37,6 +37,8 @@ public interface MethodMetadataDecoder {
 
     MethodDescriptor[] getAllReachableMethods();
 
+    long getMetadataByteLength();
+
     class MethodDescriptor {
         private final Class<?> declaringClass;
         private final String name;

substratevm/src/com.oracle.svm.graal/src/com/oracle/svm/graal/hosted/GraalFeature.java

@@ -132,6 +132,7 @@
 import com.oracle.svm.hosted.phases.StrengthenStampsPhase;
 import com.oracle.svm.hosted.phases.SubstrateClassInitializationPlugin;
 import com.oracle.svm.hosted.phases.SubstrateGraphBuilderPhase;
+import com.oracle.svm.hosted.reporting.ProgressReporter;
 import com.oracle.truffle.api.CompilerDirectives.TruffleBoundary;
 
 import jdk.vm.ci.meta.DeoptimizationAction;
@@ -647,7 +648,8 @@ public void beforeCompilation(BeforeCompilationAccess c) {
         if (Options.PrintRuntimeCompileMethods.getValue()) {
             printCallTree();
         }
-        System.out.println(methods.size() + " method(s) included for runtime compilation");
+
+        ProgressReporter.singleton().printRuntimeCompileMethods(methods.size(), config.getMethods().size());
 
         if (Options.PrintStaticTruffleBoundaries.getValue()) {
             printStaticTruffleBoundaries();
@@ -720,6 +722,7 @@ public void beforeCompilation(BeforeCompilationAccess c) {
             }
         }
 
+        ProgressReporter.singleton().setGraphEncodingByteLength(graphEncoder.getEncoding().length);
         GraalSupport.setGraphEncoding(graphEncoder.getEncoding(), graphEncoder.getObjects(), graphEncoder.getNodeClasses());
 
         objectReplacer.updateDataDuringAnalysis((AnalysisMetaAccess) hMetaAccess.getWrapped());

substratevm/src/com.oracle.svm.hosted.jdk11/src/com/oracle/svm/hosted/StringAccess.java

@@ -0,0 +1,46 @@
+/*
+ * Copyright (c) 2021, 2021, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.  Oracle designates this
+ * particular file as subject to the "Classpath" exception as provided
+ * by Oracle in the LICENSE file that accompanied this code.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+package com.oracle.svm.hosted;
+
+import java.lang.invoke.MethodHandles;
+import java.lang.invoke.VarHandle;
+
+/* TODO: Added as a temporary workaround to provide compatibility with JDK8 (GR-35238). */
+public class StringAccess {
+
+    private static final VarHandle STRING_VALUE;
+    static {
+        try {
+            MethodHandles.Lookup privateLookup = MethodHandles.privateLookupIn(String.class, MethodHandles.lookup());
+            STRING_VALUE = privateLookup.unreflectVarHandle(String.class.getDeclaredField("value"));
+        } catch (ReflectiveOperationException e) {
+            throw new ExceptionInInitializerError(e);
+        }
+    }
+
+    public static int getInternalByteArrayLength(String string) {
+        return ((byte[]) STRING_VALUE.get(string)).length;
+    }
+}

substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/FeatureHandler.java

@@ -40,6 +40,7 @@
 import org.graalvm.nativeimage.hosted.Feature;
 
 import com.oracle.graal.pointsto.reports.ReportUtils;
+import com.oracle.svm.core.ClassLoaderSupport;
 import com.oracle.svm.core.SubstrateOptions;
 import com.oracle.svm.core.annotate.AutomaticFeature;
 import com.oracle.svm.core.graal.GraalFeature;
@@ -194,4 +195,13 @@ private void registerFeature(Class<?> baseFeatureClass, Function<Class<?>, Class
 
         featureInstances.add(feature);
     }
+
+    public List<String> getUserFeatureNames() {
+        ClassLoaderSupport classLoaderSupport = ImageSingletons.lookup(ClassLoaderSupport.class);
+        return featureInstances.stream()
+                        .filter(f -> classLoaderSupport.isNativeImageClassLoader(f.getClass().getClassLoader()))
+                        .map(f -> f.getClass().getName())
+                        .sorted()
+                        .collect(Collectors.toList());
+    }
 }