Skip to content

Commit

Permalink
[GR-28898] Generate conditional configuration using the native-image-…
Browse files Browse the repository at this point in the history 
…configure tool.

PullRequest: graal/11362
  • Loading branch information
Aleksandar Gradinac committed Mar 20, 2022
2 parents dc6d58f + 2168e31 commit 1e40cb7
Show file tree
Hide file tree
Showing 39 changed files with 958 additions and 488 deletions.
52 changes: 47 additions & 5 deletions substratevm/mx.substratevm/mx_substratevm.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -495,9 +495,6 @@ def native_unittests_task(extra_build_args=None):

def conditional_config_task(native_image):
agent_path = build_native_image_agent(native_image)
config_dir = join(svmbuild_dir(), 'cond-config-test-config')
if exists(config_dir):
mx.rmtree(config_dir)
conditional_config_filter_path = join(svmbuild_dir(), 'conditional-config-filter.json')
with open(conditional_config_filter_path, 'w') as conditional_config_filter:
conditional_config_filter.write(
Expand All @@ -509,11 +506,56 @@ def conditional_config_task(native_image):
}
'''
)
agent_opts = ['config-output-dir=' + config_dir, 'experimental-conditional-config-filter-file=' + conditional_config_filter_path]

run_agent_conditional_config_test(agent_path, conditional_config_filter_path)

run_nic_conditional_config_test(agent_path, conditional_config_filter_path)


def run_nic_conditional_config_test(agent_path, conditional_config_filter_path):
test_cases = [
"createConfigPartOne",
"createConfigPartTwo",
"createConfigPartThree"
]
config_directories = []
nic_test_dir = join(svmbuild_dir(), 'nic-cond-config-test')
if exists(nic_test_dir):
mx.rmtree(nic_test_dir)
for test_case in test_cases:
config_dir = join(nic_test_dir, test_case)
config_directories.append(config_dir)

agent_opts = ['config-output-dir=' + config_dir,
'experimental-conditional-config-part']
jvm_unittest(['-agentpath:' + agent_path + '=' + ','.join(agent_opts),
'-Dcom.oracle.svm.configure.test.conditionalconfig.PartialConfigurationGenerator.enabled=true',
'com.oracle.svm.configure.test.conditionalconfig.PartialConfigurationGenerator#' + test_case])
config_output_dir = join(nic_test_dir, 'config-output')
nic_exe = mx.cmd_suffix(join(mx.JDKConfig(home=mx_sdk_vm_impl.graalvm_output()).home, 'bin', 'native-image-configure'))
nic_command = [nic_exe, 'create-conditional'] \
+ ['--user-code-filter=' + conditional_config_filter_path] \
+ ['--input-dir=' + config_dir for config_dir in config_directories] \
+ ['--output-dir=' + config_output_dir]
mx.run(nic_command)
jvm_unittest(
['-Dcom.oracle.svm.configure.test.conditionalconfig.ConfigurationVerifier.configpath=' + config_output_dir,
"-Dcom.oracle.svm.configure.test.conditionalconfig.ConfigurationVerifier.enabled=true",
'com.oracle.svm.configure.test.conditionalconfig.ConfigurationVerifier'])


def run_agent_conditional_config_test(agent_path, conditional_config_filter_path):
config_dir = join(svmbuild_dir(), 'cond-config-test-config')
if exists(config_dir):
mx.rmtree(config_dir)

agent_opts = ['config-output-dir=' + config_dir,
'experimental-conditional-config-filter-file=' + conditional_config_filter_path]
# This run generates the configuration from different test cases
jvm_unittest(['-agentpath:' + agent_path + '=' + ','.join(agent_opts),
'-Dcom.oracle.svm.configure.test.conditionalconfig.ConfigurationGenerator.enabled=true',
'com.oracle.svm.configure.test.conditionalconfig.ConfigurationGenerator'])

# This run verifies that the generated configuration matches the expected one
jvm_unittest(['-Dcom.oracle.svm.configure.test.conditionalconfig.ConfigurationVerifier.configpath=' + config_dir,
"-Dcom.oracle.svm.configure.test.conditionalconfig.ConfigurationVerifier.enabled=true",
'com.oracle.svm.configure.test.conditionalconfig.ConfigurationVerifier'])
Expand Down
126 changes: 69 additions & 57 deletions substratevm/src/com.oracle.svm.agent/src/com/oracle/svm/agent/NativeImageAgent.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,8 @@
import java.util.function.Supplier;
import java.util.regex.Pattern;

import com.oracle.svm.agent.conditionalconfig.ConditionalConfigurationPartialRunWriter;
import com.oracle.svm.agent.configwithorigins.ConfigurationWithOriginsTracer;
import org.graalvm.nativeimage.Platform;
import org.graalvm.nativeimage.ProcessProperties;
import org.graalvm.nativeimage.hosted.Feature;
Expand All @@ -65,7 +67,7 @@
import com.oracle.svm.agent.tracing.TraceFileWriter;
import com.oracle.svm.agent.tracing.core.Tracer;
import com.oracle.svm.agent.tracing.core.TracingResultWriter;
import com.oracle.svm.configure.config.ConditionalConfigurationPredicate;
import com.oracle.svm.configure.config.conditional.ConditionalConfigurationPredicate;
import com.oracle.svm.configure.config.ConfigurationFileCollection;
import com.oracle.svm.configure.config.ConfigurationSet;
import com.oracle.svm.configure.filters.ComplexFilter;
Expand Down Expand Up @@ -105,6 +107,18 @@ private static String getTokenValue(String token) {
return token.substring(token.indexOf('=') + 1);
}

private static boolean getBooleanTokenValue(String token) {
int equalsIndex = token.indexOf('=');
if (equalsIndex == -1) {
return true;
}
return Boolean.parseBoolean(token.substring(equalsIndex + 1));
}

private static boolean isBooleanOption(String token, String option) {
return token.equals(option) || token.startsWith(option + "=");
}

@Override
protected int getRequiredJvmtiVersion() {
return JvmtiInterface.JVMTI_VERSION_1_2;
Expand Down Expand Up @@ -133,6 +147,7 @@ protected int onLoadCallback(JNIJavaVM vm, JvmtiEnv jvmti, JvmtiEventCallbacks c
boolean configurationWithOrigins = false;
List<String> conditionalConfigUserPackageFilterFiles = new ArrayList<>();
List<String> conditionalConfigClassNameFilterFiles = new ArrayList<>();
boolean conditionalConfigPartialRun = false;
int configWritePeriod = -1; // in seconds
int configWritePeriodInitialDelay = 1; // in seconds
boolean trackReflectionMetadata = true;
Expand All @@ -156,42 +171,31 @@ protected int onLoadCallback(JNIJavaVM vm, JvmtiEnv jvmti, JvmtiEventCallbacks c
String omittedConfigDir = getTokenValue(token);
omittedConfigDir = transformPath(omittedConfigDir);
omittedConfigs.addDirectory(Paths.get(omittedConfigDir));
} else if (token.equals("experimental-omit-config-from-classpath")) {
experimentalOmitClasspathConfig = true;
} else if (token.startsWith("experimental-omit-config-from-classpath=")) {
experimentalOmitClasspathConfig = Boolean.parseBoolean(getTokenValue(token));
} else if (isBooleanOption(token, "experimental-omit-config-from-classpath")) {
experimentalOmitClasspathConfig = getBooleanTokenValue(token);
} else if (token.startsWith("restrict-all-dir") || token.equals("restrict") || token.startsWith("restrict=")) {
warn("restrict mode is no longer supported, ignoring option: " + token);
} else if (token.equals("no-builtin-caller-filter")) {
builtinCallerFilter = false;
} else if (token.startsWith("builtin-caller-filter=")) {
builtinCallerFilter = Boolean.parseBoolean(getTokenValue(token));
} else if (isBooleanOption(token, "builtin-caller-filter")) {
builtinCallerFilter = getBooleanTokenValue(token);
} else if (token.equals("no-builtin-heuristic-filter")) {
builtinHeuristicFilter = false;
} else if (token.startsWith("builtin-heuristic-filter=")) {
builtinHeuristicFilter = Boolean.parseBoolean(getTokenValue(token));
} else if (token.equals("no-filter")) { // legacy
builtinCallerFilter = false;
builtinHeuristicFilter = false;
} else if (token.startsWith("no-filter=")) { // legacy
builtinCallerFilter = !Boolean.parseBoolean(getTokenValue(token));
} else if (isBooleanOption(token, "builtin-heuristic-filter")) {
builtinHeuristicFilter = getBooleanTokenValue(token);
} else if (isBooleanOption(token, "no-filter")) { // legacy
builtinCallerFilter = !getBooleanTokenValue(token);
builtinHeuristicFilter = builtinCallerFilter;
} else if (token.startsWith("caller-filter-file=")) {
callerFilterFiles.add(getTokenValue(token));
} else if (token.startsWith("access-filter-file=")) {
accessFilterFiles.add(getTokenValue(token));
} else if (token.equals("experimental-class-loader-support")) {
experimentalClassLoaderSupport = true;
} else if (token.startsWith("experimental-class-loader-support=")) {
experimentalClassLoaderSupport = Boolean.parseBoolean(getTokenValue(token));
} else if (token.equals("experimental-class-define-support")) {
experimentalClassDefineSupport = true;
} else if (token.startsWith("experimental-class-define-support=")) {
experimentalClassDefineSupport = Boolean.parseBoolean(getTokenValue(token));
} else if (token.equals("experimental-unsafe-allocation-support")) {
experimentalUnsafeAllocationSupport = Boolean.parseBoolean(getTokenValue(token));
} else if (token.startsWith("experimental-unsafe-allocation-support=")) {
experimentalUnsafeAllocationSupport = Boolean.parseBoolean(getTokenValue(token));
} else if (isBooleanOption(token, "experimental-class-loader-support")) {
experimentalClassLoaderSupport = getBooleanTokenValue(token);
} else if (isBooleanOption(token, "experimental-class-define-support")) {
experimentalClassDefineSupport = getBooleanTokenValue(token);
} else if (isBooleanOption(token, "experimental-unsafe-allocation-support")) {
experimentalUnsafeAllocationSupport = getBooleanTokenValue(token);
} else if (token.startsWith("config-write-period-secs=")) {
configWritePeriod = parseIntegerOrNegative(getTokenValue(token));
if (configWritePeriod <= 0) {
Expand All @@ -202,20 +206,18 @@ protected int onLoadCallback(JNIJavaVM vm, JvmtiEnv jvmti, JvmtiEventCallbacks c
if (configWritePeriodInitialDelay < 0) {
return usage(1, "config-write-initial-delay-secs must be an integer greater or equal to 0");
}
} else if (token.equals("build")) {
build = true;
} else if (token.startsWith("build=")) {
build = Boolean.parseBoolean(getTokenValue(token));
} else if (token.equals("experimental-configuration-with-origins")) {
configurationWithOrigins = true;
} else if (isBooleanOption(token, "build")) {
build = getBooleanTokenValue(token);
} else if (isBooleanOption(token, "experimental-configuration-with-origins")) {
configurationWithOrigins = getBooleanTokenValue(token);
} else if (token.startsWith("experimental-conditional-config-filter-file=")) {
conditionalConfigUserPackageFilterFiles.add(getTokenValue(token));
} else if (token.startsWith("conditional-config-class-filter-file=")) {
conditionalConfigClassNameFilterFiles.add(getTokenValue(token));
} else if (token.equals("track-reflection-metadata")) {
trackReflectionMetadata = true;
} else if (token.startsWith("track-reflection-metadata=")) {
trackReflectionMetadata = Boolean.parseBoolean(getTokenValue(token));
} else if (isBooleanOption(token, "experimental-conditional-config-part")) {
conditionalConfigPartialRun = getBooleanTokenValue(token);
} else if (isBooleanOption(token, "track-reflection-metadata")) {
trackReflectionMetadata = getBooleanTokenValue(token);
} else {
return usage(1, "unknown option: '" + token + "'.");
}
Expand Down Expand Up @@ -264,7 +266,12 @@ protected int onLoadCallback(JNIJavaVM vm, JvmtiEnv jvmti, JvmtiEventCallbacks c
}
}

boolean shouldTraceOriginInformation = configurationWithOrigins || !conditionalConfigUserPackageFilterFiles.isEmpty();
if (!conditionalConfigUserPackageFilterFiles.isEmpty() && conditionalConfigPartialRun) {
return error(6, "The agent can generate conditional configuration either for the current run or in the partial mode but not both at the same time.");
}

boolean isConditionalConfigurationRun = !conditionalConfigUserPackageFilterFiles.isEmpty() || conditionalConfigPartialRun;
boolean shouldTraceOriginInformation = configurationWithOrigins || isConditionalConfigurationRun;
final MethodInfoRecordKeeper recordKeeper = new MethodInfoRecordKeeper(shouldTraceOriginInformation);
final Supplier<InterceptedState> interceptedStateSupplier = shouldTraceOriginInformation ? EagerlyLoadedJavaStackAccess.stackAccessSupplier()
: OnDemandJavaStackAccess.stackAccessSupplier();
Expand Down Expand Up @@ -309,29 +316,34 @@ protected int onLoadCallback(JNIJavaVM vm, JvmtiEnv jvmti, JvmtiEventCallbacks c
shouldExcludeClassesWithHash = omittedConfiguration.getPredefinedClassesConfiguration()::containsClassWithHash;
}

if (configurationWithOrigins) {
ConfigurationWithOriginsWriter writer = new ConfigurationWithOriginsWriter(processor, recordKeeper);
tracer = writer;
tracingResultWriter = writer;
} else if (!conditionalConfigUserPackageFilterFiles.isEmpty()) {
ComplexFilter userCodeFilter = new ComplexFilter(HierarchyFilterNode.createRoot());
if (!parseFilterFiles(userCodeFilter, conditionalConfigUserPackageFilterFiles)) {
return 2;
}
ComplexFilter classNameFilter;
if (!conditionalConfigClassNameFilterFiles.isEmpty()) {
classNameFilter = new ComplexFilter(HierarchyFilterNode.createRoot());
if (!parseFilterFiles(classNameFilter, conditionalConfigClassNameFilterFiles)) {
return 3;
if (shouldTraceOriginInformation) {
ConfigurationWithOriginsTracer configWithOriginsTracer = new ConfigurationWithOriginsTracer(processor, recordKeeper);
tracer = configWithOriginsTracer;

if (isConditionalConfigurationRun) {
if (conditionalConfigPartialRun) {
tracingResultWriter = new ConditionalConfigurationPartialRunWriter(configWithOriginsTracer);
} else {
ComplexFilter userCodeFilter = new ComplexFilter(HierarchyFilterNode.createRoot());
if (!parseFilterFiles(userCodeFilter, conditionalConfigUserPackageFilterFiles)) {
return 2;
}
ComplexFilter classNameFilter;
if (!conditionalConfigClassNameFilterFiles.isEmpty()) {
classNameFilter = new ComplexFilter(HierarchyFilterNode.createRoot());
if (!parseFilterFiles(classNameFilter, conditionalConfigClassNameFilterFiles)) {
return 3;
}
} else {
classNameFilter = new ComplexFilter(HierarchyFilterNode.createInclusiveRoot());
}

ConditionalConfigurationPredicate predicate = new ConditionalConfigurationPredicate(classNameFilter);
tracingResultWriter = new ConditionalConfigurationWriter(configWithOriginsTracer, userCodeFilter, predicate);
}
} else {
classNameFilter = new ComplexFilter(HierarchyFilterNode.createInclusiveRoot());
tracingResultWriter = new ConfigurationWithOriginsWriter(configWithOriginsTracer);
}

ConditionalConfigurationPredicate predicate = new ConditionalConfigurationPredicate(classNameFilter);
ConditionalConfigurationWriter writer = new ConditionalConfigurationWriter(processor, recordKeeper, userCodeFilter, predicate);
tracer = writer;
tracingResultWriter = writer;
} else {
Path[] predefinedClassDestDirs = {Files.createDirectories(configOutputDirPath.resolve(ConfigurationFile.PREDEFINED_CLASSES_AGENT_EXTRACTED_SUBDIR))};
Function<IOException, Exception> handler = e -> {
Expand Down
64 changes: 64 additions & 0 deletions .../src/com/oracle/svm/agent/conditionalconfig/ConditionalConfigurationPartialRunWriter.java
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,64 @@
/*
* Copyright (c) 2022, 2022, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
* under the terms of the GNU General Public License version 2 only, as
* published by the Free Software Foundation. Oracle designates this
* particular file as subject to the "Classpath" exception as provided
* by Oracle in the LICENSE file that accompanied this code.
*
* This code is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
* version 2 for more details (a copy is included in the LICENSE file that
* accompanied this code).
*
* You should have received a copy of the GNU General Public License version
* 2 along with this work; if not, write to the Free Software Foundation,
* Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
*
* Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
* or visit www.oracle.com if you need additional information or have any
* questions.
*/
package com.oracle.svm.agent.conditionalconfig;

import java.io.IOException;
import java.nio.file.Path;
import java.util.Collections;
import java.util.List;

import com.oracle.svm.agent.configwithorigins.ConfigurationWithOriginsTracer;
import com.oracle.svm.agent.tracing.core.TracingResultWriter;
import com.oracle.svm.configure.config.conditional.PartialConfigurationWithOrigins;
import com.oracle.svm.configure.json.JsonWriter;
import com.oracle.svm.core.configure.ConfigurationFile;

public class ConditionalConfigurationPartialRunWriter implements TracingResultWriter {

private final ConfigurationWithOriginsTracer tracer;

public ConditionalConfigurationPartialRunWriter(ConfigurationWithOriginsTracer tracer) {
this.tracer = tracer;
}

@Override
public boolean supportsPeriodicTraceWriting() {
return false;
}

@Override
public boolean supportsOnUnloadTraceWriting() {
return true;
}

@Override
public List<Path> writeToDirectory(Path directoryPath) throws IOException {
Path resolvedPath = directoryPath.resolve(ConfigurationFile.PARTIAL_CONFIGURATION_WITH_ORIGINS);
try (JsonWriter writer = new JsonWriter(resolvedPath)) {
new PartialConfigurationWithOrigins(tracer.getRootNode(), null).printJson(writer);
}
return Collections.singletonList(resolvedPath);
}
}
Loading

0 comments on commit 1e40cb7

Please sign in to comment.