Skip to content

Commit

Permalink
Implement LayerCreate subcommand package
Browse files Browse the repository at this point in the history
  • Loading branch information
@olpaw
olpaw committed Dec 11, 2024
1 parent 8459a70 commit 600350a
Show file tree
Hide file tree
Showing 9 changed files with 153 additions and 79 deletions.
13 changes: 10 additions & 3 deletions substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateOptions.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -130,11 +130,14 @@ protected void onValueUpdate(EconomicMap<OptionKey<?>, Object> values, Boolean o
}
});

// @APIOption(name = "layer-create")//
public static final String LAYER_OPTION_PREFIX = "-H:Layer"; // "--layer"
public static final String LAYER_CREATE_OPTION = LAYER_OPTION_PREFIX + "Create"; // "-create"
// @APIOption(name = LAYER_CREATE_OPTION) // use when non-experimental
@Option(help = "Experimental: Build a Native Image layer.")//
public static final HostedOptionKey<AccumulatingLocatableMultiOptionValue.Strings> LayerCreate = new HostedOptionKey<>(AccumulatingLocatableMultiOptionValue.Strings.build());

// @APIOption(name = "layer-use")//
// public static final String LAYER_USE_OPTION = LAYER_OPTION_PREFIX + "-use";
// @APIOption(name = LAYER_USE_OPTION) // use when non-experimental
@Option(help = "Experimental: Build an image based on a Native Image layer.")//
@BundleMember(role = Role.Input) //
public static final HostedOptionKey<AccumulatingLocatableMultiOptionValue.Paths> LayerUse = new HostedOptionKey<>(AccumulatingLocatableMultiOptionValue.Paths.build());
Expand Down Expand Up @@ -1286,11 +1289,15 @@ public enum ReportingMode {
@Option(help = "Include all classes, methods, fields, and resources from given paths", type = OptionType.Debug) //
public static final HostedOptionKey<AccumulatingLocatableMultiOptionValue.Strings> IncludeAllFromPath = new HostedOptionKey<>(AccumulatingLocatableMultiOptionValue.Strings.build());

@Option(help = "Include all classes, methods and fields from the given packages", type = OptionType.Debug) //
public static final HostedOptionKey<AccumulatingLocatableMultiOptionValue.Strings> IncludeAllFromPackage = new HostedOptionKey<>(
AccumulatingLocatableMultiOptionValue.Strings.buildWithCommaDelimiter());

@Option(help = "Include all classes, methods, fields, and resources from the class path", type = OptionType.Debug) //
public static final HostedOptionKey<Boolean> IncludeAllFromClassPath = new HostedOptionKey<>(false);

public static boolean includeAll() {
return IncludeAllFromModule.hasBeenSet() || IncludeAllFromPath.hasBeenSet() || IncludeAllFromClassPath.hasBeenSet();
return IncludeAllFromModule.hasBeenSet() || IncludeAllFromPath.hasBeenSet() || IncludeAllFromPackage.hasBeenSet() || IncludeAllFromClassPath.hasBeenSet();
}

@Option(help = "Force include include all public types and methods that can be reached using normal Java access rules.")//
Expand Down
23 changes: 23 additions & 0 deletions substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/CmdLineOptionHandler.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,10 +31,14 @@
import java.util.regex.Pattern;
import java.util.regex.PatternSyntaxException;

import com.oracle.svm.core.SubstrateOptions;
import com.oracle.svm.core.VM;
import com.oracle.svm.core.option.OptionOrigin;
import com.oracle.svm.core.util.ExitStatus;
import com.oracle.svm.driver.NativeImage.ArgumentQueue;
import com.oracle.svm.hosted.imagelayer.LayerArchiveSupport;
import com.oracle.svm.hosted.imagelayer.LayerOptionsSupport.ExtendedOption;
import com.oracle.svm.hosted.imagelayer.LayerOptionsSupport.LayerOption;
import com.oracle.svm.util.LogUtils;

import jdk.graal.compiler.options.OptionType;
Expand Down Expand Up @@ -138,6 +142,25 @@ private boolean consume(ArgumentQueue args, String headArg) {
return true;
}

if (headArg.startsWith(SubstrateOptions.LAYER_CREATE_OPTION)) {
String layerCreateValue = headArg.substring(SubstrateOptions.LAYER_CREATE_OPTION.length());
if (!layerCreateValue.isEmpty()) {
LayerOption layerOption = LayerOption.parse(layerCreateValue);
for (ExtendedOption option : layerOption.extendedOptions()) {
switch (option.key()) {
case LayerArchiveSupport.PACKAGE_OPTION -> {
String packageName = option.value();
String moduleName = nativeImage.systemPackagesToModules.get(packageName);
if (moduleName != null) {
nativeImage.addAddedModules(moduleName);
}
}
}
}
}
return false;
}

if (headArg.startsWith(DEBUG_ATTACH_OPTION)) {
if (useDebugAttach) {
throw NativeImage.showError("The " + DEBUG_ATTACH_OPTION + " option can only be used once.");
Expand Down
15 changes: 14 additions & 1 deletion substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImage.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -145,6 +145,19 @@ private static String flagsFileName(String versionTag) {

static final Map<String, String[]> graalCompilerFlags = getCompilerFlags();

private static Map<String, String> getSystemPackages() {
Map<String, String> res = new HashMap<>();
for (ModuleReference moduleRef : ModuleFinder.ofSystem().findAll()) {
ModuleDescriptor moduleDescriptor = moduleRef.descriptor();
for (String packageName : moduleDescriptor.packages()) {
res.put(packageName, moduleDescriptor.name());
}
}
return Map.copyOf(res);
}

final Map<String, String> systemPackagesToModules = getSystemPackages();

static String getResource(String resourceName) {
try (InputStream input = NativeImage.class.getResourceAsStream(resourceName)) {
BufferedReader reader = new BufferedReader(new InputStreamReader(input, StandardCharsets.UTF_8));
Expand Down Expand Up @@ -963,7 +976,7 @@ private void prepareImageBuildArgs() {

/*
* The presence of CDS and custom system class loaders disables the use of archived
* non-system class and and triggers a warning.
* non-system class and triggers a warning.
*/
addImageBuilderJavaArgs("-Xshare:off");

Expand Down
16 changes: 4 additions & 12 deletions substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ModuleLayerFeature.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -217,7 +217,7 @@ public void afterAnalysis(AfterAnalysisAccess access) {
* Parse explicitly added modules via --add-modules. This is done early as this information
* is required when filtering the analysis reachable module set.
*/
Set<String> extraModules = ModuleLayerFeatureUtils.parseModuleSetModifierProperty(ModuleSupport.PROPERTY_IMAGE_EXPLICITLY_ADDED_MODULES);
Set<String> extraModules = ModuleSupport.parseModuleSetModifierProperty(ModuleSupport.PROPERTY_IMAGE_EXPLICITLY_ADDED_MODULES);
extraModules.addAll(Resources.getIncludedResourcesModules());
extraModules.stream().filter(Predicate.not(ModuleSupport.nonExplicitModules::contains)).forEach(moduleName -> {
Optional<?> module = accessImpl.imageClassLoader.findModule(moduleName);
Expand Down Expand Up @@ -282,7 +282,7 @@ private Set<String> calculateRootModules(Collection<String> addModules) {
ModuleFinder upgradeModulePath = NativeImageClassLoaderSupport.finderFor("jdk.module.upgrade.path");
ModuleFinder appModulePath = moduleLayerFeatureUtils.getAppModuleFinder();
String mainModule = ModuleLayerFeatureUtils.getMainModuleName();
Set<String> limitModules = ModuleLayerFeatureUtils.parseModuleSetModifierProperty(ModuleSupport.PROPERTY_IMAGE_EXPLICITLY_LIMITED_MODULES);
Set<String> limitModules = ModuleSupport.parseModuleSetModifierProperty(ModuleSupport.PROPERTY_IMAGE_EXPLICITLY_LIMITED_MODULES);

Object systemModules = null;
ModuleFinder systemModuleFinder;
Expand Down Expand Up @@ -658,7 +658,8 @@ private static final class ModuleLayerFeatureUtils {
ModuleLayerFeatureUtils(ImageClassLoader cl) {
runtimeModules = new HashMap<>();
imageClassLoader = cl;
nativeAccessEnabled = NativeImageClassLoaderOptions.EnableNativeAccess.getValue().values().stream().flatMap(m -> Arrays.stream(SubstrateUtil.split(m, ",")))
nativeAccessEnabled = NativeImageClassLoaderOptions.EnableNativeAccess.getValue().values().stream()
.flatMap(m -> Arrays.stream(SubstrateUtil.split(m, ",")))
.collect(Collectors.toSet());

Method classGetDeclaredMethods0Method = ReflectionUtil.lookupMethod(Class.class, "getDeclaredFields0", boolean.class);
Expand Down Expand Up @@ -770,15 +771,6 @@ static String formatModule(Module module) {
}
}

static Set<String> parseModuleSetModifierProperty(String prop) {
Set<String> specifiedModules = new HashSet<>();
String args = System.getProperty(prop, "");
if (!args.isEmpty()) {
specifiedModules.addAll(Arrays.asList(SubstrateUtil.split(args, ",")));
}
return specifiedModules;
}

static int distanceFromBootModuleLayer(ModuleLayer layer) {
if (layer == ModuleLayer.boot()) {
return 0;
Expand Down
83 changes: 53 additions & 30 deletions ...vm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/NativeImageClassLoaderSupport.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@

import static com.oracle.svm.core.SubstrateOptions.IncludeAllFromClassPath;
import static com.oracle.svm.core.SubstrateOptions.IncludeAllFromModule;
import static com.oracle.svm.core.SubstrateOptions.IncludeAllFromPackage;
import static com.oracle.svm.core.SubstrateOptions.IncludeAllFromPath;
import static com.oracle.svm.core.util.VMError.guarantee;
import static com.oracle.svm.core.util.VMError.shouldNotReachHere;
Expand Down Expand Up @@ -74,14 +75,14 @@
import java.util.stream.Collectors;
import java.util.stream.Stream;

import com.oracle.svm.core.SubstrateUtil;
import org.graalvm.collections.EconomicMap;
import org.graalvm.collections.EconomicSet;
import org.graalvm.collections.MapCursor;
import org.graalvm.nativeimage.impl.AnnotationExtractor;

import com.oracle.svm.core.NativeImageClassLoaderOptions;
import com.oracle.svm.core.SubstrateOptions;
import com.oracle.svm.core.SubstrateUtil;
import com.oracle.svm.core.option.AccumulatingLocatableMultiOptionValue;
import com.oracle.svm.core.option.HostedOptionKey;
import com.oracle.svm.core.option.LocatableMultiOptionValue.ValueWithOrigin;
Expand Down Expand Up @@ -129,13 +130,15 @@ public final class NativeImageClassLoaderSupport {
public final AnnotationExtractor annotationExtractor;

private Set<String> javaModuleNamesToInclude;
private Set<String> javaPackagesToInclude;
private Set<Path> javaPathsToInclude;
private boolean includeAllFromClassPath;

private Optional<LibGraalClassLoaderBase> libGraalLoader;
private List<ClassLoader> classLoaders;

private final Set<Class<?>> classesToIncludeUnconditionally = Collections.newSetFromMap(new ConcurrentHashMap<>());
private final Set<Class<?>> classesToIncludeUnconditionally = ConcurrentHashMap.newKeySet();
private final Set<String> includedJavaPackages = ConcurrentHashMap.newKeySet();

private final Method implAddReadsAllUnnamed = ReflectionUtil.lookupMethod(Module.class, "implAddReadsAllUnnamed");
private final Method implAddEnableNativeAccess = ReflectionUtil.lookupMethod(Module.class, "implAddEnableNativeAccess");
Expand Down Expand Up @@ -244,7 +247,7 @@ private static Path stringToPath(String path) {
}

public void loadAllClasses(ForkJoinPool executor, ImageClassLoader imageClassLoader) {
guarantee(javaModuleNamesToInclude == null, "This method should be executed only once.");
guarantee(javaModuleNamesToInclude == null && javaPackagesToInclude == null, "This method should be executed only once.");
javaModuleNamesToInclude = Collections.unmodifiableSet(new HashSet<>(IncludeAllFromModule.getValue(parsedHostedOptions).values()));
/* Verify all modules are present */
final Set<String> allModules = Stream.concat(modulepathModuleFinder.findAll().stream(), upgradeAndSystemModuleFinder.findAll().stream())
Expand All @@ -254,6 +257,8 @@ public void loadAllClasses(ForkJoinPool executor, ImageClassLoader imageClassLoa
.filter(m -> !allModules.contains(m))
.findAny().ifPresent(m -> missingFromSetOfEntriesError(m, allModules, "module-path", IncludeAllFromModule));

javaPackagesToInclude = Set.copyOf(IncludeAllFromPackage.getValue(parsedHostedOptions).values());

javaPathsToInclude = IncludeAllFromPath.getValue(parsedHostedOptions).values().stream()
.map(NativeImageClassLoaderSupport::stringToPath)
.map(Path::toAbsolutePath)
Expand Down Expand Up @@ -722,32 +727,44 @@ private void run() {
System.out.println("Total processed entries: " + entriesProcessed.longValue() + ", current entry: " + currentlyProcessedEntry);
}, 5, 1, TimeUnit.MINUTES);

List<String> requiresInit = new ArrayList<>(Arrays.asList(
"jdk.internal.vm.ci", "jdk.graal.compiler", "com.oracle.graal.graal_enterprise",
var requiresInit = new HashSet<>(List.of("jdk.internal.vm.ci", "jdk.graal.compiler", "com.oracle.graal.graal_enterprise",
"org.graalvm.nativeimage", "org.graalvm.truffle", "org.graalvm.truffle.runtime",
"org.graalvm.truffle.compiler", "com.oracle.truffle.enterprise", "org.graalvm.jniutils",
"org.graalvm.nativebridge"));

Set<String> additionalSystemModules = upgradeAndSystemModuleFinder.findAll().stream().map(v -> v.descriptor().name()).collect(Collectors.toSet());
Set<String> additionalSystemModules = upgradeAndSystemModuleFinder.findAll().stream()
.map(v -> v.descriptor().name())
.collect(Collectors.toSet());
additionalSystemModules.retainAll(getJavaModuleNamesToInclude());
requiresInit.addAll(additionalSystemModules);

Set<String> explicitlyAddedModules = ModuleSupport.parseModuleSetModifierProperty(ModuleSupport.PROPERTY_IMAGE_EXPLICITLY_ADDED_MODULES);

for (ModuleReference moduleReference : upgradeAndSystemModuleFinder.findAll()) {
if (requiresInit.contains(moduleReference.descriptor().name())) {
initModule(moduleReference);
String moduleName = moduleReference.descriptor().name();
boolean moduleRequiresInit = requiresInit.contains(moduleName);
if (moduleRequiresInit || explicitlyAddedModules.contains(moduleName)) {
initModule(moduleReference, moduleRequiresInit);
}
}
for (ModuleReference moduleReference : modulepathModuleFinder.findAll()) {
initModule(moduleReference);
initModule(moduleReference, true);
}

classpath().parallelStream().forEach(this::loadClassesFromPath);
} finally {
scheduledExecutor.shutdown();
}

/* Verify all package inclusion requests were successful */
for (String packageName : javaPackagesToInclude) {
if (!includedJavaPackages.contains(packageName)) {
missingFromSetOfEntriesError(packageName, includedJavaPackages, "package", IncludeAllFromPackage);
}
}
}

private void initModule(ModuleReference moduleReference) {
private void initModule(ModuleReference moduleReference, boolean moduleRequiresInit) {
String moduleReferenceLocation = moduleReference.location().map(URI::toString).orElse("UnknownModuleReferenceLocation");
currentlyProcessedEntry = moduleReferenceLocation;
Optional<Module> optionalModule = findModule(moduleReference.descriptor().name());
Expand All @@ -766,7 +783,7 @@ private void initModule(ModuleReference moduleReference) {
String className = extractClassName(moduleResource, fileSystemSeparatorChar);
if (className != null) {
currentlyProcessedEntry = moduleReferenceLocation + fileSystemSeparatorChar + moduleResource;
executor.execute(() -> handleClassFileName(container, module, className, includeUnconditionally));
executor.execute(() -> handleClassFileName(container, module, className, includeUnconditionally, moduleRequiresInit));
}
entriesProcessed.increment();
});
Expand Down Expand Up @@ -832,7 +849,7 @@ public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) {
String className = extractClassName(fileName, fileSystemSeparatorChar);
if (className != null) {
currentlyProcessedEntry = file.toUri().toString();
executor.execute(() -> handleClassFileName(container, null, className, includeUnconditionally));
executor.execute(() -> handleClassFileName(container, null, className, includeUnconditionally, true));
}
entriesProcessed.increment();
return FileVisitResult.CONTINUE;
Expand Down Expand Up @@ -916,24 +933,26 @@ private String extractClassName(String fileName, char fileSystemSeparatorChar) {
return strippedClassFileName.equals("module-info") ? null : strippedClassFileName.replace(fileSystemSeparatorChar, '.');
}

private void handleClassFileName(URI container, Module module, String className, boolean includeUnconditionally) {
synchronized (classes) {
EconomicSet<String> classNames = classes.get(container);
if (classNames == null) {
classNames = EconomicSet.create();
classes.put(container, classNames);
private void handleClassFileName(URI container, Module module, String className, boolean includeUnconditionally, boolean classRequiresInit) {
if (classRequiresInit) {
synchronized (classes) {
EconomicSet<String> classNames = classes.get(container);
if (classNames == null) {
classNames = EconomicSet.create();
classes.put(container, classNames);
}
classNames.add(className);
}
classNames.add(className);
}
int packageSep = className.lastIndexOf('.');
String packageName = packageSep > 0 ? className.substring(0, packageSep) : "";
synchronized (packages) {
EconomicSet<String> packageNames = packages.get(container);
if (packageNames == null) {
packageNames = EconomicSet.create();
packages.put(container, packageNames);
int packageSep = className.lastIndexOf('.');
String packageName = packageSep > 0 ? className.substring(0, packageSep) : "";
synchronized (packages) {
EconomicSet<String> packageNames = packages.get(container);
if (packageNames == null) {
packageNames = EconomicSet.create();
packages.put(container, packageNames);
}
packageNames.add(packageName);
}
packageNames.add(packageName);
}

Class<?> clazz = null;
Expand All @@ -945,10 +964,14 @@ private void handleClassFileName(URI container, Module module, String className,
ImageClassLoader.handleClassLoadingError(t);
}
if (clazz != null) {
if (includeUnconditionally) {
String packageName = clazz.getPackageName();
includedJavaPackages.add(packageName);
if (includeUnconditionally || javaPackagesToInclude.contains(packageName)) {
classesToIncludeUnconditionally.add(clazz);
}
imageClassLoader.handleClass(clazz);
if (classRequiresInit) {
imageClassLoader.handleClass(clazz);
}
}
imageClassLoader.watchdog.recordActivity();
}
Expand Down
Loading

0 comments on commit 600350a

Please sign in to comment.