From 0e98eecdd56ab4081d5ec3eb9897e78ebc5e511c Mon Sep 17 00:00:00 2001 From: jimmyk Date: Wed, 30 Oct 2024 15:27:06 -0400 Subject: [PATCH] Make disable CAS and CAE options consistant Changes TR_DisableCASInlining envvar to disable inlining Unsafe compareAndSwap on all platforms. This used to only work on X and would also disable inlining compareAndExchange at the same time. Now it only disables inlining compareAndSwap. Renames TR_DisableCAEIntrinsic envvar to TR_DisableCAEIntrinsic and it disables inlining Unsafe compareAndExchange on all platforms. Envvars are now checked in fewer locations and set an option bit instead. The following methods were added to support this: getSupportsInlineUnsafeCompareAndSet setSupportsInlineUnsafeCompareAndSet getSupportsInlineUnsafeCompareAndExchange setSupportsInlineUnsafeCompareAndExchange These methods are used to indicate support for inlining CAS/CAE and also to query for suppport. Signed-off-by: jimmyk --- .../aarch64/codegen/J9CodeGenerator.cpp | 12 ++++++ .../aarch64/codegen/J9TreeEvaluator.cpp | 30 ++++++++----- runtime/compiler/codegen/J9CodeGenerator.cpp | 25 +++++------ runtime/compiler/codegen/J9CodeGenerator.hpp | 22 ++++++++++ .../compiler/optimizer/InlinerTempForJ9.cpp | 16 ++++--- .../compiler/p/codegen/J9CodeGenerator.cpp | 12 ++++++ .../compiler/p/codegen/J9TreeEvaluator.cpp | 37 ++++++++++------ .../compiler/x/codegen/J9CodeGenerator.cpp | 37 +++++++++++----- .../compiler/x/codegen/J9TreeEvaluator.cpp | 20 ++++----- .../compiler/z/codegen/J9CodeGenerator.cpp | 42 ++++++++++++++----- 10 files changed, 181 insertions(+), 72 deletions(-) diff --git a/runtime/compiler/aarch64/codegen/J9CodeGenerator.cpp b/runtime/compiler/aarch64/codegen/J9CodeGenerator.cpp index e76f8ad63ca..ba7d6354294 100644 --- a/runtime/compiler/aarch64/codegen/J9CodeGenerator.cpp +++ b/runtime/compiler/aarch64/codegen/J9CodeGenerator.cpp @@ -101,6 +101,18 @@ J9::ARM64::CodeGenerator::initialize() } if (comp->fej9()->hasFixedFrameC_CallingConvention()) cg->setHasFixedFrameC_CallingConvention(); + + static bool disableCASInlining = feGetEnv("TR_DisableCASInlining") != NULL; + if (!disableCASInlining) + { + cg->setSupportsInlineUnsafeCompareAndSet(); + } + + static bool disableCAEInlining = feGetEnv("TR_DisableCAEInlining") != NULL; + if (!disableCAEInlining) + { + cg->setSupportsInlineUnsafeCompareAndExchange(); + } } TR::Linkage * diff --git a/runtime/compiler/aarch64/codegen/J9TreeEvaluator.cpp b/runtime/compiler/aarch64/codegen/J9TreeEvaluator.cpp index f12e6f3ea21..1e06b1bf2a1 100644 --- a/runtime/compiler/aarch64/codegen/J9TreeEvaluator.cpp +++ b/runtime/compiler/aarch64/codegen/J9TreeEvaluator.cpp @@ -6863,7 +6863,8 @@ J9::ARM64::CodeGenerator::inlineDirectCall(TR::Node *node, TR::Register *&result break; } - static bool disableCAEIntrinsic = feGetEnv("TR_DisableCAEIntrinsic") != NULL; + bool disableCASInlining = !cg->getSupportsInlineUnsafeCompareAndSet(); + bool disableCAEInlining = !cg->getSupportsInlineUnsafeCompareAndExchange(); switch (methodSymbol->getRecognizedMethod()) { case TR::java_lang_Thread_onSpinWait: @@ -6957,8 +6958,11 @@ J9::ARM64::CodeGenerator::inlineDirectCall(TR::Node *node, TR::Register *&result if ((node->isUnsafeGetPutCASCallOnNonArray() || !TR::Compiler->om.canGenerateArraylets()) && node->isSafeForCGToFastPathUnsafeCall()) { - resultReg = VMinlineCompareAndSwap(node, cg, false); - return true; + if (!disableCASInlining) + { + resultReg = VMinlineCompareAndSwap(node, cg, false); + return true; + } } break; } @@ -6971,8 +6975,11 @@ J9::ARM64::CodeGenerator::inlineDirectCall(TR::Node *node, TR::Register *&result if ((node->isUnsafeGetPutCASCallOnNonArray() || !TR::Compiler->om.canGenerateArraylets()) && node->isSafeForCGToFastPathUnsafeCall()) { - resultReg = VMinlineCompareAndSwap(node, cg, true); - return true; + if (!disableCASInlining) + { + resultReg = VMinlineCompareAndSwap(node, cg, true); + return true; + } } break; } @@ -6984,8 +6991,11 @@ J9::ARM64::CodeGenerator::inlineDirectCall(TR::Node *node, TR::Register *&result if ((node->isUnsafeGetPutCASCallOnNonArray() || !TR::Compiler->om.canGenerateArraylets()) && node->isSafeForCGToFastPathUnsafeCall()) { - resultReg = VMinlineCompareAndSwapObject(node, cg); - return true; + if (!disableCASInlining) + { + resultReg = VMinlineCompareAndSwapObject(node, cg); + return true; + } } break; } @@ -6994,7 +7004,7 @@ J9::ARM64::CodeGenerator::inlineDirectCall(TR::Node *node, TR::Register *&result { if ((node->isUnsafeGetPutCASCallOnNonArray() || !TR::Compiler->om.canGenerateArraylets()) && node->isSafeForCGToFastPathUnsafeCall()) { - if (!disableCAEIntrinsic) + if (!disableCAEInlining) { resultReg = VMinlineCompareAndSwap(node, cg, false, true); return true; @@ -7007,7 +7017,7 @@ J9::ARM64::CodeGenerator::inlineDirectCall(TR::Node *node, TR::Register *&result { if ((node->isUnsafeGetPutCASCallOnNonArray() || !TR::Compiler->om.canGenerateArraylets()) && node->isSafeForCGToFastPathUnsafeCall()) { - if (!disableCAEIntrinsic) + if (!disableCAEInlining) { resultReg = VMinlineCompareAndSwap(node, cg, true, true); return true; @@ -7029,7 +7039,7 @@ J9::ARM64::CodeGenerator::inlineDirectCall(TR::Node *node, TR::Register *&result { if ((node->isUnsafeGetPutCASCallOnNonArray() || !TR::Compiler->om.canGenerateArraylets()) && node->isSafeForCGToFastPathUnsafeCall()) { - if (!disableCAEIntrinsic) + if (!disableCAEInlining) { resultReg = VMinlineCompareAndSwapObject(node, cg, true); return true; diff --git a/runtime/compiler/codegen/J9CodeGenerator.cpp b/runtime/compiler/codegen/J9CodeGenerator.cpp index 1467a8de101..8a86d55aab8 100644 --- a/runtime/compiler/codegen/J9CodeGenerator.cpp +++ b/runtime/compiler/codegen/J9CodeGenerator.cpp @@ -650,31 +650,32 @@ J9::CodeGenerator::lowerTreesPreChildrenVisit(TR::Node *parent, TR::TreeTop *tre if (parent->getOpCode().isFunctionCall()) { - // J9 - // - /* Hiding compressedref logic from CodeGen isn't a good practice, and the evaluator still needs the uncompressedref node for write barriers. + /* J9 + * + * Hiding compressedref logic from CodeGen isn't a good practice, and the evaluator still needs the uncompressedref node for write barriers. * Therefore, this part is deprecated. It can only be activated on X, P or Z with the TR_UseOldCompareAndSwapObject envvar. * - * If TR_DisableCAEIntrinsic is set to disable inlining of compareAndExchange, compressedref logic will not be hidden for compareAndExchange - * calls even if TR_UseOldCompareAndSwapObject is set. The reason is that TR_DisableCAEIntrinsic takes priority over TR_UseOldCompareAndSwapObject + * If TR_DisableCAEInlining is set to disable inlining of compareAndExchange, compressedref logic will not be hidden for compareAndExchange + * calls even if TR_UseOldCompareAndSwapObject is set. The reason is that TR_DisableCAEInlining takes priority over TR_UseOldCompareAndSwapObject * so neither the old nor new version of the inlined compareAndExchange are used and the non-inlined version expects that the compressedrefs are * not hidden. * - * Similarly, TR_DisableCASInlining (which is only supported on X) can be used to disable inlining on both compareAndSwap and compareAndExchange. - * This also takes priority over TR_UseOldCompareAndSwapObject. Once again, the compressedrefs logic will not be hidden since it is expected by - * the non-inlined version. + * Similarly, TR_DisableCASInlining can be used to disable inlining of compareAndSet. This also takes priority over TR_UseOldCompareAndSwapObject. + * Once again, the compressedrefs logic will not be hidden since it is expected by the non-inlined version. */ static bool useOldCompareAndSwapObject = (bool)feGetEnv("TR_UseOldCompareAndSwapObject"); - static bool disableCASInlining = feGetEnv("TR_DisableCASInlining") != NULL; - if (((self()->comp()->target().cpu.isX86() && !disableCASInlining) || self()->comp()->target().cpu.isPower() || self()->comp()->target().cpu.isZ()) && + if ((self()->comp()->target().cpu.isX86() || self()->comp()->target().cpu.isPower() || self()->comp()->target().cpu.isZ()) && self()->comp()->useCompressedPointers() && useOldCompareAndSwapObject) { TR::MethodSymbol *methodSymbol = parent->getSymbol()->castToMethodSymbol(); - static bool disableCAEIntrinsic = feGetEnv("TR_DisableCAEIntrinsic") != NULL; + + bool disableCASInlining = !self()->getSupportsInlineUnsafeCompareAndSet(); + bool disableCAEIntrinsic = !self()->getSupportsInlineUnsafeCompareAndExchange(); + // In Java9 Unsafe could be the jdk.internal JNI method or the sun.misc ordinary method wrapper, // while in Java8 it can only be the sun.misc package which will itself contain the JNI method. // Test for isNative to distinguish between them. - if (((methodSymbol->getRecognizedMethod() == TR::sun_misc_Unsafe_compareAndSwapObject_jlObjectJjlObjectjlObject_Z) || + if ((((methodSymbol->getRecognizedMethod() == TR::sun_misc_Unsafe_compareAndSwapObject_jlObjectJjlObjectjlObject_Z) && !disableCASInlining) || ((methodSymbol->getRecognizedMethod() == TR::jdk_internal_misc_Unsafe_compareAndExchangeObject) && !disableCAEIntrinsic) || ((methodSymbol->getRecognizedMethod() == TR::jdk_internal_misc_Unsafe_compareAndExchangeReference) && !disableCAEIntrinsic)) && methodSymbol->isNative() && diff --git a/runtime/compiler/codegen/J9CodeGenerator.hpp b/runtime/compiler/codegen/J9CodeGenerator.hpp index 5bb0a356203..6a3182e2016 100644 --- a/runtime/compiler/codegen/J9CodeGenerator.hpp +++ b/runtime/compiler/codegen/J9CodeGenerator.hpp @@ -522,6 +522,26 @@ void addMonClass(TR::Node* monNode, TR_OpaqueClassBlock* clazz); */ void setSupportsInlineVectorizedHashCode() { _j9Flags.set(SupportsInlineVectorizedHashCode); } + /** \brief + * Determines whether the code generator supports inlining of jdk/internal/misc/Unsafe.CompareAndSet[Object|Reference|Int|Long] + */ + bool getSupportsInlineUnsafeCompareAndSet() { return _j9Flags.testAny(SupportsInlineUnsafeCompareAndSet); } + + /** \brief + * The code generator supports inlining of jdk/internal/misc/Unsafe.CompareAndSet[Object|Reference|Int|Long] + */ + void setSupportsInlineUnsafeCompareAndSet() { _j9Flags.set(SupportsInlineUnsafeCompareAndSet); } + + /** \brief + * Determines whether the code generator supports inlining of jdk/internal/misc/Unsafe.CompareAndExchange[Object|Reference|Int|Long] + */ + bool getSupportsInlineUnsafeCompareAndExchange() { return _j9Flags.testAny(SupportsInlineUnsafeCompareAndExchange); } + + /** \brief + * The code generator supports inlining of jdk/internal/misc/Unsafe.CompareAndExchange[Object|Reference|Int|Long] + */ + void setSupportsInlineUnsafeCompareAndExchange() { _j9Flags.set(SupportsInlineUnsafeCompareAndExchange); } + /** * \brief * The number of nodes between a monext and the next monent before @@ -688,6 +708,8 @@ void addMonClass(TR::Node* monNode, TR_OpaqueClassBlock* clazz); SupportsInlineVectorizedMismatch = 0x00001000, SupportsInlineVectorizedHashCode = 0x00002000, SupportsInlineStringCodingHasNegatives = 0x00004000, + SupportsInlineUnsafeCompareAndSet = 0x00008000, + SupportsInlineUnsafeCompareAndExchange = 0x00010000, }; flags32_t _j9Flags; diff --git a/runtime/compiler/optimizer/InlinerTempForJ9.cpp b/runtime/compiler/optimizer/InlinerTempForJ9.cpp index 0bdc3fd0097..a71080d7ad9 100644 --- a/runtime/compiler/optimizer/InlinerTempForJ9.cpp +++ b/runtime/compiler/optimizer/InlinerTempForJ9.cpp @@ -2517,7 +2517,8 @@ TR_J9InlinerPolicy::inlineUnsafeCall(TR::ResolvedMethodSymbol *calleeSymbol, TR: !comp()->fej9()->traceableMethodsCanBeInlined())) return false; - static bool disableCAEIntrinsic = feGetEnv("TR_DisableCAEIntrinsic") != NULL; + bool disableCASInlining = !comp()->cg()->getSupportsInlineUnsafeCompareAndSet(); + bool disableCAEInlining = !comp()->cg()->getSupportsInlineUnsafeCompareAndExchange(); // I am not sure if having the same type between C/S and B/Z matters here.. ie. if the type is being used as the only distinguishing factor switch (callNode->getSymbol()->castToResolvedMethodSymbol()->getRecognizedMethod()) { @@ -2705,18 +2706,21 @@ TR_J9InlinerPolicy::inlineUnsafeCall(TR::ResolvedMethodSymbol *calleeSymbol, TR: case TR::jdk_internal_misc_Unsafe_compareAndExchangeLong: case TR::jdk_internal_misc_Unsafe_compareAndExchangeObject: case TR::jdk_internal_misc_Unsafe_compareAndExchangeReference: - if (disableCAEIntrinsic) + if (disableCAEInlining || callNode->isSafeForCGToFastPathUnsafeCall()) { - break; + return false; } - // Fallthrough if previous if condition is not met. + return createUnsafeCASCallDiamond(callNodeTreeTop, callNode); + case TR::sun_misc_Unsafe_compareAndSwapInt_jlObjectJII_Z: case TR::sun_misc_Unsafe_compareAndSwapLong_jlObjectJJJ_Z: case TR::sun_misc_Unsafe_compareAndSwapObject_jlObjectJjlObjectjlObject_Z: - if (callNode->isSafeForCGToFastPathUnsafeCall()) + if (disableCASInlining || callNode->isSafeForCGToFastPathUnsafeCall()) + { return false; + } #if defined(J9VM_GC_SPARSE_HEAP_ALLOCATION) - if(TR::Compiler->om.isOffHeapAllocationEnabled()) + if (TR::Compiler->om.isOffHeapAllocationEnabled()) return createUnsafeCASCallDiamond(callNodeTreeTop, callNode); #endif /* J9VM_GC_SPARSE_HEAP_ALLOCATION */ switch (callerSymbol->castToMethodSymbol()->getRecognizedMethod()) diff --git a/runtime/compiler/p/codegen/J9CodeGenerator.cpp b/runtime/compiler/p/codegen/J9CodeGenerator.cpp index bbb449abb73..9067c3eaed2 100644 --- a/runtime/compiler/p/codegen/J9CodeGenerator.cpp +++ b/runtime/compiler/p/codegen/J9CodeGenerator.cpp @@ -108,6 +108,18 @@ J9::Power::CodeGenerator::initialize() !disableStringInflateIntrinsic) cg->setSupportsInlineStringLatin1Inflate(); + static bool disableCASInlining = feGetEnv("TR_DisableCASInlining") != NULL; + if (!disableCASInlining) + { + cg->setSupportsInlineUnsafeCompareAndSet(); + } + + static bool disableCAEInlining = feGetEnv("TR_DisableCAEInlining") != NULL; + if (!disableCAEInlining) + { + cg->setSupportsInlineUnsafeCompareAndExchange(); + } + if (!comp->getOption(TR_DisableReadMonitors)) cg->setSupportsReadOnlyLocks(); diff --git a/runtime/compiler/p/codegen/J9TreeEvaluator.cpp b/runtime/compiler/p/codegen/J9TreeEvaluator.cpp index 1d246026c35..61c49c76840 100644 --- a/runtime/compiler/p/codegen/J9TreeEvaluator.cpp +++ b/runtime/compiler/p/codegen/J9TreeEvaluator.cpp @@ -11793,7 +11793,8 @@ J9::Power::CodeGenerator::inlineDirectCall(TR::Node *node, TR::Register *&result } else if (methodSymbol) { - static bool disableCAEIntrinsic = feGetEnv("TR_DisableCAEIntrinsic") != NULL; + bool disableCASInlining = !cg->getSupportsInlineUnsafeCompareAndSet(); + bool disableCAEInlining = !cg->getSupportsInlineUnsafeCompareAndExchange(); switch (methodSymbol->getRecognizedMethod()) { case TR::java_util_concurrent_ConcurrentLinkedQueue_tmOffer: @@ -12073,8 +12074,11 @@ J9::Power::CodeGenerator::inlineDirectCall(TR::Node *node, TR::Register *&result if ((node->isUnsafeGetPutCASCallOnNonArray() || !TR::Compiler->om.canGenerateArraylets()) && node->isSafeForCGToFastPathUnsafeCall()) { - resultReg = VMinlineCompareAndSetOrExchange(node, cg, 4, false); - return true; + if (!disableCASInlining) + { + resultReg = VMinlineCompareAndSetOrExchange(node, cg, 4, false); + return true; + } } break; @@ -12087,13 +12091,19 @@ J9::Power::CodeGenerator::inlineDirectCall(TR::Node *node, TR::Register *&result if (comp->target().is64Bit() && (node->isUnsafeGetPutCASCallOnNonArray() || !TR::Compiler->om.canGenerateArraylets()) && node->isSafeForCGToFastPathUnsafeCall()) { - resultReg = VMinlineCompareAndSetOrExchange(node, cg, 8, false); - return true; + if (!disableCASInlining) + { + resultReg = VMinlineCompareAndSetOrExchange(node, cg, 8, false); + return true; + } } else if ((node->isUnsafeGetPutCASCallOnNonArray() || !TR::Compiler->om.canGenerateArraylets()) && node->isSafeForCGToFastPathUnsafeCall()) { - resultReg = inlineAtomicOperation(node, cg, methodSymbol); - return true; + if (!disableCASInlining) + { + resultReg = inlineAtomicOperation(node, cg, methodSymbol); + return true; + } } break; @@ -12104,15 +12114,18 @@ J9::Power::CodeGenerator::inlineDirectCall(TR::Node *node, TR::Register *&result if ((node->isUnsafeGetPutCASCallOnNonArray() || !TR::Compiler->om.canGenerateArraylets()) && node->isSafeForCGToFastPathUnsafeCall()) { - resultReg = VMinlineCompareAndSetOrExchangeReference(node, cg, false); - return true; + if (!disableCASInlining) + { + resultReg = VMinlineCompareAndSetOrExchangeReference(node, cg, false); + return true; + } } break; case TR::jdk_internal_misc_Unsafe_compareAndExchangeInt: if ((node->isUnsafeGetPutCASCallOnNonArray() || !TR::Compiler->om.canGenerateArraylets()) && node->isSafeForCGToFastPathUnsafeCall()) { - if (!disableCAEIntrinsic) + if (!disableCAEInlining) { resultReg = VMinlineCompareAndSetOrExchange(node, cg, 4, true); return true; @@ -12123,7 +12136,7 @@ J9::Power::CodeGenerator::inlineDirectCall(TR::Node *node, TR::Register *&result case TR::jdk_internal_misc_Unsafe_compareAndExchangeLong: if (comp->target().is64Bit() && (node->isUnsafeGetPutCASCallOnNonArray() || !TR::Compiler->om.canGenerateArraylets()) && node->isSafeForCGToFastPathUnsafeCall()) { - if (!disableCAEIntrinsic) + if (!disableCAEInlining) { resultReg = VMinlineCompareAndSetOrExchange(node, cg, 8, true); return true; @@ -12143,7 +12156,7 @@ J9::Power::CodeGenerator::inlineDirectCall(TR::Node *node, TR::Register *&result case TR::jdk_internal_misc_Unsafe_compareAndExchangeReference: if ((node->isUnsafeGetPutCASCallOnNonArray() || !TR::Compiler->om.canGenerateArraylets()) && node->isSafeForCGToFastPathUnsafeCall()) { - if (!disableCAEIntrinsic) + if (!disableCAEInlining) { resultReg = VMinlineCompareAndSetOrExchangeReference(node, cg, true); return true; diff --git a/runtime/compiler/x/codegen/J9CodeGenerator.cpp b/runtime/compiler/x/codegen/J9CodeGenerator.cpp index 23c33ef97dd..13fb5b903ba 100644 --- a/runtime/compiler/x/codegen/J9CodeGenerator.cpp +++ b/runtime/compiler/x/codegen/J9CodeGenerator.cpp @@ -145,6 +145,18 @@ J9::X86::CodeGenerator::initialize() cg->setSupportsInlineVectorizedMismatch(); } + static bool disableCASInlining = feGetEnv("TR_DisableCASInlining") != NULL; + if (!disableCASInlining) + { + cg->setSupportsInlineUnsafeCompareAndSet(); + } + + static bool disableCAEInlining = feGetEnv("TR_DisableCAEInlining") != NULL; + if (!disableCAEInlining) + { + cg->setSupportsInlineUnsafeCompareAndExchange(); + } + // Disable fast gencon barriers for AOT compiles because relocations on // the inlined heap addresses are not available (yet). // @@ -522,21 +534,23 @@ J9::X86::CodeGenerator::supportsNonHelper(TR::SymbolReferenceTable::CommonNonhel } /* - * This method returns TRUE for all the cases we decide NOT to replace the call to CAS - * with inline assembly. The GRA and Evaluator should be consistent about whether to inline CAS natives. + * This method returns TRUE for all the cases we decide NOT to replace the call to CAS/CAE + * with inline assembly. The GRA and Evaluator should be consistent about whether to inline CAS/CAE natives. */ static bool willNotInlineCompareAndSwapNative(TR::Node *node, int8_t size, - TR::Compilation *comp) + TR::Compilation *comp, + bool isExchange) { TR::SymbolReference *callSymRef = node->getSymbolReference(); TR::MethodSymbol *methodSymbol = callSymRef->getSymbol()->castToMethodSymbol(); if (TR::Compiler->om.canGenerateArraylets() && !node->isUnsafeGetPutCASCallOnNonArray()) return true; - static char *disableCASInlining = feGetEnv("TR_DisableCASInlining"); - if (disableCASInlining) + if (!isExchange && !comp->cg()->getSupportsInlineUnsafeCompareAndSet()) + return true; + if (isExchange && !comp->cg()->getSupportsInlineUnsafeCompareAndExchange()) return true; // In Java9 the sun.misc.Unsafe JNI methods have been moved to jdk.internal, @@ -583,15 +597,18 @@ J9::X86::CodeGenerator::willBeEvaluatedAsCallByCodeGen(TR::Node *node, TR::Compi switch (methodSymbol->getRecognizedMethod()) { case TR::sun_misc_Unsafe_compareAndSwapInt_jlObjectJII_Z: - case TR::jdk_internal_misc_Unsafe_compareAndExchangeInt: - return willNotInlineCompareAndSwapNative(node, 4, comp); + return willNotInlineCompareAndSwapNative(node, 4, comp, false); case TR::sun_misc_Unsafe_compareAndSwapLong_jlObjectJJJ_Z: - case TR::jdk_internal_misc_Unsafe_compareAndExchangeLong: - return willNotInlineCompareAndSwapNative(node, 8, comp); + return willNotInlineCompareAndSwapNative(node, 8, comp, false); case TR::sun_misc_Unsafe_compareAndSwapObject_jlObjectJjlObjectjlObject_Z: + return willNotInlineCompareAndSwapNative(node, TR::Compiler->om.sizeofReferenceField(), comp, false); + case TR::jdk_internal_misc_Unsafe_compareAndExchangeInt: + return willNotInlineCompareAndSwapNative(node, 4, comp, true); + case TR::jdk_internal_misc_Unsafe_compareAndExchangeLong: + return willNotInlineCompareAndSwapNative(node, 8, comp, true); case TR::jdk_internal_misc_Unsafe_compareAndExchangeObject: case TR::jdk_internal_misc_Unsafe_compareAndExchangeReference: - return willNotInlineCompareAndSwapNative(node, TR::Compiler->om.sizeofReferenceField(), comp); + return willNotInlineCompareAndSwapNative(node, TR::Compiler->om.sizeofReferenceField(), comp, true); default: break; diff --git a/runtime/compiler/x/codegen/J9TreeEvaluator.cpp b/runtime/compiler/x/codegen/J9TreeEvaluator.cpp index e0fc5120170..52b73d6c850 100644 --- a/runtime/compiler/x/codegen/J9TreeEvaluator.cpp +++ b/runtime/compiler/x/codegen/J9TreeEvaluator.cpp @@ -9641,11 +9641,6 @@ inlineCompareAndSwapNative( if (TR::Compiler->om.canGenerateArraylets() && !node->isUnsafeGetPutCASCallOnNonArray()) return false; - static char *disableCASInlining = feGetEnv("TR_DisableCASInlining"); - - if (disableCASInlining /* || comp->useCompressedPointers() */) - return false; - // size = 4 --> CMPXCHG4 // size = 8 --> if 64-bit -> CMPXCHG8 // else if proc supports CMPXCHG8B -> CMPXCHG8B @@ -9947,7 +9942,8 @@ bool J9::X86::TreeEvaluator::VMinlineCallEvaluator( bool callWasInlined = false; TR::Compilation *comp = cg->comp(); - static bool disableCAEIntrinsic = feGetEnv("TR_DisableCAEIntrinsic") != NULL; + bool disableCASInlining = !cg->getSupportsInlineUnsafeCompareAndSet(); + bool disableCAEInlining = !cg->getSupportsInlineUnsafeCompareAndExchange(); if (methodSymbol) { @@ -10030,20 +10026,20 @@ bool J9::X86::TreeEvaluator::VMinlineCallEvaluator( return false; // Call the native version of NativeThread.current() case TR::sun_misc_Unsafe_compareAndSwapInt_jlObjectJII_Z: { - if (node->isSafeForCGToFastPathUnsafeCall()) + if (!disableCASInlining && node->isSafeForCGToFastPathUnsafeCall()) return inlineCompareAndSwapNative(node, 4, false, false, cg); } break; case TR::sun_misc_Unsafe_compareAndSwapLong_jlObjectJJJ_Z: { - if (node->isSafeForCGToFastPathUnsafeCall()) + if (!disableCASInlining && node->isSafeForCGToFastPathUnsafeCall()) return inlineCompareAndSwapNative(node, 8, false, false, cg); } break; case TR::sun_misc_Unsafe_compareAndSwapObject_jlObjectJjlObjectjlObject_Z: { static bool useOldCompareAndSwapObject = (bool)feGetEnv("TR_UseOldCompareAndSwapObject"); - if (node->isSafeForCGToFastPathUnsafeCall()) + if (!disableCASInlining && node->isSafeForCGToFastPathUnsafeCall()) { if (useOldCompareAndSwapObject) return inlineCompareAndSwapNative(node, TR::Compiler->om.sizeofReferenceField(), true, false, cg); @@ -10057,13 +10053,13 @@ bool J9::X86::TreeEvaluator::VMinlineCallEvaluator( break; case TR::jdk_internal_misc_Unsafe_compareAndExchangeInt: { - if (!disableCAEIntrinsic && node->isSafeForCGToFastPathUnsafeCall()) + if (!disableCAEInlining && node->isSafeForCGToFastPathUnsafeCall()) return inlineCompareAndSwapNative(node, 4, false, true, cg); } break; case TR::jdk_internal_misc_Unsafe_compareAndExchangeLong: { - if (!disableCAEIntrinsic && node->isSafeForCGToFastPathUnsafeCall()) + if (!disableCAEInlining && node->isSafeForCGToFastPathUnsafeCall()) return inlineCompareAndSwapNative(node, 8, false, true, cg); } break; @@ -10071,7 +10067,7 @@ bool J9::X86::TreeEvaluator::VMinlineCallEvaluator( case TR::jdk_internal_misc_Unsafe_compareAndExchangeReference: { static bool useOldCompareAndSwapObject = (bool)feGetEnv("TR_UseOldCompareAndSwapObject"); - if (!disableCAEIntrinsic && node->isSafeForCGToFastPathUnsafeCall()) + if (!disableCAEInlining && node->isSafeForCGToFastPathUnsafeCall()) { if (useOldCompareAndSwapObject) return inlineCompareAndSwapNative(node, TR::Compiler->om.sizeofReferenceField(), true, true, cg); diff --git a/runtime/compiler/z/codegen/J9CodeGenerator.cpp b/runtime/compiler/z/codegen/J9CodeGenerator.cpp index 75aea131fdc..bb45bd8a389 100644 --- a/runtime/compiler/z/codegen/J9CodeGenerator.cpp +++ b/runtime/compiler/z/codegen/J9CodeGenerator.cpp @@ -150,6 +150,18 @@ J9::Z::CodeGenerator::initialize() cg->setSupportsInlineVectorizedMismatch(); } + static bool disableCASInlining = feGetEnv("TR_DisableCASInlining") != NULL; + if (!disableCASInlining) + { + cg->setSupportsInlineUnsafeCompareAndSet(); + } + + static bool disableCAEInlining = feGetEnv("TR_DisableCAEInlining") != NULL; + if (!disableCAEInlining) + { + cg->setSupportsInlineUnsafeCompareAndExchange(); + } + // Let's turn this on. There is more work needed in the opt // to catch the case where the BNDSCHK is inserted after // @@ -3822,7 +3834,8 @@ J9::Z::CodeGenerator::inlineDirectCall( } static const char * enableTRTRE = feGetEnv("TR_enableTRTRE"); - static bool disableCAEIntrinsic = feGetEnv("TR_DisableCAEIntrinsic") != NULL; + bool disableCASInlining = !cg->getSupportsInlineUnsafeCompareAndSet(); + bool disableCAEInlining = !cg->getSupportsInlineUnsafeCompareAndExchange(); switch (methodSymbol->getRecognizedMethod()) { case TR::sun_misc_Unsafe_compareAndSwapInt_jlObjectJII_Z: @@ -3834,8 +3847,11 @@ J9::Z::CodeGenerator::inlineDirectCall( if ((!TR::Compiler->om.canGenerateArraylets() || node->isUnsafeGetPutCASCallOnNonArray()) && node->isSafeForCGToFastPathUnsafeCall()) { - resultReg = TR::TreeEvaluator::VMinlineCompareAndSwap(node, cg, TR::InstOpCode::CS, IS_NOT_OBJ); - return true; + if (!disableCASInlining) + { + resultReg = TR::TreeEvaluator::VMinlineCompareAndSwap(node, cg, TR::InstOpCode::CS, IS_NOT_OBJ); + return true; + } } break; @@ -3847,8 +3863,11 @@ J9::Z::CodeGenerator::inlineDirectCall( // Too risky to do Long-31bit version now. if (comp->target().is64Bit() && (!TR::Compiler->om.canGenerateArraylets() || node->isUnsafeGetPutCASCallOnNonArray()) && node->isSafeForCGToFastPathUnsafeCall()) { - resultReg = TR::TreeEvaluator::VMinlineCompareAndSwap(node, cg, TR::InstOpCode::CSG, IS_NOT_OBJ); - return true; + if (!disableCASInlining) + { + resultReg = TR::TreeEvaluator::VMinlineCompareAndSwap(node, cg, TR::InstOpCode::CSG, IS_NOT_OBJ); + return true; + } } break; @@ -3859,15 +3878,18 @@ J9::Z::CodeGenerator::inlineDirectCall( if ((!TR::Compiler->om.canGenerateArraylets() || node->isUnsafeGetPutCASCallOnNonArray()) && node->isSafeForCGToFastPathUnsafeCall()) { - resultReg = TR::TreeEvaluator::VMinlineCompareAndSwap(node, cg, (comp->useCompressedPointers() ? TR::InstOpCode::CS : TR::InstOpCode::getCmpAndSwapOpCode()), IS_OBJ); - return true; + if (!disableCASInlining) + { + resultReg = TR::TreeEvaluator::VMinlineCompareAndSwap(node, cg, (comp->useCompressedPointers() ? TR::InstOpCode::CS : TR::InstOpCode::getCmpAndSwapOpCode()), IS_OBJ); + return true; + } } break; case TR::jdk_internal_misc_Unsafe_compareAndExchangeInt: if ((!TR::Compiler->om.canGenerateArraylets() || node->isUnsafeGetPutCASCallOnNonArray()) && node->isSafeForCGToFastPathUnsafeCall()) { - if (!disableCAEIntrinsic) + if (!disableCAEInlining) { resultReg = TR::TreeEvaluator::VMinlineCompareAndSwap(node, cg, TR::InstOpCode::CS, IS_NOT_OBJ, true); return true; @@ -3879,7 +3901,7 @@ J9::Z::CodeGenerator::inlineDirectCall( // Too risky to do Long-31bit version now. if (comp->target().is64Bit() && (!TR::Compiler->om.canGenerateArraylets() || node->isUnsafeGetPutCASCallOnNonArray()) && node->isSafeForCGToFastPathUnsafeCall()) { - if (!disableCAEIntrinsic) + if (!disableCAEInlining) { resultReg = TR::TreeEvaluator::VMinlineCompareAndSwap(node, cg, TR::InstOpCode::CSG, IS_NOT_OBJ, true); return true; @@ -3899,7 +3921,7 @@ J9::Z::CodeGenerator::inlineDirectCall( case TR::jdk_internal_misc_Unsafe_compareAndExchangeReference: if ((!TR::Compiler->om.canGenerateArraylets() || node->isUnsafeGetPutCASCallOnNonArray()) && node->isSafeForCGToFastPathUnsafeCall()) { - if (!disableCAEIntrinsic) + if (!disableCAEInlining) { resultReg = TR::TreeEvaluator::VMinlineCompareAndSwap(node, cg, (comp->useCompressedPointers() ? TR::InstOpCode::CS : TR::InstOpCode::getCmpAndSwapOpCode()), IS_OBJ, true); return true;