diff --git a/runtime/compiler/aarch64/codegen/J9CodeGenerator.cpp b/runtime/compiler/aarch64/codegen/J9CodeGenerator.cpp index e76f8ad63ca..ba7d6354294 100644 --- a/runtime/compiler/aarch64/codegen/J9CodeGenerator.cpp +++ b/runtime/compiler/aarch64/codegen/J9CodeGenerator.cpp @@ -101,6 +101,18 @@ J9::ARM64::CodeGenerator::initialize() } if (comp->fej9()->hasFixedFrameC_CallingConvention()) cg->setHasFixedFrameC_CallingConvention(); + + static bool disableCASInlining = feGetEnv("TR_DisableCASInlining") != NULL; + if (!disableCASInlining) + { + cg->setSupportsInlineUnsafeCompareAndSet(); + } + + static bool disableCAEInlining = feGetEnv("TR_DisableCAEInlining") != NULL; + if (!disableCAEInlining) + { + cg->setSupportsInlineUnsafeCompareAndExchange(); + } } TR::Linkage * diff --git a/runtime/compiler/aarch64/codegen/J9TreeEvaluator.cpp b/runtime/compiler/aarch64/codegen/J9TreeEvaluator.cpp index f12e6f3ea21..1e06b1bf2a1 100644 --- a/runtime/compiler/aarch64/codegen/J9TreeEvaluator.cpp +++ b/runtime/compiler/aarch64/codegen/J9TreeEvaluator.cpp @@ -6863,7 +6863,8 @@ J9::ARM64::CodeGenerator::inlineDirectCall(TR::Node *node, TR::Register *&result break; } - static bool disableCAEIntrinsic = feGetEnv("TR_DisableCAEIntrinsic") != NULL; + bool disableCASInlining = !cg->getSupportsInlineUnsafeCompareAndSet(); + bool disableCAEInlining = !cg->getSupportsInlineUnsafeCompareAndExchange(); switch (methodSymbol->getRecognizedMethod()) { case TR::java_lang_Thread_onSpinWait: @@ -6957,8 +6958,11 @@ J9::ARM64::CodeGenerator::inlineDirectCall(TR::Node *node, TR::Register *&result if ((node->isUnsafeGetPutCASCallOnNonArray() || !TR::Compiler->om.canGenerateArraylets()) && node->isSafeForCGToFastPathUnsafeCall()) { - resultReg = VMinlineCompareAndSwap(node, cg, false); - return true; + if (!disableCASInlining) + { + resultReg = VMinlineCompareAndSwap(node, cg, false); + return true; + } } break; } @@ -6971,8 +6975,11 @@ J9::ARM64::CodeGenerator::inlineDirectCall(TR::Node *node, TR::Register *&result if ((node->isUnsafeGetPutCASCallOnNonArray() || !TR::Compiler->om.canGenerateArraylets()) && node->isSafeForCGToFastPathUnsafeCall()) { - resultReg = VMinlineCompareAndSwap(node, cg, true); - return true; + if (!disableCASInlining) + { + resultReg = VMinlineCompareAndSwap(node, cg, true); + return true; + } } break; } @@ -6984,8 +6991,11 @@ J9::ARM64::CodeGenerator::inlineDirectCall(TR::Node *node, TR::Register *&result if ((node->isUnsafeGetPutCASCallOnNonArray() || !TR::Compiler->om.canGenerateArraylets()) && node->isSafeForCGToFastPathUnsafeCall()) { - resultReg = VMinlineCompareAndSwapObject(node, cg); - return true; + if (!disableCASInlining) + { + resultReg = VMinlineCompareAndSwapObject(node, cg); + return true; + } } break; } @@ -6994,7 +7004,7 @@ J9::ARM64::CodeGenerator::inlineDirectCall(TR::Node *node, TR::Register *&result { if ((node->isUnsafeGetPutCASCallOnNonArray() || !TR::Compiler->om.canGenerateArraylets()) && node->isSafeForCGToFastPathUnsafeCall()) { - if (!disableCAEIntrinsic) + if (!disableCAEInlining) { resultReg = VMinlineCompareAndSwap(node, cg, false, true); return true; @@ -7007,7 +7017,7 @@ J9::ARM64::CodeGenerator::inlineDirectCall(TR::Node *node, TR::Register *&result { if ((node->isUnsafeGetPutCASCallOnNonArray() || !TR::Compiler->om.canGenerateArraylets()) && node->isSafeForCGToFastPathUnsafeCall()) { - if (!disableCAEIntrinsic) + if (!disableCAEInlining) { resultReg = VMinlineCompareAndSwap(node, cg, true, true); return true; @@ -7029,7 +7039,7 @@ J9::ARM64::CodeGenerator::inlineDirectCall(TR::Node *node, TR::Register *&result { if ((node->isUnsafeGetPutCASCallOnNonArray() || !TR::Compiler->om.canGenerateArraylets()) && node->isSafeForCGToFastPathUnsafeCall()) { - if (!disableCAEIntrinsic) + if (!disableCAEInlining) { resultReg = VMinlineCompareAndSwapObject(node, cg, true); return true; diff --git a/runtime/compiler/codegen/J9CodeGenerator.cpp b/runtime/compiler/codegen/J9CodeGenerator.cpp index 1467a8de101..8a86d55aab8 100644 --- a/runtime/compiler/codegen/J9CodeGenerator.cpp +++ b/runtime/compiler/codegen/J9CodeGenerator.cpp @@ -650,31 +650,32 @@ J9::CodeGenerator::lowerTreesPreChildrenVisit(TR::Node *parent, TR::TreeTop *tre if (parent->getOpCode().isFunctionCall()) { - // J9 - // - /* Hiding compressedref logic from CodeGen isn't a good practice, and the evaluator still needs the uncompressedref node for write barriers. + /* J9 + * + * Hiding compressedref logic from CodeGen isn't a good practice, and the evaluator still needs the uncompressedref node for write barriers. * Therefore, this part is deprecated. It can only be activated on X, P or Z with the TR_UseOldCompareAndSwapObject envvar. * - * If TR_DisableCAEIntrinsic is set to disable inlining of compareAndExchange, compressedref logic will not be hidden for compareAndExchange - * calls even if TR_UseOldCompareAndSwapObject is set. The reason is that TR_DisableCAEIntrinsic takes priority over TR_UseOldCompareAndSwapObject + * If TR_DisableCAEInlining is set to disable inlining of compareAndExchange, compressedref logic will not be hidden for compareAndExchange + * calls even if TR_UseOldCompareAndSwapObject is set. The reason is that TR_DisableCAEInlining takes priority over TR_UseOldCompareAndSwapObject * so neither the old nor new version of the inlined compareAndExchange are used and the non-inlined version expects that the compressedrefs are * not hidden. * - * Similarly, TR_DisableCASInlining (which is only supported on X) can be used to disable inlining on both compareAndSwap and compareAndExchange. - * This also takes priority over TR_UseOldCompareAndSwapObject. Once again, the compressedrefs logic will not be hidden since it is expected by - * the non-inlined version. + * Similarly, TR_DisableCASInlining can be used to disable inlining of compareAndSet. This also takes priority over TR_UseOldCompareAndSwapObject. + * Once again, the compressedrefs logic will not be hidden since it is expected by the non-inlined version. */ static bool useOldCompareAndSwapObject = (bool)feGetEnv("TR_UseOldCompareAndSwapObject"); - static bool disableCASInlining = feGetEnv("TR_DisableCASInlining") != NULL; - if (((self()->comp()->target().cpu.isX86() && !disableCASInlining) || self()->comp()->target().cpu.isPower() || self()->comp()->target().cpu.isZ()) && + if ((self()->comp()->target().cpu.isX86() || self()->comp()->target().cpu.isPower() || self()->comp()->target().cpu.isZ()) && self()->comp()->useCompressedPointers() && useOldCompareAndSwapObject) { TR::MethodSymbol *methodSymbol = parent->getSymbol()->castToMethodSymbol(); - static bool disableCAEIntrinsic = feGetEnv("TR_DisableCAEIntrinsic") != NULL; + + bool disableCASInlining = !self()->getSupportsInlineUnsafeCompareAndSet(); + bool disableCAEIntrinsic = !self()->getSupportsInlineUnsafeCompareAndExchange(); + // In Java9 Unsafe could be the jdk.internal JNI method or the sun.misc ordinary method wrapper, // while in Java8 it can only be the sun.misc package which will itself contain the JNI method. // Test for isNative to distinguish between them. - if (((methodSymbol->getRecognizedMethod() == TR::sun_misc_Unsafe_compareAndSwapObject_jlObjectJjlObjectjlObject_Z) || + if ((((methodSymbol->getRecognizedMethod() == TR::sun_misc_Unsafe_compareAndSwapObject_jlObjectJjlObjectjlObject_Z) && !disableCASInlining) || ((methodSymbol->getRecognizedMethod() == TR::jdk_internal_misc_Unsafe_compareAndExchangeObject) && !disableCAEIntrinsic) || ((methodSymbol->getRecognizedMethod() == TR::jdk_internal_misc_Unsafe_compareAndExchangeReference) && !disableCAEIntrinsic)) && methodSymbol->isNative() && diff --git a/runtime/compiler/codegen/J9CodeGenerator.hpp b/runtime/compiler/codegen/J9CodeGenerator.hpp index 5bb0a356203..6a3182e2016 100644 --- a/runtime/compiler/codegen/J9CodeGenerator.hpp +++ b/runtime/compiler/codegen/J9CodeGenerator.hpp @@ -522,6 +522,26 @@ void addMonClass(TR::Node* monNode, TR_OpaqueClassBlock* clazz); */ void setSupportsInlineVectorizedHashCode() { _j9Flags.set(SupportsInlineVectorizedHashCode); } + /** \brief + * Determines whether the code generator supports inlining of jdk/internal/misc/Unsafe.CompareAndSet[Object|Reference|Int|Long] + */ + bool getSupportsInlineUnsafeCompareAndSet() { return _j9Flags.testAny(SupportsInlineUnsafeCompareAndSet); } + + /** \brief + * The code generator supports inlining of jdk/internal/misc/Unsafe.CompareAndSet[Object|Reference|Int|Long] + */ + void setSupportsInlineUnsafeCompareAndSet() { _j9Flags.set(SupportsInlineUnsafeCompareAndSet); } + + /** \brief + * Determines whether the code generator supports inlining of jdk/internal/misc/Unsafe.CompareAndExchange[Object|Reference|Int|Long] + */ + bool getSupportsInlineUnsafeCompareAndExchange() { return _j9Flags.testAny(SupportsInlineUnsafeCompareAndExchange); } + + /** \brief + * The code generator supports inlining of jdk/internal/misc/Unsafe.CompareAndExchange[Object|Reference|Int|Long] + */ + void setSupportsInlineUnsafeCompareAndExchange() { _j9Flags.set(SupportsInlineUnsafeCompareAndExchange); } + /** * \brief * The number of nodes between a monext and the next monent before @@ -688,6 +708,8 @@ void addMonClass(TR::Node* monNode, TR_OpaqueClassBlock* clazz); SupportsInlineVectorizedMismatch = 0x00001000, SupportsInlineVectorizedHashCode = 0x00002000, SupportsInlineStringCodingHasNegatives = 0x00004000, + SupportsInlineUnsafeCompareAndSet = 0x00008000, + SupportsInlineUnsafeCompareAndExchange = 0x00010000, }; flags32_t _j9Flags; diff --git a/runtime/compiler/optimizer/InlinerTempForJ9.cpp b/runtime/compiler/optimizer/InlinerTempForJ9.cpp index 0bdc3fd0097..a71080d7ad9 100644 --- a/runtime/compiler/optimizer/InlinerTempForJ9.cpp +++ b/runtime/compiler/optimizer/InlinerTempForJ9.cpp @@ -2517,7 +2517,8 @@ TR_J9InlinerPolicy::inlineUnsafeCall(TR::ResolvedMethodSymbol *calleeSymbol, TR: !comp()->fej9()->traceableMethodsCanBeInlined())) return false; - static bool disableCAEIntrinsic = feGetEnv("TR_DisableCAEIntrinsic") != NULL; + bool disableCASInlining = !comp()->cg()->getSupportsInlineUnsafeCompareAndSet(); + bool disableCAEInlining = !comp()->cg()->getSupportsInlineUnsafeCompareAndExchange(); // I am not sure if having the same type between C/S and B/Z matters here.. ie. if the type is being used as the only distinguishing factor switch (callNode->getSymbol()->castToResolvedMethodSymbol()->getRecognizedMethod()) { @@ -2705,18 +2706,21 @@ TR_J9InlinerPolicy::inlineUnsafeCall(TR::ResolvedMethodSymbol *calleeSymbol, TR: case TR::jdk_internal_misc_Unsafe_compareAndExchangeLong: case TR::jdk_internal_misc_Unsafe_compareAndExchangeObject: case TR::jdk_internal_misc_Unsafe_compareAndExchangeReference: - if (disableCAEIntrinsic) + if (disableCAEInlining || callNode->isSafeForCGToFastPathUnsafeCall()) { - break; + return false; } - // Fallthrough if previous if condition is not met. + return createUnsafeCASCallDiamond(callNodeTreeTop, callNode); + case TR::sun_misc_Unsafe_compareAndSwapInt_jlObjectJII_Z: case TR::sun_misc_Unsafe_compareAndSwapLong_jlObjectJJJ_Z: case TR::sun_misc_Unsafe_compareAndSwapObject_jlObjectJjlObjectjlObject_Z: - if (callNode->isSafeForCGToFastPathUnsafeCall()) + if (disableCASInlining || callNode->isSafeForCGToFastPathUnsafeCall()) + { return false; + } #if defined(J9VM_GC_SPARSE_HEAP_ALLOCATION) - if(TR::Compiler->om.isOffHeapAllocationEnabled()) + if (TR::Compiler->om.isOffHeapAllocationEnabled()) return createUnsafeCASCallDiamond(callNodeTreeTop, callNode); #endif /* J9VM_GC_SPARSE_HEAP_ALLOCATION */ switch (callerSymbol->castToMethodSymbol()->getRecognizedMethod()) diff --git a/runtime/compiler/p/codegen/J9CodeGenerator.cpp b/runtime/compiler/p/codegen/J9CodeGenerator.cpp index bbb449abb73..9067c3eaed2 100644 --- a/runtime/compiler/p/codegen/J9CodeGenerator.cpp +++ b/runtime/compiler/p/codegen/J9CodeGenerator.cpp @@ -108,6 +108,18 @@ J9::Power::CodeGenerator::initialize() !disableStringInflateIntrinsic) cg->setSupportsInlineStringLatin1Inflate(); + static bool disableCASInlining = feGetEnv("TR_DisableCASInlining") != NULL; + if (!disableCASInlining) + { + cg->setSupportsInlineUnsafeCompareAndSet(); + } + + static bool disableCAEInlining = feGetEnv("TR_DisableCAEInlining") != NULL; + if (!disableCAEInlining) + { + cg->setSupportsInlineUnsafeCompareAndExchange(); + } + if (!comp->getOption(TR_DisableReadMonitors)) cg->setSupportsReadOnlyLocks(); diff --git a/runtime/compiler/p/codegen/J9TreeEvaluator.cpp b/runtime/compiler/p/codegen/J9TreeEvaluator.cpp index 1d246026c35..61c49c76840 100644 --- a/runtime/compiler/p/codegen/J9TreeEvaluator.cpp +++ b/runtime/compiler/p/codegen/J9TreeEvaluator.cpp @@ -11793,7 +11793,8 @@ J9::Power::CodeGenerator::inlineDirectCall(TR::Node *node, TR::Register *&result } else if (methodSymbol) { - static bool disableCAEIntrinsic = feGetEnv("TR_DisableCAEIntrinsic") != NULL; + bool disableCASInlining = !cg->getSupportsInlineUnsafeCompareAndSet(); + bool disableCAEInlining = !cg->getSupportsInlineUnsafeCompareAndExchange(); switch (methodSymbol->getRecognizedMethod()) { case TR::java_util_concurrent_ConcurrentLinkedQueue_tmOffer: @@ -12073,8 +12074,11 @@ J9::Power::CodeGenerator::inlineDirectCall(TR::Node *node, TR::Register *&result if ((node->isUnsafeGetPutCASCallOnNonArray() || !TR::Compiler->om.canGenerateArraylets()) && node->isSafeForCGToFastPathUnsafeCall()) { - resultReg = VMinlineCompareAndSetOrExchange(node, cg, 4, false); - return true; + if (!disableCASInlining) + { + resultReg = VMinlineCompareAndSetOrExchange(node, cg, 4, false); + return true; + } } break; @@ -12087,13 +12091,19 @@ J9::Power::CodeGenerator::inlineDirectCall(TR::Node *node, TR::Register *&result if (comp->target().is64Bit() && (node->isUnsafeGetPutCASCallOnNonArray() || !TR::Compiler->om.canGenerateArraylets()) && node->isSafeForCGToFastPathUnsafeCall()) { - resultReg = VMinlineCompareAndSetOrExchange(node, cg, 8, false); - return true; + if (!disableCASInlining) + { + resultReg = VMinlineCompareAndSetOrExchange(node, cg, 8, false); + return true; + } } else if ((node->isUnsafeGetPutCASCallOnNonArray() || !TR::Compiler->om.canGenerateArraylets()) && node->isSafeForCGToFastPathUnsafeCall()) { - resultReg = inlineAtomicOperation(node, cg, methodSymbol); - return true; + if (!disableCASInlining) + { + resultReg = inlineAtomicOperation(node, cg, methodSymbol); + return true; + } } break; @@ -12104,15 +12114,18 @@ J9::Power::CodeGenerator::inlineDirectCall(TR::Node *node, TR::Register *&result if ((node->isUnsafeGetPutCASCallOnNonArray() || !TR::Compiler->om.canGenerateArraylets()) && node->isSafeForCGToFastPathUnsafeCall()) { - resultReg = VMinlineCompareAndSetOrExchangeReference(node, cg, false); - return true; + if (!disableCASInlining) + { + resultReg = VMinlineCompareAndSetOrExchangeReference(node, cg, false); + return true; + } } break; case TR::jdk_internal_misc_Unsafe_compareAndExchangeInt: if ((node->isUnsafeGetPutCASCallOnNonArray() || !TR::Compiler->om.canGenerateArraylets()) && node->isSafeForCGToFastPathUnsafeCall()) { - if (!disableCAEIntrinsic) + if (!disableCAEInlining) { resultReg = VMinlineCompareAndSetOrExchange(node, cg, 4, true); return true; @@ -12123,7 +12136,7 @@ J9::Power::CodeGenerator::inlineDirectCall(TR::Node *node, TR::Register *&result case TR::jdk_internal_misc_Unsafe_compareAndExchangeLong: if (comp->target().is64Bit() && (node->isUnsafeGetPutCASCallOnNonArray() || !TR::Compiler->om.canGenerateArraylets()) && node->isSafeForCGToFastPathUnsafeCall()) { - if (!disableCAEIntrinsic) + if (!disableCAEInlining) { resultReg = VMinlineCompareAndSetOrExchange(node, cg, 8, true); return true; @@ -12143,7 +12156,7 @@ J9::Power::CodeGenerator::inlineDirectCall(TR::Node *node, TR::Register *&result case TR::jdk_internal_misc_Unsafe_compareAndExchangeReference: if ((node->isUnsafeGetPutCASCallOnNonArray() || !TR::Compiler->om.canGenerateArraylets()) && node->isSafeForCGToFastPathUnsafeCall()) { - if (!disableCAEIntrinsic) + if (!disableCAEInlining) { resultReg = VMinlineCompareAndSetOrExchangeReference(node, cg, true); return true; diff --git a/runtime/compiler/x/codegen/J9CodeGenerator.cpp b/runtime/compiler/x/codegen/J9CodeGenerator.cpp index 23c33ef97dd..13fb5b903ba 100644 --- a/runtime/compiler/x/codegen/J9CodeGenerator.cpp +++ b/runtime/compiler/x/codegen/J9CodeGenerator.cpp @@ -145,6 +145,18 @@ J9::X86::CodeGenerator::initialize() cg->setSupportsInlineVectorizedMismatch(); } + static bool disableCASInlining = feGetEnv("TR_DisableCASInlining") != NULL; + if (!disableCASInlining) + { + cg->setSupportsInlineUnsafeCompareAndSet(); + } + + static bool disableCAEInlining = feGetEnv("TR_DisableCAEInlining") != NULL; + if (!disableCAEInlining) + { + cg->setSupportsInlineUnsafeCompareAndExchange(); + } + // Disable fast gencon barriers for AOT compiles because relocations on // the inlined heap addresses are not available (yet). // @@ -522,21 +534,23 @@ J9::X86::CodeGenerator::supportsNonHelper(TR::SymbolReferenceTable::CommonNonhel } /* - * This method returns TRUE for all the cases we decide NOT to replace the call to CAS - * with inline assembly. The GRA and Evaluator should be consistent about whether to inline CAS natives. + * This method returns TRUE for all the cases we decide NOT to replace the call to CAS/CAE + * with inline assembly. The GRA and Evaluator should be consistent about whether to inline CAS/CAE natives. */ static bool willNotInlineCompareAndSwapNative(TR::Node *node, int8_t size, - TR::Compilation *comp) + TR::Compilation *comp, + bool isExchange) { TR::SymbolReference *callSymRef = node->getSymbolReference(); TR::MethodSymbol *methodSymbol = callSymRef->getSymbol()->castToMethodSymbol(); if (TR::Compiler->om.canGenerateArraylets() && !node->isUnsafeGetPutCASCallOnNonArray()) return true; - static char *disableCASInlining = feGetEnv("TR_DisableCASInlining"); - if (disableCASInlining) + if (!isExchange && !comp->cg()->getSupportsInlineUnsafeCompareAndSet()) + return true; + if (isExchange && !comp->cg()->getSupportsInlineUnsafeCompareAndExchange()) return true; // In Java9 the sun.misc.Unsafe JNI methods have been moved to jdk.internal, @@ -583,15 +597,18 @@ J9::X86::CodeGenerator::willBeEvaluatedAsCallByCodeGen(TR::Node *node, TR::Compi switch (methodSymbol->getRecognizedMethod()) { case TR::sun_misc_Unsafe_compareAndSwapInt_jlObjectJII_Z: - case TR::jdk_internal_misc_Unsafe_compareAndExchangeInt: - return willNotInlineCompareAndSwapNative(node, 4, comp); + return willNotInlineCompareAndSwapNative(node, 4, comp, false); case TR::sun_misc_Unsafe_compareAndSwapLong_jlObjectJJJ_Z: - case TR::jdk_internal_misc_Unsafe_compareAndExchangeLong: - return willNotInlineCompareAndSwapNative(node, 8, comp); + return willNotInlineCompareAndSwapNative(node, 8, comp, false); case TR::sun_misc_Unsafe_compareAndSwapObject_jlObjectJjlObjectjlObject_Z: + return willNotInlineCompareAndSwapNative(node, TR::Compiler->om.sizeofReferenceField(), comp, false); + case TR::jdk_internal_misc_Unsafe_compareAndExchangeInt: + return willNotInlineCompareAndSwapNative(node, 4, comp, true); + case TR::jdk_internal_misc_Unsafe_compareAndExchangeLong: + return willNotInlineCompareAndSwapNative(node, 8, comp, true); case TR::jdk_internal_misc_Unsafe_compareAndExchangeObject: case TR::jdk_internal_misc_Unsafe_compareAndExchangeReference: - return willNotInlineCompareAndSwapNative(node, TR::Compiler->om.sizeofReferenceField(), comp); + return willNotInlineCompareAndSwapNative(node, TR::Compiler->om.sizeofReferenceField(), comp, true); default: break; diff --git a/runtime/compiler/x/codegen/J9TreeEvaluator.cpp b/runtime/compiler/x/codegen/J9TreeEvaluator.cpp index e0fc5120170..52b73d6c850 100644 --- a/runtime/compiler/x/codegen/J9TreeEvaluator.cpp +++ b/runtime/compiler/x/codegen/J9TreeEvaluator.cpp @@ -9641,11 +9641,6 @@ inlineCompareAndSwapNative( if (TR::Compiler->om.canGenerateArraylets() && !node->isUnsafeGetPutCASCallOnNonArray()) return false; - static char *disableCASInlining = feGetEnv("TR_DisableCASInlining"); - - if (disableCASInlining /* || comp->useCompressedPointers() */) - return false; - // size = 4 --> CMPXCHG4 // size = 8 --> if 64-bit -> CMPXCHG8 // else if proc supports CMPXCHG8B -> CMPXCHG8B @@ -9947,7 +9942,8 @@ bool J9::X86::TreeEvaluator::VMinlineCallEvaluator( bool callWasInlined = false; TR::Compilation *comp = cg->comp(); - static bool disableCAEIntrinsic = feGetEnv("TR_DisableCAEIntrinsic") != NULL; + bool disableCASInlining = !cg->getSupportsInlineUnsafeCompareAndSet(); + bool disableCAEInlining = !cg->getSupportsInlineUnsafeCompareAndExchange(); if (methodSymbol) { @@ -10030,20 +10026,20 @@ bool J9::X86::TreeEvaluator::VMinlineCallEvaluator( return false; // Call the native version of NativeThread.current() case TR::sun_misc_Unsafe_compareAndSwapInt_jlObjectJII_Z: { - if (node->isSafeForCGToFastPathUnsafeCall()) + if (!disableCASInlining && node->isSafeForCGToFastPathUnsafeCall()) return inlineCompareAndSwapNative(node, 4, false, false, cg); } break; case TR::sun_misc_Unsafe_compareAndSwapLong_jlObjectJJJ_Z: { - if (node->isSafeForCGToFastPathUnsafeCall()) + if (!disableCASInlining && node->isSafeForCGToFastPathUnsafeCall()) return inlineCompareAndSwapNative(node, 8, false, false, cg); } break; case TR::sun_misc_Unsafe_compareAndSwapObject_jlObjectJjlObjectjlObject_Z: { static bool useOldCompareAndSwapObject = (bool)feGetEnv("TR_UseOldCompareAndSwapObject"); - if (node->isSafeForCGToFastPathUnsafeCall()) + if (!disableCASInlining && node->isSafeForCGToFastPathUnsafeCall()) { if (useOldCompareAndSwapObject) return inlineCompareAndSwapNative(node, TR::Compiler->om.sizeofReferenceField(), true, false, cg); @@ -10057,13 +10053,13 @@ bool J9::X86::TreeEvaluator::VMinlineCallEvaluator( break; case TR::jdk_internal_misc_Unsafe_compareAndExchangeInt: { - if (!disableCAEIntrinsic && node->isSafeForCGToFastPathUnsafeCall()) + if (!disableCAEInlining && node->isSafeForCGToFastPathUnsafeCall()) return inlineCompareAndSwapNative(node, 4, false, true, cg); } break; case TR::jdk_internal_misc_Unsafe_compareAndExchangeLong: { - if (!disableCAEIntrinsic && node->isSafeForCGToFastPathUnsafeCall()) + if (!disableCAEInlining && node->isSafeForCGToFastPathUnsafeCall()) return inlineCompareAndSwapNative(node, 8, false, true, cg); } break; @@ -10071,7 +10067,7 @@ bool J9::X86::TreeEvaluator::VMinlineCallEvaluator( case TR::jdk_internal_misc_Unsafe_compareAndExchangeReference: { static bool useOldCompareAndSwapObject = (bool)feGetEnv("TR_UseOldCompareAndSwapObject"); - if (!disableCAEIntrinsic && node->isSafeForCGToFastPathUnsafeCall()) + if (!disableCAEInlining && node->isSafeForCGToFastPathUnsafeCall()) { if (useOldCompareAndSwapObject) return inlineCompareAndSwapNative(node, TR::Compiler->om.sizeofReferenceField(), true, true, cg); diff --git a/runtime/compiler/z/codegen/J9CodeGenerator.cpp b/runtime/compiler/z/codegen/J9CodeGenerator.cpp index 75aea131fdc..bb45bd8a389 100644 --- a/runtime/compiler/z/codegen/J9CodeGenerator.cpp +++ b/runtime/compiler/z/codegen/J9CodeGenerator.cpp @@ -150,6 +150,18 @@ J9::Z::CodeGenerator::initialize() cg->setSupportsInlineVectorizedMismatch(); } + static bool disableCASInlining = feGetEnv("TR_DisableCASInlining") != NULL; + if (!disableCASInlining) + { + cg->setSupportsInlineUnsafeCompareAndSet(); + } + + static bool disableCAEInlining = feGetEnv("TR_DisableCAEInlining") != NULL; + if (!disableCAEInlining) + { + cg->setSupportsInlineUnsafeCompareAndExchange(); + } + // Let's turn this on. There is more work needed in the opt // to catch the case where the BNDSCHK is inserted after // @@ -3822,7 +3834,8 @@ J9::Z::CodeGenerator::inlineDirectCall( } static const char * enableTRTRE = feGetEnv("TR_enableTRTRE"); - static bool disableCAEIntrinsic = feGetEnv("TR_DisableCAEIntrinsic") != NULL; + bool disableCASInlining = !cg->getSupportsInlineUnsafeCompareAndSet(); + bool disableCAEInlining = !cg->getSupportsInlineUnsafeCompareAndExchange(); switch (methodSymbol->getRecognizedMethod()) { case TR::sun_misc_Unsafe_compareAndSwapInt_jlObjectJII_Z: @@ -3834,8 +3847,11 @@ J9::Z::CodeGenerator::inlineDirectCall( if ((!TR::Compiler->om.canGenerateArraylets() || node->isUnsafeGetPutCASCallOnNonArray()) && node->isSafeForCGToFastPathUnsafeCall()) { - resultReg = TR::TreeEvaluator::VMinlineCompareAndSwap(node, cg, TR::InstOpCode::CS, IS_NOT_OBJ); - return true; + if (!disableCASInlining) + { + resultReg = TR::TreeEvaluator::VMinlineCompareAndSwap(node, cg, TR::InstOpCode::CS, IS_NOT_OBJ); + return true; + } } break; @@ -3847,8 +3863,11 @@ J9::Z::CodeGenerator::inlineDirectCall( // Too risky to do Long-31bit version now. if (comp->target().is64Bit() && (!TR::Compiler->om.canGenerateArraylets() || node->isUnsafeGetPutCASCallOnNonArray()) && node->isSafeForCGToFastPathUnsafeCall()) { - resultReg = TR::TreeEvaluator::VMinlineCompareAndSwap(node, cg, TR::InstOpCode::CSG, IS_NOT_OBJ); - return true; + if (!disableCASInlining) + { + resultReg = TR::TreeEvaluator::VMinlineCompareAndSwap(node, cg, TR::InstOpCode::CSG, IS_NOT_OBJ); + return true; + } } break; @@ -3859,15 +3878,18 @@ J9::Z::CodeGenerator::inlineDirectCall( if ((!TR::Compiler->om.canGenerateArraylets() || node->isUnsafeGetPutCASCallOnNonArray()) && node->isSafeForCGToFastPathUnsafeCall()) { - resultReg = TR::TreeEvaluator::VMinlineCompareAndSwap(node, cg, (comp->useCompressedPointers() ? TR::InstOpCode::CS : TR::InstOpCode::getCmpAndSwapOpCode()), IS_OBJ); - return true; + if (!disableCASInlining) + { + resultReg = TR::TreeEvaluator::VMinlineCompareAndSwap(node, cg, (comp->useCompressedPointers() ? TR::InstOpCode::CS : TR::InstOpCode::getCmpAndSwapOpCode()), IS_OBJ); + return true; + } } break; case TR::jdk_internal_misc_Unsafe_compareAndExchangeInt: if ((!TR::Compiler->om.canGenerateArraylets() || node->isUnsafeGetPutCASCallOnNonArray()) && node->isSafeForCGToFastPathUnsafeCall()) { - if (!disableCAEIntrinsic) + if (!disableCAEInlining) { resultReg = TR::TreeEvaluator::VMinlineCompareAndSwap(node, cg, TR::InstOpCode::CS, IS_NOT_OBJ, true); return true; @@ -3879,7 +3901,7 @@ J9::Z::CodeGenerator::inlineDirectCall( // Too risky to do Long-31bit version now. if (comp->target().is64Bit() && (!TR::Compiler->om.canGenerateArraylets() || node->isUnsafeGetPutCASCallOnNonArray()) && node->isSafeForCGToFastPathUnsafeCall()) { - if (!disableCAEIntrinsic) + if (!disableCAEInlining) { resultReg = TR::TreeEvaluator::VMinlineCompareAndSwap(node, cg, TR::InstOpCode::CSG, IS_NOT_OBJ, true); return true; @@ -3899,7 +3921,7 @@ J9::Z::CodeGenerator::inlineDirectCall( case TR::jdk_internal_misc_Unsafe_compareAndExchangeReference: if ((!TR::Compiler->om.canGenerateArraylets() || node->isUnsafeGetPutCASCallOnNonArray()) && node->isSafeForCGToFastPathUnsafeCall()) { - if (!disableCAEIntrinsic) + if (!disableCAEInlining) { resultReg = TR::TreeEvaluator::VMinlineCompareAndSwap(node, cg, (comp->useCompressedPointers() ? TR::InstOpCode::CS : TR::InstOpCode::getCmpAndSwapOpCode()), IS_OBJ, true); return true;