From 7763f6092f0f758168a1e1afae5ed56e31357361 Mon Sep 17 00:00:00 2001 From: "Christopher L. Shannon" Date: Fri, 13 Dec 2024 09:56:15 -0500 Subject: [PATCH] Support running Yarn with JDK 17 (#305) Hadoop is using some older dependencies that still require access to internal JDK features so this enables that by adding JVM args See https://github.com/apache/accumulo/issues/5139 --- bin/impl/install/hadoop.sh | 11 +++++++++++ bin/impl/run/hadoop.sh | 8 +------- 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/bin/impl/install/hadoop.sh b/bin/impl/install/hadoop.sh index a6d7f9d..ab626bf 100755 --- a/bin/impl/install/hadoop.sh +++ b/bin/impl/install/hadoop.sh @@ -63,5 +63,16 @@ $SED "s#YARN_NM_CPU_VCORES#$YARN_NM_CPU_VCORES#g" "$hadoop_conf/yarn-site.xml" } >> "$hadoop_conf/hadoop-env.sh" [[ $HADOOP_VERSION =~ ^2\..*$ ]] && echo "export YARN_LOG_DIR=$HADOOP_LOG_DIR" >> "$hadoop_conf/yarn-env.sh" +# Yarn requires extra JVM args to start with Java 17+ +jver=$("$JAVA_HOME"/bin/java -version 2>&1 | grep version | cut -f2 -d'"' | cut -f1 -d.) + +if [[ $jver -gt 11 ]]; then + echo "Setting yarn JVM args for java $jver" + { + echo "export YARN_RESOURCEMANAGER_OPTS=\"--add-opens java.base/java.lang=ALL-UNNAMED\"" + echo "export YARN_NODEMANAGER_OPTS=\"--add-opens java.base/java.lang=ALL-UNNAMED\"" + } >> "$hadoop_conf/yarn-env.sh" +fi + true # hadoop.sh diff --git a/bin/impl/run/hadoop.sh b/bin/impl/run/hadoop.sh index e465bf2..17be5d4 100755 --- a/bin/impl/run/hadoop.sh +++ b/bin/impl/run/hadoop.sh @@ -27,13 +27,7 @@ trap 'echo "[ERROR] Error occurred at $BASH_SOURCE:$LINENO command: $BASH_COMMAN "$HADOOP_HOME"/bin/hdfs namenode -format "$HADOOP_HOME"/sbin/start-dfs.sh -# Yarn won't start on newer versions of Java -jver=$("$JAVA_HOME"/bin/java -version 2>&1 | grep version | cut -f2 -d'"' | cut -f1 -d.) -if [[ $jver -gt 11 ]]; then - echo "Skipping yarn because it doesn't start on Java $jver" -else - "$HADOOP_HOME"/sbin/start-yarn.sh -fi +"$HADOOP_HOME"/sbin/start-yarn.sh namenode_port=9870 if [[ $HADOOP_VERSION =~ ^2\..*$ ]]; then