Skip to content

Commit

Permalink
Support running Yarn with JDK 17 (#305)
Browse files Browse the repository at this point in the history
Hadoop is using some older dependencies that still require access to
internal JDK features so this enables that by adding JVM args

See apache/accumulo#5139
  • Loading branch information
cshannon authored Dec 13, 2024
1 parent 66f5bd5 commit 7763f60
Show file tree
Hide file tree
Showing 2 changed files with 12 additions and 7 deletions.
11 changes: 11 additions & 0 deletions bin/impl/install/hadoop.sh
Original file line number Diff line number Diff line change
Expand Up @@ -63,5 +63,16 @@ $SED "s#YARN_NM_CPU_VCORES#$YARN_NM_CPU_VCORES#g" "$hadoop_conf/yarn-site.xml"
} >> "$hadoop_conf/hadoop-env.sh"
[[ $HADOOP_VERSION =~ ^2\..*$ ]] && echo "export YARN_LOG_DIR=$HADOOP_LOG_DIR" >> "$hadoop_conf/yarn-env.sh"

# Yarn requires extra JVM args to start with Java 17+
jver=$("$JAVA_HOME"/bin/java -version 2>&1 | grep version | cut -f2 -d'"' | cut -f1 -d.)

if [[ $jver -gt 11 ]]; then
echo "Setting yarn JVM args for java $jver"
{
echo "export YARN_RESOURCEMANAGER_OPTS=\"--add-opens java.base/java.lang=ALL-UNNAMED\""
echo "export YARN_NODEMANAGER_OPTS=\"--add-opens java.base/java.lang=ALL-UNNAMED\""
} >> "$hadoop_conf/yarn-env.sh"
fi

true
# hadoop.sh
8 changes: 1 addition & 7 deletions bin/impl/run/hadoop.sh
Original file line number Diff line number Diff line change
Expand Up @@ -27,13 +27,7 @@ trap 'echo "[ERROR] Error occurred at $BASH_SOURCE:$LINENO command: $BASH_COMMAN

"$HADOOP_HOME"/bin/hdfs namenode -format
"$HADOOP_HOME"/sbin/start-dfs.sh
# Yarn won't start on newer versions of Java
jver=$("$JAVA_HOME"/bin/java -version 2>&1 | grep version | cut -f2 -d'"' | cut -f1 -d.)
if [[ $jver -gt 11 ]]; then
echo "Skipping yarn because it doesn't start on Java $jver"
else
"$HADOOP_HOME"/sbin/start-yarn.sh
fi
"$HADOOP_HOME"/sbin/start-yarn.sh

namenode_port=9870
if [[ $HADOOP_VERSION =~ ^2\..*$ ]]; then
Expand Down

0 comments on commit 7763f60

Please sign in to comment.