From 3cfa925515b30045a17141405bc8d7b282b3e00b Mon Sep 17 00:00:00 2001 From: Denisas Date: Thu, 3 Feb 2022 16:55:55 +0200 Subject: [PATCH 1/2] bug fix: prevent loop if yarn down --- .../com/exacaster/lighter/backend/yarn/YarnBackend.java | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/server/src/main/java/com/exacaster/lighter/backend/yarn/YarnBackend.java b/server/src/main/java/com/exacaster/lighter/backend/yarn/YarnBackend.java index edb17b6c..fb809fb5 100644 --- a/server/src/main/java/com/exacaster/lighter/backend/yarn/YarnBackend.java +++ b/server/src/main/java/com/exacaster/lighter/backend/yarn/YarnBackend.java @@ -13,7 +13,6 @@ import java.io.IOException; import java.net.URI; import java.util.Comparator; -import java.util.EnumSet; import java.util.HashMap; import java.util.Map; import java.util.Optional; @@ -21,7 +20,6 @@ import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationReport; -import org.apache.hadoop.yarn.api.records.YarnApplicationState; import org.apache.hadoop.yarn.client.api.YarnClient; import org.apache.hadoop.yarn.exceptions.YarnException; import org.slf4j.Logger; @@ -96,6 +94,7 @@ public void kill(Application application) { client.killApplication(ApplicationId.fromString(id)); } catch (YarnException | IOException e) { LOG.error("Can't kill Yarn app: {}", application, e); + throw new IllegalStateException(e); } }); } @@ -120,11 +119,10 @@ public Map getSubmitConfiguration(Application application) { } private Optional getYarnApplicationId(Application application) { - var allStates = EnumSet.allOf(YarnApplicationState.class); return Optional.ofNullable(application.getAppId()) .or(() -> { try { - var request = GetApplicationsRequest.newInstance(Set.of("SPARK")); + var request = GetApplicationsRequest.newInstance(); request.setApplicationTags(Set.of(application.getId())); return client.getApplications(request).stream() .max(Comparator.comparing(ApplicationReport::getStartTime)) @@ -132,7 +130,7 @@ private Optional getYarnApplicationId(Application application) { .map(ApplicationId::toString); } catch (YarnException | IOException e) { LOG.error("Failed to get app id for app: {}", application, e); - return Optional.empty(); + throw new IllegalStateException(e); } }); } From a12c7b1c50908603ec4c8f210b887f1bf110d723 Mon Sep 17 00:00:00 2001 From: Denisas Date: Thu, 3 Feb 2022 21:22:56 +0200 Subject: [PATCH 2/2] bug fix: core-site.xml config ignored --- .../lighter/backend/yarn/YarnConfigurationFactory.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/src/main/java/com/exacaster/lighter/backend/yarn/YarnConfigurationFactory.java b/server/src/main/java/com/exacaster/lighter/backend/yarn/YarnConfigurationFactory.java index 9bd9d0ba..d0aef42f 100644 --- a/server/src/main/java/com/exacaster/lighter/backend/yarn/YarnConfigurationFactory.java +++ b/server/src/main/java/com/exacaster/lighter/backend/yarn/YarnConfigurationFactory.java @@ -8,9 +8,9 @@ import io.micronaut.context.annotation.Property; import io.micronaut.context.annotation.Requires; import jakarta.inject.Singleton; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.hadoop.yarn.client.api.YarnClient; -import org.apache.hadoop.yarn.conf.YarnConfiguration; @Factory @Requires(beans = YarnProperties.class) @@ -19,7 +19,7 @@ public class YarnConfigurationFactory { @Singleton public YarnBackend backend(YarnProperties yarnProperties, AppConfiguration conf, @Property(name = "hadoop.conf.dir") String hadoopConfDir) { - YarnConfiguration yarnConfiguration = new YarnConfiguration(); + var yarnConfiguration = new Configuration(false); yarnConfiguration.addResource(new Path(hadoopConfDir, "core-site.xml")); yarnConfiguration.addResource(new Path(hadoopConfDir, "yarn-site.xml")); if (yarnProperties.getKerberosKeytab() != null && yarnProperties.getKerberosPrincipal() != null) {