From 20be6a3d0554bf49beb7dc209ecfac06299dd5b4 Mon Sep 17 00:00:00 2001
From: Mindaugas <Minutis@gmail.com>
Date: Fri, 14 Oct 2022 16:14:53 +0300
Subject: [PATCH] Mark application as ERROR if it is not found in YARN (#147)

* Add missing master property to launcher
Mark application as ERROR if it is not found in YARN

* Pull error handling for state fetch failures up
---
 .../lighter/backend/yarn/YarnBackend.java     | 19 +++++++++----------
 .../yarn/YarnConfigurationFactory.java        |  1 -
 2 files changed, 9 insertions(+), 11 deletions(-)

diff --git a/server/src/main/java/com/exacaster/lighter/backend/yarn/YarnBackend.java b/server/src/main/java/com/exacaster/lighter/backend/yarn/YarnBackend.java
index cdf03a2c..ba04f67e 100644
--- a/server/src/main/java/com/exacaster/lighter/backend/yarn/YarnBackend.java
+++ b/server/src/main/java/com/exacaster/lighter/backend/yarn/YarnBackend.java
@@ -7,9 +7,7 @@
 import static org.apache.spark.launcher.SparkLauncher.SPARK_MASTER;
 import static org.slf4j.LoggerFactory.getLogger;
 
-import com.exacaster.lighter.application.Application;
-import com.exacaster.lighter.application.ApplicationInfo;
-import com.exacaster.lighter.application.ApplicationState;
+import com.exacaster.lighter.application.*;
 import com.exacaster.lighter.backend.Backend;
 import com.exacaster.lighter.backend.SparkApp;
 import com.exacaster.lighter.configuration.AppConfiguration;
@@ -22,6 +20,7 @@
 import java.util.Map;
 import java.util.Optional;
 import java.util.Set;
+
 import java.util.function.Consumer;
 import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest;
 import org.apache.hadoop.yarn.api.records.ApplicationId;
@@ -47,26 +46,26 @@ public YarnBackend(YarnProperties yarnProperties, YarnClient client, AppConfigur
     @Override
     public Optional<ApplicationInfo> getInfo(Application application) {
         return getYarnApplicationId(application)
-                .map(id -> new ApplicationInfo(getState(id), id));
+                .flatMap(id -> getState(id).map(state -> new ApplicationInfo(state, id)));
     }
 
-    private ApplicationState getState(String id) {
+    private Optional<ApplicationState> getState(String id) {
         try {
             var yarnApplication = client.getApplicationReport(fromString(id));
             switch (yarnApplication.getFinalApplicationStatus()) {
                 case UNDEFINED:
-                    return ApplicationState.BUSY;
+                    return Optional.of(ApplicationState.BUSY);
                 case SUCCEEDED:
-                    return ApplicationState.SUCCESS;
+                    return Optional.of(ApplicationState.SUCCESS);
                 case FAILED:
-                    return ApplicationState.ERROR;
+                    return Optional.of(ApplicationState.ERROR);
                 case KILLED:
-                    return ApplicationState.KILLED;
+                    return Optional.of(ApplicationState.KILLED);
             }
         } catch (YarnException | IOException e) {
             LOG.error("Unexpected error for appId: {}", id, e);
         }
-        throw new IllegalStateException("Unexpected state for appId: " + id);
+        return Optional.empty();
     }
 
     @Override
diff --git a/server/src/main/java/com/exacaster/lighter/backend/yarn/YarnConfigurationFactory.java b/server/src/main/java/com/exacaster/lighter/backend/yarn/YarnConfigurationFactory.java
index 46ddb637..6b6f7c2e 100644
--- a/server/src/main/java/com/exacaster/lighter/backend/yarn/YarnConfigurationFactory.java
+++ b/server/src/main/java/com/exacaster/lighter/backend/yarn/YarnConfigurationFactory.java
@@ -17,7 +17,6 @@
 @Factory
 @Requires(beans = YarnProperties.class)
 public class YarnConfigurationFactory {
-
     @Singleton
     public YarnBackend backend(YarnProperties yarnProperties, AppConfiguration conf,
             @Property(name = "hadoop.conf.dir") String hadoopConfDir) throws IOException {