Fix external pod deletion for jobs (#2915) [ci fast]

Signed-off-by: Lukas Hejtmanek <[email protected]>
Co-authored-by: Paolo Di Tommaso <[email protected]>

modules/nextflow/src/main/groovy/nextflow/k8s/client/K8sClient.groovy

@@ -230,7 +230,15 @@ class K8sClient {
         // delete all pods in a job
         if (podList.kind == "PodList") { 
             for (item in podList.items) {
-                podDelete(((item as Map).metadata as Map).name as String)
+                try {
+                   podDelete(((item as Map).metadata as Map).name as String)
+                }
+                catch(K8sResponseException err) {
+                   if( err.response.code == 404 )
+                       log.debug("Unable to delete Pod for job $name, pod already gone")
+                   else
+                       throw err
+                }
             }
         }
 
@@ -371,7 +379,16 @@ class K8sClient {
         assert jobName
         final podName = findPodNameForJob(jobName)
         if( podName ) {
-            return podState(podName)
+            try {
+                return podState(podName)
+            } 
+            /* pod might be deleted by control plane just after findPodNameForJob() call
+             * so try fallback to jobState
+             */   
+            catch (NodeTerminationException err) {
+                log.warn1("Job $jobName's Pod not found, probably cleaned by controlplane")
+                return jobStateFallback0(jobName)           
+            }
         }
         else {
             return jobStateFallback0(jobName)
@@ -386,7 +403,7 @@ class K8sClient {
             final cond = allConditions.find { cond -> cond.type == 'Complete' }
 
             if( cond?.status == 'True' ) {
-                log.warn1("Job $jobName already completed and Pod is gone.")
+                log.warn1("Job $jobName already completed and Pod is gone")
                 final dummyPodStatus = [
                         terminated: [
                                 exitcode: 0,