wb | Nomad backend error handling fixes

IntersectMBO · Feb 5, 2024 · 88601d3 · 88601d3
1 parent 078754c
commit 88601d3
Show file tree

Hide file tree

Showing 3 changed files with 16 additions and 7 deletions.
diff --git a/nix/workbench/backend/nomad.sh b/nix/workbench/backend/nomad.sh
@@ -1415,17 +1415,17 @@ backend_nomad() {
         then
           if ! wait_kill_em_all "${jobs_array[@]}"
           then
-            # Don't use fatal here, let `start` decide!
             msg "$(red "Failed to start tracer(s)")"
-            return 1
+            backend_nomad stop-nomad-job "${dir}" || msg "$(red "Failed to stop Nomad Job")"
+            fatal "scenario.sh start-tracers failed!"
           else
             for node in ${nodes[*]}
             do
               if ! test -f "${dir}"/tracer/"${node}"/started
               then
-                # Don't use fatal here, let `start` decide!
                 msg "$(red "Tracer for \"${node}\" failed to start!")"
-                return 1
+                backend_nomad stop-nomad-job "${dir}" || msg "$(red "Failed to stop Nomad Job")"
+                fatal "scenario.sh start-tracers failed!"
               fi
             done
           fi

diff --git a/nix/workbench/backend/nomad/exec.sh b/nix/workbench/backend/nomad/exec.sh
@@ -199,8 +199,8 @@ deploy-genesis-nomadexec() {
       if test "${nomad_agents_were_already_running}" = "false"
       then
         msg "$(red "Startup of webfs failed, cleaning up ...")"
+        # `stop-nomad-job` takes care of stopping the Nomad agents.
         backend_nomad stop-nomad-job "${dir}" || msg "$(red "Failed to stop Nomad Job")"
-        wb_nomad agents stop "${server_name}" "${client_name}" "exec"
       fi
       fatal "Failed to start a local HTTP server"
     fi
@@ -213,8 +213,8 @@ deploy-genesis-nomadexec() {
     if test "${nomad_agents_were_already_running}" = "false"
     then
       msg "$(red "Startup of webfs failed, cleaning up ...")"
+      # `stop-nomad-job` takes care of stopping the Nomad agents.
       backend_nomad stop-nomad-job "${dir}" || msg "$(red "Failed to stop Nomad Job")"
-      wb_nomad agents stop "${server_name}" "${client_name}" "exec"
     fi
     fatal "Failed to add genesis file to local HTTP server"
   else
@@ -225,8 +225,8 @@ deploy-genesis-nomadexec() {
   if ! backend_nomad deploy-genesis-wget "${dir}" "${uri}"
   then
     msg "$(red "Deploy of genesis failed, cleaning up ...")"
+    # `stop-nomad-job` takes care of stopping the Nomad agents.
     backend_nomad stop-nomad-job "${dir}" || msg "$(red "Failed to stop Nomad Job")"
-    wb_nomad agents stop "${server_name}" "${client_name}" "exec"
     fatal "Deploy of genesis \"${uri}\" failed"
   else
     msg "$(green "Genesis \"${uri}\" deployed successfully")"

diff --git a/nix/workbench/scenario.sh b/nix/workbench/scenario.sh
@@ -31,7 +31,16 @@ fi
 case "$op" in
     idle )
         backend start-tracers        "$dir"
+
+        scenario_setup_exit_trap              "$dir"
+        # Trap start
+        ############
         backend start-nodes          "$dir"
+        # Trap end
+        ##########
+        scenario_cleanup_termination
+
+        backend stop-all             "$dir"
         ;;
 
     tracer-only )