i#95: Detach on Linux (#6513)

Enable the feature of detachment on Linux. 

- I setup a temporary separate stack for the nudged thread, and resume
the normal execution of this thread after detachment.
- A new variable named `nudged_sigcxt` is used to save the `sigcxt` of
nudged thread.
- And the extra code `dcontext == GLOBAL_DCONTEXT` is added to cover the
new code path that won't be executed before during the process of thread
exit.
- What's more, I turn off the option of `sigreturn_setcontext` for the
smooth resumption of nudged thread on X64 ISA.
- Finally, the frontend, automated test cases and documentation are
modified accordingly.

Issue: [#95](#95)

api/docs/deployment.dox

@@ -337,6 +337,12 @@ with this command:
 % echo 0 | sudo tee /proc/sys/kernel/yama/ptrace_scope
 \endcode
 
+Then, you can also detach DynamoRIO from the target process
+without affecting the normal execution of the application.
+\code
+% bin32/drconfig -detach <target_pid>
+\endcode
+
 Run \c drrun with no options to get a list of the options and
 environment variable shortcuts it supports.  To disable following across
 child execve calls, use the \ref op_children "-no_follow_children" runtime

core/lib/globals_shared.h

@@ -933,9 +933,9 @@ enum {
 #else
     NUDGE_NUDGER_FREE_STACK = 0x02, /* nudger will free the nudge thread's stack so the
                                      * nudge thread itself shouldn't */
-    NUDGE_FREE_ARG = 0x04,          /* nudge arg is in a separate allocation and should
-                                     * be freed by the nudge thread */
 #endif
+    NUDGE_FREE_ARG = 0x04, /* nudge arg is in a separate allocation and should
+                            * be freed by the nudge thread */
 };
 
 typedef struct {

core/nudge.c

@@ -42,6 +42,10 @@
 #else
 #endif /* WINDOWS */
 
+#ifdef LINUX
+#    include "synch.h"
+#endif
+
 #ifdef HOT_PATCHING_INTERFACE
 #    include "hotpatch.h" /* for hotp_nudge_update() */
 #endif
@@ -430,12 +434,30 @@ handle_nudge(dcontext_t *dcontext, nudge_arg_t *arg)
             SYSLOG_INTERNAL_WARNING("nudge reset ignored since resets are disabled");
         }
     }
-#ifdef WINDOWS
+#if defined(WINDOWS) || defined(LINUX)
     /* The detach handler is last since in the common case it doesn't return. */
     if (TEST(NUDGE_GENERIC(detach), nudge_action_mask)) {
+#    ifdef WINDOWS
         dcontext->free_app_stack = false;
         nudge_action_mask &= ~NUDGE_GENERIC(detach);
         detach_helper(DETACH_NORMAL_TYPE);
+#    else
+        nudge_action_mask &= ~NUDGE_GENERIC(detach);
+        /* This is not using stack_alloc() because we can't have this being cleaned up
+         * via normal cleanup paths. */
+        heap_error_code_t error_code_reserve, error_code_commit;
+        void *d_r_detachstack =
+            os_heap_reserve(NULL, DYNAMORIO_STACK_SIZE, &error_code_reserve, false);
+        /* XXX: This memory is not freed. */
+        if (!os_heap_commit(d_r_detachstack, DYNAMORIO_STACK_SIZE,
+                            MEMPROT_READ | MEMPROT_WRITE, &error_code_commit)) {
+            ASSERT_NOT_REACHED();
+        }
+        call_switch_stack(dcontext,
+                          (byte *)((ptr_uint_t)d_r_detachstack + DYNAMORIO_STACK_SIZE),
+                          (void (*)(void *))detach_externally_on_new_stack, NULL, true);
+        ASSERT_NOT_REACHED();
+#    endif
     }
 #endif
 }

core/os_shared.h

@@ -204,15 +204,21 @@ is_thread_currently_native(thread_record_t *tr);
  */
 bool
 thread_get_mcontext(thread_record_t *tr, priv_mcontext_t *mc);
+
+#ifdef LINUX
+bool
+thread_get_nudged_mcontext(thread_record_t *tr, priv_mcontext_t *mc);
+#endif
+
 bool
 thread_set_mcontext(thread_record_t *tr, priv_mcontext_t *mc);
 
 /* Takes an os-specific context. Does not return. */
 void
-thread_set_self_context(void *cxt);
+thread_set_self_context(void *cxt, bool is_detach_external);
 /* Only sets the priv_mcontext_t state.  Does not return. */
 void
-thread_set_self_mcontext(priv_mcontext_t *mc);
+thread_set_self_mcontext(priv_mcontext_t *mc, bool is_detach_external);
 
 /* Assumes target thread is suspended */
 bool