Merge branch 'master' into i7113-decode-cache-lib

DynamoRIO · Dec 19, 2024 · 2b301b5 · 2b301b5
2 parents 3042d6b + 4adbf0f
commit 2b301b5
Show file tree

Hide file tree

Showing 40 changed files with 19,042 additions and 17,395 deletions.
diff --git a/.github/workflows/ci-docs.yml b/.github/workflows/ci-docs.yml
@@ -90,7 +90,7 @@ jobs:
       # We only use a non-zero build # when making multiple manual builds in one day.
       run: |
         if test -z "${{ github.event.inputs.version }}"; then
-          export VERSION_NUMBER=11.0.$((`git log -n 1 --format=%ct` / (60*60*24)))
+          export VERSION_NUMBER=11.1.$((`git log -n 1 --format=%ct` / (60*60*24)))
         else
           export VERSION_NUMBER=${{ github.event.inputs.version }}
         fi

diff --git a/.github/workflows/ci-package.yml b/.github/workflows/ci-package.yml
@@ -103,7 +103,7 @@ jobs:
       # We only use a non-zero build # when making multiple manual builds in one day.
       run: |
         if test -z "${{ github.event.inputs.version }}"; then
-          export VERSION_NUMBER=11.0.$((`git log -n 1 --format=%ct` / (60*60*24)))
+          export VERSION_NUMBER=11.1.$((`git log -n 1 --format=%ct` / (60*60*24)))
         else
           export VERSION_NUMBER=${{ github.event.inputs.version }}
         fi
@@ -196,7 +196,7 @@ jobs:
       # XXX: See x86 job comments on sharing the default ver# with CMakeLists.txt.
       run: |
         if test -z "${{ github.event.inputs.version }}"; then
-          export VERSION_NUMBER=11.0.$((`git log -n 1 --format=%ct` / (60*60*24)))
+          export VERSION_NUMBER=11.1.$((`git log -n 1 --format=%ct` / (60*60*24)))
         else
           export VERSION_NUMBER=${{ github.event.inputs.version }}
         fi
@@ -285,7 +285,7 @@ jobs:
       # XXX: See x86 job comments on sharing the default ver# with CMakeLists.txt.
       run: |
         if test -z "${{ github.event.inputs.version }}"; then
-          export VERSION_NUMBER=11.0.$((`git log -n 1 --format=%ct` / (60*60*24)))
+          export VERSION_NUMBER=11.1.$((`git log -n 1 --format=%ct` / (60*60*24)))
         else
           export VERSION_NUMBER=${{ github.event.inputs.version }}
         fi
@@ -374,7 +374,7 @@ jobs:
       # XXX: See x86 job comments on sharing the default ver# with CMakeLists.txt.
       run: |
         if test -z "${{ github.event.inputs.version }}"; then
-          export VERSION_NUMBER=11.0.$((`git log -n 1 --format=%ct` / (60*60*24)))
+          export VERSION_NUMBER=11.1.$((`git log -n 1 --format=%ct` / (60*60*24)))
         else
           export VERSION_NUMBER=${{ github.event.inputs.version }}
         fi
@@ -457,7 +457,7 @@ jobs:
       # XXX: See x86 job comments on sharing the default ver# with CMakeLists.txt.
       run: |
         if test -z "${{ github.event.inputs.version }}"; then
-          export VERSION_NUMBER=11.0.$((`git log -n 1 --format=%ct` / (60*60*24)))
+          export VERSION_NUMBER=11.1.$((`git log -n 1 --format=%ct` / (60*60*24)))
         else
           export VERSION_NUMBER=${{ github.event.inputs.version }}
         fi
@@ -541,7 +541,7 @@ jobs:
       # XXX: See x86 job comments on sharing the default ver# with CMakeLists.txt.
       run: |
         if test -z "${{ github.event.inputs.version }}"; then
-          export VERSION_NUMBER="11.0.$((`git log -n 1 --format=%ct` / (60*60*24)))"
+          export VERSION_NUMBER="11.1.$((`git log -n 1 --format=%ct` / (60*60*24)))"
           export PREFIX="cronbuild-"
         else
           export VERSION_NUMBER=${{ github.event.inputs.version }}

diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -581,7 +581,7 @@ endif (EXISTS "${PROJECT_SOURCE_DIR}/.svn")
 
 # N.B.: When updating this, update all the default versions in ci-package.yml
 # and ci-docs.yml.  We should find a way to share (xref i#1565).
-set(VERSION_NUMBER_DEFAULT "11.0.${VERSION_NUMBER_PATCHLEVEL}")
+set(VERSION_NUMBER_DEFAULT "11.1.${VERSION_NUMBER_PATCHLEVEL}")
 # do not store the default VERSION_NUMBER in the cache to prevent a stale one
 # from preventing future version updates in a pre-existing build dir
 set(VERSION_NUMBER "" CACHE STRING "Version number: leave empty for default")

diff --git a/api/docs/code_reviews.dox b/api/docs/code_reviews.dox
@@ -198,10 +198,11 @@ appear.  Do not simply reply at the top level, as such comments are more
 difficult to follow as a conversation thread with context.
 
 After making changes in response to review comments, commit those changes
-locally as a new commit on top of your original commit.  If the reviewer
-requested changes to the commit message, use a new proposed final commit
-message as the message for this change commit so that the reviewer can
-review the new message.
+locally as a new commit on top of your original commit (do not squash the
+new changes into the original commit; use separate commits).  If the reviewer
+requested changes to the commit message or title, edit those directly in
+the pull request text boxes as those are what become the final commit
+message.
 
 Project members should then push the new commit to the pull request via:
 
@@ -215,7 +216,10 @@ separate commit (these separate commits will all be squashed together upon mergi
 When the requested changes have been pushed, request a re-review from the reviewer so they know that the pull request is ready for another round of reviewing.
 This can be done by clicking the re-review button next to the reviewer's name at the top of the right sidebar on the pull request page.
 
-The final squash-and-merge step will squash these additional commits with the original to make a single commit that will be fast-forward-merged into master (see below for more details).
+The final squash-and-merge step will squash these additional commits with the
+original to make a single commit that will be fast-forward-merged into master (see
+below for more details).  The final master branch merge commit's title and
+description come directly from the pull request title and description.
 
 # Updating from Master
 

diff --git a/api/docs/download.dox b/api/docs/download.dox
@@ -50,29 +50,29 @@ The source code is available:
 
 For the very latest changes since the last official release, you can download \ref page_weekly_builds.
 
-The [11.0.0 release](https://github.com/DynamoRIO/dynamorio/releases/tag/release_11.0.0):
+The [11.1.0 release](https://github.com/DynamoRIO/dynamorio/releases/tag/release_11.1.0):
 
-  - [DynamoRIO-Windows-11.0.0.zip](https://github.com/DynamoRIO/dynamorio/releases/download/release_11.0.0/DynamoRIO-Windows-11.0.0.zip)
+  - [DynamoRIO-Windows-11.1.0.zip](https://github.com/DynamoRIO/dynamorio/releases/download/release_11.1.0/DynamoRIO-Windows-11.1.0.zip)
 
-  - [DynamoRIO-Linux-11.0.0.tar.gz](https://github.com/DynamoRIO/dynamorio/releases/download/release_11.0.0/DynamoRIO-Linux-11.0.0.tar.gz)
+  - [DynamoRIO-Linux-11.1.0.tar.gz](https://github.com/DynamoRIO/dynamorio/releases/download/release_11.1.0/DynamoRIO-Linux-11.1.0.tar.gz)
 
-  - [DynamoRIO-ARM-Linux-EABIHF-11.0.0.tar.gz](https://github.com/DynamoRIO/dynamorio/releases/download/release_11.0.0/DynamoRIO-ARM-Linux-EABIHF-11.0.0.tar.gz)
+  - [DynamoRIO-ARM-Linux-EABIHF-11.1.0.tar.gz](https://github.com/DynamoRIO/dynamorio/releases/download/release_11.1.0/DynamoRIO-ARM-Linux-EABIHF-11.1.0.tar.gz)
 
-  - [DynamoRIO-ARM-Android-EABI-11.0.0.tar.gz](https://github.com/DynamoRIO/dynamorio/releases/download/release_11.0.0/DynamoRIO-ARM-Android-EABI-11.0.0.tar.gz)
+  - [DynamoRIO-ARM-Android-EABI-11.1.0.tar.gz](https://github.com/DynamoRIO/dynamorio/releases/download/release_11.1.0/DynamoRIO-ARM-Android-EABI-11.1.0.tar.gz)
 
-  - [DynamoRIO-AArch64-Linux-11.0.0.tar.gz](https://github.com/DynamoRIO/dynamorio/releases/download/release_11.0.0/DynamoRIO-AArch64-Linux-11.0.0.tar.gz)
+  - [DynamoRIO-AArch64-Linux-11.1.0.tar.gz](https://github.com/DynamoRIO/dynamorio/releases/download/release_11.1.0/DynamoRIO-AArch64-Linux-11.1.0.tar.gz)
 
-The prior [10.0.0 release](https://github.com/DynamoRIO/dynamorio/releases/tag/release_10.0.0):
+The [11.0.0 release](https://github.com/DynamoRIO/dynamorio/releases/tag/release_11.0.0):
 
-  - [DynamoRIO-Windows-10.0.0.zip](https://github.com/DynamoRIO/dynamorio/releases/download/release_10.0.0/DynamoRIO-Windows-10.0.0.zip)
+  - [DynamoRIO-Windows-11.0.0.zip](https://github.com/DynamoRIO/dynamorio/releases/download/release_11.0.0/DynamoRIO-Windows-11.0.0.zip)
 
-  - [DynamoRIO-Linux-10.0.0.tar.gz](https://github.com/DynamoRIO/dynamorio/releases/download/release_10.0.0/DynamoRIO-Linux-10.0.0.tar.gz)
+  - [DynamoRIO-Linux-11.0.0.tar.gz](https://github.com/DynamoRIO/dynamorio/releases/download/release_11.0.0/DynamoRIO-Linux-11.0.0.tar.gz)
 
-  - [DynamoRIO-ARM-Linux-EABIHF-10.0.0.tar.gz](https://github.com/DynamoRIO/dynamorio/releases/download/release_10.0.0/DynamoRIO-ARM-Linux-EABIHF-10.0.0.tar.gz)
+  - [DynamoRIO-ARM-Linux-EABIHF-11.0.0.tar.gz](https://github.com/DynamoRIO/dynamorio/releases/download/release_11.0.0/DynamoRIO-ARM-Linux-EABIHF-11.0.0.tar.gz)
 
-  - [DynamoRIO-ARM-Android-EABI-10.0.0.tar.gz](https://github.com/DynamoRIO/dynamorio/releases/download/release_10.0.0/DynamoRIO-ARM-Android-EABI-10.0.0.tar.gz)
+  - [DynamoRIO-ARM-Android-EABI-11.0.0.tar.gz](https://github.com/DynamoRIO/dynamorio/releases/download/release_11.0.0/DynamoRIO-ARM-Android-EABI-11.0.0.tar.gz)
 
-  - [DynamoRIO-AArch64-Linux-10.0.0.tar.gz](https://github.com/DynamoRIO/dynamorio/releases/download/release_10.0.0/DynamoRIO-AArch64-Linux-10.0.0.tar.gz)
+  - [DynamoRIO-AArch64-Linux-11.0.0.tar.gz](https://github.com/DynamoRIO/dynamorio/releases/download/release_11.0.0/DynamoRIO-AArch64-Linux-11.0.0.tar.gz)
 
 
 ***************************************************************************

diff --git a/api/docs/release.dox b/api/docs/release.dox
@@ -124,7 +124,18 @@ Dr. Memory Framework (DRMF) in the same package as DynamoRIO.  DRMF
 provides the umbra, drsyscall, and drsymcache Extensions for use by
 clients.
 
-The changes between version \DR_VERSION and 11.0.0 include the following compatibility
+The changes between version \DR_VERSION and 11.1.0 include the following compatibility
+changes:
+ - No compatibility changes yet.
+
+Further non-compatibility-affecting changes include:
+ - Added instr_get_operation_size() and instr_set_operation_size() APIs for
+   #DR_ISA_REGDEPS #instr_t.
+
+**************************************************
+<hr>
+
+The changes between version 11.1.0 and 11.0.0 include the following compatibility
 changes:
  - Removed uncompleted instructions and the corresponding memrefs from drmemtraces.
    Instructions which are fetched but not completed due to asynchronous signal or
@@ -149,6 +160,8 @@ Further non-compatibility-affecting changes include:
    on the 32/64 bit mode and the presence or absence of the address size prefix.
  - Fixed the order of operands for the vpexpandd/vpexpandq opcodes (previously the
    source and destination operands were reversed).
+ - Fixed the VEX-encoded forms of vaesdec/vaesdeclast/vaesenc/vaesenclast/vpclmulqdq
+   to obey VEX.L, which they previously ignored.
 
 **************************************************
 <hr>

diff --git a/clients/drcachesim/scheduler/scheduler.h b/clients/drcachesim/scheduler/scheduler.h
@@ -758,6 +758,9 @@ template <typename RecordType, typename ReaderType> class scheduler_tmpl_t {
          * The scheduling quantum duration for preemption, in instruction count,
          * for #QUANTUM_INSTRUCTIONS.  The time passed to next_record() is ignored
          * for purposes of quantum preempts.
+         *
+         * Instructions executed in a quantum may end up higher than the specified
+         * value to avoid interruption of the kernel system call sequence.
          */
         // We pick 10 million to match 2 instructions per nanosecond with a 5ms quantum.
         uint64_t quantum_duration_instrs = 10 * 1000 * 1000;

diff --git a/clients/drcachesim/scheduler/scheduler_dynamic.cpp b/clients/drcachesim/scheduler/scheduler_dynamic.cpp
@@ -458,7 +458,9 @@ scheduler_dynamic_tmpl_t<RecordType, ReaderType>::check_for_input_switch(
         // boundaries so we live with those being before the switch.
         // XXX: Once we insert kernel traces, we may have to try harder
         // to stop before the post-syscall records.
-        if (this->record_type_is_instr_boundary(record, outputs_[output].last_record)) {
+        if (this->record_type_is_instr_boundary(record, outputs_[output].last_record) &&
+            // We want to delay the context switch until after the injected syscall trace.
+            !outputs_[output].in_syscall_code) {
             if (input->switch_to_input != sched_type_t::INVALID_INPUT_ORDINAL) {
                 // The switch request overrides any latency threshold.
                 need_new_input = true;
@@ -506,18 +508,28 @@ scheduler_dynamic_tmpl_t<RecordType, ReaderType>::check_for_input_switch(
     }
     if (options_.quantum_unit == sched_type_t::QUANTUM_INSTRUCTIONS &&
         this->record_type_is_instr_boundary(record, outputs_[output].last_record) &&
-        !outputs_[output].in_kernel_code) {
+        !outputs_[output].in_context_switch_code) {
         ++input->instrs_in_quantum;
         if (input->instrs_in_quantum > options_.quantum_duration_instrs) {
-            // We again prefer to switch to another input even if the current
-            // input has the oldest timestamp, prioritizing context switches
-            // over timestamp ordering.
-            VPRINT(this, 4, "next_record[%d]: input %d hit end of instr quantum\n",
-                   output, input->index);
-            preempt = true;
-            need_new_input = true;
-            input->instrs_in_quantum = 0;
-            ++outputs_[output].stats[memtrace_stream_t::SCHED_STAT_QUANTUM_PREEMPTS];
+            if (outputs_[output].in_syscall_code) {
+                // XXX: Maybe this should be printed only once per-syscall-instance to
+                // reduce log spam.
+                VPRINT(this, 5,
+                       "next_record[%d]: input %d delaying context switch "
+                       "after end of instr quantum due to syscall trace\n",
+                       output, input->index);
+
+            } else {
+                // We again prefer to switch to another input even if the current
+                // input has the oldest timestamp, prioritizing context switches
+                // over timestamp ordering.
+                VPRINT(this, 4, "next_record[%d]: input %d hit end of instr quantum\n",
+                       output, input->index);
+                preempt = true;
+                need_new_input = true;
+                input->instrs_in_quantum = 0;
+                ++outputs_[output].stats[memtrace_stream_t::SCHED_STAT_QUANTUM_PREEMPTS];
+            }
         }
     } else if (options_.quantum_unit == sched_type_t::QUANTUM_TIME) {
         if (cur_time == 0 || cur_time < input->prev_time_in_quantum) {
@@ -535,14 +547,23 @@ scheduler_dynamic_tmpl_t<RecordType, ReaderType>::check_for_input_switch(
             // in between (e.g., scatter/gather long sequence of reads/writes) by
             // setting input->switching_pre_instruction.
             this->record_type_is_instr_boundary(record, outputs_[output].last_record)) {
-            VPRINT(this, 4,
-                   "next_record[%d]: input %d hit end of time quantum after %" PRIu64
-                   "\n",
-                   output, input->index, input->time_spent_in_quantum);
-            preempt = true;
-            need_new_input = true;
-            input->time_spent_in_quantum = 0;
-            ++outputs_[output].stats[memtrace_stream_t::SCHED_STAT_QUANTUM_PREEMPTS];
+            if (outputs_[output].in_syscall_code) {
+                // XXX: Maybe this should be printed only once per-syscall-instance to
+                // reduce log spam.
+                VPRINT(this, 5,
+                       "next_record[%d]: input %d delaying context switch after end of "
+                       "time quantum after %" PRIu64 " due to syscall trace\n",
+                       output, input->index, input->time_spent_in_quantum);
+            } else {
+                VPRINT(this, 4,
+                       "next_record[%d]: input %d hit end of time quantum after %" PRIu64
+                       "\n",
+                       output, input->index, input->time_spent_in_quantum);
+                preempt = true;
+                need_new_input = true;
+                input->time_spent_in_quantum = 0;
+                ++outputs_[output].stats[memtrace_stream_t::SCHED_STAT_QUANTUM_PREEMPTS];
+            }
         }
     }
     // For sched_type_t::DEPENDENCY_TIMESTAMPS: enforcing asked-for
@@ -574,16 +595,20 @@ scheduler_dynamic_tmpl_t<RecordType, ReaderType>::process_marker(
         break;
     case TRACE_MARKER_TYPE_CONTEXT_SWITCH_START:
         outputs_[output].in_context_switch_code = true;
-        ANNOTATE_FALLTHROUGH;
+        break;
     case TRACE_MARKER_TYPE_SYSCALL_TRACE_START:
-        outputs_[output].in_kernel_code = true;
+        outputs_[output].in_syscall_code = true;
         break;
     case TRACE_MARKER_TYPE_CONTEXT_SWITCH_END:
         // We have to delay until the next record.
         outputs_[output].hit_switch_code_end = true;
-        ANNOTATE_FALLTHROUGH;
+        break;
     case TRACE_MARKER_TYPE_SYSCALL_TRACE_END:
-        outputs_[output].in_kernel_code = false;
+        outputs_[output].in_syscall_code = false;
+        break;
+    case TRACE_MARKER_TYPE_TIMESTAMP:
+        // Syscall sequences are not expected to have a timestamp.
+        assert(!outputs_[output].in_syscall_code);
         break;
     case TRACE_MARKER_TYPE_DIRECT_THREAD_SWITCH: {
         if (!options_.honor_direct_switches)

diff --git a/clients/drcachesim/scheduler/scheduler_impl.h b/clients/drcachesim/scheduler/scheduler_impl.h
@@ -480,7 +480,7 @@ template <typename RecordType, typename ReaderType> class scheduler_impl_tmpl_t
         // This is accessed by other outputs for stealing and rebalancing.
         // Indirected so we can store it in our vector.
         std::unique_ptr<std::atomic<bool>> active;
-        bool in_kernel_code = false;
+        bool in_syscall_code = false;
         bool in_context_switch_code = false;
         bool hit_switch_code_end = false;
         // Used for time-based quanta.

diff --git a/clients/drcachesim/tests/burst_syscall_inject.cpp b/clients/drcachesim/tests/burst_syscall_inject.cpp
@@ -53,6 +53,10 @@
 #include <iostream>
 #include <string>
 #include <sys/syscall.h>
+#include <sys/types.h>
+#ifndef LINUX
+#    error Only Linux is supported
+#endif
 
 namespace dynamorio {
 namespace drmemtrace {
@@ -72,6 +76,12 @@ static instr_t *instr_in_gettid = nullptr;
         exit(1);                                            \
     } while (0)
 
+static pid_t
+gettid(void)
+{
+    return syscall(SYS_gettid);
+}
+
 static int
 do_some_syscalls()
 {

diff --git a/clients/drcachesim/tests/invariant_checker_test.cpp b/clients/drcachesim/tests/invariant_checker_test.cpp
@@ -3581,6 +3581,49 @@ check_regdeps(void)
 {
     std::cerr << "Testing regdeps traces\n";
 
+    // Incorrect: TRACE_MARKER_TYPE_SIGNAL_NUMBER not allowed.
+    {
+        std::vector<memref_t> memrefs = {
+            gen_marker(TID_A, TRACE_MARKER_TYPE_FILETYPE, OFFLINE_FILE_TYPE_ARCH_REGDEPS),
+            gen_marker(TID_A, TRACE_MARKER_TYPE_CACHE_LINE_SIZE, 64),
+            gen_marker(TID_A, TRACE_MARKER_TYPE_PAGE_SIZE, 4096),
+            gen_marker(TID_A, TRACE_MARKER_TYPE_SIGNAL_NUMBER, 42),
+            gen_instr(TID_A),
+            gen_exit(TID_A),
+        };
+        if (!run_checker(
+                memrefs, true,
+                { "OFFLINE_FILE_TYPE_ARCH_REGDEPS traces cannot have "
+                  "TRACE_MARKER_TYPE_SIGNAL_NUMBER markers",
+                  /*tid=*/TID_A,
+                  /*ref_ordinal=*/4, /*last_timestamp=*/0,
+                  /*instrs_since_last_timestamp=*/0 },
+                "Failed to catch non-allowed TRACE_MARKER_TYPE_SIGNAL_NUMBER marker"))
+            return false;
+    }
+
+    // Incorrect: TRACE_MARKER_TYPE_UNCOMPLETED_INSTRUCTION not allowed.
+    // XXX i#7155: Allow these markers once we update their values in record_filter.
+    {
+        std::vector<memref_t> memrefs = {
+            gen_marker(TID_A, TRACE_MARKER_TYPE_FILETYPE, OFFLINE_FILE_TYPE_ARCH_REGDEPS),
+            gen_marker(TID_A, TRACE_MARKER_TYPE_CACHE_LINE_SIZE, 64),
+            gen_marker(TID_A, TRACE_MARKER_TYPE_PAGE_SIZE, 4096),
+            gen_marker(TID_A, TRACE_MARKER_TYPE_UNCOMPLETED_INSTRUCTION, 42),
+            gen_instr(TID_A),
+            gen_exit(TID_A),
+        };
+        if (!run_checker(memrefs, true,
+                         { "OFFLINE_FILE_TYPE_ARCH_REGDEPS traces cannot have "
+                           "TRACE_MARKER_TYPE_UNCOMPLETED_INSTRUCTION markers",
+                           /*tid=*/TID_A,
+                           /*ref_ordinal=*/4, /*last_timestamp=*/0,
+                           /*instrs_since_last_timestamp=*/0 },
+                         "Failed to catch non-allowed "
+                         "TRACE_MARKER_TYPE_UNCOMPLETED_INSTRUCTION marker"))
+            return false;
+    }
+
     // Incorrect: OFFLINE_FILE_TYPE_ARCH_AARCH64 not allowed.
     {
         std::vector<memref_t> memrefs = {