Merge branch 'master' into master

CMakeLists.txt

@@ -224,10 +224,16 @@ if (TARGET_ARCH MATCHES "^arm64" OR TARGET_ARCH MATCHES "^aarch64")
   set(AARCH64 1)
   set(X64 1)
   message(STATUS "Building for AArch64")
+  if (ANDROID)
+    set(ANDROID64 1)
+  endif ()
 elseif (TARGET_ARCH MATCHES "^arm")
   set(ARM 1) # This means AArch32.
   set(X64 OFF)
   message(STATUS "Building for ARM")
+  if (ANDROID)
+    set(ANDROID32 1)
+  endif ()
 elseif (TARGET_ARCH MATCHES "^riscv64")
   set(RISCV64 1)
   set(X64 1)
@@ -692,6 +698,7 @@ if (UNIX)
   endif (NOT CMAKE_COMPILER_IS_GNUCC)
 
   check_if_linker_is_gnu_gold(LINKER_IS_GNU_GOLD)
+  check_if_linker_is_llvm_lld(LINKER_IS_LLVM_LLD)
 
   # FIXME i#2949: static 32-bit release-build linking with gcc 7.3.1 fails when
   # static C++ clients like drmemtrace or drmemtrace_raw2trace are linked in.
@@ -821,6 +828,7 @@ if (UNIX)
   # (see i#1034 where this caused problems in the past).
   set(BASE_CFLAGS "${BASE_CFLAGS} -funsigned-char")
   # Ubuntu defaults to -fstack-protector these days, which depends on app TLS.
+  CHECK_C_COMPILER_FLAG("-no-pie" no_pie_avail)
   CHECK_C_COMPILER_FLAG("-fno-stack-protector" no_stack_protector_avail)
   if (no_stack_protector_avail)
     set(BASE_CFLAGS "${BASE_CFLAGS} -fno-stack-protector")
@@ -875,14 +883,14 @@ if (UNIX)
   endif ()
   # there's no cmake warning control so we hardcode it
   set(WARN "-Wall -Werror -Wwrite-strings -Wvla")
+  CHECK_C_COMPILER_FLAG("-Wno-unused-but-set-variable" nounused_avail)
+  if (nounused_avail)
+    set(WARN "${WARN} -Wno-unused-but-set-variable")
+  endif (nounused_avail)
   if (NOT CMAKE_COMPILER_IS_CLANG)
     # Old gcc's ignore unknown -W flags, but -Wall -Werror causes clang to
     # complain that it doesn't recognize it.
     # Actually this is not true: gcc 4.1.2 aborts on unknown -W so we check
-    CHECK_C_COMPILER_FLAG("-Wno-unused-but-set-variable" nounused_avail)
-    if (nounused_avail)
-      set(WARN "${WARN} -Wno-unused-but-set-variable")
-    endif (nounused_avail)
     # XXX i#3792: DynamoRIO manages '\0' termination and error states itself in
     # too many places. In order to activate this warning, this code needs to get
     # re-factored for no good reason.
@@ -908,6 +916,21 @@ if (UNIX)
       set(WARN "${WARN} -Wno-dangling-pointer")
     endif ()
   else (NOT CMAKE_COMPILER_IS_CLANG)
+    # Clang emits this warning when a function is used without a defined
+    # prototype, however clang can't find function prototpes in #define macros.
+    # If function prototypes are moved out of #define macros then this warning
+    # can be re-enabled.
+    CHECK_C_COMPILER_FLAG("-Wno-deprecated-non-prototype" nodeprecated_prototype_avail)
+    if (nodeprecated_prototype_avail)
+      set(WARN "${WARN} -Wno-deprecated-non-prototype")
+    endif (nodeprecated_prototype_avail)
+    # Clang emits this warning due to the assembly shared between arm and
+    # aarch64 in os.c not specifying register widths. If the shared code is
+    # split into arm and aarch64 versions, then this warning can be re-enabled.
+    CHECK_C_COMPILER_FLAG("-Wno-asm-operand-widths" noasm_operand_widths)
+    if (noasm_operand_widths)
+      set(WARN "${WARN} -Wno-asm-operand-widths")
+    endif (noasm_operand_widths)
     # clang turns off color when it's writing to a pipe, but the user may still
     # wish to force color if it eventually goes to a terminal.
     option(CLANG_COLOR_DIAGNOSTICS "force colored clang diagnostics" OFF)

api/docs/release.dox

@@ -134,6 +134,10 @@ changes:
    is the encoding of the removed instruction up to a pointer's length. Added
    #OFFLINE_FILE_VERSION_RETIRED_INSTRUCTIONS_ONLY to increase the trace version
    for drmemtraces with uncompleted instructions removed.
+ - Moved module file read logic into read_module_file() in raw2trace_shared, and
+   removed raw2trace_directory_t::initialize_module_file() since the
+   read_module_file() can be directly used without having to pull in the whole
+   raw2trace_directory_t.
 
 Further non-compatibility-affecting changes include:
  - Added X64 Linux support to dr_create_memory_dump(). This API has the same
@@ -143,6 +147,8 @@ Further non-compatibility-affecting changes include:
  - Added the AArch64 FPMR register as DR_REG_FPMR.
  - Added OPSZ_addr which chooses the appropriate operand size on X86 from 2/4/8 based
    on the 32/64 bit mode and the presence or absence of the address size prefix.
+ - Fixed the order of operands for the vpexpandd/vpexpandq opcodes (previously the
+   source and destination operands were reversed).
 
 **************************************************
 <hr>

clients/drcachesim/CMakeLists.txt

@@ -161,9 +161,11 @@ add_exported_library(drmemtrace_reuse_distance STATIC tools/reuse_distance.cpp)
 add_exported_library(drmemtrace_histogram STATIC tools/histogram.cpp)
 add_exported_library(drmemtrace_reuse_time STATIC tools/reuse_time.cpp)
 add_exported_library(drmemtrace_basic_counts STATIC tools/basic_counts.cpp)
-add_exported_library(drmemtrace_opcode_mix STATIC tools/opcode_mix.cpp)
+add_exported_library(drmemtrace_opcode_mix STATIC
+                     tools/opcode_mix.cpp tracer/raw2trace_shared.cpp)
 add_exported_library(drmemtrace_syscall_mix STATIC tools/syscall_mix.cpp)
-add_exported_library(drmemtrace_view STATIC tools/view.cpp)
+add_exported_library(drmemtrace_view STATIC
+                     tools/view.cpp tracer/raw2trace_shared.cpp)
 add_exported_library(drmemtrace_func_view STATIC tools/func_view.cpp)
 add_exported_library(drmemtrace_invariant_checker STATIC tools/invariant_checker.cpp)
 add_exported_library(drmemtrace_schedule_stats STATIC tools/schedule_stats.cpp)
@@ -1202,7 +1204,7 @@ if (BUILD_TESTS)
     add_executable(tool.drcacheoff.skip_unit_tests tests/skip_unit_tests.cpp)
     configure_DynamoRIO_standalone(tool.drcacheoff.skip_unit_tests)
     target_link_libraries(tool.drcacheoff.skip_unit_tests drmemtrace_analyzer
-      drmemtrace_view drmemtrace_raw2trace test_helpers)
+      drmemtrace_view test_helpers ${zlib_libs})
     add_win32_flags(tool.drcacheoff.skip_unit_tests)
     use_DynamoRIO_extension(tool.drcacheoff.skip_unit_tests drreg_static)
     use_DynamoRIO_extension(tool.drcacheoff.skip_unit_tests drcovlib_static)

clients/drcachesim/tools/opcode_mix.cpp

@@ -1,5 +1,5 @@
 /* **********************************************************
- * Copyright (c) 2017-2023 Google, Inc.  All rights reserved.
+ * Copyright (c) 2017-2024 Google, Inc.  All rights reserved.
  * **********************************************************/
 
 /*
@@ -56,7 +56,7 @@
 #include "dr_api.h"
 #include "memref.h"
 #include "raw2trace.h"
-#include "raw2trace_directory.h"
+#include "raw2trace_shared.h"
 #include "reader.h"
 #include "trace_entry.h"
 #include "utils.h"
@@ -92,13 +92,16 @@ opcode_mix_t::initialize()
         return "";
     // Legacy trace support where binaries are needed.
     // We do not support non-module code for such traces.
-    std::string error = directory_.initialize_module_file(module_file_path_);
-    if (!error.empty())
-        return "Failed to initialize directory: " + error;
+    file_t modfile;
+    std::string error = read_module_file(module_file_path_, modfile, modfile_bytes_);
+    if (!error.empty()) {
+        return "Failed to read module file: " + error;
+    }
     module_mapper_ =
-        module_mapper_t::create(directory_.modfile_bytes_, nullptr, nullptr, nullptr,
-                                nullptr, knob_verbose_, knob_alt_module_dir_);
+        module_mapper_t::create(modfile_bytes_, nullptr, nullptr, nullptr, nullptr,
+                                knob_verbose_, knob_alt_module_dir_);
     module_mapper_->get_loaded_modules();
+    dr_close_file(modfile);
     error = module_mapper_->get_last_error();
     if (!error.empty())
         return "Failed to load binaries: " + error;
@@ -110,6 +113,9 @@ opcode_mix_t::~opcode_mix_t()
     for (auto &iter : shard_map_) {
         delete iter.second;
     }
+    if (modfile_bytes_ != nullptr) {
+        delete[] modfile_bytes_;
+    }
 }
 
 bool

clients/drcachesim/tools/opcode_mix.h

@@ -46,7 +46,6 @@
 #include "analysis_tool.h"
 #include "memref.h"
 #include "raw2trace.h"
-#include "raw2trace_directory.h"
 #include "trace_entry.h"
 
 namespace dynamorio {
@@ -185,9 +184,9 @@ class opcode_mix_t : public analysis_tool_t {
     std::string module_file_path_;
     std::unique_ptr<module_mapper_t> module_mapper_;
     std::mutex mapper_mutex_;
-    // We reference directory.modfile_bytes throughout operation, so its lifetime
-    // must match ours.
-    raw2trace_directory_t directory_;
+    // XXX: Perhaps module_mapper_t should be made to own the cleanup of
+    // modfile_bytes_.
+    char *modfile_bytes_ = nullptr;
 
     std::unordered_map<int, shard_data_t *> shard_map_;
     // This mutex is only needed in parallel_shard_init.  In all other accesses to

clients/drcachesim/tools/view.cpp

@@ -53,7 +53,7 @@
 #include "memref.h"
 #include "memtrace_stream.h"
 #include "raw2trace.h"
-#include "raw2trace_directory.h"
+#include "raw2trace_shared.h"
 #include "trace_entry.h"
 #include "utils.h"
 
@@ -124,9 +124,12 @@ view_t::initialize_stream(memtrace_stream_t *serial_stream)
     if (module_file_path_.empty()) {
         has_modules_ = false;
     } else {
-        std::string error = directory_.initialize_module_file(module_file_path_);
+        file_t modfile;
+        std::string error = read_module_file(module_file_path_, modfile, modfile_bytes_);
         if (!error.empty())
             has_modules_ = false;
+        else
+            dr_close_file(modfile);
     }
     if (!has_modules_) {
         // Continue but omit disassembly to support cases where binaries are
@@ -136,8 +139,8 @@ view_t::initialize_stream(memtrace_stream_t *serial_stream)
     // Legacy trace support where binaries are needed.
     // We do not support non-module code for such traces.
     module_mapper_ =
-        module_mapper_t::create(directory_.modfile_bytes_, nullptr, nullptr, nullptr,
-                                nullptr, knob_verbose_, knob_alt_module_dir_);
+        module_mapper_t::create(modfile_bytes_, nullptr, nullptr, nullptr, nullptr,
+                                knob_verbose_, knob_alt_module_dir_);
     module_mapper_->get_loaded_modules();
     std::string error = module_mapper_->get_last_error();
     if (!error.empty())
@@ -662,5 +665,12 @@ view_t::print_results()
     return true;
 }
 
+view_t::~view_t()
+{
+    if (modfile_bytes_ != nullptr) {
+        delete[] modfile_bytes_;
+    }
+}
+
 } // namespace drmemtrace
 } // namespace dynamorio

clients/drcachesim/tools/view.h

@@ -1,5 +1,5 @@
 /* **********************************************************
- * Copyright (c) 2018-2023 Google, Inc.  All rights reserved.
+ * Copyright (c) 2018-2024 Google, Inc.  All rights reserved.
  * **********************************************************/
 
 /*
@@ -47,7 +47,6 @@
 #include "memref.h"
 #include "memtrace_stream.h"
 #include "raw2trace.h"
-#include "raw2trace_directory.h"
 
 namespace dynamorio {
 namespace drmemtrace {
@@ -61,6 +60,7 @@ class view_t : public analysis_tool_t {
     view_t(const std::string &module_file_path, uint64_t skip_refs, uint64_t sim_refs,
            const std::string &syntax, unsigned int verbose,
            const std::string &alt_module_dir = "");
+    virtual ~view_t();
     std::string
     initialize_stream(memtrace_stream_t *serial_stream) override;
     bool
@@ -136,7 +136,9 @@ class view_t : public analysis_tool_t {
     // std::optional here.
     std::string module_file_path_;
     std::unique_ptr<module_mapper_t> module_mapper_;
-    raw2trace_directory_t directory_;
+    // XXX: Perhaps module_mapper_t should be made to own the cleanup of
+    // modfile_bytes_.
+    char *modfile_bytes_ = nullptr;
 
     unsigned int knob_verbose_;
     int trace_version_;

clients/drcachesim/tracer/instru.h

@@ -47,16 +47,14 @@
 #include "dr_allocator.h"
 #include "dr_api.h"
 #include "drvector.h"
+#include "raw2trace_shared.h"
 #include "trace_entry.h"
 
 namespace dynamorio {
 namespace drmemtrace {
 
 #define MINSERT instrlist_meta_preinsert
 
-// Versioning for our drmodtrack custom module fields.
-#define CUSTOM_MODULE_VERSION 1
-
 // A std::unordered_set, even using dr_allocator_t, raises transparency risks when
 // statically linked on Windows (from lock functions and other non-allocator
 // resources).  We thus create our own resource-isolated class to track GPR register
@@ -499,11 +497,6 @@ class offline_instru_t : public instru_t {
     label_marks_elidable(instr_t *instr, DR_PARAM_OUT int *opnd_index,
                          DR_PARAM_OUT int *memopnd_index, DR_PARAM_OUT bool *is_write,
                          DR_PARAM_OUT bool *needs_base);
-    static int
-    print_module_data_fields(char *dst, size_t max_len, const void *custom_data,
-                             size_t custom_size,
-                             int (*user_print_cb)(void *data, char *dst, size_t max_len),
-                             void *user_cb_data);
 
 private:
     struct custom_module_data_t {

clients/drcachesim/tracer/instru_offline.cpp

@@ -47,6 +47,7 @@
 #include "drreg.h"
 #include "drutil.h"
 #include "drvector.h"
+#include "raw2trace_shared.h"
 #include "trace_entry.h"
 #include "utils.h"
 #include "instru.h"
@@ -186,31 +187,6 @@ offline_instru_t::load_custom_module_data(module_data_t *module, int seg_idx)
     return nullptr;
 }
 
-int
-offline_instru_t::print_module_data_fields(
-    char *dst, size_t max_len, const void *custom_data, size_t custom_size,
-    int (*user_print_cb)(void *data, char *dst, size_t max_len), void *user_cb_data)
-{
-    char *cur = dst;
-    int len = dr_snprintf(dst, max_len, "v#%d,%zu,", CUSTOM_MODULE_VERSION, custom_size);
-    if (len < 0)
-        return -1;
-    cur += len;
-    if (cur - dst + custom_size > max_len)
-        return -1;
-    if (custom_size > 0) {
-        memcpy(cur, custom_data, custom_size);
-        cur += custom_size;
-    }
-    if (user_print_cb != nullptr) {
-        int res = (*user_print_cb)(user_cb_data, cur, max_len - (cur - dst));
-        if (res == -1)
-            return -1;
-        cur += res;
-    }
-    return (int)(cur - dst);
-}
-
 int
 offline_instru_t::print_custom_module_data(void *data, char *dst, size_t max_len)
 {