i#6495 syscall templates: Inject kernel traces in raw2trace

api/docs/release.dox

@@ -143,9 +143,9 @@ changes:
 
 Further non-compatibility-affecting changes include:
  - Added raw2trace support to inject system call kernel trace templates collected from
-   elsewhere (e.g., QEMU, Gem5) into the user-space trace at the corresponding system
-   call number marker. This is done by specifying the path to the template file via
-   the new -syscall_template_file option.
+   elsewhere (e.g., QEMU, Gem5) into the user-space drmemtrace traces at the
+   corresponding system call number marker. This is done by specifying the path to the
+   template file via the new -syscall_template_file option.
  - Added a new scheme for the modoffs field in the PC trace entry which allows L0
    filtering of non-module code; see
    #dynamorio::drmemtrace::ENCODING_FILE_TYPE_SEPARATE_NON_MOD_INSTRS. Also added

clients/drcachesim/CMakeLists.txt

@@ -216,6 +216,8 @@ set(raw2trace_srcs
   tracer/instru_offline.cpp
   reader/reader.cpp
   common/trace_entry.cpp
+  reader/record_file_reader.cpp
+  ${zlib_reader}
   )
 if (libsnappy)
   set(raw2trace_srcs ${raw2trace_srcs}
@@ -1022,7 +1024,7 @@ if (BUILD_TESTS)
       target_link_libraries(tool.drcacheoff.burst_maps test_helpers)
       add_win32_flags(tool.drcacheoff.burst_maps)
     endif ()
-    if (UNIX)
+    if (LINUX)
       add_executable(tool.drcacheoff.burst_syscall_inject tests/burst_syscall_inject.cpp)
       configure_DynamoRIO_static(tool.drcacheoff.burst_syscall_inject)
       use_DynamoRIO_static_client(tool.drcacheoff.burst_syscall_inject drmemtrace_static)

clients/drcachesim/analyzer_multi.cpp

@@ -173,7 +173,7 @@ analyzer_multi_t::analyzer_multi_t()
                 nullptr, op_verbose.get_value(), op_jobs.get_value(),
                 op_alt_module_dir.get_value(), op_chunk_instr_count.get_value(),
                 dir.in_kfiles_map_, dir.kcoredir_, dir.kallsymsdir_,
-                dir.syscall_template_file_);
+                std::move(dir.syscall_template_file_reader_));
             std::string error = raw2trace.do_conversion();
             if (!error.empty()) {
                 success_ = false;

clients/drcachesim/common/trace_entry.h

@@ -917,6 +917,9 @@ typedef enum {
      * #OFFLINE_FILE_TYPE_KERNEL_SYSCALLS trace. See the sample file written by the
      * burst_syscall_inject.cpp test for more details on the expected format for the
      * system call template file.
+     * TODO i#6495: Add support for reading a zipfile where each trace template is in
+     * a separate component. This will make it easier to manually append, update, or
+     * inspect the individual templates.
      */
     OFFLINE_FILE_TYPE_KERNEL_SYSCALL_TRACE_TEMPLATES = 0x4000,
 } offline_file_type_t;

clients/drcachesim/reader/record_file_reader.cpp

@@ -60,8 +60,12 @@ template <>
 bool
 record_file_reader_t<std::ifstream>::read_next_entry()
 {
-    if (!input_file_->read((char *)&cur_entry_, sizeof(cur_entry_)))
+    if (!input_file_->read((char *)&cur_entry_, sizeof(cur_entry_))) {
+        if (input_file_->eof()) {
+            eof_ = true;
+        }
         return false;
+    }
     VPRINT(this, 4, "Read from file: type=%s (%d), size=%d, addr=%zu\n",
            trace_type_names[cur_entry_.type], cur_entry_.type, cur_entry_.size,
            cur_entry_.addr);

clients/drcachesim/tests/burst_syscall_inject.cpp

@@ -56,8 +56,12 @@
 namespace dynamorio {
 namespace drmemtrace {
 
+static instr_t *instr_in_getpid = nullptr;
+static instr_t *instr_in_gettid = nullptr;
+
 #define PC_SYSCALL_GETPID 0xdeadbe00
 #define PC_SYSCALL_GETTID 0x8badf000
+#define READ_MEMADDR_GETTID 0xdecafbad
 
 #define FATAL_ERROR(msg, ...)                               \
     do {                                                    \
@@ -86,6 +90,9 @@ static void
 write_instr_entry(void *dr_context, std::unique_ptr<std::ostream> &writer, instr_t *instr,
                   app_pc instr_app_pc)
 {
+    if (instr == nullptr) {
+        FATAL_ERROR("Cannot write a nullptr instr.");
+    }
     unsigned short len = instr_length(dr_context, instr);
     trace_entry_t encoding = { TRACE_TYPE_ENCODING, len, 0 };
     if (len >= sizeof(encoding.encoding)) {
@@ -112,42 +119,60 @@ write_system_call_template(void *dr_context)
         std::unique_ptr<std::ostream>(new std::ofstream(syscall_trace_template_file));
 
     // Write a valid header so the trace can be used with the trace analyzer.
-    trace_entry_t header = { TRACE_TYPE_HEADER, 0, { TRACE_ENTRY_VERSION } };
-    write_trace_entry(writer, header);
+#define MAX_HEADER_ENTRIES 10
+    trace_entry_t header_buf[MAX_HEADER_ENTRIES];
+    byte *buf = reinterpret_cast<byte *>(header_buf);
     offline_file_type_t file_type = static_cast<offline_file_type_t>(
         OFFLINE_FILE_TYPE_KERNEL_SYSCALL_TRACE_TEMPLATES | OFFLINE_FILE_TYPE_ENCODINGS |
         IF_X86_ELSE(
             IF_X64_ELSE(OFFLINE_FILE_TYPE_ARCH_X86_64, OFFLINE_FILE_TYPE_ARCH_X86_32),
             IF_X64_ELSE(OFFLINE_FILE_TYPE_ARCH_AARCH64, OFFLINE_FILE_TYPE_ARCH_ARM32)));
-    trace_entry_t file_type_entry = { TRACE_TYPE_MARKER,
-                                      TRACE_MARKER_TYPE_FILETYPE,
-                                      { file_type } };
-    write_trace_entry(writer, file_type_entry);
     // We just want a sentinel other than zero for tid and pid.
-    trace_entry_t tid = { TRACE_TYPE_THREAD, 0, { 1 } };
-    write_trace_entry(writer, tid);
-    trace_entry_t pid = { TRACE_TYPE_PID, 0, { 1 } };
-    write_trace_entry(writer, pid);
+    raw2trace_t::create_essential_header_entries(buf, TRACE_ENTRY_VERSION, file_type,
+                                                 /*tid=*/1,
+                                                 /*pid=*/1);
+    if (buf >= reinterpret_cast<byte *>(header_buf + MAX_HEADER_ENTRIES)) {
+        FATAL_ERROR("Too many header entries.");
+    }
+    for (byte *buf_at = reinterpret_cast<byte *>(header_buf); buf_at < buf;
+         buf_at += sizeof(trace_entry_t)) {
+        trace_entry_t to_write = *reinterpret_cast<trace_entry_t *>(buf_at);
+        write_trace_entry(writer, to_write);
+    }
 
     // Write the trace template for SYS_getpid.
     trace_entry_t getpid_marker = { TRACE_TYPE_MARKER, TRACE_MARKER_TYPE_SYSCALL,
                                     SYS_getpid };
     write_trace_entry(writer, getpid_marker);
-    instr_t *getpid_instr = XINST_CREATE_nop(dr_context);
-    write_instr_entry(dr_context, writer, getpid_instr,
+    // Just a random instruction.
+    instr_in_getpid = XINST_CREATE_nop(dr_context);
+    write_instr_entry(dr_context, writer, instr_in_getpid,
                       reinterpret_cast<app_pc>(PC_SYSCALL_GETPID));
-    instr_destroy(dr_context, getpid_instr);
 
     // Write the trace template for SYS_gettid.
     trace_entry_t gettid_marker = { TRACE_TYPE_MARKER, TRACE_MARKER_TYPE_SYSCALL,
                                     SYS_gettid };
     write_trace_entry(writer, gettid_marker);
-    instr_t *gettid_instr = XINST_CREATE_return(dr_context);
-    write_instr_entry(dr_context, writer, gettid_instr,
+    // Just a random instruction.
+#ifdef X86
+#    define TEST_REG DR_REG_XDX
+#elif defined(ARM)
+#    define TEST_REG DR_REG_R12
+#elif defined(AARCH64)
+#    define TEST_REG DR_REG_X4
+#endif
+    instr_in_gettid =
+        XINST_CREATE_load(dr_context, opnd_create_reg(TEST_REG),
+                          opnd_create_base_disp(TEST_REG, DR_REG_NULL, 0, 0, OPSZ_PTR));
+    write_instr_entry(dr_context, writer, instr_in_gettid,
                       reinterpret_cast<app_pc>(PC_SYSCALL_GETTID));
-    instr_destroy(dr_context, gettid_instr);
+    trace_entry_t gettid_read_entry = { TRACE_TYPE_READ,
+                                        static_cast<unsigned short>(
+                                            opnd_size_in_bytes(OPSZ_PTR)),
+                                        { READ_MEMADDR_GETTID } };
+    write_trace_entry(writer, gettid_read_entry);
 
-    trace_entry_t footer = { TRACE_TYPE_THREAD_EXIT, 0, { 1 } };
+    trace_entry_t footer = { TRACE_TYPE_FOOTER, 0, { 0 } };
     write_trace_entry(writer, footer);
     std::cerr << "Done writing system call trace template\n";
     return syscall_trace_template_file;
@@ -170,14 +195,15 @@ postprocess(void *dr_context, std::string syscall_trace_template_file)
     std::string dir_err = dir.initialize(raw_dir, outdir, DEFAULT_TRACE_COMPRESSION_TYPE,
                                          syscall_trace_template_file);
     assert(dir_err.empty());
-    raw2trace_t raw2trace(
-        dir.modfile_bytes_, dir.in_files_, dir.out_files_, dir.out_archives_,
-        dir.encoding_file_, dir.serial_schedule_file_, dir.cpu_schedule_file_, dr_context,
-        /*verbosity=*/0, /*worker_count=*/-1,
-        /*alt_module_dir=*/"",
-        /*chunk_instr_count=*/10 * 1000 * 1000,
-        /*kthread_files_map=*/ {},
-        /*kcore_path=*/"", /*kallsyms_path=*/"", dir.syscall_template_file_);
+    raw2trace_t raw2trace(dir.modfile_bytes_, dir.in_files_, dir.out_files_,
+                          dir.out_archives_, dir.encoding_file_,
+                          dir.serial_schedule_file_, dir.cpu_schedule_file_, dr_context,
+                          /*verbosity=*/0, /*worker_count=*/-1,
+                          /*alt_module_dir=*/"",
+                          /*chunk_instr_count=*/10 * 1000 * 1000,
+                          /*kthread_files_map=*/ {},
+                          /*kcore_path=*/"", /*kallsyms_path=*/"",
+                          std::move(dir.syscall_template_file_reader_));
     std::string error = raw2trace.do_conversion();
     if (!error.empty())
         FATAL_ERROR("raw2trace failed: %s\n", error.c_str());
@@ -221,6 +247,29 @@ gather_trace()
     return;
 }
 
+static bool
+check_instr_same(void *dr_context, memref_t &memref, instr_t *expected_instr)
+{
+    assert(type_is_instr(memref.instr.type));
+    instr_t instr;
+    instr_init(dr_context, &instr);
+    app_pc next_pc =
+        decode_from_copy(dr_context, memref.instr.encoding,
+                         reinterpret_cast<byte *>(memref.instr.addr), &instr);
+    assert(next_pc != nullptr && instr_valid(&instr));
+    bool res = true;
+    if (!instr_same(expected_instr, &instr)) {
+        std::cerr << "Unexpected instruction: |";
+        instr_disassemble(dr_context, &instr, STDERR);
+        std::cerr << "| expected: |";
+        instr_disassemble(dr_context, expected_instr, STDERR);
+        std::cerr << "|\n";
+        res = false;
+    }
+    instr_free(dr_context, &instr);
+    return res;
+}
+
 static bool
 look_for_syscall_trace(void *dr_context, std::string trace_dir)
 {
@@ -235,14 +284,17 @@ look_for_syscall_trace(void *dr_context, std::string trace_dir)
     }
     auto *stream = scheduler.get_stream(0);
     memref_t memref;
-    bool found_getpid_trace = false;
-    bool found_gettid_trace = false;
+    bool found_getpid_instr = false;
+    bool found_gettid_instr = false;
+    bool found_gettid_read = false;
     bool have_syscall_trace_type = false;
-    int in_syscall_trace = -1;
-    bool success = true;
+    int syscall_trace_num = -1;
+    int prev_syscall_num_marker = -1;
     for (scheduler_t::stream_status_t status = stream->next_record(memref);
          status != scheduler_t::STATUS_EOF; status = stream->next_record(memref)) {
         assert(status == scheduler_t::STATUS_OK);
+        int prev_syscall_num_marker_saved = prev_syscall_num_marker;
+        prev_syscall_num_marker = -1;
         if (memref.marker.type == TRACE_TYPE_MARKER) {
             switch (memref.marker.marker_type) {
             case TRACE_MARKER_TYPE_FILETYPE:
@@ -252,66 +304,98 @@ look_for_syscall_trace(void *dr_context, std::string trace_dir)
                 }
                 break;
             case TRACE_MARKER_TYPE_SYSCALL_TRACE_START:
-                in_syscall_trace = memref.marker.marker_value;
+                syscall_trace_num = memref.marker.marker_value;
+                if (syscall_trace_num != prev_syscall_num_marker_saved ||
+                    // We assume there would not be a chunk split between the sysnum
+                    // marker and the trace start marker for this short trace.
+                    prev_syscall_num_marker_saved == -1) {
+                    std::cerr << "Found unexpected trace for system call "
+                              << syscall_trace_num
+                              << " when prev system call number marker was (-1 for not a "
+                                 "sysnum marker) "
+                              << prev_syscall_num_marker_saved << "\n";
+                    return false;
+                }
+                break;
+            case TRACE_MARKER_TYPE_SYSCALL_TRACE_END: syscall_trace_num = -1; break;
+            case TRACE_MARKER_TYPE_SYSCALL:
+                prev_syscall_num_marker = memref.marker.marker_value;
                 break;
-            case TRACE_MARKER_TYPE_SYSCALL_TRACE_END: in_syscall_trace = -1; break;
             }
             continue;
         }
-        if (in_syscall_trace == -1 || !type_is_instr(memref.instr.type)) {
+        if (syscall_trace_num == -1) {
             continue;
         }
-        instr_t instr;
-        instr_init(dr_context, &instr);
-        app_pc next_pc =
-            decode_from_copy(dr_context, memref.instr.encoding,
-                             reinterpret_cast<byte *>(memref.instr.addr), &instr);
-        assert(next_pc != nullptr && instr_valid(&instr));
-        switch (in_syscall_trace) {
+        bool is_instr = type_is_instr(memref.instr.type);
+        if (!is_instr && !type_is_data(memref.instr.type)) {
+            std::cerr << "Found unexpected memref record " << memref.instr.type
+                      << " inside inserted system call template\n";
+            return false;
+        }
+        switch (syscall_trace_num) {
         case SYS_gettid:
-            assert(!found_gettid_trace);
-            found_gettid_trace = true;
-            if (memref.instr.addr != PC_SYSCALL_GETTID) {
-                std::cerr << "Found incorrect addr (" << std::hex << memref.instr.addr
-                          << " vs expected " << PC_SYSCALL_GETTID << std::dec
-                          << ") for gettid trace instr.\n";
-                success = false;
-            }
-            if (!instr_is_return(&instr)) {
-                std::cerr << "Found unexpected instruction for gettid trace.\n";
-                success = false;
+            if (is_instr) {
+                assert(!found_gettid_instr);
+                found_gettid_instr = true;
+                if (memref.instr.addr != PC_SYSCALL_GETTID) {
+                    std::cerr << "Found incorrect addr (" << std::hex << memref.instr.addr
+                              << " vs expected " << PC_SYSCALL_GETTID << std::dec
+                              << ") for gettid trace instr.\n";
+                    return false;
+                }
+                if (!check_instr_same(dr_context, memref, instr_in_gettid)) {
+                    return false;
+                }
+            } else {
+                assert(!found_gettid_read);
+                found_gettid_read = true;
+                if (memref.data.type != TRACE_TYPE_READ ||
+                    memref.data.size != opnd_size_in_bytes(OPSZ_PTR) ||
+                    memref.data.addr != READ_MEMADDR_GETTID) {
+                    std::cerr << "Found incorrect entry (" << memref.data.type << ","
+                              << memref.data.size << "," << std::hex << memref.data.addr
+                              << ") vs expected ptr-sized read for "
+                              << READ_MEMADDR_GETTID << std::dec
+                              << ") for gettid trace.\n";
+                    return false;
+                }
             }
             break;
         case SYS_getpid:
-            assert(!found_getpid_trace);
-            found_getpid_trace = true;
-            if (memref.instr.addr != PC_SYSCALL_GETPID) {
-                std::cerr << "Found incorrect addr (" << std::hex << memref.instr.addr
-                          << " vs expected " << PC_SYSCALL_GETPID << std::dec
-                          << ") for getpid instr.\n";
-                success = false;
-            }
-            if (!instr_is_nop(&instr)) {
-                std::cerr << "Found unexpected instruction for getpid trace.\n";
-                success = false;
+            if (is_instr) {
+                assert(!found_getpid_instr);
+                found_getpid_instr = true;
+                if (memref.instr.addr != PC_SYSCALL_GETPID) {
+                    std::cerr << "Found incorrect addr (" << std::hex << memref.instr.addr
+                              << " vs expected " << PC_SYSCALL_GETPID << std::dec
+                              << ") for getpid instr.\n";
+                    return false;
+                }
+                if (!check_instr_same(dr_context, memref, instr_in_getpid)) {
+                    std::cerr << "Found unexpected instruction for getpid trace.\n";
+                    return false;
+                }
+            } else {
+                std::cerr << "Found unexpected data memref in getpid trace\n";
+                return false;
             }
             break;
         }
-        instr_free(dr_context, &instr);
     }
     if (!have_syscall_trace_type) {
-        success = false;
         std::cerr << "Trace did not have the expected file type\n";
-    } else if (!found_gettid_trace) {
-        success = false;
-        std::cerr << "Did not find trace for gettid\n";
-    } else if (!found_getpid_trace) {
-        success = false;
-        std::cerr << "Did not find trace for getpid\n";
-    } else if (success) {
+    } else if (!found_gettid_instr) {
+        std::cerr << "Did not find instr in gettid trace\n";
+    } else if (!found_getpid_instr) {
+        std::cerr << "Did not find instr in getpid trace\n";
+    } else if (!found_gettid_read) {
+        std::cerr << "Did not find read data memref in gettid trace\n";
+    } else {
         std::cerr << "Successfully completed checks\n";
+        return true;
     }
-    return success;
+    return false;
 }
 
 int
@@ -334,6 +418,8 @@ test_main(int argc, const char *argv[])
 
     std::string trace_dir = postprocess(dr_context, syscall_trace_template);
     bool success = look_for_syscall_trace(dr_context, trace_dir);
+    instr_destroy(dr_context, instr_in_getpid);
+    instr_destroy(dr_context, instr_in_gettid);
     dr_standalone_exit();
     return success ? 0 : 1;
 }