i#6495 syscall templates: Inject kernel traces in raw2trace

clients/drcachesim/common/trace_entry.h

@@ -919,7 +919,10 @@ typedef enum {
      * system call template file.
      * TODO i#6495: Add support for reading a zipfile where each trace template is in
      * a separate component. This will make it easier to manually append, update, or
-     * inspect the individual templates.
+     * inspect the individual templates, and also allow streaming the component with the
+     * required template when needed instead of reading the complete file into memory
+     * ahead of time. Note that we may drop support for non-zipfile template files in
+     * the future.
      */
     OFFLINE_FILE_TYPE_KERNEL_SYSCALL_TRACE_TEMPLATES = 0x4000,
 } offline_file_type_t;

clients/drcachesim/tests/burst_syscall_inject.cpp

@@ -43,6 +43,7 @@
 #include "dr_api.h"
 #include "drmemtrace/drmemtrace.h"
 #include "drmemtrace/raw2trace.h"
+#include "mock_reader.h"
 #include "raw2trace_directory.h"
 #include "scheduler.h"
 
@@ -101,8 +102,9 @@ write_instr_entry(void *dr_context, std::unique_ptr<std::ostream> &writer, instr
     }
     instr_encode_to_copy(dr_context, instr, encoding.encoding, instr_app_pc);
     write_trace_entry(writer, encoding);
-    trace_entry_t pc = { TRACE_TYPE_INSTR, len, reinterpret_cast<addr_t>(instr_app_pc) };
-    write_trace_entry(writer, pc);
+    write_trace_entry(
+        writer,
+        make_instr(reinterpret_cast<addr_t>(instr_app_pc), TRACE_TYPE_INSTR, len));
 }
 
 static std::string
@@ -141,18 +143,14 @@ write_system_call_template(void *dr_context)
     }
 
     // Write the trace template for SYS_getpid.
-    trace_entry_t getpid_marker = { TRACE_TYPE_MARKER, TRACE_MARKER_TYPE_SYSCALL,
-                                    SYS_getpid };
-    write_trace_entry(writer, getpid_marker);
+    write_trace_entry(writer, make_marker(TRACE_MARKER_TYPE_SYSCALL, SYS_getpid));
     // Just a random instruction.
     instr_in_getpid = XINST_CREATE_nop(dr_context);
     write_instr_entry(dr_context, writer, instr_in_getpid,
                       reinterpret_cast<app_pc>(PC_SYSCALL_GETPID));
 
     // Write the trace template for SYS_gettid.
-    trace_entry_t gettid_marker = { TRACE_TYPE_MARKER, TRACE_MARKER_TYPE_SYSCALL,
-                                    SYS_gettid };
-    write_trace_entry(writer, gettid_marker);
+    write_trace_entry(writer, make_marker(TRACE_MARKER_TYPE_SYSCALL, SYS_gettid));
     // Just a random instruction.
 #ifdef X86
 #    define TEST_REG DR_REG_XDX
@@ -166,14 +164,11 @@ write_system_call_template(void *dr_context)
                           opnd_create_base_disp(TEST_REG, DR_REG_NULL, 0, 0, OPSZ_PTR));
     write_instr_entry(dr_context, writer, instr_in_gettid,
                       reinterpret_cast<app_pc>(PC_SYSCALL_GETTID));
-    trace_entry_t gettid_read_entry = { TRACE_TYPE_READ,
-                                        static_cast<unsigned short>(
-                                            opnd_size_in_bytes(OPSZ_PTR)),
-                                        { READ_MEMADDR_GETTID } };
-    write_trace_entry(writer, gettid_read_entry);
+    write_trace_entry(
+        writer,
+        make_memref(READ_MEMADDR_GETTID, TRACE_TYPE_READ, opnd_size_in_bytes(OPSZ_PTR)));
 
-    trace_entry_t footer = { TRACE_TYPE_FOOTER, 0, { 0 } };
-    write_trace_entry(writer, footer);
+    write_trace_entry(writer, make_footer());
     std::cerr << "Done writing system call trace template\n";
     return syscall_trace_template_file;
 }
@@ -306,8 +301,6 @@ look_for_syscall_trace(void *dr_context, std::string trace_dir)
             case TRACE_MARKER_TYPE_SYSCALL_TRACE_START:
                 syscall_trace_num = memref.marker.marker_value;
                 if (syscall_trace_num != prev_syscall_num_marker_saved ||
-                    // We assume there would not be a chunk split between the sysnum
-                    // marker and the trace start marker for this short trace.
                     prev_syscall_num_marker_saved == -1) {
                     std::cerr << "Found unexpected trace for system call "
                               << syscall_trace_num

clients/drcachesim/tests/mock_reader.h

@@ -84,11 +84,21 @@ class mock_reader_t : public reader_t {
 };
 
 static inline trace_entry_t
-make_instr(addr_t pc, trace_type_t type = TRACE_TYPE_INSTR)
+make_memref(addr_t addr, trace_type_t type = TRACE_TYPE_READ, unsigned short size = 1)
 {
     trace_entry_t entry;
     entry.type = static_cast<unsigned short>(type);
-    entry.size = 1;
+    entry.size = size;
+    entry.addr = addr;
+    return entry;
+}
+
+static inline trace_entry_t
+make_instr(addr_t pc, trace_type_t type = TRACE_TYPE_INSTR, unsigned short size = 1)
+{
+    trace_entry_t entry;
+    entry.type = static_cast<unsigned short>(type);
+    entry.size = size;
     entry.addr = pc;
     return entry;
 }

clients/drcachesim/tracer/raw2trace.cpp

@@ -583,8 +583,22 @@ module_mapper_t::write_module_data(char *buf, size_t buf_size,
  * Top-level
  */
 
+app_pc
+raw2trace_t::get_first_app_pc_for_syscall_template(int syscall_num)
+{
+    auto it = syscall_trace_templates_[syscall_num].begin();
+    while (it != syscall_trace_templates_[syscall_num].end()) {
+        if (type_is_instr(static_cast<trace_type_t>(it->type))) {
+            return reinterpret_cast<app_pc>(it->addr);
+        }
+        ++it;
+    }
+    return nullptr;
+}
+
 bool
-raw2trace_t::write_syscall_template(raw2trace_thread_data_t *tdata, int syscall_num)
+raw2trace_t::write_syscall_template(raw2trace_thread_data_t *tdata, byte *&buf_in,
+                                    trace_entry_t *buf_base, int syscall_num)
 {
     // Check if we have a template for this system call.
     if (syscall_trace_templates_.find(syscall_num) == syscall_trace_templates_.end())
@@ -595,17 +609,38 @@ raw2trace_t::write_syscall_template(raw2trace_thread_data_t *tdata, int syscall_
                        "template.";
         return false;
     }
-    trace_entry_t *buf_base = get_write_buffer(tdata);
-    trace_entry_t *buf = buf_base;
+    // Write out delayed branches if any. When we're at a system call number marker
+    // any delayed branches should already be written out at the prior syscall
+    // instruction. However, on Win32 the WOW64 calls are treated as a system call
+    // (has a corresponding system call number marker) but is also a CTI so is
+    // delayed. We expect delayed branches only in that case.
+    if (delayed_branches_exist(tdata)) {
+        app_pc next_pc = get_first_app_pc_for_syscall_template(syscall_num);
+        if (next_pc == nullptr) {
+            tdata->error = "Could not find first app pc in system call template for " +
+                std::to_string(syscall_num);
+            return false;
+        }
+        if (!append_delayed_branch(tdata, next_pc))
+            return false;
+    }
+    trace_entry_t *buf = reinterpret_cast<trace_entry_t *>(buf_in);
     trace_entry_t start_entry = { TRACE_TYPE_MARKER,
                                   TRACE_MARKER_TYPE_SYSCALL_TRACE_START,
                                   { static_cast<addr_t>(syscall_num) } };
     *buf = start_entry;
     ++buf;
+    // Now write any accumulated entries from before and the start entry.
+    size_t size = buf - buf_base;
+    if ((uint)size >= WRITE_BUFFER_SIZE) {
+        tdata->error = "Too many entries";
+        return false;
+    }
     if (!write(tdata, buf_base, buf)) {
         return false;
     }
     buf = buf_base;
+    buf_in = reinterpret_cast<byte *>(buf_base); // Defensive: update the returned buf.
 
     app_pc saved_decode_pc;
     int inserted_instr_count = 0;
@@ -650,6 +685,7 @@ raw2trace_t::write_syscall_template(raw2trace_thread_data_t *tdata, int syscall_
     if (!write(tdata, buf_base, buf)) {
         return false;
     }
+    buf_in = reinterpret_cast<byte *>(buf_base);
     log(2, "Inserted %s instrs from system call trace template for sysnum %d\n",
         inserted_instr_count, syscall_num);
     return true;
@@ -712,25 +748,16 @@ raw2trace_t::process_offline_entry(raw2trace_thread_data_t *tdata,
             // process_next_thread_buffer() so there is no need to have a separate
             // check for it here.
             if (marker_type != TRACE_MARKER_TYPE_CPU_ID) {
+                if (syscall_template_file_reader_ != nullptr &&
+                    marker_type == TRACE_MARKER_TYPE_SYSCALL) {
+                    // Also writes out the delayed branches if any.
+                    if (!write_syscall_template(tdata, buf, buf_base,
+                                                static_cast<int>(marker_val)))
+                        return false;
+                }
                 if (delayed_branches_exist(tdata)) {
-                    // Any un-written delayed branches should've been written out already
-                    // at the prior syscall instruction.
-                    DR_ASSERT(marker_type != TRACE_MARKER_TYPE_SYSCALL);
                     return write_delayed_branches(tdata, buf_base,
                                                   reinterpret_cast<trace_entry_t *>(buf));
-                } else if (marker_type == TRACE_MARKER_TYPE_SYSCALL) {
-                    size_t size = reinterpret_cast<trace_entry_t *>(buf) - buf_base;
-                    if ((uint)size >= WRITE_BUFFER_SIZE) {
-                        tdata->error = "Too many entries";
-                        return false;
-                    }
-                    if (size > 0 &&
-                        !write(tdata, buf_base, reinterpret_cast<trace_entry_t *>(buf))) {
-                        return false;
-                    }
-                    buf = reinterpret_cast<byte *>(buf_base);
-                    if (!write_syscall_template(tdata, static_cast<int>(marker_val)))
-                        return false;
                 }
             }
             log(3, "Appended marker type %u value " PIFX "\n", marker_type,
@@ -1403,6 +1430,9 @@ raw2trace_t::process_tasks(std::vector<raw2trace_thread_data_t *> *tasks)
     }
 }
 
+// XXX i#6495: This assumes that all contents of the file can easily fit into memory.
+// With zipfile support we can potentially stream only the required component (the one
+// with the trace template we want) when needed in write_syscall_template().
 std::string
 raw2trace_t::read_syscall_template_file()
 {
@@ -1411,9 +1441,8 @@ raw2trace_t::read_syscall_template_file()
     int last_syscall_num = -1;
     bool first_entry_for_syscall = false;
     // This object works for the eof check with any type of record_reader_t.
-    auto record_reader_end = std::unique_ptr<dynamorio::drmemtrace::record_reader_t>(
-        new dynamorio::drmemtrace::record_file_reader_t<std::ifstream>());
-    while (*syscall_template_file_reader_ != *record_reader_end) {
+    dynamorio::drmemtrace::record_file_reader_t<std::ifstream> record_reader_end;
+    while (*syscall_template_file_reader_ != record_reader_end) {
         trace_entry_t entry = **syscall_template_file_reader_;
         ++(*syscall_template_file_reader_);
         // Track encodings for system call template instructions. We do not need the

clients/drcachesim/tracer/raw2trace.h

@@ -1265,12 +1265,20 @@ class raw2trace_t {
     std::string
     read_syscall_template_file();
 
+    /**
+     * Returns the app pc of the first instruction in the system call template
+     * read for syscall_num. Returns nullptr if it could not find it.
+     */
+    app_pc
+    get_first_app_pc_for_syscall_template(int syscall_num);
+
     /**
      * Writes the system call template to the output trace, if any was provided in
      * the system call template file for the given syscall_num.
      */
     bool
-    write_syscall_template(raw2trace_thread_data_t *tdata, int syscall_num);
+    write_syscall_template(raw2trace_thread_data_t *tdata, byte *&buf,
+                           trace_entry_t *buf_base, int syscall_num);
 
     /**
      * The pointer to the DR context.