move scratchpad to outside with TL interface, add mvout_spad instruct…

…ion that writes scaled results to spad

src/main/scala/gemmini/AccumulatorMem.scala

@@ -92,7 +92,7 @@ class AccPipeShared[T <: Data : Arithmetic](latency: Int, t: Vec[Vec[T]], banks:
 class AccumulatorMem[T <: Data, U <: Data](
   n: Int, t: Vec[Vec[T]], scale_func: (T, U) => T, scale_t: U,
   acc_singleported: Boolean, acc_sub_banks: Int,
-  use_shared_ext_mem: Boolean,
+  use_shared_ext_mem: Boolean, use_tl_ext_ram: Boolean,
   acc_latency: Int, acc_type: T, is_dummy: Boolean
 )
   (implicit ev: Arithmetic[T]) extends Module {
@@ -134,8 +134,46 @@ class AccumulatorMem[T <: Data, U <: Data](
   val mask_len = t.getWidth / 8
   val mask_elem = UInt((t.getWidth / mask_len).W)
 
+  // val ext_mem_write_q_enq = if (use_shared_ext_mem && use_tl_ext_ram) {
+  //   require(acc_sub_banks == 1)
+  //   Some(io.ext_mem.get.map { ext_mem =>
+  //     val write_q = Module(new Queue(new Bundle {
+  //       val write_addr = UInt()
+  //       val write_data = UInt()
+  //       val write_mask = UInt()
+  //     }, 8, pipe = true, flow = true))
+
+  //     write_q.io.enq.valid := false.B
+  //     write_q.io.enq.bits := DontCare
+
+  //     ext_mem.write_valid := write_q.io.deq.valid
+  //     ext_mem.write_addr := write_q.io.deq.bits.write_addr
+  //     ext_mem.write_data := write_q.io.deq.bits.write_data
+  //     ext_mem.write_mask := write_q.io.deq.bits.write_mask
+  //     write_q.io.deq.ready := ext_mem.write_ready
+  //     write_q.io.enq
+  //   })
+  // } else None
+  io.ext_mem.get.foreach(_.write_req.valid := false.B)
+  io.ext_mem.get.foreach(_.write_req.bits.addr := 0.U(io.write.bits.addr.getWidth.W))
+  io.ext_mem.get.foreach(_.write_req.bits.mask := 0.U(io.write.bits.mask.getWidth.W))
+  io.ext_mem.get.foreach(_.write_req.bits.data := 0.U(io.write.bits.data.getWidth.W))
+  io.ext_mem.get.foreach(_.read_req.bits := 0.U((mask_len * mask_elem.getWidth).W))
+  io.ext_mem.get.foreach(_.read_req.valid := false.B)
+  io.ext_mem.get.foreach(_.read_resp.ready := false.B) // no reading from external accmem
   if (!acc_singleported && !is_dummy) {
-    require(!use_shared_ext_mem)
+    // if (use_shared_ext_mem && use_tl_ext_ram) {
+    //   // duplicate write to external memory
+    //   val enq = ext_mem_write_q_enq.get(0)
+    //   enq.valid := oldest_pipelined_write.valid
+    //   enq.bits.write_addr := oldest_pipelined_write.bits.addr
+    //   enq.bits.write_data := Mux(oldest_pipelined_write.bits.acc, adder_sum.asUInt, oldest_pipelined_write.bits.data.asUInt)
+    //   enq.bits.write_mask := oldest_pipelined_write.bits.mask.asUInt
+    //   // TODO (richard): add buffer here and potentially propagate backpressure to systolic array
+    //   assert(enq.ready || !enq.valid, "accumulator external memory write dropped")
+    // } else if (use_shared_ext_mem) {
+    //   require(false, "cannot use two-port external acc mem bank")
+    // }
     val mem = TwoPortSyncMem(n, t, mask_len) // TODO We assume byte-alignment here. Use aligned_to instead
     mem.io.waddr := oldest_pipelined_write.bits.addr
     mem.io.wen := oldest_pipelined_write.valid
@@ -163,27 +201,39 @@ class AccumulatorMem[T <: Data, U <: Data](
     for (i <- 0 until acc_sub_banks) {
       def isThisBank(addr: UInt) = addr(log2Ceil(acc_sub_banks)-1,0) === i.U
       def getBankIdx(addr: UInt) = addr >> log2Ceil(acc_sub_banks)
-      val (read, write) = if (use_shared_ext_mem) {
+      val (read, write) = if (use_shared_ext_mem && !use_tl_ext_ram) {
         def read(addr: UInt, ren: Bool): Data = {
-          io.ext_mem.get(i).read_en := ren
-          io.ext_mem.get(i).read_addr := addr
-          io.ext_mem.get(i).read_data
+          io.ext_mem.get(i).read_req.valid := ren
+          io.ext_mem.get(i).read_req.bits := addr
+          io.ext_mem.get(i).read_resp.bits
         }
-        io.ext_mem.get(i).write_en := false.B
-        io.ext_mem.get(i).write_addr := DontCare
-        io.ext_mem.get(i).write_data := DontCare
-        io.ext_mem.get(i).write_mask := DontCare
+        io.ext_mem.get(i).write_req.bits := DontCare
         def write(addr: UInt, wdata: Vec[UInt], wmask: Vec[Bool]) = {
-          io.ext_mem.get(i).write_en := true.B
-          io.ext_mem.get(i).write_addr := addr
-          io.ext_mem.get(i).write_data := wdata.asUInt
-          io.ext_mem.get(i).write_mask := wmask.asUInt
+          io.ext_mem.get(i).write_req.valid := true.B
+          io.ext_mem.get(i).write_req.bits.addr := addr
+          io.ext_mem.get(i).write_req.bits.data := wdata.asUInt
+          io.ext_mem.get(i).write_req.bits.mask := wmask.asUInt
         }
         (read _, write _)
       } else {
         val mem = SyncReadMem(n / acc_sub_banks, Vec(mask_len, mask_elem))
+        io.ext_mem.get(i).read_req.bits := 0.U((mask_len * mask_elem.getWidth).W)
+        io.ext_mem.get(i).read_req.valid := false.B
+
         def read(addr: UInt, ren: Bool): Data = mem.read(addr, ren)
-        def write(addr: UInt, wdata: Vec[UInt], wmask: Vec[Bool]) = mem.write(addr, wdata, wmask)
+        def write(addr: UInt, wdata: Vec[UInt], wmask: Vec[Bool]) = if (use_tl_ext_ram) {
+          mem.write(addr, wdata, wmask)
+          // duplicate write signal to external memory
+          // val enq = ext_mem_write_q_enq.get(i)
+          // enq.valid := true.B
+          // enq.bits.write_mask := wmask.asUInt
+          // enq.bits.write_addr := addr
+          // enq.bits.write_data := wdata.asUInt
+          // // TODO (richard): propagate backpressure to systolic array, add fence ability
+          // assert(enq.ready, "accumulator external memory write dropped")
+        } else {
+          mem.write(addr, wdata, wmask)
+        }
         (read _, write _)
       }
 

src/main/scala/gemmini/Controller.scala

@@ -3,14 +3,13 @@ package gemmini
 
 import java.nio.charset.StandardCharsets
 import java.nio.file.{Files, Paths}
-
 import chisel3._
 import chisel3.util._
 import org.chipsalliance.cde.config._
 import freechips.rocketchip.diplomacy._
 import freechips.rocketchip.tile._
 import freechips.rocketchip.util.ClockGate
-import freechips.rocketchip.tilelink.TLIdentityNode
+import freechips.rocketchip.tilelink.{TLBundle, TLClientNode, TLEdgeOut, TLFragmenter, TLIdentityNode, TLManagerNode, TLMasterParameters, TLMasterPortParameters, TLMasterToSlaveTransferSizes, TLRAM, TLSlaveParameters, TLSlavePortParameters, TLWidthWidget, TLXbar}
 import GemminiISA._
 import Util._
 
@@ -35,11 +34,115 @@ class Gemmini[T <: Data : Arithmetic, U <: Data, V <: Data](val config: GemminiA
   val xLen = p(XLen)
   val spad = LazyModule(new Scratchpad(config))
 
+  val create_tl_mem = config.use_shared_ext_mem && config.use_tl_ext_mem
+
+  val num_ids = 32 // TODO (richard): move to config
+  val spad_base = 0 // 0x60000000L
+
+  val unified_mem_read_node = TLIdentityNode()
+  val spad_read_nodes = if (create_tl_mem) TLClientNode(Seq.tabulate(config.sp_banks) {i =>
+    TLMasterPortParameters.v1(Seq(TLMasterParameters.v1(name = s"spad_read_node_$i", sourceId = IdRange(0, num_ids))))
+  }) else TLIdentityNode()
+  // val acc_read_nodes = if (create_tl_mem) TLClientNode(Seq.tabulate(config.acc_banks) { i =>
+  //   TLMasterPortParameters.v1(Seq(TLMasterParameters.v1(name = s"acc_read_node_$i", sourceId = IdRange(0, numIDs))))
+  // }) else TLIdentityNode()
+
+  val unified_mem_write_node = TLIdentityNode()
+  val spad_write_nodes = if (create_tl_mem) TLClientNode(Seq.tabulate(config.sp_banks) { i =>
+    TLMasterPortParameters.v1(Seq(TLMasterParameters.v1(name = s"spad_write_node_$i", sourceId = IdRange(0, num_ids))))
+  }) else TLIdentityNode()
+
+  // val spad_dma_write_node = TLClientNode(Seq(
+  //   TLMasterPortParameters.v1(Seq(TLMasterParameters.v1(name = s"spad_dma_write_node", sourceId = IdRange(0, num_ids))))))
+  // val acc_write_nodes = if (create_tl_mem) TLClientNode(Seq.tabulate(config.acc_banks) { i =>
+  //   TLMasterPortParameters.v1(Seq(TLMasterParameters.v1(name = s"acc_write_node_$i", sourceId = IdRange(0, numIDs))))
+  // }) else TLIdentityNode()
+
+  val spad_data_len = config.sp_width / 8
+  val acc_data_len = config.sp_width / config.inputType.getWidth * config.accType.getWidth / 8
+  val max_data_len = spad_data_len // max acc_data_len
+
+  val spad_tl_ram : Seq[Seq[TLManagerNode]] = if (config.use_shared_ext_mem && config.use_tl_ext_mem) {
+    unified_mem_read_node :=* TLWidthWidget(spad_data_len) :=* spad_read_nodes
+    // unified_mem_read_node :=* TLWidthWidget(acc_data_len) :=* acc_read_nodes
+    unified_mem_write_node :=* TLWidthWidget(spad_data_len) :=* spad_write_nodes
+    // unified_mem_write_node :=* TLWidthWidget(acc_data_len) :=* acc_write_nodes
+
+    val stride_by_word = false // TODO (richard): move to config
+
+    require(isPow2(config.sp_banks))
+    val banks : Seq[Seq[TLManagerNode]] =
+    if (stride_by_word) {
+      assert(false, "TODO under construction")
+      assert((config.sp_capacity match { case CapacityInKilobytes(kb) => kb * 1024}) ==
+        config.sp_bank_entries * spad_data_len / max_data_len * config.sp_banks * max_data_len)
+      (0 until config.sp_banks).map { bank =>
+        LazyModule(new TLRAM(
+          address = AddressSet(max_data_len * bank,
+            ((config.sp_bank_entries * spad_data_len / max_data_len - 1) * config.sp_banks + bank)
+              * max_data_len + (max_data_len - 1)),
+          beatBytes = max_data_len
+        ))
+      }.map(x => Seq(x.node))
+    } else {
+      (0 until config.sp_banks).map { bank =>
+        val mem_depth = config.sp_bank_entries * spad_data_len / max_data_len
+        val mem_width = max_data_len
+
+        Seq(TLManagerNode(Seq(TLSlavePortParameters.v1(
+          managers = Seq(TLSlaveParameters.v2(
+            name = Some(f"sp_bank${bank}_read_mgr"),
+            address = Seq(AddressSet(spad_base + (mem_depth * mem_width * bank),
+              mem_depth * mem_width - 1)),
+            supports = TLMasterToSlaveTransferSizes(
+              get = TransferSizes(1, mem_width)),
+            fifoId = Some(0)
+          )),
+          beatBytes = mem_width
+        ))),
+        TLManagerNode(Seq(TLSlavePortParameters.v1(
+          managers = Seq(TLSlaveParameters.v2(
+            name = Some(f"sp_bank${bank}_write_mgr"),
+            address = Seq(AddressSet(spad_base + (mem_depth * mem_width * bank),
+              mem_depth * mem_width - 1)),
+            supports = TLMasterToSlaveTransferSizes(
+              putFull = TransferSizes(1, mem_width),
+              putPartial = TransferSizes(1, mem_width)),
+            fifoId = Some(0)
+          )),
+          beatBytes = mem_width
+        ))))
+      }
+    }
+
+    require(!config.sp_singleported)
+    if (config.sp_singleported) {
+      val xbar = TLXbar()
+      xbar :=* unified_mem_read_node
+      xbar :=* unified_mem_write_node
+      banks.foreach(_.head := xbar)
+    } else {
+      val r_xbar = TLXbar()
+      val w_xbar = TLXbar()
+      r_xbar :=* unified_mem_read_node
+      w_xbar :=* unified_mem_write_node
+      banks.foreach { mem =>
+        require(mem.length == 2)
+        mem.head := r_xbar
+        mem.last := TLFragmenter(spad_data_len, spad.maxBytes) := w_xbar
+      }
+    }
+
+    banks
+  } else Seq()
+
   override lazy val module = new GemminiModule(this)
   override val tlNode = if (config.use_dedicated_tl_port) spad.id_node else TLIdentityNode()
   override val atlNode = if (config.use_dedicated_tl_port) TLIdentityNode() else spad.id_node
 
   val node = if (config.use_dedicated_tl_port) tlNode else atlNode
+
+  unified_mem_write_node := spad.spad_writer.node
 }
 
 class GemminiModule[T <: Data: Arithmetic, U <: Data, V <: Data]
@@ -50,8 +153,121 @@ class GemminiModule[T <: Data: Arithmetic, U <: Data, V <: Data]
   import outer.config._
   import outer.spad
 
-  val ext_mem_io = if (use_shared_ext_mem) Some(IO(new ExtSpadMemIO(sp_banks, acc_banks, acc_sub_banks))) else None
-  ext_mem_io.foreach(_ <> outer.spad.module.io.ext_mem.get)
+  val ext_mem_io = if (use_shared_ext_mem && !use_tl_ext_mem)
+    Some(IO(new ExtSpadMemIO(sp_banks, acc_banks, acc_sub_banks))) else None
+
+  // we need these 2 separate signals because ext_mem_io is not writable in this module
+  val ext_mem_spad = outer.spad.module.io.ext_mem.get.spad
+  val ext_mem_acc = outer.spad.module.io.ext_mem.get.acc
+
+  // connecting to unified TL interface
+  val source_counters = Seq.fill(4)(Counter(outer.num_ids))
+
+  if (outer.create_tl_mem) {
+    def connect(ext_mem: ExtMemIO, req_size: Int, r_node: TLBundle, r_edge: TLEdgeOut, r_source: Counter,
+                w_node: TLBundle, w_edge: TLEdgeOut, w_source: Counter): Unit = {
+      r_node.a.valid := ext_mem.read_req.valid
+      r_node.a.bits := r_edge.Get(r_source.value,
+        (ext_mem.read_req.bits << req_size.U).asUInt | outer.spad_base.U,
+        req_size.U)._2
+      ext_mem.read_req.ready := r_node.a.ready
+
+      val w_shifted_addr = (ext_mem.write_req.bits.addr << req_size.U).asUInt
+      val w_mask = (ext_mem.write_req.bits.mask << (w_shifted_addr & (w_edge.manager.beatBytes - 1).U)).asUInt
+
+      w_node.a.valid := ext_mem.write_req.valid
+      w_node.a.bits := w_edge.Put(w_source.value,
+        w_shifted_addr | outer.spad_base.U,
+        req_size.U, ext_mem.write_req.bits.data, w_mask)._2
+      ext_mem.write_req.ready := w_node.a.ready
+
+      ext_mem.read_resp.valid := r_node.d.valid
+      ext_mem.read_resp.bits := r_node.d.bits.data
+      r_node.d.ready := ext_mem.read_resp.ready
+
+      w_node.d.ready := true.B // writes are not acknowledged in gemmini
+
+      when(ext_mem.read_req.fire) { r_source.inc() }
+      when(ext_mem.write_req.fire) { w_source.inc() }
+    }
+    (outer.spad_read_nodes.out zip outer.spad_write_nodes.out)
+      .zipWithIndex.foreach{ case (((r_node, r_edge), (w_node, w_edge)), i) =>
+        connect(ext_mem_spad(i), log2Up(outer.spad_data_len),
+          r_node, r_edge, source_counters(0), w_node, w_edge, source_counters(1))
+    }
+
+    outer.spad_tl_ram.foreach { case Seq(r, w) =>
+      val mem_depth = outer.config.sp_bank_entries * outer.spad_data_len / outer.max_data_len
+      val mem_width = outer.max_data_len
+
+      val mem = TwoPortSyncMem(
+        n = mem_depth,
+        t = UInt((mem_width * 8).W),
+        mask_len = mem_width // byte level mask
+      )
+
+      val (r_node, r_edge) = r.in.head
+      val (w_node, w_edge) = w.in.head
+
+      // READ
+      mem.io.ren := r_node.a.fire
+      mem.io.raddr := r_node.a.bits.address ^ outer.spad_base.U
+
+      val data_pipe_in = Wire(DecoupledIO(mem.io.rdata.cloneType))
+      data_pipe_in.valid := RegNext(mem.io.ren)
+      data_pipe_in.bits := mem.io.rdata
+
+      val metadata_pipe_in = Wire(DecoupledIO(new Bundle {
+        val source = r_node.a.bits.source.cloneType
+        val size = r_node.a.bits.size.cloneType
+      }))
+      metadata_pipe_in.valid := mem.io.ren
+      metadata_pipe_in.bits.source := r_node.a.bits.source
+      metadata_pipe_in.bits.size := r_node.a.bits.size
+
+      val data_pipe_inst = Module(new Pipeline(data_pipe_in.bits.cloneType, 1)())
+      data_pipe_inst.io.in <> data_pipe_in
+      val data_pipe = data_pipe_inst.io.out
+      val metadata_pipe = Pipeline(metadata_pipe_in, 2)
+      assert(data_pipe_in.ready || !data_pipe_in.valid)
+      assert(metadata_pipe_in.ready || !data_pipe_in.ready)
+      assert(data_pipe.valid === metadata_pipe.valid)
+
+      r_node.d.bits := r_edge.AccessAck(
+        metadata_pipe.bits.source,
+        metadata_pipe.bits.size,
+        data_pipe.bits)
+      r_node.d.valid := data_pipe.valid
+      // take new requests only we have the buffer slot open in case downstream becomes unready
+      r_node.a.ready := r_node.d.ready && !data_pipe_inst.io.busy
+      data_pipe.ready := r_node.d.ready
+      metadata_pipe.ready := r_node.d.ready
+
+      // WRITE
+      mem.io.wen := w_node.a.fire
+      mem.io.waddr := w_node.a.bits.address ^ outer.spad_base.U
+      mem.io.wdata := w_node.a.bits.data
+      mem.io.mask := w_node.a.bits.mask.asBools
+      w_node.a.ready := w_node.d.ready// && (mem.io.waddr =/= mem.io.raddr)
+      w_node.d.valid := w_node.a.valid
+      w_node.d.bits := w_edge.AccessAck(w_node.a.bits)
+    }
+
+    ext_mem_acc.foreach(_.foreach(x => {
+      x.read_resp.bits := 0.U(1.W)
+      x.read_resp.valid := false.B
+      x.read_req.ready := false.B
+      x.write_req.ready := false.B
+    }))
+    // (outer.acc_read_nodes.out zip outer.acc_write_nodes.out)
+    //   .zipWithIndex.foreach { case (((r_node, r_edge), (w_node, w_edge)), i) =>
+    //     // TODO (richard): one subbank only for now
+    //     connect(ext_mem_acc(i)(0), log2Up(outer.acc_data_len),
+    //       r_node, r_edge, source_counters(2), w_node, w_edge, source_counters(3))
+    // }
+  } else if (use_shared_ext_mem) {
+    ext_mem_io.foreach(_ <> outer.spad.module.io.ext_mem.get)
+  }
 
   val tagWidth = 32
 
@@ -66,7 +282,8 @@ class GemminiModule[T <: Data: Arithmetic, U <: Data, V <: Data]
 
   // TLB
   implicit val edge = outer.spad.id_node.edges.out.head
-  val tlb = Module(new FrontendTLB(2, tlb_size, dma_maxbytes, use_tlb_register_filter, use_firesim_simulation_counters, use_shared_tlb))
+  // TODO(richard): bypass TLB
+  val tlb = Module(new FrontendTLB(3, tlb_size, dma_maxbytes, use_tlb_register_filter, use_firesim_simulation_counters, use_shared_tlb))
   (tlb.io.clients zip outer.spad.module.io.tlb).foreach(t => t._1 <> t._2)
 
   tlb.io.exp.foreach(_.flush_skip := false.B)

src/main/scala/gemmini/CustomConfigs.scala

@@ -49,8 +49,18 @@ object GemminiCustomConfigs {
     acc_capacity = CapacityInKilobytes(128),
   )
 
+  val unifiedMemConfig = defaultConfig.copy(
+    has_training_convs = false,
+    has_max_pool = false,
+    use_tl_ext_mem = true,
+    sp_singleported = false,
+    spad_read_delay = 8,
+    use_shared_ext_mem = true,
+    acc_sub_banks = 1
+  )
+
   // Specify which of your custom configs you want to build here
-  val customConfig = baselineInferenceConfig
+  val customConfig = unifiedMemConfig
 }