Add chiplet router

README.md

@@ -19,6 +19,7 @@ BAR projects generally use these components with [chipyard](https://chipyard.rea
 
 ## ``testchipip.ctc``
 * ``CTC``: Simple, 32-bit chip-to-chip interface
+* ``CTCMem``: SerialRAM-like backing memory for CTC
 
 ## ``testchipip.dram``
 * ``SimDRAM``: DRAMSim-backed AXI-4 memory model
@@ -35,6 +36,7 @@ BAR projects generally use these components with [chipyard](https://chipyard.rea
 
 ## ``testchipip.soc``
 * ``OffchipBus``: Custom bus for interfacing with off-chip memory
+* ``OffchipRouter``: Routes requests to the die-to-die ports
 * ``Scratchpad``: TileLink SRAM-backed on-chip scratchpad memory
 * ``SimDTM``: Simulation model for interacting with on-chip debug module
 * ``TLNetwork``: DEPRECATED mechanism for creating a TileLink network-on-chip. Use Constellation NoC instead

src/main/resources/testchipip/csrc/SimTSI.cc

@@ -49,7 +49,7 @@ extern "C" int tsi_tick(
         remove_vcs_simv_opt(info.argc, info.argv);
 
         // TODO: We should somehow inspect whether or not our backing memory supports loadmem, instead of unconditionally setting it to true
-        tsis[chip_id] = new testchip_tsi_t(info.argc, info.argv, true);
+        tsis[chip_id] = new testchip_tsi_t(info.argc, info.argv, true, chip_id);
     }
 
     testchip_tsi_t* tsi = tsis[chip_id];

src/main/resources/testchipip/csrc/testchip_tsi.cc

@@ -1,7 +1,7 @@
 #include "testchip_tsi.h"
 #include <stdexcept>
 
-testchip_tsi_t::testchip_tsi_t(int argc, char** argv, bool can_have_loadmem) : tsi_t(argc, argv)
+testchip_tsi_t::testchip_tsi_t(int argc, char** argv, bool can_have_loadmem, int chip_id) : tsi_t(argc, argv)
 {
   has_loadmem = false;
   init_accesses = std::vector<init_access_t>();
@@ -14,6 +14,27 @@ testchip_tsi_t::testchip_tsi_t(int argc, char** argv, bool can_have_loadmem) : t
       has_loadmem = can_have_loadmem;
     if (arg.find("+cflush_addr=0x") == 0)
       cflush_addr = strtoull(arg.substr(15).c_str(), 0, 16);
+    // We cannot use init_write to program the chip ID directly because each chip id needs to be programmed individually,
+    // and init_write does not distinguish between chips.
+    if (arg.find("+chip_id") == 0) {
+      static const std::string prefix = "+chip_id";
+      auto eq = arg.find('=');
+      if (eq == std::string::npos || eq <= prefix.size() || arg.find("=0x", eq) != eq) {
+        throw std::invalid_argument("Improperly formatted +chip_id argument");
+      }
+      unsigned long arg_chip_id = strtoul(arg.substr(prefix.size(), eq - prefix.size()).c_str(), 0, 0);
+      if ((int)arg_chip_id == chip_id) {
+        auto d = arg.find(":0x", eq + 3);
+        if (d == std::string::npos) {
+          throw std::invalid_argument("Improperly formatted +chip_id argument");
+        }
+        uint64_t addr = strtoull(arg.substr(eq + 3, d - (eq + 3)).c_str(), 0, 16);
+        uint32_t val = strtoull(arg.substr(d + 3).c_str(), 0, 16);
+        init_access_t access = { .address=addr, .stdata=val, .store=true };
+        init_accesses.push_back(access);
+      }
+    }
+
   }
 
   testchip_htif_t::parse_htif_args(args);

src/main/resources/testchipip/csrc/testchip_tsi.h

@@ -10,7 +10,7 @@
 class testchip_tsi_t : public tsi_t, public testchip_htif_t
 {
  public:
-  testchip_tsi_t(int argc, char** argv, bool has_loadmem);
+  testchip_tsi_t(int argc, char** argv, bool has_loadmem, int chip_id);
   virtual ~testchip_tsi_t() {};
 
   void write_chunk(addr_t taddr, size_t nbytes, const void* src) override;

src/main/scala/ctc/CTC.scala

@@ -14,7 +14,6 @@ import testchipip.soc._
 
 import testchipip.serdes._
 
-
 object CTC {
   val INNER_WIDTH = 32
   val INNER_WIDTH_BYTES = INNER_WIDTH / 8
@@ -29,28 +28,141 @@ object CTCCommand {
 }
 
 case class CTCParams(
-  translationParams: AddressTranslatorParams = OutwardAddressTranslatorParams(onchipAddr = 0x1000000000L, offchipAddr = 0x0L, size = ((1L << 32) - 1)),
+  translationParams: Option[AddressTranslatorParams] = Some(OutwardAddressTranslatorParams(onchipAddr = 0x1000000000L, offchipAddr = 0x0L, size = ((1L << 32) - 1))),
   offchip: Seq[AddressSet] = Nil,
-  managerBus: Option[TLBusWrapperLocation] = Some(SBUS),
-  clientBus: Option[TLBusWrapperLocation] = Some(SBUS),
-  phyParams: Option[SerialPhyParams] = Some(CreditedSourceSyncSerialPhyParams(phitWidth = CTC.OUTER_WIDTH, flitWidth = CTC.INNER_WIDTH, freqMHz = 100, flitBufferSz = 16)) // Set to None to disable PHY
-) {
-  def offchipRange = translationParams match {
-    case OutwardAddressTranslatorParams(_,_,_) => Seq(translationParams.offchipRange)
-    case InwardAddressTranslatorParams(_,_,_) => {
-      require(offchip != Nil)
-      offchip
+  managerBus: TLBusWrapperLocation = SBUS,
+  clientBus: TLBusWrapperLocation = SBUS,
+  phyParams: Option[SerialPhyParams] = Some(CreditedSourceSyncSerialPhyParams(phitWidth = CTC.OUTER_WIDTH, flitWidth = CTC.INNER_WIDTH, freqMHz = 100, flitBufferSz = 16)) // Set to None to remove PHY
+) extends ChipletLinkParams
+ with ChipletLinkWrapperInstantiationLike 
+ {
+  def offchipRange = translationParams.map { tp =>
+    tp match {
+      case OutwardAddressTranslatorParams(_,_,_) => Seq(tp.offchipRange)
+      case InwardAddressTranslatorParams(_,_,_) => {
+        require(offchip != Nil)
+        offchip
+      }
     }
-  }
+  }.getOrElse(offchip)
+
+  def managerBusWhere = managerBus
+  def controlManagerBusWhere = None
+  def instantiate(params: OffchipSubsystemParams, id: Int)(implicit p: Parameters): ChipletLinkWrapper = LazyModule(new CTCChipletLink(this, params, id))
 }
 
+case object CTCKey extends Field[Seq[CTCParams]](Nil)
+
 // For using CTC in a chiplet firesim config with no PHY
-class CTCBridgeIO extends Bundle {
+class CTCBridgeIO extends ChipletIO {
   val client_flit = new DecoupledFlitIO(CTC.INNER_WIDTH) // Driven by client/ctc2tl
   val manager_flit = new DecoupledFlitIO(CTC.INNER_WIDTH) // Driven by manager/tl2ctc
+
+  def tieoff: Unit = {
+    manager_flit := DontCare
+    manager_flit.in.valid := false.B
+    manager_flit.out.ready := false.B
+    client_flit := DontCare
+    client_flit.in.valid := false.B
+    client_flit.out.ready := false.B
+  }
+
+  def connect(io: ChipletIO): Unit = io match {
+    case io: CTCBridgeIO => {
+      client_flit.in <> io.manager_flit.out
+      io.client_flit.in <> manager_flit.out
+      manager_flit.in <> io.client_flit.out
+      io.manager_flit.in <> client_flit.out
+    }
+    case _ => assert(false, s"IO does not match CTCBridgeIO: ${io.getClass}")
+  }
+
+  def loopback: Unit = {
+    client_flit.in <> manager_flit.out
+    manager_flit.in <> client_flit.out
+  }
 }
 
-case object CTCKey extends Field[Seq[CTCParams]](Nil)
+class CTCMemIO(phitWidth: Int, offchip: Seq[AddressSet], phyParams: SerialPhyParams) extends DecoupledInternalSyncPhitIO(phitWidth) {
+  def connectRAM(implicit p: Parameters): Unit = {
+    withClock(clock_out) {
+      val ram = Module(LazyModule(new CTCMem(offchip, phyParams)(p)).module)
+      ram.io.ser.in <> out
+      in <> ram.io.ser.out
+    }
+  }
+}
+
+case class CTCMemSerialPhyParams(
+  phitWidth: Int = CTC.OUTER_WIDTH,
+  flitWidth: Int = CTC.INNER_WIDTH,
+  flitBufferSz: Int = 16,
+  offchip: Seq[AddressSet]) extends SerialPhyParams {
+  def genIO = new CTCMemIO(phitWidth, offchip, this)
+}
+
+class CTCChipletLink(val params: CTCParams, val sys_params: OffchipSubsystemParams, val id: Int)(implicit p: Parameters) extends ChipletLinkWrapper {
+  // a TL master/client device
+  val ctc2tl = LazyModule(new CTCToTileLink(portId=id)(p))
+  // a TL slave/manager device
+  val tl2ctc = LazyModule(new TileLinkToCTC(addrRegion=sys_params.managerRegion)(p))
+
+  val client_node = ctc2tl.node
+  val manager_node = tl2ctc.node
+  val control_manager_node = None
+  val clock_node = None
+  val top_IO = params.phyParams match {
+    case Some(pP) => BundleBridgeSource(() => pP.genIO.asInstanceOf[ChipletIO])
+    case None     => BundleBridgeSource(() => new CTCBridgeIO)
+  }
+  override lazy val module = new CTCChipletLinkImpl(this)
+}
+
+class CTCChipletLinkImpl(outer: CTCChipletLink) extends LazyModuleImp(outer) {
+  val io = outer.top_IO.out(0)._1
+
+  io match {
+    case io: CreditedSourceSyncPhitIO => {
+      val outgoing_clock = clock
+      val outgoing_reset = ResetCatchAndSync(outgoing_clock, reset.asBool)
+      val incoming_clock = io.clock_in
+      val incoming_reset = ResetCatchAndSync(incoming_clock, io.reset_in.asBool)
+      io.clock_out := outgoing_clock
+      io.reset_out := outgoing_reset.asAsyncReset
+      val phy = Module(new CreditedSerialPhy(2, outer.params.phyParams.get))
+      phy.io.incoming_clock := incoming_clock
+      phy.io.incoming_reset := incoming_reset
+      phy.io.outgoing_clock := outgoing_clock
+      phy.io.outgoing_reset := outgoing_reset
+      phy.io.inner_clock := outer.ctc2tl.module.clock
+      phy.io.inner_reset := outer.ctc2tl.module.reset
+      phy.io.inner_ser(0).in <> outer.ctc2tl.module.io.flit.in
+      phy.io.inner_ser(0).out <> outer.tl2ctc.module.io.flit.out
+      phy.io.inner_ser(1).in <> outer.tl2ctc.module.io.flit.in
+      phy.io.inner_ser(1).out <> outer.ctc2tl.module.io.flit.out
+      phy.io.outer_ser <> io.viewAsSupertype(new ValidPhitIO(outer.params.phyParams.get.phitWidth))
+    }
+    case io: CTCMemIO => {
+      val outgoing_clock = clock
+      io.clock_out := outgoing_clock
+      val phy = Module(new DecoupledSerialPhy(2, outer.params.phyParams.get))
+      phy.io.outer_clock := outgoing_clock
+      phy.io.outer_reset := ResetCatchAndSync(outgoing_clock, reset.asBool)
+      phy.io.inner_clock := outer.ctc2tl.module.clock
+      phy.io.inner_reset := outer.ctc2tl.module.reset
+      phy.io.inner_ser(0).in <> outer.ctc2tl.module.io.flit.in
+      phy.io.inner_ser(0).out <> outer.tl2ctc.module.io.flit.out
+      phy.io.inner_ser(1).in <> outer.tl2ctc.module.io.flit.in
+      phy.io.inner_ser(1).out <> outer.ctc2tl.module.io.flit.out
+      phy.io.outer_ser <> io.viewAsSupertype(new DecoupledPhitIO(outer.params.phyParams.get.phitWidth))
+    }
+    case io: CTCBridgeIO => {
+      io.manager_flit <> outer.tl2ctc.module.io.flit
+      io.client_flit <> outer.ctc2tl.module.io.flit
+    }
+    case _ => assert(false, s"IO unsupported for CTC: ${io.getClass}")
+  }
+}
 
 trait CanHavePeripheryCTC { this: BaseSubsystem =>
   private val portName = "ctc"
@@ -65,15 +177,15 @@ trait CanHavePeripheryCTC { this: BaseSubsystem =>
         assert(pP.flitWidth == CTC.INNER_WIDTH)
       }
 
-      lazy val slave_bus = locateTLBusWrapper(params.managerBus.get)
-      lazy val master_bus = locateTLBusWrapper(params.clientBus.get)
+      lazy val slave_bus = locateTLBusWrapper(params.managerBus)
+      lazy val master_bus = locateTLBusWrapper(params.clientBus)
 
       val ctc_domain = LazyModule(new ClockSinkDomain(name=Some(s"CTC$id")))
       ctc_domain.clockNode := slave_bus.fixedClockNode
 
-      val translator = ctc_domain {
-        LazyModule(AddressTranslator(params.translationParams)(p))
-      }
+      val translator = ctc_domain { params.translationParams.map { tp =>
+        LazyModule(AddressTranslator(tp)(p))
+      }}
 
       require(slave_bus.dtsFrequency.isDefined,
         s"Slave bus ${slave_bus.busName} must provide a frequency")
@@ -89,14 +201,18 @@ trait CanHavePeripheryCTC { this: BaseSubsystem =>
 
       params.translationParams match {
         // Translate outgoing requests
-        case OutwardAddressTranslatorParams(_,_,_) => {
-          slave_bus.coupleTo(portName) { translator(tl2ctc.node) := TLBuffer() := _ }
+        case Some(OutwardAddressTranslatorParams(_,_,_)) => {
+          slave_bus.coupleTo(portName) { translator.get(tl2ctc.node) := TLBuffer() := _ }
           master_bus.coupleFrom(portName) { _ := TLBuffer() := ctc2tl.node }
         }
         // Translate incoming requests
-        case InwardAddressTranslatorParams(_,_,_) => {
+        case Some(InwardAddressTranslatorParams(_,_,_)) => {
+          slave_bus.coupleTo(portName) { tl2ctc.node := TLBuffer() := _ }
+          master_bus.coupleFrom(portName) { _ := TLBuffer() := translator.get(ctc2tl.node) }
+        }
+        case None => {
           slave_bus.coupleTo(portName) { tl2ctc.node := TLBuffer() := _ }
-          master_bus.coupleFrom(portName) { _ := TLBuffer() := translator(ctc2tl.node) }
+          master_bus.coupleFrom(portName) { _ := TLBuffer() := ctc2tl.node }
         }
       }
 

src/main/scala/ctc/CTCMem.scala

@@ -8,19 +8,18 @@ import freechips.rocketchip.tilelink._
 import freechips.rocketchip.devices.tilelink._
 import freechips.rocketchip.diplomacy._
 import freechips.rocketchip.util._
-import testchipip.serdes.{DecoupledPhitIO, DecoupledSerialPhy}
+import testchipip.serdes.{DecoupledPhitIO, DecoupledSerialPhy, SerialPhyParams}
 import testchipip.ctc._
 
-// A test memory like SerialRAM but for CTC
-// Takes in DecoupledFlitIO, puts it through CTCToTileLink, then connects the TileLink node to TLRAM
-class CTCMem(params: CTCParams)(implicit p: Parameters) extends LazyModule {
+// A test memory like SerialRAM but for CTC, used for modeling chiplets as RAM
+class CTCMem(offchip: Seq[AddressSet], phyParams: SerialPhyParams)(implicit p: Parameters) extends LazyModule {
     val beatBytes = 8
     val ctc2tl = LazyModule(new CTCToTileLink(sourceIds = 1, portId = 0))
 
     val xbar = TLXbar()
 
     // Use smaller memory size for testing
-    val testmems = params.offchip.map(address => LazyModule(new TLRAM(address = AddressSet(address.base, 0xFFFFFL), beatBytes = beatBytes) {override lazy val desiredName = "CTCMem"}))
+    val testmems = offchip.map(address => LazyModule(new TLRAM(address = AddressSet(address.base, 0xFFFFFL), beatBytes = beatBytes) {override lazy val desiredName = "CTCMem"}))
     testmems.foreach { s => (s.node
       := TLBuffer()
       := TLFragmenter(beatBytes, p(CacheBlockBytes), nameSuffix = Some("CTCMem"))
@@ -35,7 +34,7 @@ class CTCMem(params: CTCParams)(implicit p: Parameters) extends LazyModule {
             val ser = new DecoupledPhitIO(CTC.OUTER_WIDTH)
         })
 
-        val phy = Module(new DecoupledSerialPhy(2, params.phyParams.get))
+        val phy = Module(new DecoupledSerialPhy(2, phyParams))
         phy.io.outer_clock := clock
         phy.io.outer_reset := reset
         phy.io.inner_clock := clock