From 6596ef41983fc28663773f808034bdd056edbfc0 Mon Sep 17 00:00:00 2001 From: Ella Schwarz Date: Mon, 23 Feb 2026 13:20:34 -0800 Subject: [PATCH 01/19] [WIP] add offchip router w/ d2d port instantiation, COMPILING --- src/main/scala/soc/ChipletLink.scala | 45 +++ src/main/scala/soc/OffchipRouter.scala | 449 +++++++++++++++++++++++++ 2 files changed, 494 insertions(+) create mode 100644 src/main/scala/soc/ChipletLink.scala create mode 100644 src/main/scala/soc/OffchipRouter.scala diff --git a/src/main/scala/soc/ChipletLink.scala b/src/main/scala/soc/ChipletLink.scala new file mode 100644 index 00000000..7a21ff64 --- /dev/null +++ b/src/main/scala/soc/ChipletLink.scala @@ -0,0 +1,45 @@ +package testchipip.soc + +import chisel3._ +import chisel3.util._ +import org.chipsalliance.cde.config.{Parameters} +import freechips.rocketchip.subsystem._ +import freechips.rocketchip.tilelink._ +import freechips.rocketchip.devices.tilelink._ +import freechips.rocketchip.diplomacy._ +import freechips.rocketchip.util._ +import freechips.rocketchip.prci._ +import freechips.rocketchip.diplomacy.BundleBridgeSource + +// Link params should extend this trait +trait ChipletLinkParams{ + def managerRegion: Seq[AddressSet] + def managerBusWhere: TLBusWrapperLocation // Where the link client node is connected + def controlManagerBusWhere: Option[TLBusWrapperLocation] // Where the link control node is connected + //def instantiationFn: _ => (TLInwardNode, TLOutwardNode, Option[TLRegisterNode]) +} + +abstract class ChipletLinkWrapper(implicit p: Parameters) extends LazyModule { + //val link_name: String + val client_node: TLClientNode + val manager_node: TLManagerNode + val control_manager_node: Option[TLRegisterNode] + val top_IO: BundleBridgeSource[ChipletIO] +} + +abstract class ChipletLinkWrapperImpl(outer: ChipletLinkWrapper) extends LazyModuleImp(outer) { + val io: ChipletIO +} + +// Links should "with" this trait +trait ChipletLinkWrapperInstantiationLike { + def instantiate(name: String, id: Int)(implicit p: Parameters): ChipletLinkWrapper +} + +// Should this be called D2D IO instead? +trait ChipletIO extends Bundle { + def tieoff: Unit + def connect(io: ChipletIO): Unit + def loopback: Unit +} + diff --git a/src/main/scala/soc/OffchipRouter.scala b/src/main/scala/soc/OffchipRouter.scala new file mode 100644 index 00000000..85c0bd02 --- /dev/null +++ b/src/main/scala/soc/OffchipRouter.scala @@ -0,0 +1,449 @@ +package testchipip.soc + +import chisel3._ +import chisel3.util._ +import org.chipsalliance.cde.config.{Parameters, Field, Config} +import freechips.rocketchip.subsystem._ +import freechips.rocketchip.tilelink._ +import freechips.rocketchip.devices.debug.HasPeripheryDebug +import freechips.rocketchip.devices.tilelink._ +import freechips.rocketchip.diplomacy._ +import freechips.rocketchip.util._ +import freechips.rocketchip.prci._ +import freechips.rocketchip.regmapper._ +import testchipip.ctc.CTCBridgeIO // TODO: remove this later +import scala.math.min +import testchipip.util.{TLSwitch} + +case class ChipletRoutingParams( + clientBusWhere: TLBusWrapperLocation = SBUS, + controlBusWhere: TLBusWrapperLocation = CBUS, + routerParams: OffchipRouterParams, + translationParams: InwardAddressTranslatorParams, + ports: Seq[ChipletLinkParams] +) + +case object ChipletRoutingKey extends Field[Option[ChipletRoutingParams]](None) + +case class OffchipRouterParams( + // Need to think about what I want here + // offchipBase: BigInt, + // offchipSize: BigInt, + tableAddress: BigInt = 0x4000L, + tableEntries: Int, + beatBytes: Int // maybe not here +) + +// class OffchipRouterWrapper(params: OffchipRouterParams, context: HasTileLinkLocations)(implicit p: Parameters) extends TLBusWrapper(params, "offchip_router") { +// private val offchip_router = LazyModule(new OffchipRouter(params)) +// private val cbus = context.locateTLBusWrapper(CBUS) +// offchip_router.routing_table_node :=* cbus.coupleTo(s"offchip_routing_table") { TLBuffer(1) := TLFragmenter(cbus) := _ } + +// val inwardNode: TLInwardNode = offchip_router.manager_node :=* TLFIFOFixer(TLFIFOFixer.allVolatile) +// val outwardNode: TLOutwardNode = offchip_router.switch.node +// def busView: TLEdge = offchip_router.switch.node.edges.in.head +// val builtInDevices = BuiltInDevices.none +// val prefixNode = None +// } + +class OffchipRouter(val params: OffchipRouterParams, val nPorts: Int, val offset: BigInt)(implicit p: Parameters) extends LazyModule { + + // BORROWED FROM TLSwitch.scala START + // This function can handle simple cases only + def unifyManagers(mgrs: Seq[Seq[TLSlaveParameters]]): Seq[TLSlaveParameters] = { + mgrs.flatten.groupBy(_.sortedAddress.head).map { case (_, m) => + require(m.forall(_.address == m.head.address), "Require homogeneous address ranges") + require(m.forall(_.regionType == m.head.regionType), "Require homogeneous regionType") + require(m.forall(_.supports == m.head.supports), "Require homogeneous supported operations") + m.head + }.toSeq + } + + val node = new TLNexusNode( + clientFn = { c => + require(c.size == 1, s"Only one ClientPort supported in TLSwitch, not $c") + c.head + }, + managerFn = { m => + // unifies all the managers, its up to the user to be careful here + // TODO: Use bus error device to report problems? + require(m.flatMap(_.responseFields).size == 0, "ResponseFields not supported in TLSwitch") + require(m.flatMap(_.requestKeys).size == 0, "RequestKeys not supported in TLSwitch") + require(m.forall(_.beatBytes == m.head.beatBytes), "Homogeneous beatBytes required") + TLSlavePortParameters.v1( + beatBytes = m.head.beatBytes, + managers = unifyManagers(m.map(_.sortedSlaves)), + endSinkId = m.map(_.endSinkId).max, + minLatency = m.map(_.minLatency).min, + responseFields = Nil, + requestKeys = Nil, + ) + } + ) + // BORROWED FROM TLSwitch.scala END + + //val memDevice = new SimpleDevice("offchip", Nil) + + // "Wide" manager node + // val manager = TLManagerNode(Seq(TLSlavePortParameters.v1( + // managers = Seq(TLSlaveParameters.v2( + // address = AddressSet.misaligned(params.offchipBase, params.offchipSize), // TODO: check + // //resources = memDevice.reg, + // regionType = RegionType.UNCACHED, // for now + // supports = TLMasterToSlaveTransferSizes( + // putFull = TransferSizes(1, params.beatBytes*maxBeats), // FIXME + // get = TransferSizes(1, params.beatBytes*maxBeats) + // ), + // fifoId = Some(0))), // requests are handled in order + // beatBytes = params.beatBytes))) + + // Instantiate one client node for each offchip port + // val client_nodes = (0 until params.nOffchipPorts).map(i => TLClientNode(Seq(TLMasterPortParameters.v1(Seq(TLClientParameters( + // name = s"offchip$i", sourceId = IdRange(i, i+1))))))) // this source id looks wrong hm + + // val switch = LazyModule(new TLSwitch) + + // switch.node :=* manager_node + + // Client node + // val client = TLClientNode(Seq(TLMasterPortParameters.v1(Seq(TLClientParameters( + // name = s"offchip", sourceId = IdRange(0, params.nOffchipPorts)))))) + + // Register node for setting the routing table + val routing_table_node = TLRegisterNode( + address = AddressSet.misaligned(params.tableAddress, 0x1000), // TODO: fix size + device = new SimpleDevice("routing-table", Nil), + beatBytes = params.beatBytes + ) + + override lazy val module = new OffchipRouterImpl(this) +} + +class OffchipRouterImpl(outer: OffchipRouter) extends LazyModuleImp(outer) { + + val io = IO(new Bundle { + val chip_id = Output(UInt(log2Ceil(outer.params.tableEntries).W)) + }) + + val (bundleIn, edgeIn) = outer.node.in(0) + val bundlesOut = outer.node.out.map(_._1) + + // val (in, edgeIn) = outer.manager.in(0) + // val (out, edgeOut) = outer.client.out(0) + + val portWidth = log2Ceil(outer.nPorts) + val idWidth = log2Ceil(outer.params.tableEntries) // But maybe we want to allow this to be configurable, in case chip ids don't count up from 0 + val addressWidth = bundleIn.a.bits.address.getWidth // TODO: check + val totalWidth = idWidth + addressWidth + portWidth + 1 + assert(totalWidth <= 64, "Total width of routing table entry must be less than or equal to 64 bits") // for now :) + + val chipIdReg = RegInit(0.U(idWidth.W)) + io.chip_id := chipIdReg + + // Create a memory mapped table of offchip addresses which store chip id, base address, and port to route to + // and initialize it to all 0s using RegInit + val routing_table = VecInit(Seq.fill(outer.params.tableEntries)(RegInit(0.U.asTypeOf(new RoutingTableEntry(idWidth, addressWidth, portWidth))))) + + // Each reg should be totalWidth bits wide and maps to an entry in the routing table + // TODO: add descriptions for each field, and specify rw? + //val mapped_entries = (0 until outer.params.tableEntries).map(i => i*8 -> Seq(RegField(totalWidth, routing_table(i).asUInt))) + + // One entry = 4 separate regs (valid, chipID, baseAddress, port) + //val bytesPerReg = outer.params.beatBytes + val regsPerEntry = 4 // TODO: fixy + //val entryStride = bytesPerReg * regsPerEntry + + val mapped_entries = (0 until outer.params.tableEntries).flatMap { i => + val base = i * regsPerEntry + Seq( + (base + 0) -> Seq(RegField(1, routing_table(i).valid)), + (base + 1) -> Seq(RegField(idWidth, routing_table(i).chipID)), + (base + 2) -> Seq(RegField(portWidth, routing_table(i).port)), + ) + } + + val chip_id = Seq((outer.params.tableEntries * regsPerEntry) -> Seq(RegField(idWidth, chipIdReg))) + outer.routing_table_node.regmap(mapped_entries ++ chip_id :_*) + + // Select offchip port from routing table + val addr_top_bits = bundleIn.a.bits.address(addressWidth-1, log2Ceil(outer.offset)) + val matches = Wire(Vec(outer.params.tableEntries, Bool())) + val sel = Wire(UInt(log2Ceil(outer.nPorts).W)) + sel := 0.U + matches := VecInit(Seq.fill(outer.params.tableEntries)(false.B)) + for (i <- 0 until outer.params.tableEntries) { + when (routing_table(i).valid && (addr_top_bits === routing_table(i).chipID)) { + sel := routing_table(i).port + matches(i) := true.B + } + } + + // Assert that only one match is found + assert(!bundleIn.a.valid || ((matches.asUInt & (matches.asUInt - 1.U)) === 0.U), "Multiple matches found in routing table") // TODO: check + + // AGENT OUTPUT START ~~~~~~~~~~~~~~~~ + // Decide legality per-request. Address is stable across beats. + val illegal = Wire(Bool()) // <- your programmable table miss / tag mismatch + illegal := bundleIn.a.valid && !matches.reduce(_ || _) + + val errActive = RegInit(false.B) + val errOpcode = Reg(bundleIn.a.bits.opcode.cloneType) + val errSize = Reg(bundleIn.a.bits.size.cloneType) + val errSource = Reg(bundleIn.a.bits.source.cloneType) + + // Helpers + val a_last = edgeIn.last(bundleIn.a) + + // Start an error response when we accept the LAST beat of an illegal request + when (!errActive && bundleIn.a.fire && a_last && illegal) { + errActive := true.B + errOpcode := bundleIn.a.bits.opcode + errSize := bundleIn.a.bits.size + errSource := bundleIn.a.bits.source + } + + // Local D generator + val dErr = Wire(chiselTypeOf(bundleIn.d)) + val (_, d_last, _) = edgeIn.firstlast(dErr) + + dErr.valid := errActive + dErr.bits.opcode := TLMessages.adResponse(errOpcode) + dErr.bits.param := 0.U + dErr.bits.size := errSize + dErr.bits.source := errSource + dErr.bits.sink := 0.U + dErr.bits.denied := true.B + dErr.bits.data := 0.U + dErr.bits.corrupt := edgeIn.hasData(dErr.bits) + + // Done when last D beat fires + when (dErr.fire && d_last) { errActive := false.B } + + // A-channel ready: + // - stall all new A while returning an error + // - otherwise: if illegal, just drain A beats (no downstream backpressure) + // - if legal, use your normal routing ready + // in.a.ready := !errActive && (if (/*legal path*/ true.B) { + // Mux(illegal, true.B, /* legalReady */ true.B) + // } else true.B) + // ^^ This is super weird lol + + // Drive D upstream (when errActive, this is your response) + // I think I need to gate this inside a when statement + // when (errActive) { + // in.d <> dErr + // } + // AGENT OUTPUT END ~~~~~~~~~~~~~~~~ + + // BORROWED FROM TLSwitch.scala START + bundlesOut.zipWithIndex.foreach { case (out, i) => + val selected = i.U === sel + + out.a.valid := bundleIn.a.valid && selected + out.a.bits := bundleIn.a.bits + + //out.b.ready := bundleIn.b.ready && selected + + out.c.valid := bundleIn.c.valid && selected + out.c.bits := bundleIn.c.bits + + //out.d.ready := bundleIn.d.ready && selected + + out.e.valid := bundleIn.e.valid && selected + out.e.bits := bundleIn.e.bits + } + + bundleIn.a.ready := VecInit(bundlesOut.map(_.a.ready))(sel) && !errActive + bundleIn.c.ready := VecInit(bundlesOut.map(_.c.ready))(sel) + bundleIn.e.ready := VecInit(bundlesOut.map(_.e.ready))(sel) + // BORROWED FROM TLSwitch.scala END + + // Arbitrate responses from d channels + val response_arbiter_d = Module(new RRArbiter(chiselTypeOf(bundleIn.d.bits), outer.nPorts + 1)) + for (i <- 0 until outer.nPorts) { + response_arbiter_d.io.in(i) <> bundlesOut(i).d + } + response_arbiter_d.io.in(outer.nPorts) <> dErr + bundleIn.d <> Queue(response_arbiter_d.io.out, outer.nPorts + 1) // I think this queue is necessary?? + + // Arbitrate responses from b channels + // TODO ?? : Error checking for coherent requests + val response_arbiter_b = Module(new RRArbiter(chiselTypeOf(bundleIn.b.bits), outer.nPorts)) + for (i <- 0 until outer.nPorts) { + response_arbiter_b.io.in(i) <> bundlesOut(i).b + } + bundleIn.b <> Queue(response_arbiter_b.io.out, outer.nPorts) + + + // Arbitrate responses from the out d channels + // I may want to use a TLArbiter instead + // val response_arbiter = Module(new RRArbiter(in.d.cloneType, outer.params.nOffchipPorts)) + // for (i <- 0 until outer.params.nOffchipPorts) { + // response_arbiter.io.in(i) := out(i).d + // } + // in.d <> response_arbiter.io.out + + // for (i <- 0 until outer.params.nOffchipPorts) { + // out(i).a.bits := in.a.bits + // } + + // val addr_top_bits = in.a.bits.address(addressWidth-1, 32) + // for (i <- 0 until outer.params.tableEntries) { + // when (addr_top_bits === routing_table(i).chipID) { + // out(routing_table(i).port).a.valid := in.a.valid + // } .otherwise { + // out.a.valid := false.B + // } + // } + + // //don't support coherent responses yet + // in.b.valid := false.B + // in.c.ready := true.B + // in.e.ready := true.B + + // //don't support coherent requests yet + // for (i <- 0 until outer.params.nOffchipPorts) { + // out(i).b.ready := true.B + // out(i).c.valid := false.B + // out(i).e.valid := false.B + // } + + // assert(!in.a.valid || (in.a.valid && (out.map(_.a.valid).reduce(_ || _))), "Router dropped a packet - illegal address likely") +} + +// TODO: fix widths +class RoutingTableEntry(idWidth: Int, addressWidth: Int, portWidth: Int) extends Bundle { + val valid = Bool() + val chipID = UInt(idWidth.W) // This is probably determined by the number of entries + val port = UInt(portWidth.W) +} + +// Mostly borrowed from the Radiance AddressRewriterNode +case class ChipletAddressTranslator(val params: InwardAddressTranslatorParams, val idWidth: Int)(implicit p: Parameters) extends LazyModule { + + val node = TLAdapterNode(clientFn = c => c, managerFn = m => m) + + assert(params.offset.bitCount == 1, "Offset must only have 1 bit set") // TODO: should we enforce this in the spec? + + override lazy val module = new ChipletAddressTranslatorImpl(this) +} + +class ChipletAddressTranslatorImpl(outer: ChipletAddressTranslator) extends LazyModuleImp(outer) { + val io = IO(new Bundle { + val chip_id = Input(UInt(outer.idWidth.W)) + }) + + // If the top bits of the address match the chip id, then trim those bits off + // Top bits are determined by the offset in the params + def trimTag(address: UInt) = { + val topBits = log2Ceil(outer.params.offset) + val addressWidth = address.getWidth + val tagMatch = address(addressWidth-1, topBits) === io.chip_id + val mask = (1.U(addressWidth.W) << topBits) - 1.U + Mux(tagMatch, address & mask, address) + } + + (outer.node.in.map(_._1) zip outer.node.out.map(_._1)).foreach { case (in, out) => + out.a <> in.a + out.a.bits.address := trimTag(in.a.bits.address) + in.b <> out.b + in.b.bits.address := trimTag(out.b.bits.address) + out.c <> in.c + out.c.bits.address := trimTag(in.c.bits.address) + in.d <> out.d + out.e <> in.e + } +} + +trait CanHaveChipletRouting { this: BaseSubsystem => + val d2d_port_ios = p(ChipletRoutingKey).map { params => + + val cbus = locateTLBusWrapper(params.controlBusWhere) + val client_bus = locateTLBusWrapper(params.clientBusWhere) + + val all_freqs = { + Seq(client_bus.dtsFrequency) ++ + params.ports.map { pP => locateTLBusWrapper(pP.managerBusWhere).dtsFrequency } ++ + Seq(cbus.dtsFrequency) ++ + params.ports.flatMap { pP => + pP.controlManagerBusWhere match { + case Some(where) => Some(locateTLBusWrapper(where).dtsFrequency) + case None => None + } + } + } + require(all_freqs.forall(_.isDefined), "All buses must provide a frequency") + require(all_freqs.forall(_ == all_freqs.head), "All buses must have the same frequency") + + val router_domain = LazyModule(new ClockSinkDomain(name=Some("offchip_router"))) + router_domain.clockNode := client_bus.fixedClockNode + + val router = router_domain { LazyModule(new OffchipRouter(params.routerParams, params.ports.size, params.translationParams.offset)) } + + // The bus drives the router's manager node + client_bus.coupleTo(s"offchip_router") { router.node := TLBuffer() := _ } + + router.routing_table_node := cbus.coupleTo(s"offchip_router_mmio") { TLBuffer() := TLFragmenter(cbus) := _ } + + val port_ios = params.ports.zipWithIndex.map { case (pP, id) => + val link_manager_bus = locateTLBusWrapper(pP.managerBusWhere) + + val port = router_domain { pP.asInstanceOf[ChipletLinkWrapperInstantiationLike].instantiate(s"d2d${id}", id)(p) } + + // TODO: Translator should take in chip ID as an IO + val translator = router_domain { + LazyModule(ChipletAddressTranslator(params.translationParams, log2Ceil(params.routerParams.tableEntries))(p)) + } + + router_domain { + InModuleBody { + translator.module.io.chip_id := router.module.io.chip_id + } + } + + port.manager_node :*= router.node + port.control_manager_node.foreach { node => + cbus.coupleTo(s"${port.name}_control") { node := TLBuffer() := _ } + } + link_manager_bus.coupleFrom(s"${port.name}") { _ := TLBuffer() := translator.node :=* port.client_node } + + val port_ioSink = BundleBridgeSink[ChipletIO]() + port_ioSink := port.top_IO + + // Try tying off port_ioSink + // InModuleBody { port_ioSink.in(0)._1 match { + // case io: CTCBridgeIO => { + // io.client_flit.in.valid := false.B + // io.client_flit.in.bits := DontCare + // io.client_flit.out.ready := false.B + // io.manager_flit.in.valid := false.B + // io.manager_flit.in.bits := DontCare + // io.manager_flit.out.ready := false.B + // } + // case _ => assert(false, s"Unsupported IO type for OffchipRouter") + // }} + + val outer_io = InModuleBody { + // port_ioSink.makeIO(s"d2d${id}_port") + val outer_io = IO(chiselTypeOf(port_ioSink.in(0)._1)) + outer_io <> port_ioSink.in(0)._1 + outer_io + } + outer_io + + // val port_io = router_domain { InModuleBody { port.module.asInstanceOf[ChipletLinkWrapperImpl].io } } + + // val outer_io = InModuleBody { + // val outer_io = IO(chiselTypeOf(port_io.getWrappedValue)) + // outer_io <> port_io.getWrappedValue + // outer_io + // } + // outer_io + } + port_ios + } +} + +class WithChipletRouting(params: ChipletRoutingParams) extends Config((site, here, up) => { + case ChipletRoutingKey => Some(params) +}) From 406a4abe8e9b71d5edef66aa68b49a184b153e47 Mon Sep 17 00:00:00 2001 From: Ella Schwarz Date: Mon, 23 Feb 2026 13:21:10 -0800 Subject: [PATCH 02/19] [WIP] Add instantiable D2D port for CTC --- src/main/scala/ctc/CTC.scala | 128 ++++++++++++++++++++++++++++++++++- 1 file changed, 126 insertions(+), 2 deletions(-) diff --git a/src/main/scala/ctc/CTC.scala b/src/main/scala/ctc/CTC.scala index 71b1128e..6012db4f 100644 --- a/src/main/scala/ctc/CTC.scala +++ b/src/main/scala/ctc/CTC.scala @@ -34,7 +34,9 @@ case class CTCParams( managerBus: Option[TLBusWrapperLocation] = Some(SBUS), clientBus: Option[TLBusWrapperLocation] = Some(SBUS), phyParams: Option[SerialPhyParams] = Some(CreditedSourceSyncSerialPhyParams(phitWidth = CTC.OUTER_WIDTH, flitWidth = CTC.INNER_WIDTH, freqMHz = 100, flitBufferSz = 16)) // Set to None to disable PHY -) { +) extends ChipletLinkParams + with ChipletLinkWrapperInstantiationLike + { def offchipRange = translationParams match { case OutwardAddressTranslatorParams(_,_,_) => Seq(translationParams.offchipRange) case InwardAddressTranslatorParams(_,_,_) => { @@ -42,16 +44,138 @@ case class CTCParams( offchip } } + + def managerRegion = offchipRange + def managerBusWhere = managerBus.get + def controlManagerBusWhere = None + def instantiate(name: String, id: Int)(implicit p: Parameters): ChipletLinkWrapper = LazyModule(new CTCChipletLink(this, name, id)) } // For using CTC in a chiplet firesim config with no PHY -class CTCBridgeIO extends Bundle { +class CTCBridgeIO extends ChipletIO { val client_flit = new DecoupledFlitIO(CTC.INNER_WIDTH) // Driven by client/ctc2tl val manager_flit = new DecoupledFlitIO(CTC.INNER_WIDTH) // Driven by manager/tl2ctc + + def tieoff: Unit = { + manager_flit := DontCare + manager_flit.in.valid := false.B + manager_flit.out.ready := false.B + client_flit := DontCare + client_flit.in.valid := false.B + client_flit.out.ready := false.B + } + + def connect(io: ChipletIO): Unit = io match { + case io: CTCBridgeIO => { + client_flit.in <> io.manager_flit.out + io.client_flit.in <> manager_flit.out + manager_flit.in <> io.client_flit.out + io.manager_flit.in <> client_flit.out + } + case _ => assert(false, s"IO does not match CTCBridgeIO: ${io.getClass}") + } + + def loopback: Unit = { + client_flit.in <> manager_flit.out + manager_flit.in <> client_flit.out + } } case object CTCKey extends Field[Seq[CTCParams]](Nil) +class CTCChipletLink(val params: CTCParams, val link_name: String, val id: Int)(implicit p: Parameters) extends ChipletLinkWrapper { + // a TL master/client device + val ctc2tl = LazyModule(new CTCToTileLink(portId=id)(p)) + // a TL slave/manager device + val tl2ctc = LazyModule(new TileLinkToCTC(addrRegion=params.offchipRange)(p)) + + // val ctc_outer_io = InModuleBody { + // val outer_io = params.phyParams.map(pP => IO(pP.genIO).suggestName(name)).getOrElse(IO(new CTCBridgeIO).suggestName(name)) + + // outer_io match { + // // case io: CreditedSourceSyncPhitIO => { + // // // 3 clock domains - + // // // - ctc2tl's "Inner clock": synchronizes signals going to the digital logic + // // // - outgoing clock: synchronizes signals going out + // // // - incoming clock: synchronizes signals coming in + // // // TODO: SERIOUSLY CHECK THIS CLOCKING, I JUST GUESSED!!!!! + // // val outgoing_clock = ctc2tl.module.clock + // // val outgoing_reset = ResetCatchAndSync(outgoing_clock, ctc2tl.module.reset.asBool) + // // val incoming_clock = io.clock_in + // // val incoming_reset = ResetCatchAndSync(incoming_clock, io.reset_in.asBool) + // // io.clock_out := outgoing_clock + // // io.reset_out := outgoing_reset.asAsyncReset + // // val phy = Module(new CreditedSerialPhy(2, params.phyParams.get)) + // // phy.io.incoming_clock := incoming_clock + // // phy.io.incoming_reset := incoming_reset + // // phy.io.outgoing_clock := outgoing_clock + // // phy.io.outgoing_reset := outgoing_reset + // // phy.io.inner_clock := ctc2tl.module.clock + // // phy.io.inner_reset := ctc2tl.module.reset + // // phy.io.inner_ser(0).in <> ctc2tl.module.io.flit.in + // // phy.io.inner_ser(0).out <> tl2ctc.module.io.flit.out + // // phy.io.inner_ser(1).in <> tl2ctc.module.io.flit.in + // // phy.io.inner_ser(1).out <> ctc2tl.module.io.flit.out + + // // phy.io.outer_ser <> io.viewAsSupertype(new ValidPhitIO(params.phyParams.get.phitWidth)) + // // } + // // case io: DecoupledInternalSyncPhitIO => { + // // val outgoing_clock = ctc2tl.module.clock + // // io.clock_out := outgoing_clock + // // val phy = Module(new DecoupledSerialPhy(2, params.phyParams.get)) + // // phy.io.outer_clock := outgoing_clock + // // phy.io.outer_reset := ResetCatchAndSync(outgoing_clock, ctc2tl.module.reset.asBool) + // // phy.io.inner_clock := ctc2tl.module.clock + // // phy.io.inner_reset := ctc2tl.module.reset + // // phy.io.inner_ser(0).in <> ctc2tl.module.io.flit.in + // // phy.io.inner_ser(0).out <> tl2ctc.module.io.flit.out + // // phy.io.inner_ser(1).in <> tl2ctc.module.io.flit.in + // // phy.io.inner_ser(1).out <> ctc2tl.module.io.flit.out + // // phy.io.outer_ser <> io.viewAsSupertype(new DecoupledPhitIO(params.phyParams.get.phitWidth)) + // // } + // // case io: DecoupledExternalSyncPhitIO => { + // // val outgoing_clock = io.clock_in + // // val outgoing_reset = ResetCatchAndSync(outgoing_clock, ctc2tl.module.reset.asBool) + // // val phy = Module(new DecoupledSerialPhy(2, params.phyParams.get)) + // // phy.io.outer_clock := outgoing_clock + // // phy.io.outer_reset := ResetCatchAndSync(outgoing_clock, ctc2tl.module.reset.asBool) + // // phy.io.inner_clock := ctc2tl.module.clock + // // phy.io.inner_reset := ctc2tl.module.reset + // // phy.io.inner_ser(0).in <> ctc2tl.module.io.flit.in + // // phy.io.inner_ser(0).out <> tl2ctc.module.io.flit.out + // // phy.io.inner_ser(1).in <> tl2ctc.module.io.flit.in + // // phy.io.inner_ser(1).out <> ctc2tl.module.io.flit.out + // // phy.io.outer_ser <> io.viewAsSupertype(new DecoupledPhitIO(params.phyParams.get.phitWidth)) + // // } + // // Start with CTCBridgeIO only for now + // case io: CTCBridgeIO => { + // io.manager_flit <> tl2ctc.module.io.flit + // io.client_flit <> ctc2tl.module.io.flit + // } + // case _ => assert(false, s"Unsupported IO type for CTC ChipletLink") + // } + // outer_io + // } + + //val link_name = link_name + id.toString + val client_node = ctc2tl.node + val manager_node = tl2ctc.node + val control_manager_node = None + val top_IO = BundleBridgeSource(() => new CTCBridgeIO) + override lazy val module = new CTCChipletLinkImpl(this) + //val top_IO = InModuleBody { module.io } + // InModuleBody { + // top_IO.out(0)._1.asInstanceOf[CTCBridgeIO].manager_flit <> tl2ctc.module.io.flit + // top_IO.out(0)._1.asInstanceOf[CTCBridgeIO].client_flit <> ctc2tl.module.io.flit + // } +} + +// Maybe can move this into an InModuleBody +class CTCChipletLinkImpl(outer: CTCChipletLink) extends LazyModuleImp(outer) { + outer.top_IO.out(0)._1.asInstanceOf[CTCBridgeIO].manager_flit <> outer.tl2ctc.module.io.flit + outer.top_IO.out(0)._1.asInstanceOf[CTCBridgeIO].client_flit <> outer.ctc2tl.module.io.flit +} + trait CanHavePeripheryCTC { this: BaseSubsystem => private val portName = "ctc" From f215f924addc3211d4c8616729133bc83db5460b Mon Sep 17 00:00:00 2001 From: Ella Schwarz Date: Fri, 27 Feb 2026 13:48:44 -0800 Subject: [PATCH 03/19] [WIP] Params fixes --- src/main/scala/ctc/CTC.scala | 44 ++++++++++++++++++++---------------- 1 file changed, 25 insertions(+), 19 deletions(-) diff --git a/src/main/scala/ctc/CTC.scala b/src/main/scala/ctc/CTC.scala index 6012db4f..c379a4ed 100644 --- a/src/main/scala/ctc/CTC.scala +++ b/src/main/scala/ctc/CTC.scala @@ -29,24 +29,26 @@ object CTCCommand { } case class CTCParams( - translationParams: AddressTranslatorParams = OutwardAddressTranslatorParams(onchipAddr = 0x1000000000L, offchipAddr = 0x0L, size = ((1L << 32) - 1)), + translationParams: Option[AddressTranslatorParams] = Some(OutwardAddressTranslatorParams(onchipAddr = 0x1000000000L, offchipAddr = 0x0L, size = ((1L << 32) - 1))), offchip: Seq[AddressSet] = Nil, - managerBus: Option[TLBusWrapperLocation] = Some(SBUS), - clientBus: Option[TLBusWrapperLocation] = Some(SBUS), + managerBus: TLBusWrapperLocation = SBUS, + clientBus: TLBusWrapperLocation = SBUS, phyParams: Option[SerialPhyParams] = Some(CreditedSourceSyncSerialPhyParams(phitWidth = CTC.OUTER_WIDTH, flitWidth = CTC.INNER_WIDTH, freqMHz = 100, flitBufferSz = 16)) // Set to None to disable PHY ) extends ChipletLinkParams with ChipletLinkWrapperInstantiationLike { - def offchipRange = translationParams match { - case OutwardAddressTranslatorParams(_,_,_) => Seq(translationParams.offchipRange) - case InwardAddressTranslatorParams(_,_,_) => { - require(offchip != Nil) - offchip + def offchipRange = translationParams.map { tp => + tp match { + case OutwardAddressTranslatorParams(_,_,_) => Seq(tp.offchipRange) + case InwardAddressTranslatorParams(_,_,_) => { + require(offchip != Nil) + offchip + } } - } + }.getOrElse(offchip) def managerRegion = offchipRange - def managerBusWhere = managerBus.get + def managerBusWhere = managerBus def controlManagerBusWhere = None def instantiate(name: String, id: Int)(implicit p: Parameters): ChipletLinkWrapper = LazyModule(new CTCChipletLink(this, name, id)) } @@ -189,15 +191,15 @@ trait CanHavePeripheryCTC { this: BaseSubsystem => assert(pP.flitWidth == CTC.INNER_WIDTH) } - lazy val slave_bus = locateTLBusWrapper(params.managerBus.get) - lazy val master_bus = locateTLBusWrapper(params.clientBus.get) + lazy val slave_bus = locateTLBusWrapper(params.managerBus) + lazy val master_bus = locateTLBusWrapper(params.clientBus) val ctc_domain = LazyModule(new ClockSinkDomain(name=Some(s"CTC$id"))) ctc_domain.clockNode := slave_bus.fixedClockNode - val translator = ctc_domain { - LazyModule(AddressTranslator(params.translationParams)(p)) - } + val translator = ctc_domain { params.translationParams.map { tp => + LazyModule(AddressTranslator(tp)(p)) + }} require(slave_bus.dtsFrequency.isDefined, s"Slave bus ${slave_bus.busName} must provide a frequency") @@ -213,14 +215,18 @@ trait CanHavePeripheryCTC { this: BaseSubsystem => params.translationParams match { // Translate outgoing requests - case OutwardAddressTranslatorParams(_,_,_) => { - slave_bus.coupleTo(portName) { translator(tl2ctc.node) := TLBuffer() := _ } + case Some(OutwardAddressTranslatorParams(_,_,_)) => { + slave_bus.coupleTo(portName) { translator.get(tl2ctc.node) := TLBuffer() := _ } master_bus.coupleFrom(portName) { _ := TLBuffer() := ctc2tl.node } } // Translate incoming requests - case InwardAddressTranslatorParams(_,_,_) => { + case Some(InwardAddressTranslatorParams(_,_,_)) => { slave_bus.coupleTo(portName) { tl2ctc.node := TLBuffer() := _ } - master_bus.coupleFrom(portName) { _ := TLBuffer() := translator(ctc2tl.node) } + master_bus.coupleFrom(portName) { _ := TLBuffer() := translator.get(ctc2tl.node) } + } + case None => { + slave_bus.coupleTo(portName) { tl2ctc.node := TLBuffer() := _ } + master_bus.coupleFrom(portName) { _ := TLBuffer() := ctc2tl.node } } } From a83369bb433014e6b73fccd2611a16fb6dd669fe Mon Sep 17 00:00:00 2001 From: Ella Schwarz Date: Fri, 27 Feb 2026 13:49:22 -0800 Subject: [PATCH 04/19] [WIP] Offchip router working in io chiplet --- src/main/scala/soc/OffchipRouter.scala | 132 +------------------------ 1 file changed, 5 insertions(+), 127 deletions(-) diff --git a/src/main/scala/soc/OffchipRouter.scala b/src/main/scala/soc/OffchipRouter.scala index 85c0bd02..f067c97a 100644 --- a/src/main/scala/soc/OffchipRouter.scala +++ b/src/main/scala/soc/OffchipRouter.scala @@ -27,25 +27,11 @@ case object ChipletRoutingKey extends Field[Option[ChipletRoutingParams]](None) case class OffchipRouterParams( // Need to think about what I want here - // offchipBase: BigInt, - // offchipSize: BigInt, tableAddress: BigInt = 0x4000L, tableEntries: Int, beatBytes: Int // maybe not here ) -// class OffchipRouterWrapper(params: OffchipRouterParams, context: HasTileLinkLocations)(implicit p: Parameters) extends TLBusWrapper(params, "offchip_router") { -// private val offchip_router = LazyModule(new OffchipRouter(params)) -// private val cbus = context.locateTLBusWrapper(CBUS) -// offchip_router.routing_table_node :=* cbus.coupleTo(s"offchip_routing_table") { TLBuffer(1) := TLFragmenter(cbus) := _ } - -// val inwardNode: TLInwardNode = offchip_router.manager_node :=* TLFIFOFixer(TLFIFOFixer.allVolatile) -// val outwardNode: TLOutwardNode = offchip_router.switch.node -// def busView: TLEdge = offchip_router.switch.node.edges.in.head -// val builtInDevices = BuiltInDevices.none -// val prefixNode = None -// } - class OffchipRouter(val params: OffchipRouterParams, val nPorts: Int, val offset: BigInt)(implicit p: Parameters) extends LazyModule { // BORROWED FROM TLSwitch.scala START @@ -82,33 +68,6 @@ class OffchipRouter(val params: OffchipRouterParams, val nPorts: Int, val offset ) // BORROWED FROM TLSwitch.scala END - //val memDevice = new SimpleDevice("offchip", Nil) - - // "Wide" manager node - // val manager = TLManagerNode(Seq(TLSlavePortParameters.v1( - // managers = Seq(TLSlaveParameters.v2( - // address = AddressSet.misaligned(params.offchipBase, params.offchipSize), // TODO: check - // //resources = memDevice.reg, - // regionType = RegionType.UNCACHED, // for now - // supports = TLMasterToSlaveTransferSizes( - // putFull = TransferSizes(1, params.beatBytes*maxBeats), // FIXME - // get = TransferSizes(1, params.beatBytes*maxBeats) - // ), - // fifoId = Some(0))), // requests are handled in order - // beatBytes = params.beatBytes))) - - // Instantiate one client node for each offchip port - // val client_nodes = (0 until params.nOffchipPorts).map(i => TLClientNode(Seq(TLMasterPortParameters.v1(Seq(TLClientParameters( - // name = s"offchip$i", sourceId = IdRange(i, i+1))))))) // this source id looks wrong hm - - // val switch = LazyModule(new TLSwitch) - - // switch.node :=* manager_node - - // Client node - // val client = TLClientNode(Seq(TLMasterPortParameters.v1(Seq(TLClientParameters( - // name = s"offchip", sourceId = IdRange(0, params.nOffchipPorts)))))) - // Register node for setting the routing table val routing_table_node = TLRegisterNode( address = AddressSet.misaligned(params.tableAddress, 0x1000), // TODO: fix size @@ -128,11 +87,8 @@ class OffchipRouterImpl(outer: OffchipRouter) extends LazyModuleImp(outer) { val (bundleIn, edgeIn) = outer.node.in(0) val bundlesOut = outer.node.out.map(_._1) - // val (in, edgeIn) = outer.manager.in(0) - // val (out, edgeOut) = outer.client.out(0) - - val portWidth = log2Ceil(outer.nPorts) - val idWidth = log2Ceil(outer.params.tableEntries) // But maybe we want to allow this to be configurable, in case chip ids don't count up from 0 + val portWidth = if (outer.nPorts == 1) 1 else log2Ceil(outer.nPorts) + val idWidth = log2Ceil(outer.params.tableEntries + 1) // But maybe we want to allow this to be configurable, in case chip ids don't count up from 0 val addressWidth = bundleIn.a.bits.address.getWidth // TODO: check val totalWidth = idWidth + addressWidth + portWidth + 1 assert(totalWidth <= 64, "Total width of routing table entry must be less than or equal to 64 bits") // for now :) @@ -142,16 +98,12 @@ class OffchipRouterImpl(outer: OffchipRouter) extends LazyModuleImp(outer) { // Create a memory mapped table of offchip addresses which store chip id, base address, and port to route to // and initialize it to all 0s using RegInit - val routing_table = VecInit(Seq.fill(outer.params.tableEntries)(RegInit(0.U.asTypeOf(new RoutingTableEntry(idWidth, addressWidth, portWidth))))) + //val routing_table = VecInit(Seq.fill(outer.params.tableEntries)(RegInit(0.U.asTypeOf(new RoutingTableEntry(idWidth, addressWidth, portWidth))))) - // Each reg should be totalWidth bits wide and maps to an entry in the routing table - // TODO: add descriptions for each field, and specify rw? - //val mapped_entries = (0 until outer.params.tableEntries).map(i => i*8 -> Seq(RegField(totalWidth, routing_table(i).asUInt))) + val routing_table = RegInit(VecInit(Seq.fill(outer.params.tableEntries)(0.U.asTypeOf(new RoutingTableEntry(idWidth, addressWidth, portWidth))))) // One entry = 4 separate regs (valid, chipID, baseAddress, port) - //val bytesPerReg = outer.params.beatBytes val regsPerEntry = 4 // TODO: fixy - //val entryStride = bytesPerReg * regsPerEntry val mapped_entries = (0 until outer.params.tableEntries).flatMap { i => val base = i * regsPerEntry @@ -219,20 +171,6 @@ class OffchipRouterImpl(outer: OffchipRouter) extends LazyModuleImp(outer) { // Done when last D beat fires when (dErr.fire && d_last) { errActive := false.B } - // A-channel ready: - // - stall all new A while returning an error - // - otherwise: if illegal, just drain A beats (no downstream backpressure) - // - if legal, use your normal routing ready - // in.a.ready := !errActive && (if (/*legal path*/ true.B) { - // Mux(illegal, true.B, /* legalReady */ true.B) - // } else true.B) - // ^^ This is super weird lol - - // Drive D upstream (when errActive, this is your response) - // I think I need to gate this inside a when statement - // when (errActive) { - // in.d <> dErr - // } // AGENT OUTPUT END ~~~~~~~~~~~~~~~~ // BORROWED FROM TLSwitch.scala START @@ -242,13 +180,9 @@ class OffchipRouterImpl(outer: OffchipRouter) extends LazyModuleImp(outer) { out.a.valid := bundleIn.a.valid && selected out.a.bits := bundleIn.a.bits - //out.b.ready := bundleIn.b.ready && selected - out.c.valid := bundleIn.c.valid && selected out.c.bits := bundleIn.c.bits - //out.d.ready := bundleIn.d.ready && selected - out.e.valid := bundleIn.e.valid && selected out.e.bits := bundleIn.e.bits } @@ -274,41 +208,6 @@ class OffchipRouterImpl(outer: OffchipRouter) extends LazyModuleImp(outer) { } bundleIn.b <> Queue(response_arbiter_b.io.out, outer.nPorts) - - // Arbitrate responses from the out d channels - // I may want to use a TLArbiter instead - // val response_arbiter = Module(new RRArbiter(in.d.cloneType, outer.params.nOffchipPorts)) - // for (i <- 0 until outer.params.nOffchipPorts) { - // response_arbiter.io.in(i) := out(i).d - // } - // in.d <> response_arbiter.io.out - - // for (i <- 0 until outer.params.nOffchipPorts) { - // out(i).a.bits := in.a.bits - // } - - // val addr_top_bits = in.a.bits.address(addressWidth-1, 32) - // for (i <- 0 until outer.params.tableEntries) { - // when (addr_top_bits === routing_table(i).chipID) { - // out(routing_table(i).port).a.valid := in.a.valid - // } .otherwise { - // out.a.valid := false.B - // } - // } - - // //don't support coherent responses yet - // in.b.valid := false.B - // in.c.ready := true.B - // in.e.ready := true.B - - // //don't support coherent requests yet - // for (i <- 0 until outer.params.nOffchipPorts) { - // out(i).b.ready := true.B - // out(i).c.valid := false.B - // out(i).e.valid := false.B - // } - - // assert(!in.a.valid || (in.a.valid && (out.map(_.a.valid).reduce(_ || _))), "Router dropped a packet - illegal address likely") } // TODO: fix widths @@ -410,35 +309,14 @@ trait CanHaveChipletRouting { this: BaseSubsystem => val port_ioSink = BundleBridgeSink[ChipletIO]() port_ioSink := port.top_IO - // Try tying off port_ioSink - // InModuleBody { port_ioSink.in(0)._1 match { - // case io: CTCBridgeIO => { - // io.client_flit.in.valid := false.B - // io.client_flit.in.bits := DontCare - // io.client_flit.out.ready := false.B - // io.manager_flit.in.valid := false.B - // io.manager_flit.in.bits := DontCare - // io.manager_flit.out.ready := false.B - // } - // case _ => assert(false, s"Unsupported IO type for OffchipRouter") - // }} - val outer_io = InModuleBody { // port_ioSink.makeIO(s"d2d${id}_port") - val outer_io = IO(chiselTypeOf(port_ioSink.in(0)._1)) + val outer_io = IO(chiselTypeOf(port_ioSink.in(0)._1)).suggestName(s"d2d${id}_port") outer_io <> port_ioSink.in(0)._1 outer_io } outer_io - // val port_io = router_domain { InModuleBody { port.module.asInstanceOf[ChipletLinkWrapperImpl].io } } - - // val outer_io = InModuleBody { - // val outer_io = IO(chiselTypeOf(port_io.getWrappedValue)) - // outer_io <> port_io.getWrappedValue - // outer_io - // } - // outer_io } port_ios } From 2fd61b7f25e7541e68eb41d6ea3c67e9bc737bc0 Mon Sep 17 00:00:00 2001 From: Ella Schwarz Date: Fri, 27 Feb 2026 13:50:12 -0800 Subject: [PATCH 05/19] Add chip id plusarg for programming chip id over TSI --- src/main/resources/testchipip/csrc/SimTSI.cc | 2 +- .../resources/testchipip/csrc/testchip_tsi.cc | 23 ++++++++++++++++++- .../resources/testchipip/csrc/testchip_tsi.h | 2 +- 3 files changed, 24 insertions(+), 3 deletions(-) diff --git a/src/main/resources/testchipip/csrc/SimTSI.cc b/src/main/resources/testchipip/csrc/SimTSI.cc index 6fe37895..f56f52a8 100644 --- a/src/main/resources/testchipip/csrc/SimTSI.cc +++ b/src/main/resources/testchipip/csrc/SimTSI.cc @@ -49,7 +49,7 @@ extern "C" int tsi_tick( remove_vcs_simv_opt(info.argc, info.argv); // TODO: We should somehow inspect whether or not our backing memory supports loadmem, instead of unconditionally setting it to true - tsis[chip_id] = new testchip_tsi_t(info.argc, info.argv, true); + tsis[chip_id] = new testchip_tsi_t(info.argc, info.argv, true, chip_id); } testchip_tsi_t* tsi = tsis[chip_id]; diff --git a/src/main/resources/testchipip/csrc/testchip_tsi.cc b/src/main/resources/testchipip/csrc/testchip_tsi.cc index 8a3e45cc..ed0ce0a7 100644 --- a/src/main/resources/testchipip/csrc/testchip_tsi.cc +++ b/src/main/resources/testchipip/csrc/testchip_tsi.cc @@ -1,7 +1,7 @@ #include "testchip_tsi.h" #include -testchip_tsi_t::testchip_tsi_t(int argc, char** argv, bool can_have_loadmem) : tsi_t(argc, argv) +testchip_tsi_t::testchip_tsi_t(int argc, char** argv, bool can_have_loadmem, int chip_id) : tsi_t(argc, argv) { has_loadmem = false; init_accesses = std::vector(); @@ -14,6 +14,27 @@ testchip_tsi_t::testchip_tsi_t(int argc, char** argv, bool can_have_loadmem) : t has_loadmem = can_have_loadmem; if (arg.find("+cflush_addr=0x") == 0) cflush_addr = strtoull(arg.substr(15).c_str(), 0, 16); + if (arg.find("+chip_id") == 0) { + fprintf(stderr, "Parsing +chip_id argument: %s\n", arg.c_str()); + static const std::string prefix = "+chip_id"; + auto eq = arg.find('='); + if (eq == std::string::npos || eq <= prefix.size() || arg.find("=0x", eq) != eq) { + throw std::invalid_argument("Improperly formatted +chip_id argument"); + } + unsigned long arg_chip_id = strtoul(arg.substr(prefix.size(), eq - prefix.size()).c_str(), 0, 0); + if ((int)arg_chip_id == chip_id) { + fprintf(stderr, "Chip ID matches, parsing init access\n"); + auto d = arg.find(":0x", eq + 3); + if (d == std::string::npos) { + throw std::invalid_argument("Improperly formatted +chip_id argument"); + } + uint64_t addr = strtoull(arg.substr(eq + 3, d - (eq + 3)).c_str(), 0, 16); + uint32_t val = strtoull(arg.substr(d + 3).c_str(), 0, 16); + init_access_t access = { .address=addr, .stdata=val, .store=true }; + init_accesses.push_back(access); + } + } + } testchip_htif_t::parse_htif_args(args); diff --git a/src/main/resources/testchipip/csrc/testchip_tsi.h b/src/main/resources/testchipip/csrc/testchip_tsi.h index 7195139b..53c21f34 100644 --- a/src/main/resources/testchipip/csrc/testchip_tsi.h +++ b/src/main/resources/testchipip/csrc/testchip_tsi.h @@ -10,7 +10,7 @@ class testchip_tsi_t : public tsi_t, public testchip_htif_t { public: - testchip_tsi_t(int argc, char** argv, bool has_loadmem); + testchip_tsi_t(int argc, char** argv, bool has_loadmem, int chip_id); virtual ~testchip_tsi_t() {}; void write_chunk(addr_t taddr, size_t nbytes, const void* src) override; From 4daf68970f9b945c34172c034570fb22fe3f3098 Mon Sep 17 00:00:00 2001 From: Ella Schwarz Date: Wed, 4 Mar 2026 13:10:53 -0800 Subject: [PATCH 06/19] [WIP] Clean up --- src/main/scala/ctc/CTC.scala | 8 +-- src/main/scala/soc/ChipletLink.scala | 8 ++- src/main/scala/soc/Configs.scala | 22 +++++++ src/main/scala/soc/OffchipRouter.scala | 88 +++++++++++++------------- 4 files changed, 76 insertions(+), 50 deletions(-) diff --git a/src/main/scala/ctc/CTC.scala b/src/main/scala/ctc/CTC.scala index c379a4ed..1348d88f 100644 --- a/src/main/scala/ctc/CTC.scala +++ b/src/main/scala/ctc/CTC.scala @@ -47,10 +47,10 @@ case class CTCParams( } }.getOrElse(offchip) - def managerRegion = offchipRange + //def managerRegion = offchipRange def managerBusWhere = managerBus def controlManagerBusWhere = None - def instantiate(name: String, id: Int)(implicit p: Parameters): ChipletLinkWrapper = LazyModule(new CTCChipletLink(this, name, id)) + def instantiate(manager_region: Seq[AddressSet], id: Int)(implicit p: Parameters): ChipletLinkWrapper = LazyModule(new CTCChipletLink(this, manager_region, id)) } // For using CTC in a chiplet firesim config with no PHY @@ -85,11 +85,11 @@ class CTCBridgeIO extends ChipletIO { case object CTCKey extends Field[Seq[CTCParams]](Nil) -class CTCChipletLink(val params: CTCParams, val link_name: String, val id: Int)(implicit p: Parameters) extends ChipletLinkWrapper { +class CTCChipletLink(val params: CTCParams, val manager_region: Seq[AddressSet], val id: Int)(implicit p: Parameters) extends ChipletLinkWrapper { // a TL master/client device val ctc2tl = LazyModule(new CTCToTileLink(portId=id)(p)) // a TL slave/manager device - val tl2ctc = LazyModule(new TileLinkToCTC(addrRegion=params.offchipRange)(p)) + val tl2ctc = LazyModule(new TileLinkToCTC(addrRegion=manager_region)(p)) // val ctc_outer_io = InModuleBody { // val outer_io = params.phyParams.map(pP => IO(pP.genIO).suggestName(name)).getOrElse(IO(new CTCBridgeIO).suggestName(name)) diff --git a/src/main/scala/soc/ChipletLink.scala b/src/main/scala/soc/ChipletLink.scala index 7a21ff64..177e8bc6 100644 --- a/src/main/scala/soc/ChipletLink.scala +++ b/src/main/scala/soc/ChipletLink.scala @@ -2,7 +2,7 @@ package testchipip.soc import chisel3._ import chisel3.util._ -import org.chipsalliance.cde.config.{Parameters} +import org.chipsalliance.cde.config.{Parameters, Field} import freechips.rocketchip.subsystem._ import freechips.rocketchip.tilelink._ import freechips.rocketchip.devices.tilelink._ @@ -11,9 +11,11 @@ import freechips.rocketchip.util._ import freechips.rocketchip.prci._ import freechips.rocketchip.diplomacy.BundleBridgeSource +case object MaxOffchipAddressRange extends Field[Seq[AddressSet]](AddressSet.misaligned(0x100000000L, 0x1000000000L)) + // Link params should extend this trait trait ChipletLinkParams{ - def managerRegion: Seq[AddressSet] + //def managerRegion: Seq[AddressSet] def managerBusWhere: TLBusWrapperLocation // Where the link client node is connected def controlManagerBusWhere: Option[TLBusWrapperLocation] // Where the link control node is connected //def instantiationFn: _ => (TLInwardNode, TLOutwardNode, Option[TLRegisterNode]) @@ -33,7 +35,7 @@ abstract class ChipletLinkWrapperImpl(outer: ChipletLinkWrapper) extends LazyMod // Links should "with" this trait trait ChipletLinkWrapperInstantiationLike { - def instantiate(name: String, id: Int)(implicit p: Parameters): ChipletLinkWrapper + def instantiate(manager_region: Seq[AddressSet], id: Int)(implicit p: Parameters): ChipletLinkWrapper } // Should this be called D2D IO instead? diff --git a/src/main/scala/soc/Configs.scala b/src/main/scala/soc/Configs.scala index e65e9029..2505d451 100644 --- a/src/main/scala/soc/Configs.scala +++ b/src/main/scala/soc/Configs.scala @@ -5,6 +5,7 @@ import org.chipsalliance.cde.config.{Parameters, Config} import freechips.rocketchip.subsystem._ import freechips.rocketchip.tilelink.{TLBusWrapperTopology} import freechips.rocketchip.diplomacy.{BufferParams, AddressSet} +import testchipip.ctc.CTCParams //------------------------- // Scratchpad Configs @@ -98,3 +99,24 @@ class WithRingSystemBus( } ) }) + +//------------------------- +// Chiplet Routing Configs +//------------------------- + +// Adds a router and translator for a single CTC port +class WithChipletRouting(params: ChipletRoutingParams = ChipletRoutingParams(ports=Seq(CTCParams(phyParams = None)))) extends Config((site, here, up) => { + case ChipletRoutingKey => Some(params) +}) + +class WithD2DPorts(ports: Seq[ChipletLinkParams]) extends Config((site, here, up) => { + case ChipletRoutingKey => Some(up(ChipletRoutingKey).getOrElse(ChipletRoutingParams(ports=Nil)).copy(ports=ports)) +}) + +class WithND2DPorts(n: Int, portParams: ChipletLinkParams) extends Config((site, here, up) => { + case ChipletRoutingKey => Some(up(ChipletRoutingKey).getOrElse(ChipletRoutingParams(ports=Nil)).copy(ports=Seq.fill(n)(portParams))) +}) + +class WithMaxOffchipAddressRange(range: Seq[AddressSet]) extends Config((site, here, up) => { + case MaxOffchipAddressRange => range +}) \ No newline at end of file diff --git a/src/main/scala/soc/OffchipRouter.scala b/src/main/scala/soc/OffchipRouter.scala index f067c97a..72a4b103 100644 --- a/src/main/scala/soc/OffchipRouter.scala +++ b/src/main/scala/soc/OffchipRouter.scala @@ -18,21 +18,20 @@ import testchipip.util.{TLSwitch} case class ChipletRoutingParams( clientBusWhere: TLBusWrapperLocation = SBUS, controlBusWhere: TLBusWrapperLocation = CBUS, - routerParams: OffchipRouterParams, - translationParams: InwardAddressTranslatorParams, + routerParams: OffchipRouterParams = OffchipRouterParams(), ports: Seq[ChipletLinkParams] -) +) { + def idWidth = log2Ceil(routerParams.tableEntries) +} case object ChipletRoutingKey extends Field[Option[ChipletRoutingParams]](None) case class OffchipRouterParams( - // Need to think about what I want here tableAddress: BigInt = 0x4000L, - tableEntries: Int, - beatBytes: Int // maybe not here + tableEntries: Int = 16 ) -class OffchipRouter(val params: OffchipRouterParams, val nPorts: Int, val offset: BigInt)(implicit p: Parameters) extends LazyModule { +class OffchipRouter(val params: ChipletRoutingParams, val beatBytes: Int = 8)(implicit p: Parameters) extends LazyModule { // BORROWED FROM TLSwitch.scala START // This function can handle simple cases only @@ -70,9 +69,9 @@ class OffchipRouter(val params: OffchipRouterParams, val nPorts: Int, val offset // Register node for setting the routing table val routing_table_node = TLRegisterNode( - address = AddressSet.misaligned(params.tableAddress, 0x1000), // TODO: fix size + address = AddressSet.misaligned(params.routerParams.tableAddress, 0x1000), // TODO: fix size device = new SimpleDevice("routing-table", Nil), - beatBytes = params.beatBytes + beatBytes = beatBytes ) override lazy val module = new OffchipRouterImpl(this) @@ -81,15 +80,18 @@ class OffchipRouter(val params: OffchipRouterParams, val nPorts: Int, val offset class OffchipRouterImpl(outer: OffchipRouter) extends LazyModuleImp(outer) { val io = IO(new Bundle { - val chip_id = Output(UInt(log2Ceil(outer.params.tableEntries).W)) + val chip_id = Output(UInt(outer.params.idWidth.W)) }) val (bundleIn, edgeIn) = outer.node.in(0) val bundlesOut = outer.node.out.map(_._1) - val portWidth = if (outer.nPorts == 1) 1 else log2Ceil(outer.nPorts) - val idWidth = log2Ceil(outer.params.tableEntries + 1) // But maybe we want to allow this to be configurable, in case chip ids don't count up from 0 - val addressWidth = bundleIn.a.bits.address.getWidth // TODO: check + val nPorts = bundlesOut.size + + val tableEntries = outer.params.routerParams.tableEntries + val idWidth = outer.params.idWidth + val addressWidth = bundleIn.a.bits.address.getWidth + val portWidth = if (nPorts == 1) 1 else log2Ceil(nPorts) val totalWidth = idWidth + addressWidth + portWidth + 1 assert(totalWidth <= 64, "Total width of routing table entry must be less than or equal to 64 bits") // for now :) @@ -100,30 +102,30 @@ class OffchipRouterImpl(outer: OffchipRouter) extends LazyModuleImp(outer) { // and initialize it to all 0s using RegInit //val routing_table = VecInit(Seq.fill(outer.params.tableEntries)(RegInit(0.U.asTypeOf(new RoutingTableEntry(idWidth, addressWidth, portWidth))))) - val routing_table = RegInit(VecInit(Seq.fill(outer.params.tableEntries)(0.U.asTypeOf(new RoutingTableEntry(idWidth, addressWidth, portWidth))))) + val routing_table = RegInit(VecInit(Seq.fill(tableEntries)(0.U.asTypeOf(new RoutingTableEntry(idWidth, portWidth))))) // One entry = 4 separate regs (valid, chipID, baseAddress, port) val regsPerEntry = 4 // TODO: fixy - val mapped_entries = (0 until outer.params.tableEntries).flatMap { i => + val mapped_entries = (0 until tableEntries).flatMap { i => val base = i * regsPerEntry Seq( - (base + 0) -> Seq(RegField(1, routing_table(i).valid)), - (base + 1) -> Seq(RegField(idWidth, routing_table(i).chipID)), - (base + 2) -> Seq(RegField(portWidth, routing_table(i).port)), + (base + 0) -> Seq(RegField(1, routing_table(i).valid.asUInt, RegFieldDesc(s"entry_${i}_valid", "Valid bit"))), + (base + 1) -> Seq(RegField(idWidth, routing_table(i).chipID, RegFieldDesc(s"entry_${i}_chipID", "Chip ID"))), + (base + 2) -> Seq(RegField(portWidth, routing_table(i).port, RegFieldDesc(s"entry_${i}_port", "Port"))), ) } - val chip_id = Seq((outer.params.tableEntries * regsPerEntry) -> Seq(RegField(idWidth, chipIdReg))) + val chip_id = Seq((tableEntries * regsPerEntry) -> Seq(RegField(idWidth, chipIdReg, RegFieldDesc("chip_id", "Chip ID for this chip")))) outer.routing_table_node.regmap(mapped_entries ++ chip_id :_*) // Select offchip port from routing table - val addr_top_bits = bundleIn.a.bits.address(addressWidth-1, log2Ceil(outer.offset)) - val matches = Wire(Vec(outer.params.tableEntries, Bool())) - val sel = Wire(UInt(log2Ceil(outer.nPorts).W)) + val addr_top_bits = bundleIn.a.bits.address(addressWidth-1, log2Ceil(p(MaxOffchipAddressRange).map(_.base).min)) + val matches = Wire(Vec(tableEntries, Bool())) + val sel = Wire(UInt(log2Ceil(nPorts).W)) sel := 0.U - matches := VecInit(Seq.fill(outer.params.tableEntries)(false.B)) - for (i <- 0 until outer.params.tableEntries) { + matches := VecInit(Seq.fill(tableEntries)(false.B)) + for (i <- 0 until tableEntries) { when (routing_table(i).valid && (addr_top_bits === routing_table(i).chipID)) { sel := routing_table(i).port matches(i) := true.B @@ -193,49 +195,51 @@ class OffchipRouterImpl(outer: OffchipRouter) extends LazyModuleImp(outer) { // BORROWED FROM TLSwitch.scala END // Arbitrate responses from d channels - val response_arbiter_d = Module(new RRArbiter(chiselTypeOf(bundleIn.d.bits), outer.nPorts + 1)) - for (i <- 0 until outer.nPorts) { + val response_arbiter_d = Module(new RRArbiter(chiselTypeOf(bundleIn.d.bits), nPorts + 1)) + for (i <- 0 until nPorts) { response_arbiter_d.io.in(i) <> bundlesOut(i).d } - response_arbiter_d.io.in(outer.nPorts) <> dErr - bundleIn.d <> Queue(response_arbiter_d.io.out, outer.nPorts + 1) // I think this queue is necessary?? + response_arbiter_d.io.in(nPorts) <> dErr + bundleIn.d <> Queue(response_arbiter_d.io.out, nPorts + 1) // I think this queue is necessary?? // Arbitrate responses from b channels // TODO ?? : Error checking for coherent requests - val response_arbiter_b = Module(new RRArbiter(chiselTypeOf(bundleIn.b.bits), outer.nPorts)) - for (i <- 0 until outer.nPorts) { + val response_arbiter_b = Module(new RRArbiter(chiselTypeOf(bundleIn.b.bits), nPorts)) + for (i <- 0 until nPorts) { response_arbiter_b.io.in(i) <> bundlesOut(i).b } - bundleIn.b <> Queue(response_arbiter_b.io.out, outer.nPorts) + bundleIn.b <> Queue(response_arbiter_b.io.out, nPorts) } // TODO: fix widths -class RoutingTableEntry(idWidth: Int, addressWidth: Int, portWidth: Int) extends Bundle { +class RoutingTableEntry(idWidth: Int, portWidth: Int) extends Bundle { val valid = Bool() val chipID = UInt(idWidth.W) // This is probably determined by the number of entries val port = UInt(portWidth.W) } // Mostly borrowed from the Radiance AddressRewriterNode -case class ChipletAddressTranslator(val params: InwardAddressTranslatorParams, val idWidth: Int)(implicit p: Parameters) extends LazyModule { +case class ChipletAddressTranslator(val params: ChipletRoutingParams)(implicit p: Parameters) extends LazyModule { val node = TLAdapterNode(clientFn = c => c, managerFn = m => m) - assert(params.offset.bitCount == 1, "Offset must only have 1 bit set") // TODO: should we enforce this in the spec? + val offset = p(MaxOffchipAddressRange).map(_.base).min + + assert(offset.bitCount == 1, "Offset must only have 1 bit set") // TODO: should we enforce this in the spec? override lazy val module = new ChipletAddressTranslatorImpl(this) } class ChipletAddressTranslatorImpl(outer: ChipletAddressTranslator) extends LazyModuleImp(outer) { val io = IO(new Bundle { - val chip_id = Input(UInt(outer.idWidth.W)) + val chip_id = Input(UInt(outer.params.idWidth.W)) }) // If the top bits of the address match the chip id, then trim those bits off // Top bits are determined by the offset in the params def trimTag(address: UInt) = { - val topBits = log2Ceil(outer.params.offset) + val topBits = log2Ceil(outer.offset) val addressWidth = address.getWidth val tagMatch = address(addressWidth-1, topBits) === io.chip_id val mask = (1.U(addressWidth.W) << topBits) - 1.U @@ -257,6 +261,8 @@ class ChipletAddressTranslatorImpl(outer: ChipletAddressTranslator) extends Lazy trait CanHaveChipletRouting { this: BaseSubsystem => val d2d_port_ios = p(ChipletRoutingKey).map { params => + require(params.ports.nonEmpty, "At least one D2D port must be specified") + val cbus = locateTLBusWrapper(params.controlBusWhere) val client_bus = locateTLBusWrapper(params.clientBusWhere) @@ -277,7 +283,7 @@ trait CanHaveChipletRouting { this: BaseSubsystem => val router_domain = LazyModule(new ClockSinkDomain(name=Some("offchip_router"))) router_domain.clockNode := client_bus.fixedClockNode - val router = router_domain { LazyModule(new OffchipRouter(params.routerParams, params.ports.size, params.translationParams.offset)) } + val router = router_domain { LazyModule(new OffchipRouter(params, beatBytes=cbus.beatBytes)) } // The bus drives the router's manager node client_bus.coupleTo(s"offchip_router") { router.node := TLBuffer() := _ } @@ -287,11 +293,11 @@ trait CanHaveChipletRouting { this: BaseSubsystem => val port_ios = params.ports.zipWithIndex.map { case (pP, id) => val link_manager_bus = locateTLBusWrapper(pP.managerBusWhere) - val port = router_domain { pP.asInstanceOf[ChipletLinkWrapperInstantiationLike].instantiate(s"d2d${id}", id)(p) } + val port = router_domain { pP.asInstanceOf[ChipletLinkWrapperInstantiationLike].instantiate(p(MaxOffchipAddressRange), id)(p).suggestName(s"d2d${id}_port") } // TODO: Translator should take in chip ID as an IO val translator = router_domain { - LazyModule(ChipletAddressTranslator(params.translationParams, log2Ceil(params.routerParams.tableEntries))(p)) + LazyModule(ChipletAddressTranslator(params)) } router_domain { @@ -310,7 +316,6 @@ trait CanHaveChipletRouting { this: BaseSubsystem => port_ioSink := port.top_IO val outer_io = InModuleBody { - // port_ioSink.makeIO(s"d2d${id}_port") val outer_io = IO(chiselTypeOf(port_ioSink.in(0)._1)).suggestName(s"d2d${id}_port") outer_io <> port_ioSink.in(0)._1 outer_io @@ -322,6 +327,3 @@ trait CanHaveChipletRouting { this: BaseSubsystem => } } -class WithChipletRouting(params: ChipletRoutingParams) extends Config((site, here, up) => { - case ChipletRoutingKey => Some(params) -}) From 046f6bdaadbbe2cbf88648c523a2347b90130fb3 Mon Sep 17 00:00:00 2001 From: Ella Schwarz Date: Thu, 12 Mar 2026 13:58:16 -0700 Subject: [PATCH 07/19] Add router unit tests, passing --- src/main/scala/soc/OffchipRouter.scala | 18 +- src/main/scala/test/Unittests.scala | 283 ++++++++++++++++++++++++- 2 files changed, 293 insertions(+), 8 deletions(-) diff --git a/src/main/scala/soc/OffchipRouter.scala b/src/main/scala/soc/OffchipRouter.scala index 72a4b103..94c88f83 100644 --- a/src/main/scala/soc/OffchipRouter.scala +++ b/src/main/scala/soc/OffchipRouter.scala @@ -21,7 +21,7 @@ case class ChipletRoutingParams( routerParams: OffchipRouterParams = OffchipRouterParams(), ports: Seq[ChipletLinkParams] ) { - def idWidth = log2Ceil(routerParams.tableEntries) + def idWidth = log2Ceil(routerParams.tableEntries) + 1 } case object ChipletRoutingKey extends Field[Option[ChipletRoutingParams]](None) @@ -98,6 +98,8 @@ class OffchipRouterImpl(outer: OffchipRouter) extends LazyModuleImp(outer) { val chipIdReg = RegInit(0.U(idWidth.W)) io.chip_id := chipIdReg + //val offsetReg = RegInit(p(MaxOffchipAddressRange).map(_.base).min.U(addressWidth.W)) + // Create a memory mapped table of offchip addresses which store chip id, base address, and port to route to // and initialize it to all 0s using RegInit //val routing_table = VecInit(Seq.fill(outer.params.tableEntries)(RegInit(0.U.asTypeOf(new RoutingTableEntry(idWidth, addressWidth, portWidth))))) @@ -108,16 +110,18 @@ class OffchipRouterImpl(outer: OffchipRouter) extends LazyModuleImp(outer) { val regsPerEntry = 4 // TODO: fixy val mapped_entries = (0 until tableEntries).flatMap { i => - val base = i * regsPerEntry + val base = i * regsPerEntry * outer.beatBytes Seq( - (base + 0) -> Seq(RegField(1, routing_table(i).valid.asUInt, RegFieldDesc(s"entry_${i}_valid", "Valid bit"))), - (base + 1) -> Seq(RegField(idWidth, routing_table(i).chipID, RegFieldDesc(s"entry_${i}_chipID", "Chip ID"))), - (base + 2) -> Seq(RegField(portWidth, routing_table(i).port, RegFieldDesc(s"entry_${i}_port", "Port"))), + (base + 0 * outer.beatBytes) -> Seq(RegField(1, routing_table(i).valid.asUInt, RegFieldDesc(s"entry_${i}_valid", "Valid bit"))), + (base + 1 * outer.beatBytes) -> Seq(RegField(idWidth, routing_table(i).chipID, RegFieldDesc(s"entry_${i}_chipID", "Chip ID"))), + (base + 2 * outer.beatBytes) -> Seq(RegField(portWidth, routing_table(i).port, RegFieldDesc(s"entry_${i}_port", "Port"))), ) } - - val chip_id = Seq((tableEntries * regsPerEntry) -> Seq(RegField(idWidth, chipIdReg, RegFieldDesc("chip_id", "Chip ID for this chip")))) + + val chip_id = Seq((tableEntries * regsPerEntry * outer.beatBytes) -> Seq(RegField(idWidth, chipIdReg, RegFieldDesc("chip_id", "Chip ID for this chip")))) outer.routing_table_node.regmap(mapped_entries ++ chip_id :_*) + // val offset = Seq((tableEntries * regsPerEntry) + 1 -> Seq(RegField(addressWidth, 0.U, RegFieldDesc("offset", "Offchip offset for this chip")))) + // outer.routing_table_node.regmap(mapped_entries ++ chip_id ++ offset :_*) // Select offchip port from routing table val addr_top_bits = bundleIn.a.bits.address(addressWidth-1, log2Ceil(p(MaxOffchipAddressRange).map(_.base).min)) diff --git a/src/main/scala/test/Unittests.scala b/src/main/scala/test/Unittests.scala index ea3ce92f..137192ef 100644 --- a/src/main/scala/test/Unittests.scala +++ b/src/main/scala/test/Unittests.scala @@ -677,6 +677,285 @@ class NetworkXbarTest extends UnitTest { io.finished := finished.reduce(_ && _) } +case class TLRequestDescriptor( + address: BigInt, + isWrite: Boolean, + data: BigInt = 0, + size: Int = 3 // log2(8) = 3 => 8 bytes by default +) + +class OffchipRouterTestDriver(reqs: Seq[TLRequestDescriptor])(implicit p: Parameters) extends LazyModule { + val node = TLClientNode(Seq(TLMasterPortParameters.v1(Seq(TLClientParameters( + name = "offchip_router_test_driver", sourceId = IdRange(0, 1)))))) + + lazy val module = new Impl + class Impl extends LazyModuleImp(this) { + val io = IO(new Bundle { + val start = Input(Bool()) + val finished = Output(Bool()) + }) + + val (tl, edge) = node.out(0) + val nReqs = reqs.size + + val reqIdx = RegInit(0.U(log2Ceil(nReqs + 1).W)) + + val addresses = VecInit(reqs.map(r => r.address.U(edge.bundle.addressBits.W))) + val writes = VecInit(reqs.map(r => r.isWrite.B)) + val datas = VecInit(reqs.map(r => r.data.U(edge.bundle.dataBits.W))) + val sizes = VecInit(reqs.map(r => r.size.U(edge.bundle.sizeBits.W))) + + val (s_idle :: s_req :: s_resp :: s_done :: Nil) = Enum(4) + val state = RegInit(s_idle) + + when (state === s_idle && io.start) { state := s_req } + + // A channel: send requests + val currAddr = addresses(reqIdx) + val currWrite = writes(reqIdx) + val currData = datas(reqIdx) + val currSize = sizes(reqIdx) + + val putBits = edge.Put(0.U, currAddr, currSize, currData)._2 + val getBits = edge.Get(0.U, currAddr, currSize)._2 + + tl.a.valid := state === s_req + tl.a.bits := Mux(currWrite, putBits, getBits) + + // D channel: accept responses + tl.d.ready := state === s_resp + + when (tl.a.fire) { state := s_resp } + when (tl.d.fire) { + reqIdx := reqIdx + 1.U + when (reqIdx === (nReqs - 1).U) { + state := s_done + }.otherwise { + state := s_req + } + } + + tl.b.ready := false.B + tl.c.valid := false.B + tl.c.bits := DontCare + tl.e.valid := false.B + tl.e.bits := DontCare + + io.finished := state === s_done + } +} + +class OffchipRouterTestChecker(portId: Int, expectedReqs: Seq[TLRequestDescriptor])(implicit p: Parameters) extends LazyModule { + val node = TLManagerNode(Seq(TLSlavePortParameters.v1( + managers = Seq(TLSlaveParameters.v1( + address = p(MaxOffchipAddressRange), + supportsGet = TransferSizes(1, 64), + supportsPutFull = TransferSizes(1, 64))), + beatBytes = 8 + ))) + + lazy val module = new Impl + class Impl extends LazyModuleImp(this) { + val io = IO(new Bundle { + val finished = Output(Bool()) + }) + + val (tl, edge) = node.in(0) + val nExpected = expectedReqs.size + + if (nExpected == 0) { + // No requests expected on this port — just tie off and assert nothing arrives + tl.a.ready := false.B + tl.d.valid := false.B + tl.d.bits := DontCare + tl.b.valid := false.B + tl.b.bits := DontCare + tl.c.ready := false.B + tl.e.ready := false.B + assert(!tl.a.valid, s"Checker $portId: unexpected request on port with no expected traffic") + io.finished := true.B + } else { + val reqIdx = RegInit(0.U(log2Ceil(nExpected + 1).W)) + + val expAddrs = VecInit(expectedReqs.map(r => r.address.U(edge.bundle.addressBits.W))) + val expWrites = VecInit(expectedReqs.map(r => r.isWrite.B)) + val expDatas = VecInit(expectedReqs.map(r => r.data.U(edge.bundle.dataBits.W))) + val expSizes = VecInit(expectedReqs.map(r => r.size.U(edge.bundle.sizeBits.W))) + + val (s_wait_req :: s_send_resp :: s_done :: Nil) = Enum(3) + val state = RegInit(s_wait_req) + + // Capture request info for response + val savedOpcode = Reg(chiselTypeOf(tl.a.bits.opcode)) + val savedSize = Reg(chiselTypeOf(tl.a.bits.size)) + val savedSource = Reg(chiselTypeOf(tl.a.bits.source)) + + // A channel: accept and check requests + tl.a.ready := state === s_wait_req + + when (tl.a.fire) { + // Check address + assert(tl.a.bits.address === expAddrs(reqIdx), + s"Checker $portId: address mismatch") + // Check opcode (PutFullData = 0, Get = 4) + when (expWrites(reqIdx)) { + assert(tl.a.bits.opcode === TLMessages.PutFullData, + s"Checker $portId: expected Put") + assert(tl.a.bits.data === expDatas(reqIdx), + s"Checker $portId: write data mismatch") + }.otherwise { + assert(tl.a.bits.opcode === TLMessages.Get, + s"Checker $portId: expected Get") + } + // Check size + assert(tl.a.bits.size === expSizes(reqIdx), + s"Checker $portId: size mismatch") + + savedOpcode := tl.a.bits.opcode + savedSize := tl.a.bits.size + savedSource := tl.a.bits.source + state := s_send_resp + } + + // D channel: send response + tl.d.valid := state === s_send_resp + tl.d.bits := edge.AccessAck(savedSource, savedSize) + // Override for Get responses (need AccessAckData) + when (savedOpcode === TLMessages.Get) { + tl.d.bits := edge.AccessAck(savedSource, savedSize, 0.U) + } + + when (tl.d.fire) { + reqIdx := reqIdx + 1.U + when (reqIdx === (nExpected - 1).U) { + state := s_done + }.otherwise { + state := s_wait_req + } + } + + tl.b.valid := false.B + tl.b.bits := DontCare + tl.c.ready := false.B + tl.e.ready := false.B + + io.finished := state === s_done + } + } +} + +class OffchipRouterTest(val nChips: Int, val nPorts: Int, val reqsPerChip: Int = 4)(implicit p: Parameters) extends LazyModule { + + require(Integer.bitCount(nChips) == 1, "nChips must be a power of 2") + require(nPorts > 0, "must have at least one port") + + val params = new ChipletRoutingParams( + routerParams = OffchipRouterParams(tableAddress = 0x1000L, tableEntries = nChips), + ports = Nil + ) + val beatBytes = 8 + val tableBase = params.routerParams.tableAddress + val tableEntries = params.routerParams.tableEntries + val regsPerEntry = 4 + + val router = LazyModule(new OffchipRouter(params, beatBytes = beatBytes)) + + // Build MMIO request sequence: program chip ID, then program routing table + val rand = new Random() + val chipId = rand.nextInt(nChips) + 1 // this chip's ID (1 to nChips) + // All other chip IDs (1 to nChips, excluding this chip), each mapped to a random port + val otherChipIds = (1 to nChips).filter(_ != chipId) + val tableData = otherChipIds.map { cid => (cid, rand.nextInt(nPorts)) } // (chipID, port) + + val mmioReqs: Seq[TLRequestDescriptor] = { + // Each regmap field is beat-aligned (offset * beatBytes), so full-word writes work + + // Program chip ID register + val chipIdAddr = tableBase + (tableEntries * regsPerEntry) * beatBytes + val progChipId = Seq(TLRequestDescriptor(chipIdAddr, isWrite = true, data = chipId)) + + // Program routing table entries (valid, chipID, port for each remote chip) + val progTable = tableData.zipWithIndex.flatMap { case ((cid, port), idx) => + val baseAddr = tableBase + (idx * regsPerEntry) * beatBytes + Seq( + TLRequestDescriptor(baseAddr + 0 * beatBytes, isWrite = true, data = 1), // valid + TLRequestDescriptor(baseAddr + 1 * beatBytes, isWrite = true, data = cid), // chipID + TLRequestDescriptor(baseAddr + 2 * beatBytes, isWrite = true, data = port), // port + ) + } + + progChipId ++ progTable + } + + // Build data request sequence: reqsPerChip random accesses to each remote chip, shuffled + val dataReqs: Seq[TLRequestDescriptor] = { + val offset = p(MaxOffchipAddressRange).map(_.base).min + val offsetBits = log2Ceil(offset) + val reqs = otherChipIds.flatMap { cid => + Seq.fill(reqsPerChip) { + val baseAddr = BigInt(cid) << offsetBits + val addr = baseAddr + (BigInt(offsetBits, rand) & ~BigInt(7)) // align to 8 bytes + TLRequestDescriptor(addr, isWrite = rand.nextBoolean(), data = BigInt(64, rand)) + } + } + rand.shuffle(reqs) + } + + // Build chipID -> port lookup from tableData + val chipIdToPort: Map[Int, Int] = tableData.map { case (cid, port) => cid -> port }.toMap + + // Determine expected port for each dataReq based on its address tag + val offset = p(MaxOffchipAddressRange).map(_.base).min + val offsetBits = log2Ceil(offset) + + def reqPort(r: TLRequestDescriptor): Int = { + val addrChipId = (r.address >> offsetBits).toInt + chipIdToPort(addrChipId) + } + + // Partition dataReqs by port, preserving order within each port + val reqsByPort: Seq[Seq[TLRequestDescriptor]] = Seq.tabulate(nPorts) { port => + dataReqs.filter(r => reqPort(r) == port) + } + + val mmio_driver = LazyModule(new OffchipRouterTestDriver(mmioReqs)) + val driver = LazyModule(new OffchipRouterTestDriver(dataReqs)) + + val checkers = Seq.tabulate(nPorts) { id => + LazyModule(new OffchipRouterTestChecker(id, reqsByPort(id))) + } + + router.node := TLBuffer() := driver.node + router.routing_table_node := TLBuffer() := mmio_driver.node + checkers.foreach(_.node := TLBuffer() := router.node) + + lazy val module = new Impl + class Impl extends LazyModuleImp(this) { + val io = IO(new Bundle { + val finished = Output(Bool()) + }) + + // Phase 1: MMIO driver programs chip ID + routing table + // Phase 2: Data driver sends requests through the router + val mmioFinished = mmio_driver.module.io.finished + mmio_driver.module.io.start := true.B + + // Verify chip ID was programmed correctly + when (mmioFinished) { + assert(router.module.io.chip_id === chipId.U, "Chip ID mismatch after programming") + } + + driver.module.io.start := mmioFinished + io.finished := driver.module.io.finished + + } +} + +class OffchipRouterTestWrapper(nChips: Int, nPorts: Int, reqsPerChip: Int = 4, timeout: Int = 8192)(implicit p: Parameters) extends UnitTest(timeout) { + val test = Module(LazyModule(new OffchipRouterTest(nChips, nPorts, reqsPerChip)).module) + io.finished := test.io.finished +} + object TestChipUnitTests { def apply(implicit p: Parameters): Seq[UnitTest] = Seq( @@ -688,6 +967,8 @@ object TestChipUnitTests { Module(new StreamWidthAdapterTest), Module(new NetworkXbarTest), Module(new TLRingNetworkTestWrapper), - Module(new TLCTCTestWrapper(CreditedSourceSyncSerialPhyParams(flitWidth = 32, phitWidth = 4), 20000)) + Module(new TLCTCTestWrapper(CreditedSourceSyncSerialPhyParams(flitWidth = 32, phitWidth = 4), 20000)), + Module(new OffchipRouterTestWrapper(nChips = 4, nPorts = 2)), + Module(new OffchipRouterTestWrapper(nChips = 8, nPorts = 3, reqsPerChip = 8)) ) } From a99061fdc2a4fd64619a9fc046b1fa38e5966f2f Mon Sep 17 00:00:00 2001 From: Ella Schwarz Date: Thu, 12 Mar 2026 17:12:02 -0700 Subject: [PATCH 08/19] [WIP] Add more ChipletIO, except decoupled doesn't sim correctly yet --- src/main/scala/ctc/CTC.scala | 141 +++++++++++-------------- src/main/scala/serdes/Bundles.scala | 63 ++++++++++- src/main/scala/serdes/Parameters.scala | 3 +- 3 files changed, 123 insertions(+), 84 deletions(-) diff --git a/src/main/scala/ctc/CTC.scala b/src/main/scala/ctc/CTC.scala index 1348d88f..9d4f1865 100644 --- a/src/main/scala/ctc/CTC.scala +++ b/src/main/scala/ctc/CTC.scala @@ -89,93 +89,74 @@ class CTCChipletLink(val params: CTCParams, val manager_region: Seq[AddressSet], // a TL master/client device val ctc2tl = LazyModule(new CTCToTileLink(portId=id)(p)) // a TL slave/manager device - val tl2ctc = LazyModule(new TileLinkToCTC(addrRegion=manager_region)(p)) - - // val ctc_outer_io = InModuleBody { - // val outer_io = params.phyParams.map(pP => IO(pP.genIO).suggestName(name)).getOrElse(IO(new CTCBridgeIO).suggestName(name)) - - // outer_io match { - // // case io: CreditedSourceSyncPhitIO => { - // // // 3 clock domains - - // // // - ctc2tl's "Inner clock": synchronizes signals going to the digital logic - // // // - outgoing clock: synchronizes signals going out - // // // - incoming clock: synchronizes signals coming in - // // // TODO: SERIOUSLY CHECK THIS CLOCKING, I JUST GUESSED!!!!! - // // val outgoing_clock = ctc2tl.module.clock - // // val outgoing_reset = ResetCatchAndSync(outgoing_clock, ctc2tl.module.reset.asBool) - // // val incoming_clock = io.clock_in - // // val incoming_reset = ResetCatchAndSync(incoming_clock, io.reset_in.asBool) - // // io.clock_out := outgoing_clock - // // io.reset_out := outgoing_reset.asAsyncReset - // // val phy = Module(new CreditedSerialPhy(2, params.phyParams.get)) - // // phy.io.incoming_clock := incoming_clock - // // phy.io.incoming_reset := incoming_reset - // // phy.io.outgoing_clock := outgoing_clock - // // phy.io.outgoing_reset := outgoing_reset - // // phy.io.inner_clock := ctc2tl.module.clock - // // phy.io.inner_reset := ctc2tl.module.reset - // // phy.io.inner_ser(0).in <> ctc2tl.module.io.flit.in - // // phy.io.inner_ser(0).out <> tl2ctc.module.io.flit.out - // // phy.io.inner_ser(1).in <> tl2ctc.module.io.flit.in - // // phy.io.inner_ser(1).out <> ctc2tl.module.io.flit.out - - // // phy.io.outer_ser <> io.viewAsSupertype(new ValidPhitIO(params.phyParams.get.phitWidth)) - // // } - // // case io: DecoupledInternalSyncPhitIO => { - // // val outgoing_clock = ctc2tl.module.clock - // // io.clock_out := outgoing_clock - // // val phy = Module(new DecoupledSerialPhy(2, params.phyParams.get)) - // // phy.io.outer_clock := outgoing_clock - // // phy.io.outer_reset := ResetCatchAndSync(outgoing_clock, ctc2tl.module.reset.asBool) - // // phy.io.inner_clock := ctc2tl.module.clock - // // phy.io.inner_reset := ctc2tl.module.reset - // // phy.io.inner_ser(0).in <> ctc2tl.module.io.flit.in - // // phy.io.inner_ser(0).out <> tl2ctc.module.io.flit.out - // // phy.io.inner_ser(1).in <> tl2ctc.module.io.flit.in - // // phy.io.inner_ser(1).out <> ctc2tl.module.io.flit.out - // // phy.io.outer_ser <> io.viewAsSupertype(new DecoupledPhitIO(params.phyParams.get.phitWidth)) - // // } - // // case io: DecoupledExternalSyncPhitIO => { - // // val outgoing_clock = io.clock_in - // // val outgoing_reset = ResetCatchAndSync(outgoing_clock, ctc2tl.module.reset.asBool) - // // val phy = Module(new DecoupledSerialPhy(2, params.phyParams.get)) - // // phy.io.outer_clock := outgoing_clock - // // phy.io.outer_reset := ResetCatchAndSync(outgoing_clock, ctc2tl.module.reset.asBool) - // // phy.io.inner_clock := ctc2tl.module.clock - // // phy.io.inner_reset := ctc2tl.module.reset - // // phy.io.inner_ser(0).in <> ctc2tl.module.io.flit.in - // // phy.io.inner_ser(0).out <> tl2ctc.module.io.flit.out - // // phy.io.inner_ser(1).in <> tl2ctc.module.io.flit.in - // // phy.io.inner_ser(1).out <> ctc2tl.module.io.flit.out - // // phy.io.outer_ser <> io.viewAsSupertype(new DecoupledPhitIO(params.phyParams.get.phitWidth)) - // // } - // // Start with CTCBridgeIO only for now - // case io: CTCBridgeIO => { - // io.manager_flit <> tl2ctc.module.io.flit - // io.client_flit <> ctc2tl.module.io.flit - // } - // case _ => assert(false, s"Unsupported IO type for CTC ChipletLink") - // } - // outer_io - // } - - //val link_name = link_name + id.toString + val tl2ctc = LazyModule(new TileLinkToCTC(addrRegion=manager_region)(p)) + val client_node = ctc2tl.node val manager_node = tl2ctc.node val control_manager_node = None - val top_IO = BundleBridgeSource(() => new CTCBridgeIO) + val top_IO = params.phyParams match { + case Some(pP) => BundleBridgeSource(() => pP.genIO.asInstanceOf[ChipletIO]) + case None => BundleBridgeSource(() => new CTCBridgeIO) + } override lazy val module = new CTCChipletLinkImpl(this) - //val top_IO = InModuleBody { module.io } - // InModuleBody { - // top_IO.out(0)._1.asInstanceOf[CTCBridgeIO].manager_flit <> tl2ctc.module.io.flit - // top_IO.out(0)._1.asInstanceOf[CTCBridgeIO].client_flit <> ctc2tl.module.io.flit - // } } -// Maybe can move this into an InModuleBody class CTCChipletLinkImpl(outer: CTCChipletLink) extends LazyModuleImp(outer) { - outer.top_IO.out(0)._1.asInstanceOf[CTCBridgeIO].manager_flit <> outer.tl2ctc.module.io.flit - outer.top_IO.out(0)._1.asInstanceOf[CTCBridgeIO].client_flit <> outer.ctc2tl.module.io.flit + val io = outer.top_IO.out(0)._1 + + io match { + case io: CreditedSourceSyncPhitIO => { + val outgoing_clock = clock + val outgoing_reset = ResetCatchAndSync(outgoing_clock, reset.asBool) + val incoming_clock = io.clock_in + val incoming_reset = ResetCatchAndSync(incoming_clock, io.reset_in.asBool) + io.clock_out := outgoing_clock + io.reset_out := outgoing_reset.asAsyncReset + val phy = Module(new CreditedSerialPhy(2, outer.params.phyParams.get)) + phy.io.incoming_clock := incoming_clock + phy.io.incoming_reset := incoming_reset + phy.io.outgoing_clock := outgoing_clock + phy.io.outgoing_reset := outgoing_reset + phy.io.inner_clock := outer.ctc2tl.module.clock + phy.io.inner_reset := outer.ctc2tl.module.reset + phy.io.inner_ser(0).in <> outer.ctc2tl.module.io.flit.in + phy.io.inner_ser(0).out <> outer.tl2ctc.module.io.flit.out + phy.io.inner_ser(1).in <> outer.tl2ctc.module.io.flit.in + phy.io.inner_ser(1).out <> outer.ctc2tl.module.io.flit.out + phy.io.outer_ser <> io.viewAsSupertype(new ValidPhitIO(outer.params.phyParams.get.phitWidth)) + } + case io: DecoupledInternalSyncPhitIO => { + val outgoing_clock = clock + io.clock_out := outgoing_clock + val phy = Module(new DecoupledSerialPhy(2, outer.params.phyParams.get)) + phy.io.outer_clock := outgoing_clock + phy.io.outer_reset := ResetCatchAndSync(outgoing_clock, reset.asBool) + phy.io.inner_clock := outer.ctc2tl.module.clock + phy.io.inner_reset := outer.ctc2tl.module.reset + phy.io.inner_ser(0).in <> outer.ctc2tl.module.io.flit.in + phy.io.inner_ser(0).out <> outer.tl2ctc.module.io.flit.out + phy.io.inner_ser(1).in <> outer.tl2ctc.module.io.flit.in + phy.io.inner_ser(1).out <> outer.ctc2tl.module.io.flit.out + phy.io.outer_ser <> io.viewAsSupertype(new DecoupledPhitIO(outer.params.phyParams.get.phitWidth)) + } + case io: DecoupledExternalSyncPhitIO => { + val outgoing_clock = io.clock_in + val phy = Module(new DecoupledSerialPhy(2, outer.params.phyParams.get)) + phy.io.outer_clock := outgoing_clock + phy.io.outer_reset := ResetCatchAndSync(outgoing_clock, reset.asBool) + phy.io.inner_clock := outer.ctc2tl.module.clock + phy.io.inner_reset := outer.ctc2tl.module.reset + phy.io.inner_ser(0).in <> outer.ctc2tl.module.io.flit.in + phy.io.inner_ser(0).out <> outer.tl2ctc.module.io.flit.out + phy.io.inner_ser(1).in <> outer.tl2ctc.module.io.flit.in + phy.io.inner_ser(1).out <> outer.ctc2tl.module.io.flit.out + phy.io.outer_ser <> io.viewAsSupertype(new DecoupledPhitIO(outer.params.phyParams.get.phitWidth)) + } + case io: CTCBridgeIO => { + io.manager_flit <> outer.tl2ctc.module.io.flit + io.client_flit <> outer.ctc2tl.module.io.flit + } + } } trait CanHavePeripheryCTC { this: BaseSubsystem => diff --git a/src/main/scala/serdes/Bundles.scala b/src/main/scala/serdes/Bundles.scala index 7b440855..2ee86a82 100644 --- a/src/main/scala/serdes/Bundles.scala +++ b/src/main/scala/serdes/Bundles.scala @@ -4,6 +4,7 @@ import chisel3._ import chisel3.util._ import freechips.rocketchip.diplomacy._ import org.chipsalliance.cde.config._ +import testchipip.soc.ChipletIO class Flit(val flitWidth: Int) extends Bundle { val flit = UInt(flitWidth.W) @@ -34,11 +35,44 @@ trait HasClockIn { this: Bundle => // A decoupled flow-control serial interface where all signals are synchronous to // a locally-produced clock -class DecoupledInternalSyncPhitIO(phitWidth: Int) extends DecoupledPhitIO(phitWidth) with HasClockOut +class DecoupledInternalSyncPhitIO(phitWidth: Int) extends DecoupledPhitIO(phitWidth) with HasClockOut with ChipletIO { + def tieoff: Unit = { + out.ready := false.B + in.valid := false.B + in.bits := DontCare + } + def connect(io: ChipletIO): Unit = io match { + case that: DecoupledExternalSyncPhitIO => + that.clock_in := this.clock_out + that.out <> this.in + this.out <> that.in + case _ => assert(false, s"IO does not match DecoupledExternalSyncPhitIO: ${io.getClass}") + } + def loopback: Unit = { + out <> in + } +} // A decoupled flow-control serial interface where all signals are synchronous to // an externally produced clock -class DecoupledExternalSyncPhitIO(phitWidth: Int) extends DecoupledPhitIO(phitWidth) with HasClockIn +class DecoupledExternalSyncPhitIO(phitWidth: Int) extends DecoupledPhitIO(phitWidth) with HasClockIn with ChipletIO { + def tieoff: Unit = { + clock_in := false.B.asClock + out.ready := false.B + in.valid := false.B + in.bits := DontCare + } + def connect(io: ChipletIO): Unit = io match { + case that: DecoupledInternalSyncPhitIO => + that.clock_out := this.clock_in + that.out <> this.in + this.out <> that.in + case _ => assert(false, s"IO does not match DecoupledInternalSyncPhitIO: ${io.getClass}") + } + def loopback: Unit = { + out <> in + } +} class ValidPhitIO(val phitWidth: Int) extends Bundle { val in = Input(Valid(new Phit(phitWidth))) @@ -52,11 +86,34 @@ class ValidFlitIO(val flitWidth: Int) extends Bundle { // A credited flow-control serial interface where all signals are synchronous to // a slock provided by the transmitter of that signal -class CreditedSourceSyncPhitIO(phitWidth: Int) extends ValidPhitIO(phitWidth) { +class CreditedSourceSyncPhitIO(phitWidth: Int) extends ValidPhitIO(phitWidth) with ChipletIO { val clock_in = Input(Clock()) val reset_out = Output(AsyncReset()) val clock_out = Output(Clock()) val reset_in = Input(AsyncReset()) + + def tieoff: Unit = { + clock_in := false.B.asClock + reset_in := false.B.asAsyncReset + in := DontCare + } + + def connect(io: ChipletIO): Unit = io match { + case that: CreditedSourceSyncPhitIO => + this.in := that.out + that.in := this.out + this.clock_in := that.clock_out + that.clock_in := this.clock_out + this.reset_in := that.reset_out + that.reset_in := this.reset_out + case _ => assert(false, s"IO does not match CreditedSourceSyncPhitIO: ${io.getClass}") + } + + def loopback: Unit = { + in := out + clock_in := clock_out + reset_in := reset_out + } } class SerialIO(val w: Int) extends Bundle { diff --git a/src/main/scala/serdes/Parameters.scala b/src/main/scala/serdes/Parameters.scala index 88c2b8d1..4613836a 100644 --- a/src/main/scala/serdes/Parameters.scala +++ b/src/main/scala/serdes/Parameters.scala @@ -5,12 +5,13 @@ import chisel3.util._ import freechips.rocketchip.diplomacy._ import org.chipsalliance.cde.config._ import freechips.rocketchip.tilelink._ +import testchipip.soc.ChipletIO trait SerialPhyParams { val phitWidth: Int val flitWidth: Int val flitBufferSz: Int - def genIO: Bundle + def genIO: ChipletIO } case class DecoupledInternalSyncSerialPhyParams( From 12f9315be68e29d2225d16639b991fa8d8cacf3a Mon Sep 17 00:00:00 2001 From: Ella Schwarz Date: Thu, 12 Mar 2026 20:43:13 -0700 Subject: [PATCH 09/19] [WIIIIP] added sertl chiplet link, compiles but does not sim --- src/main/scala/ctc/CTC.scala | 7 +- src/main/scala/serdes/PeripheryTLSerial.scala | 153 +++++++++++++++++- src/main/scala/soc/ChipletLink.scala | 10 +- src/main/scala/soc/OffchipRouter.scala | 19 ++- 4 files changed, 183 insertions(+), 6 deletions(-) diff --git a/src/main/scala/ctc/CTC.scala b/src/main/scala/ctc/CTC.scala index 9d4f1865..662e8b41 100644 --- a/src/main/scala/ctc/CTC.scala +++ b/src/main/scala/ctc/CTC.scala @@ -50,7 +50,7 @@ case class CTCParams( //def managerRegion = offchipRange def managerBusWhere = managerBus def controlManagerBusWhere = None - def instantiate(manager_region: Seq[AddressSet], id: Int)(implicit p: Parameters): ChipletLinkWrapper = LazyModule(new CTCChipletLink(this, manager_region, id)) + def instantiate(params: OffchipSubsystemParams, id: Int)(implicit p: Parameters): ChipletLinkWrapper = LazyModule(new CTCChipletLink(this, params, id)) } // For using CTC in a chiplet firesim config with no PHY @@ -85,11 +85,12 @@ class CTCBridgeIO extends ChipletIO { case object CTCKey extends Field[Seq[CTCParams]](Nil) -class CTCChipletLink(val params: CTCParams, val manager_region: Seq[AddressSet], val id: Int)(implicit p: Parameters) extends ChipletLinkWrapper { +// TODO: derive beatbytes from offchip subsystem params +class CTCChipletLink(val params: CTCParams, val sys_params: OffchipSubsystemParams, val id: Int)(implicit p: Parameters) extends ChipletLinkWrapper { // a TL master/client device val ctc2tl = LazyModule(new CTCToTileLink(portId=id)(p)) // a TL slave/manager device - val tl2ctc = LazyModule(new TileLinkToCTC(addrRegion=manager_region)(p)) + val tl2ctc = LazyModule(new TileLinkToCTC(addrRegion=sys_params.managerRegion)(p)) val client_node = ctc2tl.node val manager_node = tl2ctc.node diff --git a/src/main/scala/serdes/PeripheryTLSerial.scala b/src/main/scala/serdes/PeripheryTLSerial.scala index 55bf519c..ffc70af9 100644 --- a/src/main/scala/serdes/PeripheryTLSerial.scala +++ b/src/main/scala/serdes/PeripheryTLSerial.scala @@ -11,7 +11,7 @@ import freechips.rocketchip.diplomacy._ import freechips.rocketchip.util._ import freechips.rocketchip.prci._ import testchipip.util.{ClockedIO} -import testchipip.soc.{OBUS} +import testchipip.soc._ // Parameters for a read-only-memory that appears over serial-TL case class ManagerROMParams( @@ -55,6 +55,157 @@ case class SerialTLParams( manager: Option[SerialTLManagerParams] = None, phyParams: SerialPhyParams = DecoupledExternalSyncSerialPhyParams(), bundleParams: TLBundleParameters = TLSerdesser.STANDARD_TLBUNDLE_PARAMS) +extends ChipletLinkParams with ChipletLinkWrapperInstantiationLike { + def managerBusWhere = client.map(_.masterWhere).getOrElse(SBUS) // Apologies for overloaded terminology + def controlManagerBusWhere = None + def instantiate(params: OffchipSubsystemParams, id: Int)(implicit p: Parameters): ChipletLinkWrapper = LazyModule(new SerialTLChipletLink(this, params, id)) +} + +class SerialTLWrapper(val params: SerialTLParams, val sys_params: OffchipSubsystemParams, val id: Int)(implicit p: Parameters) extends LazyModule { + val tlChannels = 5 + + val portName = s"serial_tl_$id" + val clientPortParams = params.client.map { c => TLMasterPortParameters.v1( + clients = Seq.tabulate(1 << c.cacheIdBits){ i => TLMasterParameters.v1( + name = s"${portName}_${i}", + sourceId = IdRange(i << (c.totalIdBits - c.cacheIdBits), (i + 1) << (c.totalIdBits - c.cacheIdBits)), + supportsProbe = if (c.supportsProbe) TransferSizes(sys_params.clientBlockBytes, sys_params.clientBlockBytes) else TransferSizes.none + )} + )} + + val managerPortParams = params.manager.map { m => + val memParams = m.memParams + val romParams = m.romParams + val cohParams = m.cohParams + val memDevice = if (m.isMemoryDevice) new MemoryDevice else new SimpleDevice("lbwif-readwrite", Nil) + val romDevice = new SimpleDevice("lbwif-readonly", Nil) + val blockBytes = sys_params.managerBlockBytes + TLSlavePortParameters.v1( + managers = memParams.map { memParams => TLSlaveParameters.v1( + address = AddressSet.misaligned(memParams.address, memParams.size), + resources = memDevice.reg, + regionType = RegionType.UNCACHED, // cacheable + executable = true, + supportsGet = TransferSizes(1, blockBytes), + supportsPutFull = TransferSizes(1, blockBytes), + supportsPutPartial = TransferSizes(1, blockBytes) + )} ++ romParams.map { romParams => TLSlaveParameters.v1( + address = List(AddressSet(romParams.address, romParams.size-1)), + resources = romDevice.reg, + regionType = RegionType.UNCACHED, // cacheable + executable = true, + supportsGet = TransferSizes(1, blockBytes), + fifoId = Some(0) + )} ++ cohParams.map { cohParams => TLSlaveParameters.v1( + address = AddressSet.misaligned(cohParams.address, cohParams.size), + regionType = RegionType.TRACKED, // cacheable + executable = true, + supportsAcquireT = TransferSizes(1, blockBytes), + supportsAcquireB = TransferSizes(1, blockBytes), + supportsGet = TransferSizes(1, blockBytes), + supportsPutFull = TransferSizes(1, blockBytes), + supportsPutPartial = TransferSizes(1, blockBytes) + )}, + beatBytes = sys_params.managerBeatBytes, + endSinkId = if (cohParams.isEmpty) 0 else (1 << m.sinkIdBits), + minLatency = 1 + ) + } + + val serdesser = LazyModule(new TLSerdesser( + flitWidth = params.phyParams.flitWidth, + clientPortParams = clientPortParams, + managerPortParams = managerPortParams, + bundleParams = params.bundleParams, + nameSuffix = Some(portName) + )) + + // If we provide a clock, generate a clock domain for the outgoing clock + val serial_tl_clock_freqMHz = params.phyParams match { + case params: DecoupledInternalSyncSerialPhyParams => Some(params.freqMHz) + case params: DecoupledExternalSyncSerialPhyParams => None + case params: CreditedSourceSyncSerialPhyParams => Some(params.freqMHz) + } + val serial_tl_clock_node = serial_tl_clock_freqMHz.map { f => + ClockSinkNode(Seq(ClockSinkParameters(take=Some(ClockParameters(f))))) + } + + // Expose IO via BundleBridge + val top_IO = BundleBridgeSource(() => params.phyParams.genIO.asInstanceOf[ChipletIO]) + val debug_IO = BundleBridgeSource(() => new SerdesDebugIO) + + override lazy val module = new SerialTLWrapperModule(this) +} + +class SerialTLWrapperModule(outer: SerialTLWrapper) extends LazyModuleImp(outer) { + val io = outer.top_IO.out(0)._1 + val debug = outer.debug_IO.out(0)._1 + + debug := outer.serdesser.module.io.debug + + io match { + case io: DecoupledInternalSyncPhitIO => { + val outer_clock = outer.serial_tl_clock_node.get.in.head._1.clock + io.clock_out := outer_clock + val phy = Module(new DecoupledSerialPhy(outer.tlChannels, outer.params.phyParams)) + phy.io.outer_clock := outer_clock + phy.io.outer_reset := ResetCatchAndSync(outer_clock, outer.serdesser.module.reset.asBool) + phy.io.inner_clock := outer.serdesser.module.clock + phy.io.inner_reset := outer.serdesser.module.reset + phy.io.outer_ser <> io.viewAsSupertype(new DecoupledPhitIO(io.phitWidth)) + phy.io.inner_ser <> outer.serdesser.module.io.ser + } + case io: DecoupledExternalSyncPhitIO => { + val outer_clock = io.clock_in + val phy = Module(new DecoupledSerialPhy(outer.tlChannels, outer.params.phyParams)) + phy.io.outer_clock := outer_clock + phy.io.outer_reset := ResetCatchAndSync(outer_clock, outer.serdesser.module.reset.asBool) + phy.io.inner_clock := outer.serdesser.module.clock + phy.io.inner_reset := outer.serdesser.module.reset + phy.io.outer_ser <> io.viewAsSupertype(new DecoupledPhitIO(outer.params.phyParams.phitWidth)) + phy.io.inner_ser <> outer.serdesser.module.io.ser + } + case io: CreditedSourceSyncPhitIO => { + val outgoing_clock = outer.serial_tl_clock_node.get.in.head._1.clock + val outgoing_reset = ResetCatchAndSync(outgoing_clock, outer.serdesser.module.reset.asBool) + val incoming_clock = io.clock_in + val incoming_reset = ResetCatchAndSync(incoming_clock, io.reset_in.asBool) + io.clock_out := outgoing_clock + io.reset_out := outgoing_reset.asAsyncReset + val phy = Module(new CreditedSerialPhy(outer.tlChannels, outer.params.phyParams)) + phy.io.incoming_clock := incoming_clock + phy.io.incoming_reset := incoming_reset + phy.io.outgoing_clock := outgoing_clock + phy.io.outgoing_reset := outgoing_reset + phy.io.inner_clock := outer.serdesser.module.clock + phy.io.inner_reset := outer.serdesser.module.reset + phy.io.inner_ser <> outer.serdesser.module.io.ser + phy.io.outer_ser <> io.viewAsSupertype(new ValidPhitIO(outer.params.phyParams.phitWidth)) + } + } +} + +class SerialTLChipletLink(val params: SerialTLParams, val sys_params: OffchipSubsystemParams, val id: Int)(implicit p: Parameters) extends ChipletLinkWrapper { + require(params.manager.isDefined, "Chip-to-chip SerialTL must have a manager") + require(params.client.isDefined, "Chip-to-chip SerialTL must have a client") + + // Override memParams with address regions from the offchip subsystem + val chipletParams = params.copy( + manager = params.manager.map(_.copy( + memParams = sys_params.managerRegion.map(as => ManagerRAMParams(address = as.base, size = as.mask + 1)) + )) + ) + val wrapper = LazyModule(new SerialTLWrapper(chipletParams, sys_params, id)) + + val client_node = wrapper.serdesser.clientNode.get + val manager_node = wrapper.serdesser.managerNode.get + val control_manager_node = None + val top_IO = wrapper.top_IO + val debug_IO = wrapper.debug_IO + val serial_tl_clock_node = wrapper.serial_tl_clock_node + + override lazy val module = new LazyModuleImp(this) { } +} case object SerialTLKey extends Field[Seq[SerialTLParams]](Nil) diff --git a/src/main/scala/soc/ChipletLink.scala b/src/main/scala/soc/ChipletLink.scala index 177e8bc6..29c13c9c 100644 --- a/src/main/scala/soc/ChipletLink.scala +++ b/src/main/scala/soc/ChipletLink.scala @@ -21,6 +21,14 @@ trait ChipletLinkParams{ //def instantiationFn: _ => (TLInwardNode, TLOutwardNode, Option[TLRegisterNode]) } +case class OffchipSubsystemParams( + val managerRegion: Seq[AddressSet], + val clientBeatBytes: Int, + val clientBlockBytes: Int, + val managerBeatBytes: Int, + val managerBlockBytes: Int +) + abstract class ChipletLinkWrapper(implicit p: Parameters) extends LazyModule { //val link_name: String val client_node: TLClientNode @@ -35,7 +43,7 @@ abstract class ChipletLinkWrapperImpl(outer: ChipletLinkWrapper) extends LazyMod // Links should "with" this trait trait ChipletLinkWrapperInstantiationLike { - def instantiate(manager_region: Seq[AddressSet], id: Int)(implicit p: Parameters): ChipletLinkWrapper + def instantiate(params: OffchipSubsystemParams, id: Int)(implicit p: Parameters): ChipletLinkWrapper } // Should this be called D2D IO instead? diff --git a/src/main/scala/soc/OffchipRouter.scala b/src/main/scala/soc/OffchipRouter.scala index 94c88f83..c5ceabfb 100644 --- a/src/main/scala/soc/OffchipRouter.scala +++ b/src/main/scala/soc/OffchipRouter.scala @@ -297,7 +297,15 @@ trait CanHaveChipletRouting { this: BaseSubsystem => val port_ios = params.ports.zipWithIndex.map { case (pP, id) => val link_manager_bus = locateTLBusWrapper(pP.managerBusWhere) - val port = router_domain { pP.asInstanceOf[ChipletLinkWrapperInstantiationLike].instantiate(p(MaxOffchipAddressRange), id)(p).suggestName(s"d2d${id}_port") } + val sys_params = OffchipSubsystemParams( + managerRegion = p(MaxOffchipAddressRange), + clientBeatBytes = client_bus.beatBytes, + clientBlockBytes = client_bus.blockBytes, + managerBeatBytes = link_manager_bus.beatBytes, + managerBlockBytes = link_manager_bus.blockBytes + ) + + val port = router_domain { pP.asInstanceOf[ChipletLinkWrapperInstantiationLike].instantiate(sys_params, id)(p).suggestName(s"d2d${id}_port") } // TODO: Translator should take in chip ID as an IO val translator = router_domain { @@ -310,6 +318,15 @@ trait CanHaveChipletRouting { this: BaseSubsystem => } } + // Connect PHY clock node and sink debug IO if the port has one (e.g. SerialTL) + port match { + case sertl: testchipip.serdes.SerialTLChipletLink => + sertl.serial_tl_clock_node.foreach(_ := ClockGroup()(p, ValName(s"d2d${id}_clock")) := allClockGroupsNode) + val debug_ioSink = BundleBridgeSink[testchipip.serdes.SerdesDebugIO]() + debug_ioSink := sertl.debug_IO + case _ => + } + port.manager_node :*= router.node port.control_manager_node.foreach { node => cbus.coupleTo(s"${port.name}_control") { node := TLBuffer() := _ } From 5514545e487aa524969816f3ca5fa0d1e207b67f Mon Sep 17 00:00:00 2001 From: Ella Schwarz Date: Mon, 16 Mar 2026 10:10:15 -0700 Subject: [PATCH 10/19] Add source shrinker to manager nodes of ports --- src/main/scala/soc/OffchipRouter.scala | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/main/scala/soc/OffchipRouter.scala b/src/main/scala/soc/OffchipRouter.scala index c5ceabfb..d781a388 100644 --- a/src/main/scala/soc/OffchipRouter.scala +++ b/src/main/scala/soc/OffchipRouter.scala @@ -326,8 +326,9 @@ trait CanHaveChipletRouting { this: BaseSubsystem => debug_ioSink := sertl.debug_IO case _ => } - - port.manager_node :*= router.node + + val shrinker = router_domain { TLSourceShrinker(1 << 8) } + port.manager_node :*= shrinker := router.node port.control_manager_node.foreach { node => cbus.coupleTo(s"${port.name}_control") { node := TLBuffer() := _ } } From de095c7bf62279be8a98b1ce8c68630ebc82bd84 Mon Sep 17 00:00:00 2001 From: Ella Schwarz Date: Tue, 17 Mar 2026 16:41:10 -0700 Subject: [PATCH 11/19] Better naming for NoC connections --- src/main/scala/serdes/PeripheryTLSerial.scala | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/src/main/scala/serdes/PeripheryTLSerial.scala b/src/main/scala/serdes/PeripheryTLSerial.scala index ffc70af9..66996a2d 100644 --- a/src/main/scala/serdes/PeripheryTLSerial.scala +++ b/src/main/scala/serdes/PeripheryTLSerial.scala @@ -61,10 +61,10 @@ extends ChipletLinkParams with ChipletLinkWrapperInstantiationLike { def instantiate(params: OffchipSubsystemParams, id: Int)(implicit p: Parameters): ChipletLinkWrapper = LazyModule(new SerialTLChipletLink(this, params, id)) } -class SerialTLWrapper(val params: SerialTLParams, val sys_params: OffchipSubsystemParams, val id: Int)(implicit p: Parameters) extends LazyModule { +class SerialTLWrapper(val params: SerialTLParams, val sys_params: OffchipSubsystemParams, val id: Int, val namePrefix: String = "serial_tl")(implicit p: Parameters) extends LazyModule { val tlChannels = 5 - val portName = s"serial_tl_$id" + val portName = s"${namePrefix}_$id" val clientPortParams = params.client.map { c => TLMasterPortParameters.v1( clients = Seq.tabulate(1 << c.cacheIdBits){ i => TLMasterParameters.v1( name = s"${portName}_${i}", @@ -81,7 +81,7 @@ class SerialTLWrapper(val params: SerialTLParams, val sys_params: OffchipSubsyst val romDevice = new SimpleDevice("lbwif-readonly", Nil) val blockBytes = sys_params.managerBlockBytes TLSlavePortParameters.v1( - managers = memParams.map { memParams => TLSlaveParameters.v1( + managers = memParams.zipWithIndex.map { case (memParams, i) => TLSlaveParameters.v1( address = AddressSet.misaligned(memParams.address, memParams.size), resources = memDevice.reg, regionType = RegionType.UNCACHED, // cacheable @@ -89,14 +89,16 @@ class SerialTLWrapper(val params: SerialTLParams, val sys_params: OffchipSubsyst supportsGet = TransferSizes(1, blockBytes), supportsPutFull = TransferSizes(1, blockBytes), supportsPutPartial = TransferSizes(1, blockBytes) - )} ++ romParams.map { romParams => TLSlaveParameters.v1( + ).v2copy(name = Some(s"${portName}_mem_${i}")) + } ++ romParams.zipWithIndex.map { case (romParams, i) => TLSlaveParameters.v1( address = List(AddressSet(romParams.address, romParams.size-1)), resources = romDevice.reg, regionType = RegionType.UNCACHED, // cacheable executable = true, supportsGet = TransferSizes(1, blockBytes), fifoId = Some(0) - )} ++ cohParams.map { cohParams => TLSlaveParameters.v1( + ).v2copy(name = Some(s"${portName}_rom_${i}")) + } ++ cohParams.zipWithIndex.map { case (cohParams, i) => TLSlaveParameters.v1( address = AddressSet.misaligned(cohParams.address, cohParams.size), regionType = RegionType.TRACKED, // cacheable executable = true, @@ -105,7 +107,8 @@ class SerialTLWrapper(val params: SerialTLParams, val sys_params: OffchipSubsyst supportsGet = TransferSizes(1, blockBytes), supportsPutFull = TransferSizes(1, blockBytes), supportsPutPartial = TransferSizes(1, blockBytes) - )}, + ).v2copy(name = Some(s"${portName}_coh_${i}")) + }, beatBytes = sys_params.managerBeatBytes, endSinkId = if (cohParams.isEmpty) 0 else (1 << m.sinkIdBits), minLatency = 1 @@ -195,7 +198,7 @@ class SerialTLChipletLink(val params: SerialTLParams, val sys_params: OffchipSub memParams = sys_params.managerRegion.map(as => ManagerRAMParams(address = as.base, size = as.mask + 1)) )) ) - val wrapper = LazyModule(new SerialTLWrapper(chipletParams, sys_params, id)) + val wrapper = LazyModule(new SerialTLWrapper(chipletParams, sys_params, id, namePrefix = "d2d_serial_tl")) val client_node = wrapper.serdesser.clientNode.get val manager_node = wrapper.serdesser.managerNode.get From da3349fc840f39ee54aa764c5f78c37d445483cf Mon Sep 17 00:00:00 2001 From: Ella Schwarz Date: Mon, 30 Mar 2026 15:15:02 -0700 Subject: [PATCH 12/19] Update CTCMem to work with router --- src/main/scala/ctc/CTC.scala | 34 +++++++++++++++++++-------------- src/main/scala/ctc/CTCMem.scala | 8 ++++---- 2 files changed, 24 insertions(+), 18 deletions(-) diff --git a/src/main/scala/ctc/CTC.scala b/src/main/scala/ctc/CTC.scala index 662e8b41..61e368b5 100644 --- a/src/main/scala/ctc/CTC.scala +++ b/src/main/scala/ctc/CTC.scala @@ -83,6 +83,24 @@ class CTCBridgeIO extends ChipletIO { } } +class CTCMemIO(phitWidth: Int, offchip: Seq[AddressSet], phyParams: SerialPhyParams) extends DecoupledInternalSyncPhitIO(phitWidth) { + def connectRAM(implicit p: Parameters): Unit = { + withClock(clock_out) { + val ram = Module(LazyModule(new CTCMem(offchip, phyParams)(p)).module) + ram.io.ser.in <> out + in <> ram.io.ser.out + } + } +} + +case class CTCMemSerialPhyParams( + phitWidth: Int = CTC.OUTER_WIDTH, + flitWidth: Int = CTC.INNER_WIDTH, + flitBufferSz: Int = 16, + offchip: Seq[AddressSet]) extends SerialPhyParams { + def genIO = new CTCMemIO(phitWidth, offchip, this) +} + case object CTCKey extends Field[Seq[CTCParams]](Nil) // TODO: derive beatbytes from offchip subsystem params @@ -126,7 +144,7 @@ class CTCChipletLinkImpl(outer: CTCChipletLink) extends LazyModuleImp(outer) { phy.io.inner_ser(1).out <> outer.ctc2tl.module.io.flit.out phy.io.outer_ser <> io.viewAsSupertype(new ValidPhitIO(outer.params.phyParams.get.phitWidth)) } - case io: DecoupledInternalSyncPhitIO => { + case io: CTCMemIO => { val outgoing_clock = clock io.clock_out := outgoing_clock val phy = Module(new DecoupledSerialPhy(2, outer.params.phyParams.get)) @@ -140,23 +158,11 @@ class CTCChipletLinkImpl(outer: CTCChipletLink) extends LazyModuleImp(outer) { phy.io.inner_ser(1).out <> outer.ctc2tl.module.io.flit.out phy.io.outer_ser <> io.viewAsSupertype(new DecoupledPhitIO(outer.params.phyParams.get.phitWidth)) } - case io: DecoupledExternalSyncPhitIO => { - val outgoing_clock = io.clock_in - val phy = Module(new DecoupledSerialPhy(2, outer.params.phyParams.get)) - phy.io.outer_clock := outgoing_clock - phy.io.outer_reset := ResetCatchAndSync(outgoing_clock, reset.asBool) - phy.io.inner_clock := outer.ctc2tl.module.clock - phy.io.inner_reset := outer.ctc2tl.module.reset - phy.io.inner_ser(0).in <> outer.ctc2tl.module.io.flit.in - phy.io.inner_ser(0).out <> outer.tl2ctc.module.io.flit.out - phy.io.inner_ser(1).in <> outer.tl2ctc.module.io.flit.in - phy.io.inner_ser(1).out <> outer.ctc2tl.module.io.flit.out - phy.io.outer_ser <> io.viewAsSupertype(new DecoupledPhitIO(outer.params.phyParams.get.phitWidth)) - } case io: CTCBridgeIO => { io.manager_flit <> outer.tl2ctc.module.io.flit io.client_flit <> outer.ctc2tl.module.io.flit } + case _ => assert(false, s"IO unsupported for CTC: ${io.getClass}") } } diff --git a/src/main/scala/ctc/CTCMem.scala b/src/main/scala/ctc/CTCMem.scala index a85a0e69..6be0a941 100644 --- a/src/main/scala/ctc/CTCMem.scala +++ b/src/main/scala/ctc/CTCMem.scala @@ -8,19 +8,19 @@ import freechips.rocketchip.tilelink._ import freechips.rocketchip.devices.tilelink._ import freechips.rocketchip.diplomacy._ import freechips.rocketchip.util._ -import testchipip.serdes.{DecoupledPhitIO, DecoupledSerialPhy} +import testchipip.serdes.{DecoupledPhitIO, DecoupledSerialPhy, SerialPhyParams} import testchipip.ctc._ // A test memory like SerialRAM but for CTC // Takes in DecoupledFlitIO, puts it through CTCToTileLink, then connects the TileLink node to TLRAM -class CTCMem(params: CTCParams)(implicit p: Parameters) extends LazyModule { +class CTCMem(offchip: Seq[AddressSet], phyParams: SerialPhyParams)(implicit p: Parameters) extends LazyModule { val beatBytes = 8 val ctc2tl = LazyModule(new CTCToTileLink(sourceIds = 1, portId = 0)) val xbar = TLXbar() // Use smaller memory size for testing - val testmems = params.offchip.map(address => LazyModule(new TLRAM(address = AddressSet(address.base, 0xFFFFFL), beatBytes = beatBytes) {override lazy val desiredName = "CTCMem"})) + val testmems = offchip.map(address => LazyModule(new TLRAM(address = AddressSet(address.base, 0xFFFFFL), beatBytes = beatBytes) {override lazy val desiredName = "CTCMem"})) testmems.foreach { s => (s.node := TLBuffer() := TLFragmenter(beatBytes, p(CacheBlockBytes), nameSuffix = Some("CTCMem")) @@ -35,7 +35,7 @@ class CTCMem(params: CTCParams)(implicit p: Parameters) extends LazyModule { val ser = new DecoupledPhitIO(CTC.OUTER_WIDTH) }) - val phy = Module(new DecoupledSerialPhy(2, params.phyParams.get)) + val phy = Module(new DecoupledSerialPhy(2, phyParams)) phy.io.outer_clock := clock phy.io.outer_reset := reset phy.io.inner_clock := clock From 4b8d9b15edcfd673fe39e00d10d6e2da9c62583b Mon Sep 17 00:00:00 2001 From: Ella Schwarz Date: Mon, 30 Mar 2026 15:15:22 -0700 Subject: [PATCH 13/19] Add mmio reg for bypassing --- src/main/scala/soc/OffchipRouter.scala | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/src/main/scala/soc/OffchipRouter.scala b/src/main/scala/soc/OffchipRouter.scala index d781a388..4923e5a2 100644 --- a/src/main/scala/soc/OffchipRouter.scala +++ b/src/main/scala/soc/OffchipRouter.scala @@ -97,6 +97,9 @@ class OffchipRouterImpl(outer: OffchipRouter) extends LazyModuleImp(outer) { val chipIdReg = RegInit(0.U(idWidth.W)) io.chip_id := chipIdReg + + val routing_mode_reg = RegInit(true.B) // true: routing mode, false: bypass mode + val bypass_port_reg = RegInit(0.U(log2Ceil(nPorts).W)) //val offsetReg = RegInit(p(MaxOffchipAddressRange).map(_.base).min.U(addressWidth.W)) @@ -118,31 +121,35 @@ class OffchipRouterImpl(outer: OffchipRouter) extends LazyModuleImp(outer) { ) } - val chip_id = Seq((tableEntries * regsPerEntry * outer.beatBytes) -> Seq(RegField(idWidth, chipIdReg, RegFieldDesc("chip_id", "Chip ID for this chip")))) - outer.routing_table_node.regmap(mapped_entries ++ chip_id :_*) + val chip_id = Seq((tableEntries * regsPerEntry * outer.beatBytes) -> Seq(RegField(idWidth, chipIdReg, RegFieldDesc("chip_id", "Chip ID for this chip")))) + val routing_mode = Seq((tableEntries * regsPerEntry * outer.beatBytes + (1 * outer.beatBytes)) -> Seq(RegField(1, routing_mode_reg, RegFieldDesc("routing_mode", "Routing mode")))) + val bypass_port = Seq((tableEntries * regsPerEntry * outer.beatBytes + (2 * outer.beatBytes)) -> Seq(RegField(log2Ceil(nPorts), bypass_port_reg, RegFieldDesc("bypass_port", "Bypass port")))) + outer.routing_table_node.regmap(mapped_entries ++ chip_id ++ routing_mode ++ bypass_port :_*) // val offset = Seq((tableEntries * regsPerEntry) + 1 -> Seq(RegField(addressWidth, 0.U, RegFieldDesc("offset", "Offchip offset for this chip")))) // outer.routing_table_node.regmap(mapped_entries ++ chip_id ++ offset :_*) // Select offchip port from routing table val addr_top_bits = bundleIn.a.bits.address(addressWidth-1, log2Ceil(p(MaxOffchipAddressRange).map(_.base).min)) val matches = Wire(Vec(tableEntries, Bool())) - val sel = Wire(UInt(log2Ceil(nPorts).W)) - sel := 0.U + val port_match = Wire(UInt(log2Ceil(nPorts).W)) + port_match := 0.U matches := VecInit(Seq.fill(tableEntries)(false.B)) for (i <- 0 until tableEntries) { when (routing_table(i).valid && (addr_top_bits === routing_table(i).chipID)) { - sel := routing_table(i).port + port_match := routing_table(i).port matches(i) := true.B } } + val sel = Mux(routing_mode_reg, port_match, bypass_port_reg) + // Assert that only one match is found - assert(!bundleIn.a.valid || ((matches.asUInt & (matches.asUInt - 1.U)) === 0.U), "Multiple matches found in routing table") // TODO: check + assert(!routing_mode_reg || !bundleIn.a.valid || ((matches.asUInt & (matches.asUInt - 1.U)) === 0.U), "Multiple matches found in routing table") // TODO: check // AGENT OUTPUT START ~~~~~~~~~~~~~~~~ // Decide legality per-request. Address is stable across beats. val illegal = Wire(Bool()) // <- your programmable table miss / tag mismatch - illegal := bundleIn.a.valid && !matches.reduce(_ || _) + illegal := bundleIn.a.valid && !matches.reduce(_ || _) && routing_mode_reg val errActive = RegInit(false.B) val errOpcode = Reg(bundleIn.a.bits.opcode.cloneType) From 49235f47f96c062106cd7d78a9fcec5926abcf5e Mon Sep 17 00:00:00 2001 From: Ella Schwarz Date: Thu, 2 Apr 2026 21:01:46 -0700 Subject: [PATCH 14/19] add clock node to ChipletLink --- src/main/scala/ctc/CTC.scala | 1 + src/main/scala/serdes/PeripheryTLSerial.scala | 2 +- src/main/scala/soc/ChipletLink.scala | 1 + src/main/scala/soc/OffchipRouter.scala | 3 ++- 4 files changed, 5 insertions(+), 2 deletions(-) diff --git a/src/main/scala/ctc/CTC.scala b/src/main/scala/ctc/CTC.scala index 662e8b41..89560675 100644 --- a/src/main/scala/ctc/CTC.scala +++ b/src/main/scala/ctc/CTC.scala @@ -95,6 +95,7 @@ class CTCChipletLink(val params: CTCParams, val sys_params: OffchipSubsystemPara val client_node = ctc2tl.node val manager_node = tl2ctc.node val control_manager_node = None + val clock_node = None val top_IO = params.phyParams match { case Some(pP) => BundleBridgeSource(() => pP.genIO.asInstanceOf[ChipletIO]) case None => BundleBridgeSource(() => new CTCBridgeIO) diff --git a/src/main/scala/serdes/PeripheryTLSerial.scala b/src/main/scala/serdes/PeripheryTLSerial.scala index 66996a2d..72c752bb 100644 --- a/src/main/scala/serdes/PeripheryTLSerial.scala +++ b/src/main/scala/serdes/PeripheryTLSerial.scala @@ -203,9 +203,9 @@ class SerialTLChipletLink(val params: SerialTLParams, val sys_params: OffchipSub val client_node = wrapper.serdesser.clientNode.get val manager_node = wrapper.serdesser.managerNode.get val control_manager_node = None + val clock_node = wrapper.serial_tl_clock_node val top_IO = wrapper.top_IO val debug_IO = wrapper.debug_IO - val serial_tl_clock_node = wrapper.serial_tl_clock_node override lazy val module = new LazyModuleImp(this) { } } diff --git a/src/main/scala/soc/ChipletLink.scala b/src/main/scala/soc/ChipletLink.scala index 29c13c9c..a66c47af 100644 --- a/src/main/scala/soc/ChipletLink.scala +++ b/src/main/scala/soc/ChipletLink.scala @@ -34,6 +34,7 @@ abstract class ChipletLinkWrapper(implicit p: Parameters) extends LazyModule { val client_node: TLClientNode val manager_node: TLManagerNode val control_manager_node: Option[TLRegisterNode] + val clock_node: Option[ClockNode] val top_IO: BundleBridgeSource[ChipletIO] } diff --git a/src/main/scala/soc/OffchipRouter.scala b/src/main/scala/soc/OffchipRouter.scala index d781a388..10089ed9 100644 --- a/src/main/scala/soc/OffchipRouter.scala +++ b/src/main/scala/soc/OffchipRouter.scala @@ -321,7 +321,6 @@ trait CanHaveChipletRouting { this: BaseSubsystem => // Connect PHY clock node and sink debug IO if the port has one (e.g. SerialTL) port match { case sertl: testchipip.serdes.SerialTLChipletLink => - sertl.serial_tl_clock_node.foreach(_ := ClockGroup()(p, ValName(s"d2d${id}_clock")) := allClockGroupsNode) val debug_ioSink = BundleBridgeSink[testchipip.serdes.SerdesDebugIO]() debug_ioSink := sertl.debug_IO case _ => @@ -334,6 +333,8 @@ trait CanHaveChipletRouting { this: BaseSubsystem => } link_manager_bus.coupleFrom(s"${port.name}") { _ := TLBuffer() := translator.node :=* port.client_node } + port.clock_node.foreach(_ := ClockGroup()(p, ValName(s"d2d${id}_clock")) := allClockGroupsNode) + val port_ioSink = BundleBridgeSink[ChipletIO]() port_ioSink := port.top_IO From cb901ea82f1ab77682d81f9d5d729be151a69998 Mon Sep 17 00:00:00 2001 From: Ella Schwarz Date: Thu, 2 Apr 2026 21:01:46 -0700 Subject: [PATCH 15/19] add clock node to ChipletLink --- src/main/scala/ctc/CTC.scala | 1 + src/main/scala/serdes/PeripheryTLSerial.scala | 2 +- src/main/scala/soc/ChipletLink.scala | 1 + src/main/scala/soc/OffchipRouter.scala | 3 ++- 4 files changed, 5 insertions(+), 2 deletions(-) diff --git a/src/main/scala/ctc/CTC.scala b/src/main/scala/ctc/CTC.scala index 61e368b5..5e5c25f8 100644 --- a/src/main/scala/ctc/CTC.scala +++ b/src/main/scala/ctc/CTC.scala @@ -113,6 +113,7 @@ class CTCChipletLink(val params: CTCParams, val sys_params: OffchipSubsystemPara val client_node = ctc2tl.node val manager_node = tl2ctc.node val control_manager_node = None + val clock_node = None val top_IO = params.phyParams match { case Some(pP) => BundleBridgeSource(() => pP.genIO.asInstanceOf[ChipletIO]) case None => BundleBridgeSource(() => new CTCBridgeIO) diff --git a/src/main/scala/serdes/PeripheryTLSerial.scala b/src/main/scala/serdes/PeripheryTLSerial.scala index ffc70af9..b4b15da5 100644 --- a/src/main/scala/serdes/PeripheryTLSerial.scala +++ b/src/main/scala/serdes/PeripheryTLSerial.scala @@ -200,9 +200,9 @@ class SerialTLChipletLink(val params: SerialTLParams, val sys_params: OffchipSub val client_node = wrapper.serdesser.clientNode.get val manager_node = wrapper.serdesser.managerNode.get val control_manager_node = None + val clock_node = wrapper.serial_tl_clock_node val top_IO = wrapper.top_IO val debug_IO = wrapper.debug_IO - val serial_tl_clock_node = wrapper.serial_tl_clock_node override lazy val module = new LazyModuleImp(this) { } } diff --git a/src/main/scala/soc/ChipletLink.scala b/src/main/scala/soc/ChipletLink.scala index 29c13c9c..a66c47af 100644 --- a/src/main/scala/soc/ChipletLink.scala +++ b/src/main/scala/soc/ChipletLink.scala @@ -34,6 +34,7 @@ abstract class ChipletLinkWrapper(implicit p: Parameters) extends LazyModule { val client_node: TLClientNode val manager_node: TLManagerNode val control_manager_node: Option[TLRegisterNode] + val clock_node: Option[ClockNode] val top_IO: BundleBridgeSource[ChipletIO] } diff --git a/src/main/scala/soc/OffchipRouter.scala b/src/main/scala/soc/OffchipRouter.scala index 4923e5a2..7d783776 100644 --- a/src/main/scala/soc/OffchipRouter.scala +++ b/src/main/scala/soc/OffchipRouter.scala @@ -328,7 +328,6 @@ trait CanHaveChipletRouting { this: BaseSubsystem => // Connect PHY clock node and sink debug IO if the port has one (e.g. SerialTL) port match { case sertl: testchipip.serdes.SerialTLChipletLink => - sertl.serial_tl_clock_node.foreach(_ := ClockGroup()(p, ValName(s"d2d${id}_clock")) := allClockGroupsNode) val debug_ioSink = BundleBridgeSink[testchipip.serdes.SerdesDebugIO]() debug_ioSink := sertl.debug_IO case _ => @@ -341,6 +340,8 @@ trait CanHaveChipletRouting { this: BaseSubsystem => } link_manager_bus.coupleFrom(s"${port.name}") { _ := TLBuffer() := translator.node :=* port.client_node } + port.clock_node.foreach(_ := ClockGroup()(p, ValName(s"d2d${id}_clock")) := allClockGroupsNode) + val port_ioSink = BundleBridgeSink[ChipletIO]() port_ioSink := port.top_IO From b9942fff6b3fb146e928215621ba18bb34fd38a1 Mon Sep 17 00:00:00 2001 From: Ella Schwarz Date: Wed, 8 Apr 2026 16:56:59 -0700 Subject: [PATCH 16/19] Add table translation mode --- src/main/scala/soc/OffchipRouter.scala | 26 +++++++++++++++++--------- src/main/scala/test/Unittests.scala | 2 +- 2 files changed, 18 insertions(+), 10 deletions(-) diff --git a/src/main/scala/soc/OffchipRouter.scala b/src/main/scala/soc/OffchipRouter.scala index 7d783776..a03a3f37 100644 --- a/src/main/scala/soc/OffchipRouter.scala +++ b/src/main/scala/soc/OffchipRouter.scala @@ -79,27 +79,31 @@ class OffchipRouter(val params: ChipletRoutingParams, val beatBytes: Int = 8)(im class OffchipRouterImpl(outer: OffchipRouter) extends LazyModuleImp(outer) { - val io = IO(new Bundle { - val chip_id = Output(UInt(outer.params.idWidth.W)) - }) - val (bundleIn, edgeIn) = outer.node.in(0) val bundlesOut = outer.node.out.map(_._1) val nPorts = bundlesOut.size + val io = IO(new Bundle { + val chip_id = Output(Vec(nPorts, UInt(outer.params.idWidth.W))) + }) + val tableEntries = outer.params.routerParams.tableEntries val idWidth = outer.params.idWidth val addressWidth = bundleIn.a.bits.address.getWidth val portWidth = if (nPorts == 1) 1 else log2Ceil(nPorts) val totalWidth = idWidth + addressWidth + portWidth + 1 assert(totalWidth <= 64, "Total width of routing table entry must be less than or equal to 64 bits") // for now :) - - val chipIdReg = RegInit(0.U(idWidth.W)) - io.chip_id := chipIdReg val routing_mode_reg = RegInit(true.B) // true: routing mode, false: bypass mode val bypass_port_reg = RegInit(0.U(log2Ceil(nPorts).W)) + val translation_mode_reg = RegInit(false.B) // true: per-port translation mode, false: chip id translation mode + val translation_table_reg = RegInit(VecInit(Seq.fill(nPorts)(0.U(idWidth.W)))) // Index with port id + + val chipIdReg = RegInit(0.U(idWidth.W)) + for (i <- 0 until nPorts) { + io.chip_id(i) := Mux(translation_mode_reg, translation_table_reg(i), chipIdReg) + } //val offsetReg = RegInit(p(MaxOffchipAddressRange).map(_.base).min.U(addressWidth.W)) @@ -124,7 +128,11 @@ class OffchipRouterImpl(outer: OffchipRouter) extends LazyModuleImp(outer) { val chip_id = Seq((tableEntries * regsPerEntry * outer.beatBytes) -> Seq(RegField(idWidth, chipIdReg, RegFieldDesc("chip_id", "Chip ID for this chip")))) val routing_mode = Seq((tableEntries * regsPerEntry * outer.beatBytes + (1 * outer.beatBytes)) -> Seq(RegField(1, routing_mode_reg, RegFieldDesc("routing_mode", "Routing mode")))) val bypass_port = Seq((tableEntries * regsPerEntry * outer.beatBytes + (2 * outer.beatBytes)) -> Seq(RegField(log2Ceil(nPorts), bypass_port_reg, RegFieldDesc("bypass_port", "Bypass port")))) - outer.routing_table_node.regmap(mapped_entries ++ chip_id ++ routing_mode ++ bypass_port :_*) + val translation_mode = Seq((tableEntries * regsPerEntry * outer.beatBytes + (3 * outer.beatBytes)) -> Seq(RegField(1, translation_mode_reg, RegFieldDesc("translation_mode", "Translation mode")))) + val translation_table = (0 until nPorts).map { i => + (tableEntries * regsPerEntry * outer.beatBytes + ((4 + i) * outer.beatBytes)) -> Seq(RegField(idWidth, translation_table_reg(i), RegFieldDesc(s"translation_table_$i", s"Translation table entry $i"))) + } + outer.routing_table_node.regmap(mapped_entries ++ chip_id ++ routing_mode ++ bypass_port ++ translation_mode ++ translation_table :_*) // val offset = Seq((tableEntries * regsPerEntry) + 1 -> Seq(RegField(addressWidth, 0.U, RegFieldDesc("offset", "Offchip offset for this chip")))) // outer.routing_table_node.regmap(mapped_entries ++ chip_id ++ offset :_*) @@ -321,7 +329,7 @@ trait CanHaveChipletRouting { this: BaseSubsystem => router_domain { InModuleBody { - translator.module.io.chip_id := router.module.io.chip_id + translator.module.io.chip_id := router.module.io.chip_id(id) } } diff --git a/src/main/scala/test/Unittests.scala b/src/main/scala/test/Unittests.scala index 137192ef..f1d3c59b 100644 --- a/src/main/scala/test/Unittests.scala +++ b/src/main/scala/test/Unittests.scala @@ -942,7 +942,7 @@ class OffchipRouterTest(val nChips: Int, val nPorts: Int, val reqsPerChip: Int = // Verify chip ID was programmed correctly when (mmioFinished) { - assert(router.module.io.chip_id === chipId.U, "Chip ID mismatch after programming") + assert(router.module.io.chip_id(0) === chipId.U, "Chip ID mismatch after programming") } driver.module.io.start := mmioFinished From f773c0fb4e90b819fae34b6f5aa400f23c2c089f Mon Sep 17 00:00:00 2001 From: Ella Schwarz Date: Fri, 22 May 2026 18:02:45 -0700 Subject: [PATCH 17/19] Add WidthWidget on control bus --- src/main/scala/soc/OffchipRouter.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/scala/soc/OffchipRouter.scala b/src/main/scala/soc/OffchipRouter.scala index a03a3f37..1dbdb166 100644 --- a/src/main/scala/soc/OffchipRouter.scala +++ b/src/main/scala/soc/OffchipRouter.scala @@ -344,7 +344,7 @@ trait CanHaveChipletRouting { this: BaseSubsystem => val shrinker = router_domain { TLSourceShrinker(1 << 8) } port.manager_node :*= shrinker := router.node port.control_manager_node.foreach { node => - cbus.coupleTo(s"${port.name}_control") { node := TLBuffer() := _ } + cbus.coupleTo(s"${port.name}_control") { node := TLWidthWidget(cbus.beatBytes) := TLBuffer() := _ } } link_manager_bus.coupleFrom(s"${port.name}") { _ := TLBuffer() := translator.node :=* port.client_node } From 5b0ae05e57b970aee86a90eaa8e35dd83884d1f6 Mon Sep 17 00:00:00 2001 From: Ella Schwarz Date: Fri, 29 May 2026 13:56:03 -0700 Subject: [PATCH 18/19] Clean up --- .../resources/testchipip/csrc/testchip_tsi.cc | 4 +- src/main/scala/ctc/CTC.scala | 9 +-- src/main/scala/ctc/CTCMem.scala | 3 +- src/main/scala/serdes/Bundles.scala | 4 +- src/main/scala/serdes/PeripheryTLSerial.scala | 2 +- src/main/scala/soc/ChipletLink.scala | 10 +-- src/main/scala/soc/Configs.scala | 4 +- src/main/scala/soc/OffchipRouter.scala | 73 +++++-------------- src/main/scala/test/Unittests.scala | 8 +- 9 files changed, 38 insertions(+), 79 deletions(-) diff --git a/src/main/resources/testchipip/csrc/testchip_tsi.cc b/src/main/resources/testchipip/csrc/testchip_tsi.cc index ed0ce0a7..17eaee0b 100644 --- a/src/main/resources/testchipip/csrc/testchip_tsi.cc +++ b/src/main/resources/testchipip/csrc/testchip_tsi.cc @@ -14,8 +14,9 @@ testchip_tsi_t::testchip_tsi_t(int argc, char** argv, bool can_have_loadmem, int has_loadmem = can_have_loadmem; if (arg.find("+cflush_addr=0x") == 0) cflush_addr = strtoull(arg.substr(15).c_str(), 0, 16); + // We cannot use init_write to program the chip ID directly because each chip id needs to be programmed individually, + // and init_write does not distinguish between chips. if (arg.find("+chip_id") == 0) { - fprintf(stderr, "Parsing +chip_id argument: %s\n", arg.c_str()); static const std::string prefix = "+chip_id"; auto eq = arg.find('='); if (eq == std::string::npos || eq <= prefix.size() || arg.find("=0x", eq) != eq) { @@ -23,7 +24,6 @@ testchip_tsi_t::testchip_tsi_t(int argc, char** argv, bool can_have_loadmem, int } unsigned long arg_chip_id = strtoul(arg.substr(prefix.size(), eq - prefix.size()).c_str(), 0, 0); if ((int)arg_chip_id == chip_id) { - fprintf(stderr, "Chip ID matches, parsing init access\n"); auto d = arg.find(":0x", eq + 3); if (d == std::string::npos) { throw std::invalid_argument("Improperly formatted +chip_id argument"); diff --git a/src/main/scala/ctc/CTC.scala b/src/main/scala/ctc/CTC.scala index 5e5c25f8..7ea4a25f 100644 --- a/src/main/scala/ctc/CTC.scala +++ b/src/main/scala/ctc/CTC.scala @@ -14,7 +14,6 @@ import testchipip.soc._ import testchipip.serdes._ - object CTC { val INNER_WIDTH = 32 val INNER_WIDTH_BYTES = INNER_WIDTH / 8 @@ -33,7 +32,7 @@ case class CTCParams( offchip: Seq[AddressSet] = Nil, managerBus: TLBusWrapperLocation = SBUS, clientBus: TLBusWrapperLocation = SBUS, - phyParams: Option[SerialPhyParams] = Some(CreditedSourceSyncSerialPhyParams(phitWidth = CTC.OUTER_WIDTH, flitWidth = CTC.INNER_WIDTH, freqMHz = 100, flitBufferSz = 16)) // Set to None to disable PHY + phyParams: Option[SerialPhyParams] = Some(CreditedSourceSyncSerialPhyParams(phitWidth = CTC.OUTER_WIDTH, flitWidth = CTC.INNER_WIDTH, freqMHz = 100, flitBufferSz = 16)) // Set to None to remove PHY ) extends ChipletLinkParams with ChipletLinkWrapperInstantiationLike { @@ -47,12 +46,13 @@ case class CTCParams( } }.getOrElse(offchip) - //def managerRegion = offchipRange def managerBusWhere = managerBus def controlManagerBusWhere = None def instantiate(params: OffchipSubsystemParams, id: Int)(implicit p: Parameters): ChipletLinkWrapper = LazyModule(new CTCChipletLink(this, params, id)) } +case object CTCKey extends Field[Seq[CTCParams]](Nil) + // For using CTC in a chiplet firesim config with no PHY class CTCBridgeIO extends ChipletIO { val client_flit = new DecoupledFlitIO(CTC.INNER_WIDTH) // Driven by client/ctc2tl @@ -101,9 +101,6 @@ case class CTCMemSerialPhyParams( def genIO = new CTCMemIO(phitWidth, offchip, this) } -case object CTCKey extends Field[Seq[CTCParams]](Nil) - -// TODO: derive beatbytes from offchip subsystem params class CTCChipletLink(val params: CTCParams, val sys_params: OffchipSubsystemParams, val id: Int)(implicit p: Parameters) extends ChipletLinkWrapper { // a TL master/client device val ctc2tl = LazyModule(new CTCToTileLink(portId=id)(p)) diff --git a/src/main/scala/ctc/CTCMem.scala b/src/main/scala/ctc/CTCMem.scala index 6be0a941..827df9be 100644 --- a/src/main/scala/ctc/CTCMem.scala +++ b/src/main/scala/ctc/CTCMem.scala @@ -11,8 +11,7 @@ import freechips.rocketchip.util._ import testchipip.serdes.{DecoupledPhitIO, DecoupledSerialPhy, SerialPhyParams} import testchipip.ctc._ -// A test memory like SerialRAM but for CTC -// Takes in DecoupledFlitIO, puts it through CTCToTileLink, then connects the TileLink node to TLRAM +// A test memory like SerialRAM but for CTC, used for modeling chiplets as RAM class CTCMem(offchip: Seq[AddressSet], phyParams: SerialPhyParams)(implicit p: Parameters) extends LazyModule { val beatBytes = 8 val ctc2tl = LazyModule(new CTCToTileLink(sourceIds = 1, portId = 0)) diff --git a/src/main/scala/serdes/Bundles.scala b/src/main/scala/serdes/Bundles.scala index 2ee86a82..19b60c26 100644 --- a/src/main/scala/serdes/Bundles.scala +++ b/src/main/scala/serdes/Bundles.scala @@ -49,7 +49,7 @@ class DecoupledInternalSyncPhitIO(phitWidth: Int) extends DecoupledPhitIO(phitWi case _ => assert(false, s"IO does not match DecoupledExternalSyncPhitIO: ${io.getClass}") } def loopback: Unit = { - out <> in + assert(false, "DecoupledInternalSyncPhitIO does not support loopback") } } @@ -70,7 +70,7 @@ class DecoupledExternalSyncPhitIO(phitWidth: Int) extends DecoupledPhitIO(phitWi case _ => assert(false, s"IO does not match DecoupledInternalSyncPhitIO: ${io.getClass}") } def loopback: Unit = { - out <> in + assert(false, "DecoupledExternalSyncPhitIO does not support loopback") } } diff --git a/src/main/scala/serdes/PeripheryTLSerial.scala b/src/main/scala/serdes/PeripheryTLSerial.scala index 72c752bb..1b9cad15 100644 --- a/src/main/scala/serdes/PeripheryTLSerial.scala +++ b/src/main/scala/serdes/PeripheryTLSerial.scala @@ -56,7 +56,7 @@ case class SerialTLParams( phyParams: SerialPhyParams = DecoupledExternalSyncSerialPhyParams(), bundleParams: TLBundleParameters = TLSerdesser.STANDARD_TLBUNDLE_PARAMS) extends ChipletLinkParams with ChipletLinkWrapperInstantiationLike { - def managerBusWhere = client.map(_.masterWhere).getOrElse(SBUS) // Apologies for overloaded terminology + def managerBusWhere = client.map(_.masterWhere).getOrElse(SBUS) // This is the bus driven by the SerialTL port client node def controlManagerBusWhere = None def instantiate(params: OffchipSubsystemParams, id: Int)(implicit p: Parameters): ChipletLinkWrapper = LazyModule(new SerialTLChipletLink(this, params, id)) } diff --git a/src/main/scala/soc/ChipletLink.scala b/src/main/scala/soc/ChipletLink.scala index a66c47af..ac1fc832 100644 --- a/src/main/scala/soc/ChipletLink.scala +++ b/src/main/scala/soc/ChipletLink.scala @@ -11,14 +11,12 @@ import freechips.rocketchip.util._ import freechips.rocketchip.prci._ import freechips.rocketchip.diplomacy.BundleBridgeSource -case object MaxOffchipAddressRange extends Field[Seq[AddressSet]](AddressSet.misaligned(0x100000000L, 0x1000000000L)) +case object OffchipAddressRange extends Field[Seq[AddressSet]](AddressSet.misaligned(0x100000000L, 0x1000000000L)) // Link params should extend this trait trait ChipletLinkParams{ - //def managerRegion: Seq[AddressSet] - def managerBusWhere: TLBusWrapperLocation // Where the link client node is connected - def controlManagerBusWhere: Option[TLBusWrapperLocation] // Where the link control node is connected - //def instantiationFn: _ => (TLInwardNode, TLOutwardNode, Option[TLRegisterNode]) + def managerBusWhere: TLBusWrapperLocation // Where the link client node is connected + def controlManagerBusWhere: Option[TLBusWrapperLocation] // Where the link control node is connected } case class OffchipSubsystemParams( @@ -30,7 +28,6 @@ case class OffchipSubsystemParams( ) abstract class ChipletLinkWrapper(implicit p: Parameters) extends LazyModule { - //val link_name: String val client_node: TLClientNode val manager_node: TLManagerNode val control_manager_node: Option[TLRegisterNode] @@ -47,7 +44,6 @@ trait ChipletLinkWrapperInstantiationLike { def instantiate(params: OffchipSubsystemParams, id: Int)(implicit p: Parameters): ChipletLinkWrapper } -// Should this be called D2D IO instead? trait ChipletIO extends Bundle { def tieoff: Unit def connect(io: ChipletIO): Unit diff --git a/src/main/scala/soc/Configs.scala b/src/main/scala/soc/Configs.scala index 2505d451..dd32769e 100644 --- a/src/main/scala/soc/Configs.scala +++ b/src/main/scala/soc/Configs.scala @@ -117,6 +117,6 @@ class WithND2DPorts(n: Int, portParams: ChipletLinkParams) extends Config((site, case ChipletRoutingKey => Some(up(ChipletRoutingKey).getOrElse(ChipletRoutingParams(ports=Nil)).copy(ports=Seq.fill(n)(portParams))) }) -class WithMaxOffchipAddressRange(range: Seq[AddressSet]) extends Config((site, here, up) => { - case MaxOffchipAddressRange => range +class WithOffchipAddressRange(range: Seq[AddressSet]) extends Config((site, here, up) => { + case OffchipAddressRange => range }) \ No newline at end of file diff --git a/src/main/scala/soc/OffchipRouter.scala b/src/main/scala/soc/OffchipRouter.scala index 1dbdb166..8816123e 100644 --- a/src/main/scala/soc/OffchipRouter.scala +++ b/src/main/scala/soc/OffchipRouter.scala @@ -31,10 +31,8 @@ case class OffchipRouterParams( tableEntries: Int = 16 ) +// The offchip router uses a table to route requests to the correct die-to-die port class OffchipRouter(val params: ChipletRoutingParams, val beatBytes: Int = 8)(implicit p: Parameters) extends LazyModule { - - // BORROWED FROM TLSwitch.scala START - // This function can handle simple cases only def unifyManagers(mgrs: Seq[Seq[TLSlaveParameters]]): Seq[TLSlaveParameters] = { mgrs.flatten.groupBy(_.sortedAddress.head).map { case (_, m) => require(m.forall(_.address == m.head.address), "Require homogeneous address ranges") @@ -50,8 +48,6 @@ class OffchipRouter(val params: ChipletRoutingParams, val beatBytes: Int = 8)(im c.head }, managerFn = { m => - // unifies all the managers, its up to the user to be careful here - // TODO: Use bus error device to report problems? require(m.flatMap(_.responseFields).size == 0, "ResponseFields not supported in TLSwitch") require(m.flatMap(_.requestKeys).size == 0, "RequestKeys not supported in TLSwitch") require(m.forall(_.beatBytes == m.head.beatBytes), "Homogeneous beatBytes required") @@ -65,11 +61,9 @@ class OffchipRouter(val params: ChipletRoutingParams, val beatBytes: Int = 8)(im ) } ) - // BORROWED FROM TLSwitch.scala END - // Register node for setting the routing table val routing_table_node = TLRegisterNode( - address = AddressSet.misaligned(params.routerParams.tableAddress, 0x1000), // TODO: fix size + address = AddressSet.misaligned(params.routerParams.tableAddress, 0x1000), device = new SimpleDevice("routing-table", Nil), beatBytes = beatBytes ) @@ -92,29 +86,21 @@ class OffchipRouterImpl(outer: OffchipRouter) extends LazyModuleImp(outer) { val idWidth = outer.params.idWidth val addressWidth = bundleIn.a.bits.address.getWidth val portWidth = if (nPorts == 1) 1 else log2Ceil(nPorts) - val totalWidth = idWidth + addressWidth + portWidth + 1 - assert(totalWidth <= 64, "Total width of routing table entry must be less than or equal to 64 bits") // for now :) - val routing_mode_reg = RegInit(true.B) // true: routing mode, false: bypass mode - val bypass_port_reg = RegInit(0.U(log2Ceil(nPorts).W)) - val translation_mode_reg = RegInit(false.B) // true: per-port translation mode, false: chip id translation mode - val translation_table_reg = RegInit(VecInit(Seq.fill(nPorts)(0.U(idWidth.W)))) // Index with port id + val routing_mode_reg = RegInit(true.B) // true: routing mode, false: bypass mode + val bypass_port_reg = RegInit(0.U(log2Ceil(nPorts).W)) // The port to bypass to when routing mode is disabled + val translation_mode_reg = RegInit(false.B) // true: per-port translation mode, false: chip id translation mode + val translation_table_reg = RegInit(VecInit(Seq.fill(nPorts)(0.U(idWidth.W)))) // Index with port id val chipIdReg = RegInit(0.U(idWidth.W)) for (i <- 0 until nPorts) { io.chip_id(i) := Mux(translation_mode_reg, translation_table_reg(i), chipIdReg) } - //val offsetReg = RegInit(p(MaxOffchipAddressRange).map(_.base).min.U(addressWidth.W)) - - // Create a memory mapped table of offchip addresses which store chip id, base address, and port to route to - // and initialize it to all 0s using RegInit - //val routing_table = VecInit(Seq.fill(outer.params.tableEntries)(RegInit(0.U.asTypeOf(new RoutingTableEntry(idWidth, addressWidth, portWidth))))) - val routing_table = RegInit(VecInit(Seq.fill(tableEntries)(0.U.asTypeOf(new RoutingTableEntry(idWidth, portWidth))))) - // One entry = 4 separate regs (valid, chipID, baseAddress, port) - val regsPerEntry = 4 // TODO: fixy + // 4 because 3 is an ugly number + val regsPerEntry = 4 val mapped_entries = (0 until tableEntries).flatMap { i => val base = i * regsPerEntry * outer.beatBytes @@ -133,11 +119,9 @@ class OffchipRouterImpl(outer: OffchipRouter) extends LazyModuleImp(outer) { (tableEntries * regsPerEntry * outer.beatBytes + ((4 + i) * outer.beatBytes)) -> Seq(RegField(idWidth, translation_table_reg(i), RegFieldDesc(s"translation_table_$i", s"Translation table entry $i"))) } outer.routing_table_node.regmap(mapped_entries ++ chip_id ++ routing_mode ++ bypass_port ++ translation_mode ++ translation_table :_*) - // val offset = Seq((tableEntries * regsPerEntry) + 1 -> Seq(RegField(addressWidth, 0.U, RegFieldDesc("offset", "Offchip offset for this chip")))) - // outer.routing_table_node.regmap(mapped_entries ++ chip_id ++ offset :_*) // Select offchip port from routing table - val addr_top_bits = bundleIn.a.bits.address(addressWidth-1, log2Ceil(p(MaxOffchipAddressRange).map(_.base).min)) + val addr_top_bits = bundleIn.a.bits.address(addressWidth-1, log2Ceil(p(OffchipAddressRange).map(_.base).min)) val matches = Wire(Vec(tableEntries, Bool())) val port_match = Wire(UInt(log2Ceil(nPorts).W)) port_match := 0.U @@ -154,9 +138,8 @@ class OffchipRouterImpl(outer: OffchipRouter) extends LazyModuleImp(outer) { // Assert that only one match is found assert(!routing_mode_reg || !bundleIn.a.valid || ((matches.asUInt & (matches.asUInt - 1.U)) === 0.U), "Multiple matches found in routing table") // TODO: check - // AGENT OUTPUT START ~~~~~~~~~~~~~~~~ - // Decide legality per-request. Address is stable across beats. - val illegal = Wire(Bool()) // <- your programmable table miss / tag mismatch + // Send an error response if no match is found + val illegal = Wire(Bool()) illegal := bundleIn.a.valid && !matches.reduce(_ || _) && routing_mode_reg val errActive = RegInit(false.B) @@ -164,7 +147,6 @@ class OffchipRouterImpl(outer: OffchipRouter) extends LazyModuleImp(outer) { val errSize = Reg(bundleIn.a.bits.size.cloneType) val errSource = Reg(bundleIn.a.bits.source.cloneType) - // Helpers val a_last = edgeIn.last(bundleIn.a) // Start an error response when we accept the LAST beat of an illegal request @@ -175,7 +157,7 @@ class OffchipRouterImpl(outer: OffchipRouter) extends LazyModuleImp(outer) { errSource := bundleIn.a.bits.source } - // Local D generator + // Generate an error response on the D channel val dErr = Wire(chiselTypeOf(bundleIn.d)) val (_, d_last, _) = edgeIn.firstlast(dErr) @@ -192,9 +174,7 @@ class OffchipRouterImpl(outer: OffchipRouter) extends LazyModuleImp(outer) { // Done when last D beat fires when (dErr.fire && d_last) { errActive := false.B } - // AGENT OUTPUT END ~~~~~~~~~~~~~~~~ - - // BORROWED FROM TLSwitch.scala START + // Switch the request to the correct port bundlesOut.zipWithIndex.foreach { case (out, i) => val selected = i.U === sel @@ -211,18 +191,12 @@ class OffchipRouterImpl(outer: OffchipRouter) extends LazyModuleImp(outer) { bundleIn.a.ready := VecInit(bundlesOut.map(_.a.ready))(sel) && !errActive bundleIn.c.ready := VecInit(bundlesOut.map(_.c.ready))(sel) bundleIn.e.ready := VecInit(bundlesOut.map(_.e.ready))(sel) - // BORROWED FROM TLSwitch.scala END - // Arbitrate responses from d channels - val response_arbiter_d = Module(new RRArbiter(chiselTypeOf(bundleIn.d.bits), nPorts + 1)) - for (i <- 0 until nPorts) { - response_arbiter_d.io.in(i) <> bundlesOut(i).d - } - response_arbiter_d.io.in(nPorts) <> dErr - bundleIn.d <> Queue(response_arbiter_d.io.out, nPorts + 1) // I think this queue is necessary?? + // Arbitrate responses from d channels (locks for the duration of multi-beat messages) + TLArbiter.robin(edgeIn, bundleIn.d, (bundlesOut.map(_.d) :+ dErr):_*) // Arbitrate responses from b channels - // TODO ?? : Error checking for coherent requests + // TODO: Error checking for coherent requests (coherent requests are currently unsupported) val response_arbiter_b = Module(new RRArbiter(chiselTypeOf(bundleIn.b.bits), nPorts)) for (i <- 0 until nPorts) { response_arbiter_b.io.in(i) <> bundlesOut(i).b @@ -231,21 +205,17 @@ class OffchipRouterImpl(outer: OffchipRouter) extends LazyModuleImp(outer) { } -// TODO: fix widths class RoutingTableEntry(idWidth: Int, portWidth: Int) extends Bundle { val valid = Bool() - val chipID = UInt(idWidth.W) // This is probably determined by the number of entries + val chipID = UInt(idWidth.W) val port = UInt(portWidth.W) } -// Mostly borrowed from the Radiance AddressRewriterNode case class ChipletAddressTranslator(val params: ChipletRoutingParams)(implicit p: Parameters) extends LazyModule { - val node = TLAdapterNode(clientFn = c => c, managerFn = m => m) + val offset = p(OffchipAddressRange).map(_.base).min - val offset = p(MaxOffchipAddressRange).map(_.base).min - - assert(offset.bitCount == 1, "Offset must only have 1 bit set") // TODO: should we enforce this in the spec? + assert(offset.bitCount == 1, "Offset must only have 1 bit set") override lazy val module = new ChipletAddressTranslatorImpl(this) } @@ -255,8 +225,6 @@ class ChipletAddressTranslatorImpl(outer: ChipletAddressTranslator) extends Lazy val chip_id = Input(UInt(outer.params.idWidth.W)) }) - // If the top bits of the address match the chip id, then trim those bits off - // Top bits are determined by the offset in the params def trimTag(address: UInt) = { val topBits = log2Ceil(outer.offset) val addressWidth = address.getWidth @@ -313,7 +281,7 @@ trait CanHaveChipletRouting { this: BaseSubsystem => val link_manager_bus = locateTLBusWrapper(pP.managerBusWhere) val sys_params = OffchipSubsystemParams( - managerRegion = p(MaxOffchipAddressRange), + managerRegion = p(OffchipAddressRange), clientBeatBytes = client_bus.beatBytes, clientBlockBytes = client_bus.blockBytes, managerBeatBytes = link_manager_bus.beatBytes, @@ -322,7 +290,6 @@ trait CanHaveChipletRouting { this: BaseSubsystem => val port = router_domain { pP.asInstanceOf[ChipletLinkWrapperInstantiationLike].instantiate(sys_params, id)(p).suggestName(s"d2d${id}_port") } - // TODO: Translator should take in chip ID as an IO val translator = router_domain { LazyModule(ChipletAddressTranslator(params)) } diff --git a/src/main/scala/test/Unittests.scala b/src/main/scala/test/Unittests.scala index f1d3c59b..c8b8f3e9 100644 --- a/src/main/scala/test/Unittests.scala +++ b/src/main/scala/test/Unittests.scala @@ -748,7 +748,7 @@ class OffchipRouterTestDriver(reqs: Seq[TLRequestDescriptor])(implicit p: Parame class OffchipRouterTestChecker(portId: Int, expectedReqs: Seq[TLRequestDescriptor])(implicit p: Parameters) extends LazyModule { val node = TLManagerNode(Seq(TLSlavePortParameters.v1( managers = Seq(TLSlaveParameters.v1( - address = p(MaxOffchipAddressRange), + address = p(OffchipAddressRange), supportsGet = TransferSizes(1, 64), supportsPutFull = TransferSizes(1, 64))), beatBytes = 8 @@ -889,7 +889,7 @@ class OffchipRouterTest(val nChips: Int, val nPorts: Int, val reqsPerChip: Int = // Build data request sequence: reqsPerChip random accesses to each remote chip, shuffled val dataReqs: Seq[TLRequestDescriptor] = { - val offset = p(MaxOffchipAddressRange).map(_.base).min + val offset = p(OffchipAddressRange).map(_.base).min val offsetBits = log2Ceil(offset) val reqs = otherChipIds.flatMap { cid => Seq.fill(reqsPerChip) { @@ -905,7 +905,7 @@ class OffchipRouterTest(val nChips: Int, val nPorts: Int, val reqsPerChip: Int = val chipIdToPort: Map[Int, Int] = tableData.map { case (cid, port) => cid -> port }.toMap // Determine expected port for each dataReq based on its address tag - val offset = p(MaxOffchipAddressRange).map(_.base).min + val offset = p(OffchipAddressRange).map(_.base).min val offsetBits = log2Ceil(offset) def reqPort(r: TLRequestDescriptor): Int = { @@ -936,7 +936,6 @@ class OffchipRouterTest(val nChips: Int, val nPorts: Int, val reqsPerChip: Int = }) // Phase 1: MMIO driver programs chip ID + routing table - // Phase 2: Data driver sends requests through the router val mmioFinished = mmio_driver.module.io.finished mmio_driver.module.io.start := true.B @@ -945,6 +944,7 @@ class OffchipRouterTest(val nChips: Int, val nPorts: Int, val reqsPerChip: Int = assert(router.module.io.chip_id(0) === chipId.U, "Chip ID mismatch after programming") } + // Phase 2: Data driver sends requests through the router driver.module.io.start := mmioFinished io.finished := driver.module.io.finished From fc8013c30e098f46402f1c98543b9d4b7b9b36d2 Mon Sep 17 00:00:00 2001 From: Ella Schwarz Date: Fri, 29 May 2026 14:29:16 -0700 Subject: [PATCH 19/19] Add router modules to README --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 22c7fe98..f0d624a0 100644 --- a/README.md +++ b/README.md @@ -19,6 +19,7 @@ BAR projects generally use these components with [chipyard](https://chipyard.rea ## ``testchipip.ctc`` * ``CTC``: Simple, 32-bit chip-to-chip interface +* ``CTCMem``: SerialRAM-like backing memory for CTC ## ``testchipip.dram`` * ``SimDRAM``: DRAMSim-backed AXI-4 memory model @@ -35,6 +36,7 @@ BAR projects generally use these components with [chipyard](https://chipyard.rea ## ``testchipip.soc`` * ``OffchipBus``: Custom bus for interfacing with off-chip memory +* ``OffchipRouter``: Routes requests to the die-to-die ports * ``Scratchpad``: TileLink SRAM-backed on-chip scratchpad memory * ``SimDTM``: Simulation model for interacting with on-chip debug module * ``TLNetwork``: DEPRECATED mechanism for creating a TileLink network-on-chip. Use Constellation NoC instead