Support clustered-cores

.github/scripts/defaults.sh

@@ -33,7 +33,7 @@ grouping["group-peripherals"]="chipyard-dmirocket chipyard-dmiboom chipyard-spif
 grouping["group-accels"]="chipyard-mempress chipyard-sha3 chipyard-hwacha chipyard-gemmini chipyard-manymmioaccels chipyard-nvdla chipyard-aes256ecb"
 grouping["group-constellation"]="chipyard-constellation"
 grouping["group-tracegen"]="tracegen tracegen-boom"
-grouping["group-other"]="icenet testchipip constellation rocketchip-amba rocketchip-tlsimple rocketchip-tlwidth rocketchip-tlxbar"
+grouping["group-other"]="icenet testchipip constellation rocketchip-amba rocketchip-tlsimple rocketchip-tlwidth rocketchip-tlxbar chipyard-clusters"
 grouping["group-fpga"]="arty35t arty100t nexysvideo vc707 vcu118"
 
 # key value store to get the build strings
@@ -67,6 +67,7 @@ mapping["chipyard-shuttle"]=" CONFIG=ShuttleConfig"
 mapping["chipyard-multiclock-rocket"]=" CONFIG=MulticlockRocketConfig"
 mapping["chipyard-nomem-scratchpad"]=" CONFIG=MMIOScratchpadOnlyRocketConfig"
 mapping["chipyard-constellation"]=" CONFIG=SharedNoCConfig"
+mapping["chipyard-clusters"]=" CONFIG=ClusteredRocketConfig verilog"
 mapping["chipyard-aes256ecb"]=" CONFIG=AES256ECBRocketConfig"
 
 mapping["constellation"]=" SUB_PROJECT=constellation"

build.sbt

@@ -95,7 +95,7 @@ lazy val chiselSettings = Seq(
 
 // -- Rocket Chip --
 
-lazy val hardfloat = freshProject("hardfloat", rocketChipDir / "hardfloat/hardfloat")
+lazy val hardfloat = freshProject("hardfloat", file("generators/hardfloat/hardfloat"))
   .settings(chiselSettings)
   .dependsOn(midasTargetUtils)
   .settings(commonSettings)

fpga/src/main/scala/arty/Configs.scala

@@ -30,6 +30,7 @@ class WithArtyTweaks extends Config(
   new chipyard.config.WithFrontBusFrequency(32) ++
   new chipyard.config.WithControlBusFrequency(32) ++
   new chipyard.config.WithPeripheryBusFrequency(32) ++
+  new chipyard.config.WithControlBusFrequency(32) ++
   new testchipip.serdes.WithNoSerialTL ++
   new testchipip.soc.WithNoScratchpads
 )

fpga/src/main/scala/arty100t/Configs.scala

@@ -18,7 +18,7 @@ import chipyard.{BuildSystem}
 
 // don't use FPGAShell's DesignKey
 class WithNoDesignKey extends Config((site, here, up) => {
-  case DesignKey => (p: Parameters) => new SimpleLazyModule()(p)
+  case DesignKey => (p: Parameters) => new SimpleLazyRawModule()(p)
 })
 
 // By default, this uses the on-board USB-UART for the TSI-over-UART link

fpga/src/main/scala/arty100t/Harness.scala

@@ -5,12 +5,12 @@ import chisel3.util._
 import freechips.rocketchip.diplomacy._
 import org.chipsalliance.cde.config.{Parameters}
 import freechips.rocketchip.tilelink._
-import freechips.rocketchip.prci.{ClockBundle, ClockBundleParameters}
+import freechips.rocketchip.prci._
 import freechips.rocketchip.subsystem.{SystemBusKey}
 
 import sifive.fpgashells.shell.xilinx._
 import sifive.fpgashells.shell._
-import sifive.fpgashells.clocks.{ClockGroup, ClockSinkNode, PLLFactoryKey, ResetWrangler}
+import sifive.fpgashells.clocks._
 import sifive.fpgashells.ip.xilinx.{IBUF, PowerOnResetFPGAOnly}
 
 import sifive.blocks.devices.uart._

fpga/src/main/scala/nexysvideo/Configs.scala

@@ -18,7 +18,7 @@ import chipyard.{BuildSystem}
 
 // don't use FPGAShell's DesignKey
 class WithNoDesignKey extends Config((site, here, up) => {
-  case DesignKey => (p: Parameters) => new SimpleLazyModule()(p)
+  case DesignKey => (p: Parameters) => new SimpleLazyRawModule()(p)
 })
 
 // DOC include start: WithNexysVideoTweaks and Rocket
@@ -60,6 +60,7 @@ class WithTinyNexysVideoTweaks extends Config(
   new chipyard.config.WithFrontBusFrequency(50.0) ++
   new chipyard.config.WithSystemBusFrequency(50.0) ++
   new chipyard.config.WithPeripheryBusFrequency(50.0) ++
+  new chipyard.config.WithControlBusFrequency(50.0) ++
   new chipyard.harness.WithAllClocksFromHarnessClockInstantiator ++
   new chipyard.clocking.WithPassthroughClockGenerator ++
   new chipyard.config.WithNoDebug ++ // no jtag

fpga/src/main/scala/nexysvideo/Harness.scala

@@ -7,10 +7,10 @@ import freechips.rocketchip.diplomacy._
 import org.chipsalliance.cde.config.{Parameters}
 import freechips.rocketchip.tilelink._
 import freechips.rocketchip.subsystem.{SystemBusKey}
-
+import freechips.rocketchip.prci._
 import sifive.fpgashells.shell.xilinx._
 import sifive.fpgashells.shell._
-import sifive.fpgashells.clocks.{ClockGroup, ClockSinkNode, PLLFactoryKey, ResetWrangler}
+import sifive.fpgashells.clocks._
 
 import sifive.blocks.devices.uart._
 

fpga/src/main/scala/vc707/TestHarness.scala

@@ -7,11 +7,12 @@ import org.chipsalliance.cde.config.{Parameters}
 import freechips.rocketchip.tilelink._
 import freechips.rocketchip.subsystem.{SystemBusKey}
 import freechips.rocketchip.diplomacy.{IdRange, TransferSizes}
+import freechips.rocketchip.prci._
 
 import sifive.fpgashells.shell.xilinx.{VC707Shell, UARTVC707ShellPlacer, PCIeVC707ShellPlacer, ChipLinkVC707PlacedOverlay}
 import sifive.fpgashells.ip.xilinx.{IBUF, PowerOnResetFPGAOnly}
 import sifive.fpgashells.shell._
-import sifive.fpgashells.clocks.{ClockGroup, ClockSinkNode, PLLFactoryKey, ResetWrangler}
+import sifive.fpgashells.clocks.{PLLFactoryKey}
 import sifive.fpgashells.devices.xilinx.xilinxvc707pciex1.{XilinxVC707PCIeX1IO}
 
 import sifive.blocks.devices.uart.{PeripheryUARTKey, UARTPortIO}
@@ -87,6 +88,7 @@ class VC707FPGATestHarness(override implicit val p: Parameters) extends VC707She
 }
 
 class VC707FPGATestHarnessImp(_outer: VC707FPGATestHarness) extends LazyRawModuleImp(_outer) with HasHarnessInstantiators {
+  override def provideImplicitClockToLazyChildren = true
   val vc707Outer = _outer
 
   val reset = IO(Input(Bool())).suggestName("reset")

fpga/src/main/scala/vcu118/Configs.scala

@@ -48,6 +48,7 @@ class WithVCU118Tweaks extends Config(
   new chipyard.config.WithSystemBusFrequency(100) ++
   new chipyard.config.WithControlBusFrequency(100) ++
   new chipyard.config.WithPeripheryBusFrequency(100) ++
+  new chipyard.config.WithControlBusFrequency(100) ++
   new WithFPGAFrequency(100) ++ // default 100MHz freq
   // harness binders
   new WithUART ++

fpga/src/main/scala/vcu118/CustomOverlays.scala

@@ -5,7 +5,7 @@ import chisel3._
 import freechips.rocketchip.diplomacy._
 import org.chipsalliance.cde.config.{Parameters, Field}
 import freechips.rocketchip.tilelink.{TLInwardNode, TLAsyncCrossingSink}
-
+import freechips.rocketchip.prci._
 import sifive.fpgashells.shell._
 import sifive.fpgashells.ip.xilinx._
 import sifive.fpgashells.shell.xilinx._
@@ -79,7 +79,7 @@ class DDR2VCU118PlacedOverlay(val shell: VCU118FPGATestHarness, name: String, va
     ui.reset := /*!port.mmcm_locked ||*/ port.c0_ddr4_ui_clk_sync_rst
     port.c0_sys_clk_i := sys.clock.asUInt
     port.sys_rst := sys.reset // pllReset
-    port.c0_ddr4_aresetn := !ar.reset
+    port.c0_ddr4_aresetn := !(ar.reset.asBool)
 
     // This was just copied from the SiFive example, but it's hard to follow.
     // The pins are emitted in the following order:

fpga/src/main/scala/vcu118/TestHarness.scala

@@ -8,11 +8,11 @@ import org.chipsalliance.cde.config.{Parameters}
 import freechips.rocketchip.tilelink._
 import freechips.rocketchip.diplomacy.{IdRange, TransferSizes}
 import freechips.rocketchip.subsystem.{SystemBusKey}
-
+import freechips.rocketchip.prci._
 import sifive.fpgashells.shell.xilinx._
 import sifive.fpgashells.ip.xilinx.{IBUF, PowerOnResetFPGAOnly}
 import sifive.fpgashells.shell._
-import sifive.fpgashells.clocks.{ClockGroup, ClockSinkNode, PLLFactoryKey, ResetWrangler}
+import sifive.fpgashells.clocks._
 
 import sifive.blocks.devices.uart.{PeripheryUARTKey, UARTPortIO}
 import sifive.blocks.devices.spi.{PeripherySPIKey, SPIPortIO}
@@ -90,6 +90,7 @@ class VCU118FPGATestHarness(override implicit val p: Parameters) extends VCU118S
 }
 
 class VCU118FPGATestHarnessImp(_outer: VCU118FPGATestHarness) extends LazyRawModuleImp(_outer) with HasHarnessInstantiators {
+  override def provideImplicitClockToLazyChildren = true
   val vcu118Outer = _outer
 
   val reset = IO(Input(Bool())).suggestName("reset")

fpga/src/main/scala/vcu118/bringup/TestHarness.scala

@@ -5,7 +5,7 @@ import freechips.rocketchip.diplomacy._
 import org.chipsalliance.cde.config._
 import freechips.rocketchip.subsystem._
 import freechips.rocketchip.tilelink._
-
+import freechips.rocketchip.prci._
 import sifive.fpgashells.shell.xilinx._
 import sifive.fpgashells.ip.xilinx._
 import sifive.fpgashells.shell._

generators/chipyard/src/main/scala/ChipTop.scala

@@ -5,7 +5,6 @@ import chisel3._
 import scala.collection.mutable.{ArrayBuffer}
 
 import freechips.rocketchip.prci.{ClockGroupIdentityNode, ClockSinkParameters, ClockSinkNode, ClockGroup}
-import freechips.rocketchip.subsystem.{BaseSubsystem, SubsystemDriveAsyncClockGroupsKey}
 import org.chipsalliance.cde.config.{Parameters, Field}
 import freechips.rocketchip.diplomacy.{LazyModule, LazyModuleImp, LazyRawModuleImp, LazyModuleImpLike, BindingScope}
 import freechips.rocketchip.util.{DontTouch}

generators/chipyard/src/main/scala/DigitalTop.scala

@@ -40,7 +40,6 @@ class DigitalTop(implicit p: Parameters) extends ChipyardSystem
 }
 
 class DigitalTopModule[+L <: DigitalTop](l: L) extends ChipyardSystemModule(l)
-  with testchipip.cosim.CanHaveTraceIOModuleImp
   with sifive.blocks.devices.i2c.HasPeripheryI2CModuleImp
   with sifive.blocks.devices.pwm.HasPeripheryPWMModuleImp
   with sifive.blocks.devices.uart.HasPeripheryUARTModuleImp

generators/chipyard/src/main/scala/SpikeTile.scala

@@ -77,22 +77,23 @@ case class SpikeTileAttachParams(
 }
 
 case class SpikeTileParams(
-  hartId: Int = 0,
+  tileId: Int = 0,
   val core: SpikeCoreParams = SpikeCoreParams(),
   icacheParams: ICacheParams = ICacheParams(nWays = 32),
   dcacheParams: DCacheParams = DCacheParams(nWays = 32),
   tcmParams: Option[MasterPortParams] = None // tightly coupled memory
 ) extends InstantiableTileParams[SpikeTile]
 {
-  val name = Some("spike_tile")
+  val baseName = "spike_tile"
+  val uniqueName = s"${baseName}_$tileId"
   val beuAddr = None
   val blockerCtrlAddr = None
   val btb = None
   val boundaryBuffers = false
   val dcache = Some(dcacheParams)
   val icache = Some(icacheParams)
   val clockSinkParams = ClockSinkParameters()
-  def instantiate(crossing: TileCrossingParamsLike, lookup: LookupByHartIdImpl)(implicit p: Parameters): SpikeTile = {
+  def instantiate(crossing: HierarchicalElementCrossingParamsLike, lookup: LookupByHartIdImpl)(implicit p: Parameters): SpikeTile = {
     new SpikeTile(this, crossing, lookup)
   }
 }
@@ -106,11 +107,11 @@ class SpikeTile(
     with SourcesExternalNotifications
 {
   // Private constructor ensures altered LazyModule.p is used implicitly
-  def this(params: SpikeTileParams, crossing: TileCrossingParamsLike, lookup: LookupByHartIdImpl)(implicit p: Parameters) =
+  def this(params: SpikeTileParams, crossing: HierarchicalElementCrossingParamsLike, lookup: LookupByHartIdImpl)(implicit p: Parameters) =
     this(params, crossing.crossingType, lookup, p)
 
   // Required TileLink nodes
-  val intOutwardNode = IntIdentityNode()
+  val intOutwardNode = None
   val masterNode = visibilityNode
   val slaveNode = TLIdentityNode()
 
@@ -129,21 +130,21 @@ class SpikeTile(
   }
 
   ResourceBinding {
-    Resource(cpuDevice, "reg").bind(ResourceAddress(hartId))
+    Resource(cpuDevice, "reg").bind(ResourceAddress(tileId))
   }
 
 
   val icacheNode = TLClientNode(Seq(TLMasterPortParameters.v1(Seq(TLMasterParameters.v1(
     sourceId = IdRange(0, 1),
-    name = s"Core ${staticIdForMetadataUseOnly} ICache")))))
+    name = s"Core ${tileId} ICache")))))
 
   val dcacheNode = TLClientNode(Seq(TLMasterPortParameters.v1(Seq(TLMasterParameters.v1(
-    name          = s"Core ${staticIdForMetadataUseOnly} DCache",
+    name          = s"Core ${tileId} DCache",
     sourceId      = IdRange(0, tileParams.dcache.get.nMSHRs),
     supportsProbe = TransferSizes(p(CacheBlockBytes), p(CacheBlockBytes)))))))
 
   val mmioNode = TLClientNode((Seq(TLMasterPortParameters.v1(Seq(TLMasterParameters.v1(
-    name          = s"Core ${staticIdForMetadataUseOnly} MMIO",
+    name          = s"Core ${tileId} MMIO",
     sourceId      = IdRange(0, 1),
     requestFifo   = true))))))
 
@@ -313,7 +314,7 @@ class SpikeBlackBox(
 }
 
 class SpikeTileModuleImp(outer: SpikeTile) extends BaseTileModuleImp(outer) {
-
+  val tileParams = outer.tileParams
   // We create a bundle here and decode the interrupt.
   val int_bundle = Wire(new TileInterrupts())
   outer.decodeCoreInterrupts(int_bundle)
@@ -337,7 +338,7 @@ class SpikeTileModuleImp(outer: SpikeTile) extends BaseTileModuleImp(outer) {
   // then the DTM-based bringup with SimDTM will be used. This isn't required to be
   // true, but it usually is
   val useDTM = p(ExportDebug).protocols.contains(DMI)
-  val spike = Module(new SpikeBlackBox(hartId, isaDTS, tileParams.core.nPMPs,
+  val spike = Module(new SpikeBlackBox(outer.tileId, outer.isaDTS, tileParams.core.nPMPs,
     tileParams.icache.get.nSets, tileParams.icache.get.nWays,
     tileParams.dcache.get.nSets, tileParams.dcache.get.nWays,
     tileParams.dcache.get.nMSHRs,
@@ -467,19 +468,21 @@ class SpikeTileModuleImp(outer: SpikeTile) extends BaseTileModuleImp(outer) {
   }
 }
 
-class WithNSpikeCores(n: Int = 1, tileParams: SpikeTileParams = SpikeTileParams(),
-  overrideIdOffset: Option[Int] = None) extends Config((site, here, up) => {
+class WithNSpikeCores(n: Int = 1, tileParams: SpikeTileParams = SpikeTileParams()
+) extends Config((site, here, up) => {
   case TilesLocated(InSubsystem) => {
     // Calculate the next available hart ID (since hart ID cannot be duplicated)
     val prev = up(TilesLocated(InSubsystem), site)
-    val idOffset = overrideIdOffset.getOrElse(prev.size)
+    val idOffset = up(NumTiles)
     // Create TileAttachParams for every core to be instantiated
     (0 until n).map { i =>
       SpikeTileAttachParams(
-        tileParams = tileParams.copy(hartId = i + idOffset)
+        tileParams = tileParams.copy(tileId = i + idOffset)
       )
     } ++ prev
   }
+  case NumTiles => up(NumTiles) + n
+
 })
 
 class WithSpikeTCM extends Config((site, here, up) => {
@@ -492,5 +495,5 @@ class WithSpikeTCM extends Config((site, here, up) => {
     )))
   }
   case ExtMem => None
-  case BankedL2Key => up(BankedL2Key).copy(nBanks = 0)
+  case SubsystemBankedCoherenceKey => up(SubsystemBankedCoherenceKey).copy(nBanks = 0)
 })

generators/chipyard/src/main/scala/Subsystem.scala

@@ -71,43 +71,45 @@ trait CanHaveChosenInDTS { this: BaseSubsystem =>
 }
 
 class ChipyardSubsystem(implicit p: Parameters) extends BaseSubsystem
-  with HasTiles
-  with HasPeripheryDebug
-  with CanHaveHTIF
-  with CanHaveChosenInDTS
+    with InstantiatesHierarchicalElements
+    with HasTileNotificationSinks
+    with HasTileInputConstants
+    with CanHavePeripheryCLINT
+    with CanHavePeripheryPLIC
+    with HasPeripheryDebug
+    with HasHierarchicalElementsRootContext
+    with HasHierarchicalElements
+    with CanHaveHTIF
+    with CanHaveChosenInDTS
 {
-  def coreMonitorBundles = tiles.map {
+  def coreMonitorBundles = totalTiles.values.map {
     case r: RocketTile => r.module.core.rocketImpl.coreMonitorBundle
     case b: BoomTile => b.module.core.coreMonitorBundle
   }.toList
 
   // No-tile configs have to be handled specially.
-  if (tiles.size == 0) {
+  if (totalTiles.size == 0) {
     // no PLIC, so sink interrupts to nowhere
     require(!p(PLICKey).isDefined)
     val intNexus = IntNexusNode(sourceFn = x => x.head, sinkFn = x => x.head)
     val intSink = IntSinkNode(IntSinkPortSimple())
     intSink := intNexus :=* ibus.toPLIC
 
     // avoids a bug when there are no interrupt sources
-    ibus.fromAsync := NullIntSource()
+    ibus { ibus.fromAsync := NullIntSource() }
 
     // Need to have at least 1 driver to the tile notification sinks
     tileHaltXbarNode := IntSourceNode(IntSourcePortSimple())
     tileWFIXbarNode := IntSourceNode(IntSourcePortSimple())
     tileCeaseXbarNode := IntSourceNode(IntSourcePortSimple())
-
-    // Sink reset vectors to nowhere
-    val resetVectorSink = BundleBridgeSink[UInt](Some(() => UInt(28.W)))
-    resetVectorSink := tileResetVectorNode
   }
 
   // Relying on [[TLBusWrapperConnection]].driveClockFromMaster for
   // bus-couplings that are not asynchronous strips the bus name from the sink
   // ClockGroup. This makes it impossible to determine which clocks are driven
   // by which bus based on the member names, which is problematic when there is
   // a rational crossing between two buses. Instead, provide all bus clocks
-  // directly from the asyncClockGroupsNode in the subsystem to ensure bus
+  // directly from the allClockGroupsNode in the subsystem to ensure bus
   // names are always preserved in the top-level clock names.
   //
   // For example, using a RationalCrossing between the Sbus and Cbus, and
@@ -116,12 +118,12 @@ class ChipyardSubsystem(implicit p: Parameters) extends BaseSubsystem
   // Conversly, if an async crossing is used, they instead receive names of the
   // form "subsystem_cbus_[0-9]*". The assignment below provides the latter names in all cases.
   Seq(PBUS, FBUS, MBUS, CBUS).foreach { loc =>
-    tlBusWrapperLocationMap.lift(loc).foreach { _.clockGroupNode := asyncClockGroupsNode }
+    tlBusWrapperLocationMap.lift(loc).foreach { _.clockGroupNode := allClockGroupsNode }
   }
   override lazy val module = new ChipyardSubsystemModuleImp(this)
 }
 
 class ChipyardSubsystemModuleImp[+L <: ChipyardSubsystem](_outer: L) extends BaseSubsystemModuleImp(_outer)
-  with HasTilesModuleImp
+    with HasHierarchicalElementsRootContextModuleImp
 {
 }

generators/chipyard/src/main/scala/System.scala

@@ -32,13 +32,6 @@ class ChipyardSystem(implicit p: Parameters) extends ChipyardSubsystem
   val bootROM  = p(BootROMLocated(location)).map { BootROM.attach(_, this, CBUS) }
   val maskROMs = p(MaskROMLocated(location)).map { MaskROM.attach(_, this, CBUS) }
 
-  // If there is no bootrom, the tile reset vector bundle will be tied to zero
-  if (bootROM.isEmpty) {
-    val fakeResetVectorSourceNode = BundleBridgeSource[UInt]()
-    InModuleBody { fakeResetVectorSourceNode.bundle := 0.U }
-    tileResetVectorNexusNode := fakeResetVectorSourceNode
-  }
-
   override lazy val module = new ChipyardSystemModule(this)
 }