From 92cd111e011ee0e57f8df3759e4e5226f91a9d5d Mon Sep 17 00:00:00 2001
From: Henry Cook <henry@sifive.com>
Date: Fri, 5 Jun 2020 12:05:35 -0700
Subject: [PATCH 01/15] tile: BaseTile HasLogicalTreeNode and RocketTile
 overrides

---
 .../scala/subsystem/RocketSubsystem.scala     | 24 +++++++++++--------
 src/main/scala/tile/BaseTile.scala            |  6 +++++
 src/main/scala/tile/RocketTile.scala          | 20 +++++++++-------
 3 files changed, 31 insertions(+), 19 deletions(-)

diff --git a/src/main/scala/subsystem/RocketSubsystem.scala b/src/main/scala/subsystem/RocketSubsystem.scala
index 789a883ae7e..483ede008a0 100644
--- a/src/main/scala/subsystem/RocketSubsystem.scala
+++ b/src/main/scala/subsystem/RocketSubsystem.scala
@@ -30,6 +30,17 @@ case class RocketCrossingParams(
   master: TileMasterPortParams = TileMasterPortParams(),
   slave: TileSlavePortParams = TileSlavePortParams())
 
+case class RocketAttachParams
+  tile: RocketTileParams,
+  crossing: RocketCrossingParams,
+  lookup: LookupByHartIdImpl
+) {
+  def instantiate(context: Attachable): RocketTile = {
+    val rocket = LazyModule(new RocketTile(tile, crossing, lookup, context.logicalTreeNode))
+    rocket
+  }
+}
+
 case object RocketTilesKey extends Field[Seq[RocketTileParams]](Nil)
 case object RocketCrossingKey extends Field[Seq[RocketCrossingParams]](List(RocketCrossingParams()))
 
@@ -46,20 +57,13 @@ trait HasRocketTiles extends HasTiles
   // according to the specified type of clock crossing.
   // Note that we also inject new nodes into the tile itself,
   // also based on the crossing type.
-  val rocketTiles = rocketTileParams.zip(crossings).map { case (tp, crossing) =>
-    val rocket = LazyModule(new RocketTile(tp, crossing, PriorityMuxHartIdFromSeq(rocketTileParams), logicalTreeNode))
+  val rocketTiles = tiles.collect { case r: RocketTile => r }
 
+  rocketTiles.foreach { rocket =>
     connectMasterPortsToSBus(rocket, crossing)
     connectSlavePortsToCBus(rocket, crossing)
     connectInterrupts(rocket, debugOpt, clintOpt, plicOpt)
-
-    rocket
-  }
-
-  rocketTiles.map {
-    r =>
-      def treeNode: RocketTileLogicalTreeNode = new RocketTileLogicalTreeNode(r.rocketLogicalTree.getOMInterruptTargets)
-      LogicalModuleTree.add(logicalTreeNode, r.rocketLogicalTree)
+    LogicalModuleTree.add(logicalTreeNode, rocket.rocketLogicalTree)
   }
 
   def coreMonitorBundles = (rocketTiles map { t =>
diff --git a/src/main/scala/tile/BaseTile.scala b/src/main/scala/tile/BaseTile.scala
index df658ddcd8c..87f43637613 100644
--- a/src/main/scala/tile/BaseTile.scala
+++ b/src/main/scala/tile/BaseTile.scala
@@ -7,6 +7,9 @@ import Chisel._
 import freechips.rocketchip.config._
 import freechips.rocketchip.subsystem._
 import freechips.rocketchip.diplomacy._
+import freechips.rocketchip.diplomaticobjectmodel.{HasLogicalTreeNode}
+import freechips.rocketchip.diplomaticobjectmodel.logicaltree.{GenericLogicalTreeNode, LogicalTreeNode}
+
 import freechips.rocketchip.interrupts._
 import freechips.rocketchip.rocket._
 import freechips.rocketchip.tilelink._
@@ -148,6 +151,7 @@ abstract class BaseTile private (val crossing: ClockCrossingType, q: Parameters)
     extends LazyModule()(q)
     with CrossesToOnlyOneClockDomain
     with HasNonDiplomaticTileParameters
+    with HasLogicalTreeNode
 {
   // Public constructor alters Parameters to supply some legacy compatibility keys
   def this(tileParams: TileParams, crossing: ClockCrossingType, lookup: LookupByHartIdImpl, p: Parameters) = {
@@ -251,6 +255,8 @@ abstract class BaseTile private (val crossing: ClockCrossingType, q: Parameters)
   def crossIntIn(): IntInwardNode = crossIntIn(intInwardNode)
   def crossIntOut(): IntOutwardNode = crossIntOut(intOutwardNode)
 
+  val logicalTreeNode: LogicalTreeNode = new GenericLogicalTreeNode
+
   this.suggestName(tileParams.name)
 }
 
diff --git a/src/main/scala/tile/RocketTile.scala b/src/main/scala/tile/RocketTile.scala
index c743c2df000..9feebb0483a 100644
--- a/src/main/scala/tile/RocketTile.scala
+++ b/src/main/scala/tile/RocketTile.scala
@@ -25,17 +25,19 @@ case class RocketTileParams(
     beuAddr: Option[BigInt] = None,
     blockerCtrlAddr: Option[BigInt] = None,
     boundaryBuffers: Boolean = false // if synthesized with hierarchical PnR, cut feed-throughs?
-    ) extends TileParams {
+    ) extends InstantiatableTileParams[RocketTile] {
   require(icache.isDefined)
   require(dcache.isDefined)
+   def instantiate(crossing: TileCrossingParamsLike, lookup: LookupByHartIdImpl)(implicit p: Parameters): RocketTile = {
+    new RocketTile(this, crossing, lookup)
+  }
 }
 
 class RocketTile private(
       val rocketParams: RocketTileParams,
       crossing: ClockCrossingType,
       lookup: LookupByHartIdImpl,
-      q: Parameters,
-      logicalTreeNode: LogicalTreeNode)
+      q: Parameters)
     extends BaseTile(rocketParams, crossing, lookup, q)
     with SinksExternalInterrupts
     with SourcesExternalNotifications
@@ -44,14 +46,14 @@ class RocketTile private(
     with HasICacheFrontend
 {
   // Private constructor ensures altered LazyModule.p is used implicitly
-  def this(params: RocketTileParams, crossing: RocketCrossingParams, lookup: LookupByHartIdImpl, logicalTreeNode: LogicalTreeNode)(implicit p: Parameters) =
-    this(params, crossing.crossingType, lookup, p, logicalTreeNode)
+  def this(params: RocketTileParams, crossing: RocketCrossingParams, lookup: LookupByHartIdImpl)(implicit p: Parameters) =
+    this(params, crossing.crossingType, lookup, p)
 
   val intOutwardNode = IntIdentityNode()
   val slaveNode = TLIdentityNode()
   val masterNode = visibilityNode
 
-  val rocketLogicalTree = new RocketLogicalTreeNode(this, p(XLen), pgLevels)
+  override val logicalTreeNode = new RocketLogicalTreeNode(this, p(XLen), pgLevels)
 
   val dtim_adapter = tileParams.dcache.flatMap { d => d.scratch.map { s =>
     val coreParams = {
@@ -63,7 +65,7 @@ class RocketTile private(
   dtim_adapter.foreach(lm => connectTLSlave(lm.node, lm.node.portParams.head.beatBytes))
 
   val bus_error_unit = rocketParams.beuAddr map { a =>
-    val beu = LazyModule(new BusErrorUnit(new L1BusErrors, BusErrorUnitParams(a), rocketLogicalTree))
+    val beu = LazyModule(new BusErrorUnit(new L1BusErrors, BusErrorUnitParams(a), logicalTreeNode))
     intOutwardNode := beu.intNode
     connectTLSlave(beu.node, xBytes)
     beu
@@ -117,8 +119,8 @@ class RocketTile private(
   }
 
   val dCacheLogicalTreeNode = new DCacheLogicalTreeNode(dcache, dtim_adapter.map(_.device), rocketParams.dcache.get)
-  LogicalModuleTree.add(rocketLogicalTree, iCacheLogicalTreeNode)
-  LogicalModuleTree.add(rocketLogicalTree, dCacheLogicalTreeNode)
+  LogicalModuleTree.add(logicalTreeNode, iCacheLogicalTreeNode)
+  LogicalModuleTree.add(logicalTreeNode, dCacheLogicalTreeNode)
 }
 
 class RocketTileModuleImp(outer: RocketTile) extends BaseTileModuleImp(outer)

From 0d4cf5999620edc58fc2bef476858950a01623fa Mon Sep 17 00:00:00 2001
From: Henry Cook <henry@sifive.com>
Date: Fri, 5 Jun 2020 12:07:16 -0700
Subject: [PATCH 02/15] Tiles are instantiated in HasTiles via TilesLocated
 Field

---
 src/main/scala/subsystem/Configs.scala        |  16 ++
 src/main/scala/subsystem/HasTiles.scala       | 201 +++++++++++++-----
 .../scala/subsystem/RocketSubsystem.scala     |  57 +----
 src/main/scala/tile/BaseTile.scala            |   5 +
 src/main/scala/tile/RocketTile.scala          |   6 +-
 src/main/scala/util/package.scala             |   6 +
 6 files changed, 192 insertions(+), 99 deletions(-)

diff --git a/src/main/scala/subsystem/Configs.scala b/src/main/scala/subsystem/Configs.scala
index 62764d6bc8b..2a5b675726a 100644
--- a/src/main/scala/subsystem/Configs.scala
+++ b/src/main/scala/subsystem/Configs.scala
@@ -41,6 +41,8 @@ class BaseSubsystemConfig extends Config ((site, here, up) => {
   case DebugModuleKey => Some(DefaultDebugModuleParams(site(XLen)))
   case CLINTKey => Some(CLINTParams())
   case PLICKey => Some(PLICParams())
+  case TilesLocated(InSubsystem) => 
+    LegacyTileFieldHelper(site(RocketTilesKey), site(RocketCrossingKey), RocketTileAttachParams.apply _)
 })
 
 /* Composable partial function Configs to set individual parameters */
@@ -410,3 +412,17 @@ class WithScratchpadsOnly extends Config((site, here, up) => {
         scratch = Some(0x80000000L))))
   }
 })
+
+/** Boilerplate code for translating between the old XTilesParamsKey/XTilesCrossingKey pattern and new TilesLocated pattern */
+object LegacyTileFieldHelper {
+  def apply[TPT <: InstantiatableTileParams[_], TCT <: TileCrossingParamsLike, TAP <: CanAttachTile](
+    tileParams: Seq[TPT],
+    tcp: Seq[TCT],
+    apply: (TPT, TCT, LookupByHartIdImpl) => TAP): Seq[TAP] =
+  {
+    val crossingParams = heterogeneousOrGlobalSetting(tcp, tileParams.size)
+    tileParams.zip(crossingParams).map { case (t, c) =>
+      apply(t, c, PriorityMuxHartIdFromSeq(tileParams))
+    }
+  }
+}
diff --git a/src/main/scala/subsystem/HasTiles.scala b/src/main/scala/subsystem/HasTiles.scala
index 399933c81aa..f5e1a7b2687 100644
--- a/src/main/scala/subsystem/HasTiles.scala
+++ b/src/main/scala/subsystem/HasTiles.scala
@@ -4,24 +4,77 @@ package freechips.rocketchip.subsystem
 
 import Chisel._
 import chisel3.dontTouch
-import freechips.rocketchip.config.Parameters
-import freechips.rocketchip.devices.debug.TLDebugModule
-import freechips.rocketchip.devices.tilelink.{BasicBusBlocker, BasicBusBlockerParams, CLINT, CLINTConsts, TLPLIC, PLICKey}
+import freechips.rocketchip.config.{Field, Parameters}
+import freechips.rocketchip.devices.debug.{HasPeripheryDebug, HasPeripheryDebugModuleImp}
+import freechips.rocketchip.devices.tilelink.{BasicBusBlocker, BasicBusBlockerParams, CLINTConsts, PLICKey, CanHavePeripheryPLIC, CanHavePeripheryCLINT}
 import freechips.rocketchip.diplomacy._
+import freechips.rocketchip.diplomaticobjectmodel.logicaltree.{LogicalModuleTree}
 import freechips.rocketchip.interrupts._
-import freechips.rocketchip.tile.{BaseTile, LookupByHartId, LookupByHartIdImpl, TileParams, HasExternallyDrivenTileConstants}
+import freechips.rocketchip.tile.{BaseTile, LookupByHartIdImpl, TileParams, HasExternallyDrivenTileConstants, InstantiatableTileParams, PriorityMuxHartIdFromSeq}
 import freechips.rocketchip.tilelink._
 import freechips.rocketchip.util._
 
-trait HasTiles extends HasCoreMonitorBundles { this: BaseSubsystem =>
-  implicit val p: Parameters
-  val tiles: Seq[BaseTile]
-  protected def tileParams: Seq[TileParams] = tiles.map(_.tileParams)
-  def nTiles: Int = tileParams.size
-  def hartIdList: Seq[Int] = tileParams.map(_.hartId)
-  def localIntCounts: Seq[Int] = tileParams.map(_.core.nLocalInterrupts)
+/** Entry point for Config-uring the presence of Tiles */
+case class TilesLocated(loc: HierarchicalLocation) extends Field[Seq[CanAttachTile]](Nil)
+
+/** An interface for describing the parameteization of how Tiles are connected to interconnects */
+trait TileCrossingParamsLike {
+  val crossingType: ClockCrossingType
+  val master: TilePortParamsLike
+  val slave: TilePortParamsLike
+}
+
+/** An interface for describing the parameterization of how a particular tile port is connected to an interconnect */
+trait TilePortParamsLike {
+  def where: TLBusWrapperLocation
+  def injectNode(context: Attachable)(implicit p: Parameters): TLNode // allows port-specific adapters to be injected
+}
 
-  // define some nodes that are useful for collecting or driving tile interrupts
+/** A default implementation of parameterizing the connectivity of the port where the tile is the master.
+  *   Optional timing buffers and/or an optional CacheCork can be inserted in the interconnect's clock domain.
+  */
+case class TileMasterPortParams(
+  buffers: Int = 0,
+  cork: Option[Boolean] = None,
+  where: TLBusWrapperLocation = SBUS
+) extends TilePortParamsLike {
+  def injectNode(context: Attachable)(implicit p: Parameters): TLNode = {
+    (TLBuffer(buffers) :=* cork.map { u => TLCacheCork(unsafe = u) } .getOrElse { TLTempNode() })
+  }
+}
+
+/** A default implementation of parameterizing the connectivity of the port giving access to slaves inside the tile.
+  *   Optional timing buffers and/or an optional BusBlocker adapter can be inserted in the interconnect's clock domain.
+  */
+case class TileSlavePortParams(
+  buffers: Int = 0,
+  blockerCtrlAddr: Option[BigInt] = None,
+  blockerCtrlWhere: TLBusWrapperLocation = CBUS,
+  where: TLBusWrapperLocation = CBUS
+) extends TilePortParamsLike {
+  def injectNode(context: Attachable)(implicit p: Parameters): TLNode = {
+    val controlBus = context.locateTLBusWrapper(where)
+    val blockerBus = context.locateTLBusWrapper(blockerCtrlWhere)
+    blockerCtrlAddr
+      .map { BasicBusBlockerParams(_, blockerBus.beatBytes, controlBus.beatBytes) }
+      .map { bbbp =>
+        val blocker = LazyModule(new BasicBusBlocker(bbbp))
+        blockerBus.coupleTo("tile_slave_port_bus_blocker") { blocker.controlNode := TLFragmenter(blockerBus) := _ }
+        blocker.node
+      } .getOrElse { TLTempNode() }
+  }
+}
+
+/** These are sources of interrupts that are driven into the tile.
+  * They need to be instantiated before tiles are attached to the subsystem
+  * containing them.
+  */
+trait HasTileInterruptSources
+  extends CanHavePeripheryPLIC
+  with CanHavePeripheryCLINT
+  with HasPeripheryDebug
+{ this: BaseSubsystem => // TODO ideally this bound would be softened to LazyModule
+  // meipNode is used to create a subsystem IO in Configs where there is no PLIC
   val meipNode = p(PLICKey) match {
     case Some(_) => None
     case None    => Some(IntNexusNode(
@@ -30,7 +83,13 @@ trait HasTiles extends HasCoreMonitorBundles { this: BaseSubsystem =>
       outputRequiresInput = false,
       inputRequiresOutput = false))
   }
+}
 
+/** These are sink of notifications that are driven out from the tile.
+  * They need to be instantiated before tiles are attached to the subsystem
+  * containing them.
+  */
+trait HasTileNotificationSinks { this: LazyModule =>
   val tileHaltXbarNode = IntXbar(p)
   val tileHaltSinkNode = IntSinkNode(IntSinkPortSimple())
   tileHaltSinkNode := tileHaltXbarNode
@@ -42,41 +101,90 @@ trait HasTiles extends HasCoreMonitorBundles { this: BaseSubsystem =>
   val tileCeaseXbarNode = IntXbar(p)
   val tileCeaseSinkNode = IntSinkNode(IntSinkPortSimple())
   tileCeaseSinkNode := tileCeaseXbarNode
+}
+
+/** HasTiles adds a Config-urable sequence of tiles of any type
+  *   to the subsystem class into which it is mixed.
+  */
+trait HasTiles extends HasCoreMonitorBundles with DefaultTileContextType
+{ this: BaseSubsystem => // TODO: ideally this bound would be softened to Attachable
+  implicit val p: Parameters
+  val tileAttachParams: Seq[CanAttachTile] = p(TilesLocated(location))
+  val tiles: Seq[BaseTile] = tileAttachParams.map(_.instantiate(p)) // actual tile creation
+  val tileParams: Seq[TileParams] = tiles.map(_.tileParams)
+  val tileCrossingTypes = tileAttachParams.map(_.crossingParams.crossingType)
+  def nTiles: Int = tileParams.size
+  def hartIdList: Seq[Int] = tileParams.map(_.hartId)
+  def localIntCounts: Seq[Int] = tileParams.map(_.core.nLocalInterrupts)
 
-  protected def connectMasterPortsToSBus(tile: BaseTile, crossing: RocketCrossingParams) {
-    locateTLBusWrapper(crossing.master.where).coupleFrom(tile.tileParams.name.getOrElse("tile")) { bus =>
-      (bus :=*
-        TLBuffer(crossing.master.buffers) :=*
-        crossing.master.cork
-          .map { u => TLCacheCork(unsafe = u) }
-          .map { _ :=* tile.crossMasterPort() }
-          .getOrElse { tile.crossMasterPort() })
+  // connect all the tiles to interconnect attachment points made available in this subsystem context
+  tileAttachParams.zip(tiles).foreach { case (params, t) =>
+    params.connect(t.asInstanceOf[params.TileType], this.asInstanceOf[params.TileContextType])
+  }
+}
+
+/** Most tile types require only these traits in order for their standardized connect functions to operate.
+  *    BaseTiles subtypes with different needs can extends this trait to provide themselves with
+  *    additional external connection points.
+  */
+trait DefaultTileContextType
+  extends Attachable
+  with HasTileInterruptSources
+  with HasTileNotificationSinks
+{ this: BaseSubsystem => } // TODO: ideally this bound would be softened to LazyModule
+
+/** Standardized interface by which parameterized tiles can be attached to contexts. */
+trait CanAttachTile {
+  type TileType <: BaseTile
+  type TileContextType <: DefaultTileContextType
+  def tileParams: InstantiatableTileParams[TileType]
+  def crossingParams: TileCrossingParamsLike
+  def lookup: LookupByHartIdImpl
+
+  // narrow waist through which all tiles are intended to pass while being instantiated
+  def instantiate(implicit p: Parameters): TileType = {
+    val tile = LazyModule(tileParams.instantiate(crossingParams, lookup))
+    tile
+  }
+
+  // a default set of connections that need to occur for most tile types
+  def connect(tile: TileType, context: TileContextType): Unit = {
+    connectMasterPorts(tile, context)
+    connectSlavePorts(tile, context)
+    connectInterrupts(tile, context)
+    LogicalModuleTree.add(context.logicalTreeNode, tile.logicalTreeNode)
+  }
+
+  // connect the port where the tile is the master to a TileLink interconnect
+  def connectMasterPorts(tile: TileType, context: Attachable): Unit = {
+    implicit val p = context.p
+    val dataBus = context.locateTLBusWrapper(crossingParams.master.where)
+    dataBus.coupleFrom(tileParams.name.getOrElse("tile")) { bus =>
+      bus :=* crossingParams.master.injectNode(context) :=* tile.crossMasterPort()
     }
   }
 
-  protected def connectSlavePortsToCBus(tile: BaseTile, crossing: RocketCrossingParams)(implicit valName: ValName) {
+  // connect the port where the tile is the slave to a TileLink interconnect
+  def connectSlavePorts(tile: TileType, context: Attachable): Unit = {
+    implicit val p = context.p
     DisableMonitors { implicit p =>
-      locateTLBusWrapper(crossing.slave.where).coupleTo(tile.tileParams.name.getOrElse("tile")) { bus =>
-        crossing.slave.blockerCtrlAddr
-          .map { BasicBusBlockerParams(_, pbus.beatBytes, sbus.beatBytes) }
-          .map { bbbp => LazyModule(new BasicBusBlocker(bbbp)) }
-          .map { bbb =>
-            cbus.coupleTo("bus_blocker") { bbb.controlNode := TLFragmenter(cbus) := _ }
-            tile.crossSlavePort() :*= bbb.node
-          } .getOrElse { tile.crossSlavePort() } :*= bus
+      val controlBus = context.locateTLBusWrapper(crossingParams.slave.where)
+      controlBus.coupleTo(tileParams.name.getOrElse("tile")) { bus =>
+        tile.crossSlavePort() :*= crossingParams.slave.injectNode(context) :*= bus
       }
     }
   }
 
-  protected def connectInterrupts(tile: BaseTile, debugOpt: Option[TLDebugModule], clintOpt: Option[CLINT], plicOpt: Option[TLPLIC]) {
-    // Handle all the different types of interrupts crossing to or from the tile:
+  // connect the various interrupt and notification wires going to and from the tile
+  def connectInterrupts(tile: TileType, context: TileContextType): Unit = {
+    implicit val p = context.p
     // NOTE: The order of calls to := matters! They must match how interrupts
     //       are decoded from tile.intInwardNode inside the tile. For this reason,
     //       we stub out missing interrupts with constant sources here.
 
     // 1. Debug interrupt is definitely asynchronous in all cases.
     tile.intInwardNode :=
-      debugOpt
+      context.debugOpt
         .map { tile { IntSyncAsyncCrossingSink(3) } := _.intnode }
         .getOrElse { NullIntSource() }
 
@@ -85,18 +193,18 @@ trait HasTiles extends HasCoreMonitorBundles { this: BaseSubsystem =>
 
     //    From CLINT: "msip" and "mtip"
     tile.crossIntIn() :=
-      clintOpt.map { _.intnode }
+      context.clintOpt.map { _.intnode }
         .getOrElse { NullIntSource(sources = CLINTConsts.ints) }
 
     //    From PLIC: "meip"
     tile.crossIntIn() :=
-      plicOpt .map { _.intnode }
-        .getOrElse { meipNode.get }
+      context.plicOpt .map { _.intnode }
+        .getOrElse { context.meipNode.get }
 
     //    From PLIC: "seip" (only if supervisor mode is enabled)
     if (tile.tileParams.core.hasSupervisorMode) {
       tile.crossIntIn() :=
-        plicOpt .map { _.intnode }
+        context.plicOpt .map { _.intnode }
           .getOrElse { NullIntSource() }
     }
 
@@ -105,27 +213,22 @@ trait HasTiles extends HasCoreMonitorBundles { this: BaseSubsystem =>
 
     // 4. Interrupts coming out of the tile are sent to the PLIC,
     //    so might need to be synchronized depending on the Tile's crossing type.
-    plicOpt.foreach { plic =>
+    context.plicOpt.foreach { plic =>
       FlipRendering { implicit p =>
         plic.intnode :=* tile.crossIntOut()
       }
     }
 
-    // 5. Reports of tile status are collected without needing to be clock-crossed
-    tileHaltXbarNode := tile.haltNode
-    tileWFIXbarNode := tile.wfiNode
-    tileCeaseXbarNode := tile.ceaseNode
-  }
-
-  protected def perTileOrGlobalSetting[T](in: Seq[T], n: Int): Seq[T] = in.size match {
-    case 1 => List.fill(n)(in.head)
-    case x if x == n => in
-    case _ => throw new Exception("must provide exactly 1 or #tiles of this key")
+    // 5. Notifications of tile status are collected without needing to be clock-crossed
+    context.tileHaltXbarNode := tile.haltNode
+    context.tileWFIXbarNode := tile.wfiNode
+    context.tileCeaseXbarNode := tile.ceaseNode
   }
 }
 
-trait HasTilesModuleImp extends LazyModuleImp {
-  val outer: HasTiles
+/** Provides some Chisel connectivity to certain tile IOs */
+trait HasTilesModuleImp extends LazyModuleImp with HasPeripheryDebugModuleImp {
+  val outer: HasTiles with HasTileInterruptSources
 
   def resetVectorBits: Int = {
     // Consider using the minimum over all widths, rather than enforcing homogeneity
diff --git a/src/main/scala/subsystem/RocketSubsystem.scala b/src/main/scala/subsystem/RocketSubsystem.scala
index 483ede008a0..4c2984d3124 100644
--- a/src/main/scala/subsystem/RocketSubsystem.scala
+++ b/src/main/scala/subsystem/RocketSubsystem.scala
@@ -14,74 +14,37 @@ import freechips.rocketchip.tile._
 
 case object HartPrefixKey extends Field[Boolean](false)
 
-// TODO: how specific are these to RocketTiles?
-case class TileMasterPortParams(
-  buffers: Int = 0,
-  cork: Option[Boolean] = None,
-  where: TLBusWrapperLocation = SBUS)
-
-case class TileSlavePortParams(
-  buffers: Int = 0,
-  blockerCtrlAddr: Option[BigInt] = None,
-  where: TLBusWrapperLocation = CBUS)
-
 case class RocketCrossingParams(
   crossingType: ClockCrossingType = SynchronousCrossing(),
   master: TileMasterPortParams = TileMasterPortParams(),
-  slave: TileSlavePortParams = TileSlavePortParams())
+  slave: TileSlavePortParams = TileSlavePortParams()
+) extends TileCrossingParamsLike
 
-case class RocketAttachParams
-  tile: RocketTileParams,
-  crossing: RocketCrossingParams,
+case class RocketTileAttachParams(
+  tileParams: RocketTileParams,
+  crossingParams: RocketCrossingParams,
   lookup: LookupByHartIdImpl
-) {
-  def instantiate(context: Attachable): RocketTile = {
-    val rocket = LazyModule(new RocketTile(tile, crossing, lookup, context.logicalTreeNode))
-    rocket
-  }
+) extends CanAttachTile {
+  type TileType = RocketTile
+  type TileContextType = DefaultTileContextType
 }
 
 case object RocketTilesKey extends Field[Seq[RocketTileParams]](Nil)
 case object RocketCrossingKey extends Field[Seq[RocketCrossingParams]](List(RocketCrossingParams()))
 
-trait HasRocketTiles extends HasTiles
-    with CanHavePeripheryPLIC
-    with CanHavePeripheryCLINT
-    with HasPeripheryDebug { this: BaseSubsystem =>
-  val module: HasRocketTilesModuleImp
-
-  protected val rocketTileParams = p(RocketTilesKey)
-  private val crossings = perTileOrGlobalSetting(p(RocketCrossingKey), rocketTileParams.size)
-
-  // Make a tile and wire its nodes into the system,
-  // according to the specified type of clock crossing.
-  // Note that we also inject new nodes into the tile itself,
-  // also based on the crossing type.
+trait HasRocketTiles extends HasTiles { this: BaseSubsystem =>
   val rocketTiles = tiles.collect { case r: RocketTile => r }
 
-  rocketTiles.foreach { rocket =>
-    connectMasterPortsToSBus(rocket, crossing)
-    connectSlavePortsToCBus(rocket, crossing)
-    connectInterrupts(rocket, debugOpt, clintOpt, plicOpt)
-    LogicalModuleTree.add(logicalTreeNode, rocket.rocketLogicalTree)
-  }
-
   def coreMonitorBundles = (rocketTiles map { t =>
     t.module.core.rocketImpl.coreMonitorBundle
   }).toList
 }
 
-trait HasRocketTilesModuleImp extends HasTilesModuleImp
-    with HasPeripheryDebugModuleImp {
-  val outer: HasRocketTiles
-}
-
 // Field for specifying MaskROM addition to subsystem
 case object PeripheryMaskROMKey extends Field[Seq[MaskROMParams]](Nil)
 
 class RocketSubsystem(implicit p: Parameters) extends BaseSubsystem
     with HasRocketTiles {
-  val tiles = rocketTiles
         
   // add Mask ROM devices
   val maskROMs = p(PeripheryMaskROMKey).map { MaskROM.attach(_, cbus) }
@@ -103,7 +66,7 @@ class RocketSubsystem(implicit p: Parameters) extends BaseSubsystem
 
 class RocketSubsystemModuleImp[+L <: RocketSubsystem](_outer: L) extends BaseSubsystemModuleImp(_outer)
     with HasResetVectorWire
-    with HasRocketTilesModuleImp {
+    with HasTilesModuleImp {
 
   for (i <- 0 until outer.tiles.size) {
     val wire = tile_inputs(i)
diff --git a/src/main/scala/tile/BaseTile.scala b/src/main/scala/tile/BaseTile.scala
index 87f43637613..cfebad6f8cd 100644
--- a/src/main/scala/tile/BaseTile.scala
+++ b/src/main/scala/tile/BaseTile.scala
@@ -32,6 +32,11 @@ trait TileParams {
   val name: Option[String]
 }
 
+abstract class InstantiatableTileParams[TileType <: BaseTile] extends TileParams {
+  def instantiate(crossing: TileCrossingParamsLike, lookup: LookupByHartIdImpl)
+                 (implicit p: Parameters): TileType
+}
+
 /** These parameters values are not computed based on diplomacy negotiation
   * and so are safe to use while diplomacy itself is running.
   */
diff --git a/src/main/scala/tile/RocketTile.scala b/src/main/scala/tile/RocketTile.scala
index 9feebb0483a..a9dcc873b7d 100644
--- a/src/main/scala/tile/RocketTile.scala
+++ b/src/main/scala/tile/RocketTile.scala
@@ -11,7 +11,7 @@ import freechips.rocketchip.diplomaticobjectmodel.logicaltree.{DCacheLogicalTree
 import freechips.rocketchip.interrupts._
 import freechips.rocketchip.tilelink._
 import freechips.rocketchip.rocket._
-import freechips.rocketchip.subsystem.{SubsystemResetSchemeKey, ResetSynchronous, RocketCrossingParams, HartPrefixKey}
+import freechips.rocketchip.subsystem.{SubsystemResetSchemeKey, ResetSynchronous, HartPrefixKey, TileCrossingParamsLike}
 import freechips.rocketchip.util._
 
 case class RocketTileParams(
@@ -28,7 +28,7 @@ case class RocketTileParams(
     ) extends InstantiatableTileParams[RocketTile] {
   require(icache.isDefined)
   require(dcache.isDefined)
-   def instantiate(crossing: TileCrossingParamsLike, lookup: LookupByHartIdImpl)(implicit p: Parameters): RocketTile = {
+  def instantiate(crossing: TileCrossingParamsLike, lookup: LookupByHartIdImpl)(implicit p: Parameters): RocketTile = {
     new RocketTile(this, crossing, lookup)
   }
 }
@@ -46,7 +46,7 @@ class RocketTile private(
     with HasICacheFrontend
 {
   // Private constructor ensures altered LazyModule.p is used implicitly
-  def this(params: RocketTileParams, crossing: RocketCrossingParams, lookup: LookupByHartIdImpl)(implicit p: Parameters) =
+  def this(params: RocketTileParams, crossing: TileCrossingParamsLike, lookup: LookupByHartIdImpl)(implicit p: Parameters) =
     this(params, crossing.crossingType, lookup, p)
 
   val intOutwardNode = IntIdentityNode()
diff --git a/src/main/scala/util/package.scala b/src/main/scala/util/package.scala
index 5aec9fcfa51..aabee6525e9 100644
--- a/src/main/scala/util/package.scala
+++ b/src/main/scala/util/package.scala
@@ -253,6 +253,12 @@ package object util {
     map.view.map({ case (k, vs) => k -> vs.toList }).toList
   }
 
+  def heterogeneousOrGlobalSetting[T](in: Seq[T], n: Int): Seq[T] = in.size match {
+    case 1 => List.fill(n)(in.head)
+    case x if x == n => in
+    case _ => throw new Exception(s"must provide exactly 1 or $n of some field, but got:\n$in")
+  }
+
 /** provides operators useful for working with bidirectional [[Bundle]]s
   * 
   * In terms of [[Flipped]] with a producer 'p' and 'consumer' c:

From 9e795259ec96dc056bc8c9e3bd80f627179c0b27 Mon Sep 17 00:00:00 2001
From: Henry Cook <henry@sifive.com>
Date: Mon, 8 Jun 2020 14:54:48 -0700
Subject: [PATCH 03/15] om: make tree root require more verbose

---
 .../diplomaticobjectmodel/logicaltree/LogicalTreeNode.scala     | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/main/scala/diplomaticobjectmodel/logicaltree/LogicalTreeNode.scala b/src/main/scala/diplomaticobjectmodel/logicaltree/LogicalTreeNode.scala
index f2f596c2a6a..71d701a9289 100644
--- a/src/main/scala/diplomaticobjectmodel/logicaltree/LogicalTreeNode.scala
+++ b/src/main/scala/diplomaticobjectmodel/logicaltree/LogicalTreeNode.scala
@@ -36,7 +36,7 @@ object LogicalModuleTree {
 
   def rootLogicalTreeNode: LogicalTreeNode = {
     val roots = tree.collect { case (k, _) if !tree.exists(_._2.contains(k)) => k }
-    assert(roots.size == 1, "Logical Tree contains more than one root.")
+    require(roots.size == 1, s"Logical Tree contains more than one root:\n$roots")
     roots.head
   }
 

From 6646b5d5cc871aff49612a2598ab5de02288e9d6 Mon Sep 17 00:00:00 2001
From: Henry Cook <henry@sifive.com>
Date: Mon, 8 Jun 2020 15:30:49 -0700
Subject: [PATCH 04/15] subsystem: RocketTileAttachParams can rely on inherited
 bound for TileContextType

---
 src/main/scala/subsystem/RocketSubsystem.scala | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/src/main/scala/subsystem/RocketSubsystem.scala b/src/main/scala/subsystem/RocketSubsystem.scala
index 4c2984d3124..e26ea2361c9 100644
--- a/src/main/scala/subsystem/RocketSubsystem.scala
+++ b/src/main/scala/subsystem/RocketSubsystem.scala
@@ -24,10 +24,7 @@ case class RocketTileAttachParams(
   tileParams: RocketTileParams,
   crossingParams: RocketCrossingParams,
   lookup: LookupByHartIdImpl
-) extends CanAttachTile {
-  type TileType = RocketTile
-  type TileContextType = DefaultTileContextType
-}
+) extends CanAttachTile { type TileType = RocketTile }
 
 case object RocketTilesKey extends Field[Seq[RocketTileParams]](Nil)
 case object RocketCrossingKey extends Field[Seq[RocketCrossingParams]](List(RocketCrossingParams()))

From 3a593ad648aaa7cdb289868c816fef649f764ab7 Mon Sep 17 00:00:00 2001
From: Henry Cook <henry@sifive.com>
Date: Mon, 8 Jun 2020 15:31:09 -0700
Subject: [PATCH 05/15] subsystem: more HasTiles comments

---
 src/main/scala/subsystem/HasTiles.scala | 24 ++++++++++++++----------
 1 file changed, 14 insertions(+), 10 deletions(-)

diff --git a/src/main/scala/subsystem/HasTiles.scala b/src/main/scala/subsystem/HasTiles.scala
index f5e1a7b2687..9bb74a8b95f 100644
--- a/src/main/scala/subsystem/HasTiles.scala
+++ b/src/main/scala/subsystem/HasTiles.scala
@@ -26,8 +26,10 @@ trait TileCrossingParamsLike {
 
 /** An interface for describing the parameterization of how a particular tile port is connected to an interconnect */
 trait TilePortParamsLike {
+  // the subnetwork location of the interconnect to which this tile port should be connected
   def where: TLBusWrapperLocation
-  def injectNode(context: Attachable)(implicit p: Parameters): TLNode // allows port-specific adapters to be injected
+  // allows port-specific adapters to be injected into the interconnect side of the attachment point
+  def injectNode(context: Attachable)(implicit p: Parameters): TLNode
 }
 
 /** A default implementation of parameterizing the connectivity of the port where the tile is the master.
@@ -66,15 +68,14 @@ case class TileSlavePortParams(
 }
 
 /** These are sources of interrupts that are driven into the tile.
-  * They need to be instantiated before tiles are attached to the subsystem
-  * containing them.
+  * They need to be instantiated before tiles are attached to the subsystem containing them.
   */
 trait HasTileInterruptSources
   extends CanHavePeripheryPLIC
   with CanHavePeripheryCLINT
   with HasPeripheryDebug
 { this: BaseSubsystem => // TODO ideally this bound would be softened to LazyModule
-  // meipNode is used to create a subsystem IO in Configs where there is no PLIC
+  // meipNode is used to create a single bit subsystem input in Configs without a PLIC
   val meipNode = p(PLICKey) match {
     case Some(_) => None
     case None    => Some(IntNexusNode(
@@ -85,9 +86,8 @@ trait HasTileInterruptSources
   }
 }
 
-/** These are sink of notifications that are driven out from the tile.
-  * They need to be instantiated before tiles are attached to the subsystem
-  * containing them.
+/** These are sinks of notifications that are driven out from the tile.
+  * They need to be instantiated before tiles are attached to the subsystem containing them.
   */
 trait HasTileNotificationSinks { this: LazyModule =>
   val tileHaltXbarNode = IntXbar(p)
@@ -123,8 +123,8 @@ trait HasTiles extends HasCoreMonitorBundles with DefaultTileContextType
   }
 }
 
-/** Most tile types require only these traits in order for their standardized connect functions to operate.
-  *    BaseTiles subtypes with different needs can extends this trait to provide themselves with
+/** Most tile types require only these traits in order for their standardized connect functions to apply.
+  *    BaseTiles subtypes with different needs can extend this trait to provide themselves with
   *    additional external connection points.
   */
 trait DefaultTileContextType
@@ -133,7 +133,11 @@ trait DefaultTileContextType
   with HasTileNotificationSinks
 { this: BaseSubsystem => } // TODO: ideally this bound would be softened to LazyModule
 
-/** Standardized interface by which parameterized tiles can be attached to contexts. */
+/** Standardized interface by which parameterized tiles can be attached to contexts containing interconnect resources.
+  *   Sub-classes of this trait can optionally override the individual connect functions in order to specialize
+  *   their attachment behaviors, but most use cases should be be handled simply by changing the implementation
+  *   of the injectNode functions in crossingParams.
+  */
 trait CanAttachTile {
   type TileType <: BaseTile
   type TileContextType <: DefaultTileContextType

From 66d56c5b7f102359db46a29472a6169317a8b2af Mon Sep 17 00:00:00 2001
From: Henry Cook <henry@sifive.com>
Date: Tue, 9 Jun 2020 16:50:11 -0700
Subject: [PATCH 06/15] subsystem: always add a WW to tile slave ports, usually
 passthru

---
 src/main/scala/subsystem/HasTiles.scala | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/main/scala/subsystem/HasTiles.scala b/src/main/scala/subsystem/HasTiles.scala
index 9bb74a8b95f..dd2af56e179 100644
--- a/src/main/scala/subsystem/HasTiles.scala
+++ b/src/main/scala/subsystem/HasTiles.scala
@@ -174,7 +174,7 @@ trait CanAttachTile {
     DisableMonitors { implicit p =>
       val controlBus = context.locateTLBusWrapper(crossingParams.slave.where)
       controlBus.coupleTo(tileParams.name.getOrElse("tile")) { bus =>
-        tile.crossSlavePort() :*= crossingParams.slave.injectNode(context) :*= bus
+        tile.crossSlavePort() :*= crossingParams.slave.injectNode(context) :*= TLWidthWidget(controlBus.beatBytes) :*= bus
       }
     }
   }

From 49d1363778319d082c1e07180769d8b7456ef411 Mon Sep 17 00:00:00 2001
From: Henry Cook <henry@sifive.com>
Date: Tue, 9 Jun 2020 16:53:09 -0700
Subject: [PATCH 07/15] config: MaxHartIdBits derived from TilesLocated

---
 src/main/scala/subsystem/Configs.scala | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/main/scala/subsystem/Configs.scala b/src/main/scala/subsystem/Configs.scala
index 2a5b675726a..e99f06630f3 100644
--- a/src/main/scala/subsystem/Configs.scala
+++ b/src/main/scala/subsystem/Configs.scala
@@ -17,7 +17,7 @@ class BaseSubsystemConfig extends Config ((site, here, up) => {
   // Tile parameters
   case PgLevels => if (site(XLen) == 64) 3 /* Sv39 */ else 2 /* Sv32 */
   case XLen => 64 // Applies to all cores
-  case MaxHartIdBits => log2Up(site(RocketTilesKey).size)
+  case MaxHartIdBits => log2Up(site(TilesLocated(InSubsystem)).map(_.tileParams.hartId).max)
   // Interconnect parameters
   case SystemBusKey => SystemBusParams(
     beatBytes = site(XLen)/8,

From 615b58e4c8227cc45cb617d79f7249e7711dd390 Mon Sep 17 00:00:00 2001
From: Henry Cook <henry@sifive.com>
Date: Wed, 10 Jun 2020 10:42:07 -0700
Subject: [PATCH 08/15] subsystem: always instantiate tiles in the order of
 their static hartids

---
 src/main/scala/subsystem/HasTiles.scala | 14 ++++++++++----
 1 file changed, 10 insertions(+), 4 deletions(-)

diff --git a/src/main/scala/subsystem/HasTiles.scala b/src/main/scala/subsystem/HasTiles.scala
index dd2af56e179..44ed59b12e6 100644
--- a/src/main/scala/subsystem/HasTiles.scala
+++ b/src/main/scala/subsystem/HasTiles.scala
@@ -109,14 +109,20 @@ trait HasTileNotificationSinks { this: LazyModule =>
 trait HasTiles extends HasCoreMonitorBundles with DefaultTileContextType
 { this: BaseSubsystem => // TODO: ideally this bound would be softened to Attachable
   implicit val p: Parameters
-  val tileAttachParams: Seq[CanAttachTile] = p(TilesLocated(location))
-  val tiles: Seq[BaseTile] = tileAttachParams.map(_.instantiate(p)) // actual tile creation
-  val tileParams: Seq[TileParams] = tiles.map(_.tileParams)
+
+  // Actually instantiate all tiles, in order based on statically-assigned hartids
+  val tileAttachParams: Seq[CanAttachTile] = p(TilesLocated(location)).sortBy(_.tileParams.hartId)
+  val tiles: Seq[BaseTile] = tileAttachParams.map(_.instantiate(p))
+
+  // Helper functions for accessing certain parameters the are popular to refer to in subsystem code
+  val tileParams: Seq[TileParams] = tileAttachParams.map(_.tileParams)
   val tileCrossingTypes = tileAttachParams.map(_.crossingParams.crossingType)
-  def nTiles: Int = tileParams.size
+  def nTiles: Int = tileAttachParams.size
   def hartIdList: Seq[Int] = tileParams.map(_.hartId)
   def localIntCounts: Seq[Int] = tileParams.map(_.core.nLocalInterrupts)
 
+  require(hartIdList.distinct.size == tiles.size, s"Every tile must be statically assigned a unique id, but got:\n${hartIdList}")
+
   // connect all the tiles to interconnect attachment points made available in this subsystem context
   tileAttachParams.zip(tiles).foreach { case (params, t) =>
     params.connect(t.asInstanceOf[params.TileType], this.asInstanceOf[params.TileContextType])

From 6e8d3877acc747701c2873714ce88a442fb49cae Mon Sep 17 00:00:00 2001
From: Henry Cook <henry@sifive.com>
Date: Wed, 10 Jun 2020 14:16:03 -0700
Subject: [PATCH 09/15] groundtest: update to use HasTiles and TilesLocated

---
 src/main/scala/groundtest/Configs.scala       | 64 ++++++++++++-------
 .../groundtest/GroundTestSubsystem.scala      | 40 +++++-------
 src/main/scala/groundtest/Status.scala        |  3 -
 src/main/scala/groundtest/Tile.scala          | 20 +++---
 src/main/scala/groundtest/TraceGen.scala      | 46 ++++++++++---
 5 files changed, 104 insertions(+), 69 deletions(-)

diff --git a/src/main/scala/groundtest/Configs.scala b/src/main/scala/groundtest/Configs.scala
index 2f95e81e1dc..4e1211bd1e7 100644
--- a/src/main/scala/groundtest/Configs.scala
+++ b/src/main/scala/groundtest/Configs.scala
@@ -5,36 +5,56 @@ package freechips.rocketchip.groundtest
 
 import Chisel._
 import freechips.rocketchip.config.Config
+import freechips.rocketchip.devices.tilelink.{CLINTKey, PLICKey}
+import freechips.rocketchip.devices.debug.{DebugModuleKey}
 import freechips.rocketchip.subsystem._
 import freechips.rocketchip.system.BaseConfig
 import freechips.rocketchip.rocket.{DCacheParams}
-import freechips.rocketchip.tile.{MaxHartIdBits, XLen}
+import freechips.rocketchip.tile.{XLen}
 
 /** Actual testing target Configs */
 
-class TraceGenConfig extends Config(new WithTraceGen(List.fill(2){ DCacheParams(nSets = 16, nWays = 1) }) ++ new BaseConfig)
+class TraceGenConfig extends Config(
+  new WithTraceGen(List.fill(2){ DCacheParams(nSets = 16, nWays = 1) }) ++
+  new GroundTestBaseConfig
+)
 
-class TraceGenBufferlessConfig extends Config(new WithBufferlessBroadcastHub ++ new TraceGenConfig)
+class TraceGenBufferlessConfig extends Config(
+  new WithBufferlessBroadcastHub ++
+  new TraceGenConfig
+)
 
 /* Composable Configs to set individual parameters */
 
-class WithTraceGen(params: Seq[DCacheParams], nReqs: Int = 8192) extends Config((site, here, up) => {
-  case GroundTestTilesKey => params.map { dcp => TraceGenParams(
-    dcache = Some(dcp),
-    wordBits = site(XLen),
-    addrBits = 32,
-    addrBag = {
-      val nSets = dcp.nSets
-      val nWays = dcp.nWays
-      val blockOffset = site(SystemBusKey).blockOffset
-      val nBeats = site(SystemBusKey).blockBeats
-      List.tabulate(nWays) { i =>
-        Seq.tabulate(nBeats) { j => BigInt((j * 8) + ((i * nSets) << blockOffset)) }
-      }.flatten
-    },
-    maxRequests = nReqs,
-    memStart = site(ExtMem).get.master.base,
-    numGens = params.size)
-  }
-  case MaxHartIdBits => log2Up(params.size)
+class GroundTestBaseConfig extends Config(
+  new BaseConfig().alter((site,here,up) => {
+    case DebugModuleKey => None
+    case CLINTKey => None
+    case PLICKey => None
+  })
+)
+
+class WithTraceGen(params: Seq[DCacheParams], nReqs: Int = 8192, idOffset: Int = 0) extends Config((site, here, up) => {
+  case TilesLocated(InSubsystem) => params.zipWithIndex.map { case (dcp, i) =>
+    TraceGenTileAttachParams(
+      tileParams = TraceGenParams(
+        hartId = i + idOffset,
+        dcache = Some(dcp),
+        wordBits = site(XLen),
+        addrBits = 32,
+        addrBag = {
+          val nSets = dcp.nSets
+          val nWays = dcp.nWays
+          val blockOffset = site(SystemBusKey).blockOffset
+          val nBeats = site(SystemBusKey).blockBeats
+          List.tabulate(nWays) { i =>
+            Seq.tabulate(nBeats) { j => BigInt((j * 8) + ((i * nSets) << blockOffset)) }
+          }.flatten
+        },
+        maxRequests = nReqs,
+        memStart = site(ExtMem).get.master.base,
+        numGens = params.size),
+      crossingParams = RocketCrossingParams()
+    )
+  } ++ up(TilesLocated(InSubsystem), site)
 })
diff --git a/src/main/scala/groundtest/GroundTestSubsystem.scala b/src/main/scala/groundtest/GroundTestSubsystem.scala
index f5d3881eeeb..bb330031357 100644
--- a/src/main/scala/groundtest/GroundTestSubsystem.scala
+++ b/src/main/scala/groundtest/GroundTestSubsystem.scala
@@ -3,30 +3,24 @@
 package freechips.rocketchip.groundtest
 
 import Chisel._
-import freechips.rocketchip.config.{Field, Parameters}
-import freechips.rocketchip.diplomacy._
-import freechips.rocketchip.diplomaticobjectmodel.model.OMInterrupt
-import freechips.rocketchip.interrupts._
-import freechips.rocketchip.subsystem._
-import freechips.rocketchip.tilelink._
-import freechips.rocketchip.tile._
-
-import scala.math.max
-
-case object TileId extends Field[Int]
-
-class GroundTestSubsystem(implicit p: Parameters) extends BaseSubsystem
-    with CanHaveMasterAXI4MemPort {
-  val tileParams = p(GroundTestTilesKey)
-  val tiles = tileParams.zipWithIndex.map { case(c, i) => LazyModule(c.build(i, p)) }
-
-  tiles.map(_.masterNode).foreach { m =>
-    sbus.fromTile(None, buffer = BufferParams.default){ m }
-  }
-
+import chisel3.dontTouch
+import freechips.rocketchip.config.{Parameters}
+import freechips.rocketchip.diplomacy.{AddressSet, LazyModule}
+import freechips.rocketchip.interrupts.{IntSinkNode, IntSinkPortSimple}
+import freechips.rocketchip.subsystem.{BaseSubsystem, BaseSubsystemModuleImp, HasTiles, CanHaveMasterAXI4MemPort}
+import freechips.rocketchip.tilelink.{TLRAM, TLFragmenter}
+
+class GroundTestSubsystem(implicit p: Parameters)
+  extends BaseSubsystem
+  with HasTiles
+  with CanHaveMasterAXI4MemPort
+{
   val testram = LazyModule(new TLRAM(AddressSet(0x52000000, 0xfff), beatBytes=pbus.beatBytes))
   pbus.coupleTo("TestRAM") { testram.node := TLFragmenter(pbus) := _ }
 
+  // No cores to monitor
+  def coreMonitorBundles = Nil
+
   // No PLIC in ground test; so just sink the interrupts to nowhere
   IntSinkNode(IntSinkPortSimple()) :=* ibus.toPLIC
 
@@ -38,6 +32,6 @@ class GroundTestSubsystemModuleImp[+L <: GroundTestSubsystem](_outer: L) extends
 
   outer.tiles.zipWithIndex.map { case(t, i) => t.module.constants.hartid := UInt(i) }
 
-  val status = DebugCombiner(outer.tiles.map(_.module.status))
-  success := status.finished
+  val status = dontTouch(DebugCombiner(outer.tiles.collect { case t: GroundTestTile => t.module.status }))
+  success := outer.tileCeaseSinkNode.in.head._1.asUInt.andR
 }
diff --git a/src/main/scala/groundtest/Status.scala b/src/main/scala/groundtest/Status.scala
index 4601593ada8..c1059f06c22 100644
--- a/src/main/scala/groundtest/Status.scala
+++ b/src/main/scala/groundtest/Status.scala
@@ -6,7 +6,6 @@ import Chisel._
 import freechips.rocketchip.util.ValidMux
 
 class GroundTestStatus extends Bundle {
-  val finished = Bool(OUTPUT)
   val timeout = Valid(UInt(width = 4))
   val error = Valid(UInt(width = 4))
 }
@@ -14,10 +13,8 @@ class GroundTestStatus extends Bundle {
 object DebugCombiner {
   def apply(debugs: Seq[GroundTestStatus]): GroundTestStatus = {
     val out = Wire(new GroundTestStatus)
-    out.finished := debugs.map(_.finished).reduce(_ && _)
     out.timeout  := ValidMux(debugs.map(_.timeout))
     out.error    := ValidMux(debugs.map(_.error))
     out
   }
 }
-
diff --git a/src/main/scala/groundtest/Tile.scala b/src/main/scala/groundtest/Tile.scala
index b4f25ba3afc..230bf58552b 100644
--- a/src/main/scala/groundtest/Tile.scala
+++ b/src/main/scala/groundtest/Tile.scala
@@ -17,8 +17,6 @@ trait GroundTestTileParams extends TileParams {
   val memStart: BigInt
   val maxRequests: Int
   val numGens: Int
-
-  def build(i: Int, p: Parameters): GroundTestTile
   
   val icache = Some(ICacheParams())
   val btb = None
@@ -28,18 +26,18 @@ trait GroundTestTileParams extends TileParams {
   val dataScratchpadBytes = 0
 }
 
-case object GroundTestTilesKey extends Field[Seq[GroundTestTileParams]]
-
-abstract class GroundTestTile private (params: GroundTestTileParams, x: ClockCrossingType, q: Parameters)
-    extends BaseTile(params, x, HartsWontDeduplicate(params), q)
+abstract class GroundTestTile(
+  params: GroundTestTileParams,
+  crossing: ClockCrossingType,
+  lookup: LookupByHartIdImpl,
+  q: Parameters
+) extends BaseTile(params, crossing, lookup, q)
+  with SinksExternalInterrupts
+  with SourcesExternalNotifications
 {
-  def this(params: GroundTestTileParams)(implicit p: Parameters) = this(params, SynchronousCrossing(), p)
-  val intInwardNode: IntInwardNode = IntIdentityNode()
+  val cpuDevice: SimpleDevice = new SimpleDevice("groundtest", Nil)
   val intOutwardNode: IntOutwardNode = IntIdentityNode()
   val slaveNode: TLInwardNode = TLIdentityNode()
-  val ceaseNode: IntOutwardNode = IntIdentityNode()
-  val haltNode: IntOutwardNode = IntIdentityNode()
-  val wfiNode: IntOutwardNode = IntIdentityNode()
 
   val dcacheOpt = params.dcache.map { dc => LazyModule(
     if (dc.nMSHRs == 0) new DCache(hartId, crossing)
diff --git a/src/main/scala/groundtest/TraceGen.scala b/src/main/scala/groundtest/TraceGen.scala
index 4cefaca2f0c..37efd3bc447 100644
--- a/src/main/scala/groundtest/TraceGen.scala
+++ b/src/main/scala/groundtest/TraceGen.scala
@@ -20,10 +20,12 @@
 package freechips.rocketchip.groundtest
  
 import Chisel._
-import freechips.rocketchip.config.{Field, Parameters}
+import freechips.rocketchip.config.{Parameters}
+import freechips.rocketchip.diplomacy.{ClockCrossingType}
 import freechips.rocketchip.rocket._
 import freechips.rocketchip.tile._
 import freechips.rocketchip.tilelink._
+import freechips.rocketchip.subsystem.{TileCrossingParamsLike, CanAttachTile}
 import freechips.rocketchip.util._
 
 // =======
@@ -57,15 +59,19 @@ import freechips.rocketchip.util._
 //     to repeatedly recompile with a different address bag.)
 
 case class TraceGenParams(
-    dcache: Option[DCacheParams] = Some(DCacheParams()),
     wordBits: Int, // p(XLen) 
     addrBits: Int, // p(PAddrBits)
     addrBag: List[BigInt], // p(AddressBag)
     maxRequests: Int,
     memStart: BigInt, //p(ExtMem).base
-    numGens: Int) extends GroundTestTileParams {
-  def build(i: Int, p: Parameters): GroundTestTile = new TraceGenTile(i, this)(p)
-  val hartId = 0
+    numGens: Int,
+    dcache: Option[DCacheParams] = Some(DCacheParams()),
+    hartId: Int = 0
+) extends InstantiatableTileParams[TraceGenTile] with GroundTestTileParams
+{
+  def instantiate(crossing: TileCrossingParamsLike, lookup: LookupByHartIdImpl)(implicit p: Parameters): TraceGenTile = {
+    new TraceGenTile(this, crossing, lookup)
+  }
   val beuAddr = None
   val blockerCtrlAddr = None
   val name = None
@@ -94,6 +100,15 @@ trait HasTraceGenParams {
   require((1 << logAddressBagLen) == addressBagLen)
 }
 
+case class TraceGenTileAttachParams(
+  tileParams: TraceGenParams,
+  crossingParams: TileCrossingParamsLike
+) extends CanAttachTile {
+  type TileType = TraceGenTile
+  val lookup: LookupByHartIdImpl = HartsWontDeduplicate(tileParams)
+}
+
+
 // ============
 // Trace format
 // ============
@@ -578,9 +593,18 @@ class TraceGenerator(val params: TraceGenParams)(implicit val p: Parameters) ext
 // Trace-generator wrapper
 // =======================
 
-class TraceGenTile(hack: Int, val id: Int, val params: TraceGenParams, q: Parameters) extends GroundTestTile(params)(q) {
-  def this(id: Int, params: TraceGenParams)(implicit p: Parameters) = this(0, id, params, p)
-  val masterNode: TLOutwardNode = TLIdentityNode() := visibilityNode := dcacheOpt.map(_.node).getOrElse(TLIdentityNode())
+class TraceGenTile private(
+  val params: TraceGenParams,
+  crossing: ClockCrossingType,
+  lookup: LookupByHartIdImpl,
+  q: Parameters
+) extends GroundTestTile(params, crossing, lookup, q)
+{
+  def this(params: TraceGenParams, crossing: TileCrossingParamsLike, lookup: LookupByHartIdImpl)(implicit p: Parameters) =
+    this(params, crossing.crossingType, lookup, p)
+
+  val masterNode: TLOutwardNode = TLIdentityNode() := visibilityNode := dcacheOpt.map(_.node).getOrElse(TLTempNode())
+
   override lazy val module = new TraceGenTileModuleImp(this)
 }
 
@@ -595,10 +619,12 @@ class TraceGenTileModuleImp(outer: TraceGenTile) extends GroundTestTileModuleImp
     dcache.module.io.cpu <> dcacheIF.io.cache
   }
 
-  status.finished := tracegen.io.finished
+  outer.reportCease(Some(tracegen.io.finished))
+  outer.reportHalt(Some(tracegen.io.timeout))
+  outer.reportWFI(None)
   status.timeout.valid := tracegen.io.timeout
   status.timeout.bits := UInt(0)
   status.error.valid := Bool(false)
 
-  assert(!tracegen.io.timeout, s"TraceGen tile ${outer.id}: request timed out")
+  assert(!tracegen.io.timeout, s"TraceGen tile ${outer.tileParams.hartId}: request timed out")
 }

From 76ff01ff0df3858fd822d465096cf48cc061c74d Mon Sep 17 00:00:00 2001
From: Henry Cook <henry@sifive.com>
Date: Wed, 10 Jun 2020 14:55:27 -0700
Subject: [PATCH 10/15] config: MaxHartIdBits fix for pow2 max id

---
 src/main/scala/subsystem/Configs.scala | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/main/scala/subsystem/Configs.scala b/src/main/scala/subsystem/Configs.scala
index e99f06630f3..c00a313363a 100644
--- a/src/main/scala/subsystem/Configs.scala
+++ b/src/main/scala/subsystem/Configs.scala
@@ -17,7 +17,7 @@ class BaseSubsystemConfig extends Config ((site, here, up) => {
   // Tile parameters
   case PgLevels => if (site(XLen) == 64) 3 /* Sv39 */ else 2 /* Sv32 */
   case XLen => 64 // Applies to all cores
-  case MaxHartIdBits => log2Up(site(TilesLocated(InSubsystem)).map(_.tileParams.hartId).max)
+  case MaxHartIdBits => log2Up(site(TilesLocated(InSubsystem)).map(_.tileParams.hartId).max+1)
   // Interconnect parameters
   case SystemBusKey => SystemBusParams(
     beatBytes = site(XLen)/8,

From a340cd3cef3479f4be93bdecb1f6b68272298b83 Mon Sep 17 00:00:00 2001
From: Henry Cook <henry@sifive.com>
Date: Wed, 10 Jun 2020 14:56:02 -0700
Subject: [PATCH 11/15] config: make WithN<>Cores pattern composable

- add idOffset parameter to control hart offsets
- make repeated usages append to list of tiles
- use in HeterogeneousTileExampleConfig
---
 src/main/scala/groundtest/Configs.scala |  7 +++++--
 src/main/scala/subsystem/Configs.scala  | 12 ++++++------
 src/main/scala/system/Configs.scala     | 11 ++++++++++-
 3 files changed, 21 insertions(+), 9 deletions(-)

diff --git a/src/main/scala/groundtest/Configs.scala b/src/main/scala/groundtest/Configs.scala
index 4e1211bd1e7..0e0dcfd50c7 100644
--- a/src/main/scala/groundtest/Configs.scala
+++ b/src/main/scala/groundtest/Configs.scala
@@ -15,7 +15,7 @@ import freechips.rocketchip.tile.{XLen}
 /** Actual testing target Configs */
 
 class TraceGenConfig extends Config(
-  new WithTraceGen(List.fill(2){ DCacheParams(nSets = 16, nWays = 1) }) ++
+  new WithTraceGen(2)() ++
   new GroundTestBaseConfig
 )
 
@@ -34,7 +34,10 @@ class GroundTestBaseConfig extends Config(
   })
 )
 
-class WithTraceGen(params: Seq[DCacheParams], nReqs: Int = 8192, idOffset: Int = 0) extends Config((site, here, up) => {
+class WithTraceGen(n: Int = 2, idOffset: Int = 0)(
+  params: Seq[DCacheParams] = List.fill(n){ DCacheParams(nSets = 16, nWays = 1) },
+  nReqs: Int = 8192
+) extends Config((site, here, up) => {
   case TilesLocated(InSubsystem) => params.zipWithIndex.map { case (dcp, i) =>
     TraceGenTileAttachParams(
       tileParams = TraceGenParams(
diff --git a/src/main/scala/subsystem/Configs.scala b/src/main/scala/subsystem/Configs.scala
index c00a313363a..5f73b5c6302 100644
--- a/src/main/scala/subsystem/Configs.scala
+++ b/src/main/scala/subsystem/Configs.scala
@@ -77,7 +77,7 @@ class WithCoherentBusTopology extends Config((site, here, up) => {
       l2 = site(BankedL2Key)))
 })
 
-class WithNBigCores(n: Int) extends Config((site, here, up) => {
+class WithNBigCores(n: Int, idOffset: Int = 0) extends Config((site, here, up) => {
   case RocketTilesKey => {
     val big = RocketTileParams(
       core   = RocketCoreParams(mulDiv = Some(MulDivParams(
@@ -91,11 +91,11 @@ class WithNBigCores(n: Int) extends Config((site, here, up) => {
       icache = Some(ICacheParams(
         rowBits = site(SystemBusKey).beatBits,
         blockBytes = site(CacheBlockBytes))))
-    List.tabulate(n)(i => big.copy(hartId = i))
+    List.tabulate(n)(i => big.copy(hartId = i + idOffset)) ++ up(RocketTilesKey, site)
   }
 })
 
-class WithNMedCores(n: Int) extends Config((site, here, up) => {
+class WithNMedCores(n: Int, idOffset: Int = 0) extends Config((site, here, up) => {
   case RocketTilesKey => {
     val med = RocketTileParams(
       core = RocketCoreParams(fpu = None),
@@ -113,11 +113,11 @@ class WithNMedCores(n: Int) extends Config((site, here, up) => {
         nWays = 1,
         nTLBEntries = 4,
         blockBytes = site(CacheBlockBytes))))
-    List.tabulate(n)(i => med.copy(hartId = i))
+    List.tabulate(n)(i => med.copy(hartId = i + idOffset)) ++ up(RocketTilesKey, site)
   }
 })
 
-class WithNSmallCores(n: Int) extends Config((site, here, up) => {
+class WithNSmallCores(n: Int, idOffset: Int = 0) extends Config((site, here, up) => {
   case RocketTilesKey => {
     val small = RocketTileParams(
       core = RocketCoreParams(useVM = false, fpu = None),
@@ -135,7 +135,7 @@ class WithNSmallCores(n: Int) extends Config((site, here, up) => {
         nWays = 1,
         nTLBEntries = 4,
         blockBytes = site(CacheBlockBytes))))
-    List.tabulate(n)(i => small.copy(hartId = i))
+    List.tabulate(n)(i => small.copy(hartId = i + idOffset)) ++ up(RocketTilesKey, site)
   }
 })
 
diff --git a/src/main/scala/system/Configs.scala b/src/main/scala/system/Configs.scala
index b76507bb07a..b8ce1423803 100644
--- a/src/main/scala/system/Configs.scala
+++ b/src/main/scala/system/Configs.scala
@@ -6,7 +6,7 @@ package freechips.rocketchip.system
 import Chisel._
 import freechips.rocketchip.config.Config
 import freechips.rocketchip.subsystem._
-import freechips.rocketchip.diplomacy._
+import freechips.rocketchip.groundtest.WithTraceGen
 
 class WithJtagDTMSystem extends freechips.rocketchip.subsystem.WithJtagDTM
 class WithDebugSBASystem extends freechips.rocketchip.subsystem.WithDebugSBA
@@ -39,6 +39,15 @@ class DualChannelDualBankConfig extends Config(
 
 class RoccExampleConfig extends Config(new WithRoccExample ++ new DefaultConfig)
 
+class HeterogeneousTileExampleConfig extends Config(
+  new WithTraceGen (n = 2, idOffset = 3)() ++
+  new WithNBigCores(n = 1, idOffset = 2) ++
+  new WithNMedCores(n = 1, idOffset = 1) ++
+  new WithNSmallCores(n = 1) ++
+  new WithCoherentBusTopology ++
+  new BaseConfig
+)
+
 class Edge128BitConfig extends Config(
   new WithEdgeDataBits(128) ++ new DefaultConfig)
 class Edge32BitConfig extends Config(

From 0286604068bda858902cd917e1013bc484c68289 Mon Sep 17 00:00:00 2001
From: Henry Cook <henry@sifive.com>
Date: Thu, 11 Jun 2020 11:16:22 -0700
Subject: [PATCH 12/15] subsystem: allow tiles to select notification arity

---
 src/main/scala/subsystem/HasTiles.scala | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/main/scala/subsystem/HasTiles.scala b/src/main/scala/subsystem/HasTiles.scala
index 44ed59b12e6..3223f367c2a 100644
--- a/src/main/scala/subsystem/HasTiles.scala
+++ b/src/main/scala/subsystem/HasTiles.scala
@@ -230,9 +230,9 @@ trait CanAttachTile {
     }
 
     // 5. Notifications of tile status are collected without needing to be clock-crossed
-    context.tileHaltXbarNode := tile.haltNode
-    context.tileWFIXbarNode := tile.wfiNode
-    context.tileCeaseXbarNode := tile.ceaseNode
+    context.tileHaltXbarNode :=* tile.haltNode
+    context.tileWFIXbarNode :=* tile.wfiNode
+    context.tileCeaseXbarNode :=* tile.ceaseNode
   }
 }
 

From a43a2af7287d276d96be8dddf98f6e879a14b60f Mon Sep 17 00:00:00 2001
From: Henry Cook <henry@sifive.com>
Date: Wed, 17 Jun 2020 10:44:30 -0700
Subject: [PATCH 13/15] subsystem: dont drop buffers arg to TileSlavePortParams

---
 src/main/scala/subsystem/HasTiles.scala | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/main/scala/subsystem/HasTiles.scala b/src/main/scala/subsystem/HasTiles.scala
index 3223f367c2a..66cea43d150 100644
--- a/src/main/scala/subsystem/HasTiles.scala
+++ b/src/main/scala/subsystem/HasTiles.scala
@@ -62,8 +62,8 @@ case class TileSlavePortParams(
       .map { bbbp =>
         val blocker = LazyModule(new BasicBusBlocker(bbbp))
         blockerBus.coupleTo("tile_slave_port_bus_blocker") { blocker.controlNode := TLFragmenter(blockerBus) := _ }
-        blocker.node
-      } .getOrElse { TLTempNode() }
+        blocker.node :*= TLBuffer(buffers)
+      } .getOrElse { TLBuffer(buffers) }
   }
 }
 

From ece6b145ba0d2f84c928dbb2eae31dc98c52155a Mon Sep 17 00:00:00 2001
From: Henry Cook <henry@sifive.com>
Date: Wed, 17 Jun 2020 15:57:52 -0700
Subject: [PATCH 14/15] InstantiableTileParams -> InstantiableTileParams

---
 src/main/scala/groundtest/TraceGen.scala | 2 +-
 src/main/scala/subsystem/Configs.scala   | 2 +-
 src/main/scala/subsystem/HasTiles.scala  | 4 ++--
 src/main/scala/tile/BaseTile.scala       | 2 +-
 src/main/scala/tile/RocketTile.scala     | 2 +-
 5 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/src/main/scala/groundtest/TraceGen.scala b/src/main/scala/groundtest/TraceGen.scala
index 37efd3bc447..27d295fa8fc 100644
--- a/src/main/scala/groundtest/TraceGen.scala
+++ b/src/main/scala/groundtest/TraceGen.scala
@@ -67,7 +67,7 @@ case class TraceGenParams(
     numGens: Int,
     dcache: Option[DCacheParams] = Some(DCacheParams()),
     hartId: Int = 0
-) extends InstantiatableTileParams[TraceGenTile] with GroundTestTileParams
+) extends InstantiableTileParams[TraceGenTile] with GroundTestTileParams
 {
   def instantiate(crossing: TileCrossingParamsLike, lookup: LookupByHartIdImpl)(implicit p: Parameters): TraceGenTile = {
     new TraceGenTile(this, crossing, lookup)
diff --git a/src/main/scala/subsystem/Configs.scala b/src/main/scala/subsystem/Configs.scala
index 5f73b5c6302..e90a1a2cdec 100644
--- a/src/main/scala/subsystem/Configs.scala
+++ b/src/main/scala/subsystem/Configs.scala
@@ -415,7 +415,7 @@ class WithScratchpadsOnly extends Config((site, here, up) => {
 
 /** Boilerplate code for translating between the old XTilesParamsKey/XTilesCrossingKey pattern and new TilesLocated pattern */
 object LegacyTileFieldHelper {
-  def apply[TPT <: InstantiatableTileParams[_], TCT <: TileCrossingParamsLike, TAP <: CanAttachTile](
+  def apply[TPT <: InstantiableTileParams[_], TCT <: TileCrossingParamsLike, TAP <: CanAttachTile](
     tileParams: Seq[TPT],
     tcp: Seq[TCT],
     apply: (TPT, TCT, LookupByHartIdImpl) => TAP): Seq[TAP] =
diff --git a/src/main/scala/subsystem/HasTiles.scala b/src/main/scala/subsystem/HasTiles.scala
index 66cea43d150..22ba7eae6be 100644
--- a/src/main/scala/subsystem/HasTiles.scala
+++ b/src/main/scala/subsystem/HasTiles.scala
@@ -10,7 +10,7 @@ import freechips.rocketchip.devices.tilelink.{BasicBusBlocker, BasicBusBlockerPa
 import freechips.rocketchip.diplomacy._
 import freechips.rocketchip.diplomaticobjectmodel.logicaltree.{LogicalModuleTree}
 import freechips.rocketchip.interrupts._
-import freechips.rocketchip.tile.{BaseTile, LookupByHartIdImpl, TileParams, HasExternallyDrivenTileConstants, InstantiatableTileParams, PriorityMuxHartIdFromSeq}
+import freechips.rocketchip.tile.{BaseTile, LookupByHartIdImpl, TileParams, HasExternallyDrivenTileConstants, InstantiableTileParams, PriorityMuxHartIdFromSeq}
 import freechips.rocketchip.tilelink._
 import freechips.rocketchip.util._
 
@@ -147,7 +147,7 @@ trait DefaultTileContextType
 trait CanAttachTile {
   type TileType <: BaseTile
   type TileContextType <: DefaultTileContextType
-  def tileParams: InstantiatableTileParams[TileType]
+  def tileParams: InstantiableTileParams[TileType]
   def crossingParams: TileCrossingParamsLike
   def lookup: LookupByHartIdImpl
 
diff --git a/src/main/scala/tile/BaseTile.scala b/src/main/scala/tile/BaseTile.scala
index cfebad6f8cd..5bb6b2582b4 100644
--- a/src/main/scala/tile/BaseTile.scala
+++ b/src/main/scala/tile/BaseTile.scala
@@ -32,7 +32,7 @@ trait TileParams {
   val name: Option[String]
 }
 
-abstract class InstantiatableTileParams[TileType <: BaseTile] extends TileParams {
+abstract class InstantiableTileParams[TileType <: BaseTile] extends TileParams {
   def instantiate(crossing: TileCrossingParamsLike, lookup: LookupByHartIdImpl)
                  (implicit p: Parameters): TileType
 }
diff --git a/src/main/scala/tile/RocketTile.scala b/src/main/scala/tile/RocketTile.scala
index a9dcc873b7d..e5a713445df 100644
--- a/src/main/scala/tile/RocketTile.scala
+++ b/src/main/scala/tile/RocketTile.scala
@@ -25,7 +25,7 @@ case class RocketTileParams(
     beuAddr: Option[BigInt] = None,
     blockerCtrlAddr: Option[BigInt] = None,
     boundaryBuffers: Boolean = false // if synthesized with hierarchical PnR, cut feed-throughs?
-    ) extends InstantiatableTileParams[RocketTile] {
+    ) extends InstantiableTileParams[RocketTile] {
   require(icache.isDefined)
   require(dcache.isDefined)
   def instantiate(crossing: TileCrossingParamsLike, lookup: LookupByHartIdImpl)(implicit p: Parameters): RocketTile = {

From 77532ca46352a3d6c5aa6f2891bca5444b84e4c6 Mon Sep 17 00:00:00 2001
From: Henry Cook <henry@sifive.com>
Date: Wed, 17 Jun 2020 16:43:17 -0700
Subject: [PATCH 15/15] config: use optional override offsets

---
 src/main/scala/groundtest/Configs.scala | 54 ++++++++++++++-----------
 src/main/scala/subsystem/Configs.scala  | 18 ++++++---
 src/main/scala/system/Configs.scala     |  6 +--
 3 files changed, 46 insertions(+), 32 deletions(-)

diff --git a/src/main/scala/groundtest/Configs.scala b/src/main/scala/groundtest/Configs.scala
index 0e0dcfd50c7..8288f2e9474 100644
--- a/src/main/scala/groundtest/Configs.scala
+++ b/src/main/scala/groundtest/Configs.scala
@@ -34,30 +34,38 @@ class GroundTestBaseConfig extends Config(
   })
 )
 
-class WithTraceGen(n: Int = 2, idOffset: Int = 0)(
+class WithTraceGen(
+  n: Int = 2,
+  overrideIdOffset: Option[Int] = None,
+  overrideMemOffset: Option[BigInt] = None)(
   params: Seq[DCacheParams] = List.fill(n){ DCacheParams(nSets = 16, nWays = 1) },
   nReqs: Int = 8192
 ) extends Config((site, here, up) => {
-  case TilesLocated(InSubsystem) => params.zipWithIndex.map { case (dcp, i) =>
-    TraceGenTileAttachParams(
-      tileParams = TraceGenParams(
-        hartId = i + idOffset,
-        dcache = Some(dcp),
-        wordBits = site(XLen),
-        addrBits = 32,
-        addrBag = {
-          val nSets = dcp.nSets
-          val nWays = dcp.nWays
-          val blockOffset = site(SystemBusKey).blockOffset
-          val nBeats = site(SystemBusKey).blockBeats
-          List.tabulate(nWays) { i =>
-            Seq.tabulate(nBeats) { j => BigInt((j * 8) + ((i * nSets) << blockOffset)) }
-          }.flatten
-        },
-        maxRequests = nReqs,
-        memStart = site(ExtMem).get.master.base,
-        numGens = params.size),
-      crossingParams = RocketCrossingParams()
-    )
-  } ++ up(TilesLocated(InSubsystem), site)
+  case TilesLocated(InSubsystem) => {
+    val prev = up(TilesLocated(InSubsystem), site)
+    val idOffset = overrideIdOffset.getOrElse(prev.size)
+    val memOffset: BigInt = overrideMemOffset.orElse(site(ExtMem).map(_.master.base)).getOrElse(0x0L)
+    params.zipWithIndex.map { case (dcp, i) =>
+      TraceGenTileAttachParams(
+        tileParams = TraceGenParams(
+          hartId = i + idOffset,
+          dcache = Some(dcp),
+          wordBits = site(XLen),
+          addrBits = 32,
+          addrBag = {
+            val nSets = dcp.nSets
+            val nWays = dcp.nWays
+            val blockOffset = site(SystemBusKey).blockOffset
+            val nBeats = site(SystemBusKey).blockBeats
+            List.tabulate(nWays) { i =>
+              Seq.tabulate(nBeats) { j => BigInt((j * 8) + ((i * nSets) << blockOffset)) }
+            }.flatten
+          },
+          maxRequests = nReqs,
+          memStart = memOffset,
+          numGens = params.size),
+        crossingParams = RocketCrossingParams()
+      )
+    } ++ prev
+  }
 })
diff --git a/src/main/scala/subsystem/Configs.scala b/src/main/scala/subsystem/Configs.scala
index e90a1a2cdec..78ed31dcf18 100644
--- a/src/main/scala/subsystem/Configs.scala
+++ b/src/main/scala/subsystem/Configs.scala
@@ -77,8 +77,10 @@ class WithCoherentBusTopology extends Config((site, here, up) => {
       l2 = site(BankedL2Key)))
 })
 
-class WithNBigCores(n: Int, idOffset: Int = 0) extends Config((site, here, up) => {
+class WithNBigCores(n: Int, overrideIdOffset: Option[Int] = None) extends Config((site, here, up) => {
   case RocketTilesKey => {
+    val prev = up(RocketTilesKey, site)
+    val idOffset = overrideIdOffset.getOrElse(prev.size)
     val big = RocketTileParams(
       core   = RocketCoreParams(mulDiv = Some(MulDivParams(
         mulUnroll = 8,
@@ -91,12 +93,14 @@ class WithNBigCores(n: Int, idOffset: Int = 0) extends Config((site, here, up) =
       icache = Some(ICacheParams(
         rowBits = site(SystemBusKey).beatBits,
         blockBytes = site(CacheBlockBytes))))
-    List.tabulate(n)(i => big.copy(hartId = i + idOffset)) ++ up(RocketTilesKey, site)
+    List.tabulate(n)(i => big.copy(hartId = i + idOffset)) ++ prev
   }
 })
 
-class WithNMedCores(n: Int, idOffset: Int = 0) extends Config((site, here, up) => {
+class WithNMedCores(n: Int, overrideIdOffset: Option[Int] = None) extends Config((site, here, up) => {
   case RocketTilesKey => {
+    val prev = up(RocketTilesKey, site)
+    val idOffset = overrideIdOffset.getOrElse(prev.size)
     val med = RocketTileParams(
       core = RocketCoreParams(fpu = None),
       btb = None,
@@ -113,12 +117,14 @@ class WithNMedCores(n: Int, idOffset: Int = 0) extends Config((site, here, up) =
         nWays = 1,
         nTLBEntries = 4,
         blockBytes = site(CacheBlockBytes))))
-    List.tabulate(n)(i => med.copy(hartId = i + idOffset)) ++ up(RocketTilesKey, site)
+    List.tabulate(n)(i => med.copy(hartId = i + idOffset)) ++ prev
   }
 })
 
-class WithNSmallCores(n: Int, idOffset: Int = 0) extends Config((site, here, up) => {
+class WithNSmallCores(n: Int, overrideIdOffset: Option[Int] = None) extends Config((site, here, up) => {
   case RocketTilesKey => {
+    val prev = up(RocketTilesKey, site)
+    val idOffset = overrideIdOffset.getOrElse(prev.size)
     val small = RocketTileParams(
       core = RocketCoreParams(useVM = false, fpu = None),
       btb = None,
@@ -135,7 +141,7 @@ class WithNSmallCores(n: Int, idOffset: Int = 0) extends Config((site, here, up)
         nWays = 1,
         nTLBEntries = 4,
         blockBytes = site(CacheBlockBytes))))
-    List.tabulate(n)(i => small.copy(hartId = i + idOffset)) ++ up(RocketTilesKey, site)
+    List.tabulate(n)(i => small.copy(hartId = i + idOffset)) ++ prev
   }
 })
 
diff --git a/src/main/scala/system/Configs.scala b/src/main/scala/system/Configs.scala
index b8ce1423803..543a064c508 100644
--- a/src/main/scala/system/Configs.scala
+++ b/src/main/scala/system/Configs.scala
@@ -40,9 +40,9 @@ class DualChannelDualBankConfig extends Config(
 class RoccExampleConfig extends Config(new WithRoccExample ++ new DefaultConfig)
 
 class HeterogeneousTileExampleConfig extends Config(
-  new WithTraceGen (n = 2, idOffset = 3)() ++
-  new WithNBigCores(n = 1, idOffset = 2) ++
-  new WithNMedCores(n = 1, idOffset = 1) ++
+  new WithTraceGen (n = 2, overrideMemOffset = Some(0x90000000L))() ++
+  new WithNBigCores(n = 1) ++
+  new WithNMedCores(n = 1) ++
   new WithNSmallCores(n = 1) ++
   new WithCoherentBusTopology ++
   new BaseConfig