From bf89a7e1a64227a66552a75283c3df10d8de665a Mon Sep 17 00:00:00 2001
From: Henry Cook <henry@sifive.com>
Date: Wed, 24 Jun 2020 15:00:40 -0700
Subject: [PATCH] Clone of 2521 for action/cache@v2 investigation purposes
 (#2531)

* subsystem: use bundlebridge to propagate tile constants

* bundlebridge: restore separate makeIO implementation to avoid extra _0 on verilog signal names

* tile: use bridged constants inside diplomatic tile units

* tile: further apply lazy val requirement checking pattern

* tile: remove obviated HasExternallyDrivenTileConstants mixin

* subsystem: continue to standardize ROM attachment

* subsystem: use BundleBridgeNexus.safeRegNext in hart prefix node

* tile: BaseTile.traceAuxNode supplied default

* groundtest: update to deal with tile input constant defaults

* subsystem: import spelling error rebase

* Update src/main/scala/devices/tilelink/BootROM.scala

Co-authored-by: Megan Wachs <megan@sifive.com>

* Update src/main/scala/subsystem/HasTiles.scala

Co-authored-by: Megan Wachs <megan@sifive.com>

* Update src/main/scala/devices/tilelink/BootROM.scala

Co-authored-by: Megan Wachs <megan@sifive.com>

* Update src/main/scala/subsystem/HasTiles.scala

Co-authored-by: Megan Wachs <megan@sifive.com>

* Update src/main/scala/system/ExampleRocketSystem.scala

Co-authored-by: Megan Wachs <megan@sifive.com>

* Update src/main/scala/tile/BaseTile.scala

Co-authored-by: Megan Wachs <megan@sifive.com>

* Update src/main/scala/tile/BaseTile.scala

Co-authored-by: Megan Wachs <megan@sifive.com>

* Update src/main/scala/tile/BaseTile.scala

Co-authored-by: Megan Wachs <megan@sifive.com>

* BootROM.attach scaladoc comment

* HasTiles: more scaladoc and better scaladoc style

* bundlebridge: expose inputRequiresOutput

* HasTiles: inputRequiresOutput=true for both new nexus nodes

* staticIdForMetadata => staticIdForMetadataUseOnly

* tile: hartIdLen and resetVectorLen are now members of HasCoreParams not HasTileParams

* subsystem: tile port params can insert chains of buffers

* util: DataToAugmentedData.getElements

* bundlebridge: better dir inference on makeIO

* tile: rely on bundlebridge direction inference for reset vector

* BaseTile: scaladoc for staticIdForMetadataUseOnly

* Update src/main/scala/subsystem/HasTiles.scala

Co-authored-by: Megan Wachs <megan@sifive.com>

* Update src/main/scala/subsystem/HasTiles.scala

Co-authored-by: Megan Wachs <megan@sifive.com>

* HartPrefixKey => InsertTimingClosureRegistersOnHartIds

* bootrom: connect reset vector to nexus, not just first tile

Co-authored-by: Megan Wachs <megan@sifive.com>
---
 src/main/scala/devices/tilelink/BootROM.scala |  52 +++--
 src/main/scala/devices/tilelink/MaskROM.scala |   7 +-
 src/main/scala/diplomacy/BundleBridge.scala   |  55 +++--
 src/main/scala/groundtest/Configs.scala       |   1 +
 .../groundtest/GroundTestSubsystem.scala      |   3 -
 src/main/scala/groundtest/Tile.scala          |   2 +
 src/main/scala/groundtest/TraceGen.scala      |   2 +-
 src/main/scala/rocket/DCache.scala            |   4 +-
 src/main/scala/rocket/Frontend.scala          |  16 +-
 src/main/scala/rocket/HellaCache.scala        |  15 +-
 src/main/scala/rocket/ICache.scala            |   9 +-
 src/main/scala/rocket/NBDcache.scala          |   2 +-
 src/main/scala/subsystem/Configs.scala        |   5 +-
 src/main/scala/subsystem/HasTiles.scala       | 200 ++++++++++++++----
 src/main/scala/subsystem/ResetVector.scala    |  11 -
 .../scala/subsystem/RocketSubsystem.scala     |  35 +--
 .../scala/system/ExampleRocketSystem.scala    |   8 +-
 src/main/scala/tile/BaseTile.scala            |  69 +++---
 src/main/scala/tile/Core.scala                |  13 +-
 src/main/scala/tile/RocketTile.scala          |  15 +-
 src/main/scala/util/package.scala             |   5 +
 21 files changed, 329 insertions(+), 200 deletions(-)
 delete mode 100644 src/main/scala/subsystem/ResetVector.scala

diff --git a/src/main/scala/devices/tilelink/BootROM.scala b/src/main/scala/devices/tilelink/BootROM.scala
index 0425d21056d..dd4d88fbeb0 100644
--- a/src/main/scala/devices/tilelink/BootROM.scala
+++ b/src/main/scala/devices/tilelink/BootROM.scala
@@ -4,7 +4,7 @@ package freechips.rocketchip.devices.tilelink
 
 import Chisel._
 import freechips.rocketchip.config.{Field, Parameters}
-import freechips.rocketchip.subsystem.{BaseSubsystem, HasResetVectorWire}
+import freechips.rocketchip.subsystem.{BaseSubsystem, HierarchicalLocation, HasTiles, TLBusWrapperLocation, CBUS}
 import freechips.rocketchip.diplomacy._
 import freechips.rocketchip.tilelink._
 import freechips.rocketchip.util._
@@ -16,9 +16,8 @@ import java.nio.file.{Files, Paths}
 case class BootROMParams(
   address: BigInt = 0x10000,
   size: Int = 0x10000,
-  hang: BigInt = 0x10040,
+  hang: BigInt = 0x10040, // The hang parameter is used as the power-on reset vector
   contentFileName: String)
-case object BootROMParams extends Field[BootROMParams]
 
 class TLROM(val base: BigInt, val size: Int, contentsDelayed: => Seq[Byte], executable: Boolean = true, beatBytes: Int = 4,
   resources: Seq[Resource] = new SimpleDevice("rom", Seq("sifive,rom0")).reg("mem"))(implicit p: Parameters) extends LazyModule
@@ -58,25 +57,36 @@ class TLROM(val base: BigInt, val size: Int, contentsDelayed: => Seq[Byte], exec
   }
 }
 
-/** Adds a boot ROM that contains the DTB describing the system's subsystem. */
-trait HasPeripheryBootROM { this: BaseSubsystem =>
-  val dtb: DTB
-  private val params = p(BootROMParams)
-  private lazy val contents = {
-    val romdata = Files.readAllBytes(Paths.get(params.contentFileName))
-    val rom = ByteBuffer.wrap(romdata)
-    rom.array() ++ dtb.contents
-  }
-  def resetVector: BigInt = params.hang
+case class BootROMLocated(loc: HierarchicalLocation) extends Field[Option[BootROMParams]](None)
 
-  val bootrom = LazyModule(new TLROM(params.address, params.size, contents, true, cbus.beatBytes))
+object BootROM {
+  /** BootROM.attach not only instantiates a TLROM and attaches it to the tilelink interconnect
+    *    at a configurable location, but also drives the tiles' reset vectors to point
+    *    at its 'hang' address parameter value.
+    */
+  def attach(params: BootROMParams, subsystem: BaseSubsystem with HasTiles, where: TLBusWrapperLocation)
+            (implicit p: Parameters): TLROM = {
+    val cbus = subsystem.locateTLBusWrapper(where)
+    val bootROMResetVectorSourceNode = BundleBridgeSource[UInt]()
+    lazy val contents = {
+      val romdata = Files.readAllBytes(Paths.get(params.contentFileName))
+      val rom = ByteBuffer.wrap(romdata)
+      rom.array() ++ subsystem.dtb.contents
+    }
 
-  bootrom.node := cbus.coupleTo("bootrom"){ TLFragmenter(cbus) := _ }
-}
+    val bootrom = subsystem {
+      LazyModule(new TLROM(params.address, params.size, contents, true, cbus.beatBytes))
+    }
 
-/** Subsystem will power-on running at 0x10040 (BootROM) */
-trait HasPeripheryBootROMModuleImp extends LazyModuleImp
-    with HasResetVectorWire {
-  val outer: HasPeripheryBootROM
-  global_reset_vector := outer.resetVector.U
+    bootrom.node := cbus.coupleTo("bootrom"){ TLFragmenter(cbus) := _ }
+    // Drive the `subsystem` reset vector to the `hang` address of this Boot ROM.
+    subsystem.tileResetVectorNexusNode := bootROMResetVectorSourceNode
+    InModuleBody {
+      val reset_vector_source = bootROMResetVectorSourceNode.bundle
+      require(reset_vector_source.getWidth >= params.hang.bitLength,
+        s"BootROM defined with a reset vector (${params.hang})too large for physical address space (${reset_vector_source.getWidth})")
+      bootROMResetVectorSourceNode.bundle := params.hang.U
+    }
+    bootrom
+  }
 }
diff --git a/src/main/scala/devices/tilelink/MaskROM.scala b/src/main/scala/devices/tilelink/MaskROM.scala
index f0df6a00318..8fb2b43573d 100644
--- a/src/main/scala/devices/tilelink/MaskROM.scala
+++ b/src/main/scala/devices/tilelink/MaskROM.scala
@@ -5,6 +5,7 @@ package freechips.rocketchip.devices.tilelink
 import Chisel._
 import freechips.rocketchip.config.{Field, Parameters}
 import freechips.rocketchip.diplomacy._
+import freechips.rocketchip.subsystem.{Attachable, HierarchicalLocation, TLBusWrapperLocation, CBUS}
 import freechips.rocketchip.tilelink._
 import freechips.rocketchip.util._
 
@@ -56,8 +57,12 @@ class TLMaskROM(c: MaskROMParams)(implicit p: Parameters) extends LazyModule {
   }
 }
 
+case class MaskROMLocated(loc: HierarchicalLocation) extends Field[Seq[MaskROMParams]](Nil)
+
 object MaskROM {
-  def attach(params: MaskROMParams, bus: TLBusWrapper)(implicit p: Parameters): TLMaskROM = {
+  def attach(params: MaskROMParams, subsystem: Attachable, where: TLBusWrapperLocation)
+            (implicit p: Parameters): TLMaskROM = {
+    val bus = subsystem.locateTLBusWrapper(where)
     val maskROM = LazyModule(new TLMaskROM(params))
     maskROM.node := bus.coupleTo("MaskROM") {
       TLFragmenter(maskROM.beatBytes, bus.blockBytes) :*= TLWidthWidget(bus) := _
diff --git a/src/main/scala/diplomacy/BundleBridge.scala b/src/main/scala/diplomacy/BundleBridge.scala
index 9f0b97ce0ad..518eb1ce7c0 100644
--- a/src/main/scala/diplomacy/BundleBridge.scala
+++ b/src/main/scala/diplomacy/BundleBridge.scala
@@ -5,7 +5,9 @@ package freechips.rocketchip.diplomacy
 import chisel3._
 import chisel3.internal.sourceinfo.SourceInfo
 import chisel3.experimental.{DataMirror,IO}
+import chisel3.experimental.DataMirror.internal.chiselTypeClone
 import freechips.rocketchip.config.{Parameters,Field}
+import freechips.rocketchip.util.DataToAugmentedData
 
 case class BundleBridgeParams[T <: Data](genOpt: Option[() => T])
 
@@ -42,16 +44,34 @@ case class BundleBridgeSink[T <: Data](genOpt: Option[() => T] = None)
 {
   def bundle: T = in(0)._1
 
-  def makeIO()(implicit valName: ValName): T = makeIOs()(valName).head
-  def makeIO(name: String): T = makeIOs()(ValName(name)).head
+  private def inferOutput = bundle.getElements.forall { elt =>
+    DataMirror.directionOf(elt) == ActualDirection.Unspecified
+  }
+
+  def makeIO()(implicit valName: ValName): T = {
+    val io: T = IO(if (inferOutput) Output(chiselTypeOf(bundle)) else chiselTypeClone(bundle))
+    io.suggestName(valName.name)
+    io <> bundle
+    io
+  }
+  def makeIO(name: String): T = makeIO()(ValName(name))
 }
 
 case class BundleBridgeSource[T <: Data](genOpt: Option[() => T] = None)(implicit valName: ValName) extends SourceNode(new BundleBridgeImp[T])(Seq(BundleBridgeParams(genOpt)))
 {
   def bundle: T = out(0)._1
 
-  def makeIO()(implicit valName: ValName): T = makeIOs()(valName).head
-  def makeIO(name: String): T = makeIOs()(ValName(name)).head
+  private def inferInput = bundle.getElements.forall { elt =>
+    DataMirror.directionOf(elt) == ActualDirection.Unspecified
+  }
+
+  def makeIO()(implicit valName: ValName): T = {
+    val io: T = IO(if (inferInput) Input(chiselTypeOf(bundle)) else Flipped(chiselTypeClone(bundle)))
+    io.suggestName(valName.name)
+    bundle <> io
+    io
+  }
+  def makeIO(name: String): T = makeIO()(ValName(name))
 
   private var doneSink = false
   def makeSink()(implicit p: Parameters) = {
@@ -72,21 +92,23 @@ case object BundleBridgeSource {
 case class BundleBridgeIdentityNode[T <: Data]()(implicit valName: ValName) extends IdentityNode(new BundleBridgeImp[T])()
 case class BundleBridgeEphemeralNode[T <: Data]()(implicit valName: ValName) extends EphemeralNode(new BundleBridgeImp[T])()
 
-case class BundleBridgeNexusNode[T <: Data](default: Option[() => T] = None)
+case class BundleBridgeNexusNode[T <: Data](default: Option[() => T] = None,
+                                            inputRequiresOutput: Boolean = false) // when false, connecting a source does not mandate connecting a sink
                                            (implicit valName: ValName)
   extends NexusNode(new BundleBridgeImp[T])(
     dFn = seq => seq.headOption.getOrElse(BundleBridgeParams(default)),
     uFn = seq => seq.headOption.getOrElse(BundleBridgeParams(default)),
-    inputRequiresOutput = false, // enables publishers with no subscribers
+    inputRequiresOutput = inputRequiresOutput,
     outputRequiresInput = !default.isDefined)
 
 class BundleBridgeNexus[T <: Data](
   inputFn: Seq[T] => T,
   outputFn: (T, Int) => Seq[T],
-  default: Option[() => T] = None
+  default: Option[() => T] = None,
+  inputRequiresOutput: Boolean = false
 ) (implicit p: Parameters) extends LazyModule
 {
-  val node = BundleBridgeNexusNode[T](default)
+  val node = BundleBridgeNexusNode[T](default, inputRequiresOutput)
 
   lazy val module = new LazyModuleImp(this) {
     val defaultWireOpt = default.map(_())
@@ -95,11 +117,7 @@ class BundleBridgeNexus[T <: Data](
     inputs.foreach { i => require(DataMirror.checkTypeEquivalence(i, inputs.head),
       s"${node.context} requires all inputs have equivalent Chisel Data types, but got\n$i\nvs\n${inputs.head}")
     }
-    def getElements(x: Data): Seq[Element] = x match {
-      case e: Element => Seq(e)
-      case a: Aggregate => a.getElements.flatMap(getElements)
-    }
-    inputs.flatMap(getElements).foreach { elt => DataMirror.directionOf(elt) match {
+    inputs.flatMap(_.getElements).foreach { elt => DataMirror.directionOf(elt) match {
       case ActualDirection.Output => ()
       case ActualDirection.Unspecified => ()
       case _ => require(false, s"${node.context} can only be used with Output-directed Bundles")
@@ -145,9 +163,10 @@ object BundleBridgeNexus {
   def apply[T <: Data](
     inputFn: Seq[T] => T = orReduction[T](false) _,
     outputFn: (T, Int) => Seq[T] = fillN[T](false) _,
-    default: Option[() => T] = None
+    default: Option[() => T] = None,
+    inputRequiresOutput: Boolean = false
   )(implicit p: Parameters, valName: ValName): BundleBridgeNexusNode[T] = {
-    val broadcast = LazyModule(new BundleBridgeNexus[T](inputFn, outputFn, default))
+    val broadcast = LazyModule(new BundleBridgeNexus[T](inputFn, outputFn, default, inputRequiresOutput))
     broadcast.node
   }
 }
@@ -155,12 +174,14 @@ object BundleBridgeNexus {
 object BundleBroadcast {
   def apply[T <: Data](
     name: Option[String] = None,
-    registered: Boolean = false
+    registered: Boolean = false,
+    inputRequiresOutput: Boolean = false // when false, connecting a source does not mandate connecting a sink
   )(implicit p: Parameters, valName: ValName): BundleBridgeNexusNode[T] = {
     val finalName = name.map(ValName(_)).getOrElse(valName)
     BundleBridgeNexus.apply[T](
       inputFn = BundleBridgeNexus.requireOne[T](registered) _,
-      outputFn = BundleBridgeNexus.fillN[T](registered) _)(
+      outputFn = BundleBridgeNexus.fillN[T](registered) _,
+      inputRequiresOutput = inputRequiresOutput)(
       p, finalName)
   }
 }
diff --git a/src/main/scala/groundtest/Configs.scala b/src/main/scala/groundtest/Configs.scala
index 8288f2e9474..35837df6037 100644
--- a/src/main/scala/groundtest/Configs.scala
+++ b/src/main/scala/groundtest/Configs.scala
@@ -31,6 +31,7 @@ class GroundTestBaseConfig extends Config(
     case DebugModuleKey => None
     case CLINTKey => None
     case PLICKey => None
+    case SubsystemExternalResetVectorKey => true
   })
 )
 
diff --git a/src/main/scala/groundtest/GroundTestSubsystem.scala b/src/main/scala/groundtest/GroundTestSubsystem.scala
index bb330031357..0423998da9f 100644
--- a/src/main/scala/groundtest/GroundTestSubsystem.scala
+++ b/src/main/scala/groundtest/GroundTestSubsystem.scala
@@ -29,9 +29,6 @@ class GroundTestSubsystem(implicit p: Parameters)
 
 class GroundTestSubsystemModuleImp[+L <: GroundTestSubsystem](_outer: L) extends BaseSubsystemModuleImp(_outer) {
   val success = IO(Bool(OUTPUT))
-
-  outer.tiles.zipWithIndex.map { case(t, i) => t.module.constants.hartid := UInt(i) }
-
   val status = dontTouch(DebugCombiner(outer.tiles.collect { case t: GroundTestTile => t.module.status }))
   success := outer.tileCeaseSinkNode.in.head._1.asUInt.andR
 }
diff --git a/src/main/scala/groundtest/Tile.scala b/src/main/scala/groundtest/Tile.scala
index 230bf58552b..19b82466665 100644
--- a/src/main/scala/groundtest/Tile.scala
+++ b/src/main/scala/groundtest/Tile.scala
@@ -44,6 +44,8 @@ abstract class GroundTestTile(
     else new NonBlockingDCache(hartId))
   }
 
+  dcacheOpt.foreach { _.hartIdSinkNode := hartIdNode }
+
   override lazy val module = new GroundTestTileModuleImp(this)
 }
 
diff --git a/src/main/scala/groundtest/TraceGen.scala b/src/main/scala/groundtest/TraceGen.scala
index 27d295fa8fc..96b9e43ff95 100644
--- a/src/main/scala/groundtest/TraceGen.scala
+++ b/src/main/scala/groundtest/TraceGen.scala
@@ -611,7 +611,7 @@ class TraceGenTile private(
 class TraceGenTileModuleImp(outer: TraceGenTile) extends GroundTestTileModuleImp(outer) {
 
   val tracegen = Module(new TraceGenerator(outer.params))
-  tracegen.io.hartid := constants.hartid
+  tracegen.io.hartid := outer.hartIdSinkNode.bundle
 
   outer.dcacheOpt foreach { dcache =>
     val dcacheIF = Module(new SimpleHellaCacheIF())
diff --git a/src/main/scala/rocket/DCache.scala b/src/main/scala/rocket/DCache.scala
index 7d83e7822ec..fc7a6ddae7c 100644
--- a/src/main/scala/rocket/DCache.scala
+++ b/src/main/scala/rocket/DCache.scala
@@ -81,7 +81,7 @@ class DCacheMetadataReq(implicit p: Parameters) extends L1HellaCacheBundle()(p)
   val data = UInt(width = cacheParams.tagCode.width(new L1Metadata().getWidth))
 }
 
-class DCache(hartid: Int, val crossing: ClockCrossingType)(implicit p: Parameters) extends HellaCache(hartid)(p) {
+class DCache(staticIdForMetadataUseOnly: Int, val crossing: ClockCrossingType)(implicit p: Parameters) extends HellaCache(staticIdForMetadataUseOnly)(p) {
   override lazy val module = new DCacheModule(this)
   override def getOMSRAMs(): Seq[OMSRAM] = Seq(module.dcacheImpl.omSRAM) ++ module.dcacheImpl.data.data_arrays.map(_._2)
 }
@@ -251,7 +251,7 @@ class DCacheModule(outer: DCache) extends HellaCacheModule(outer) {
   val s1_victim_way = Wire(init = replacer.way)
   val (s1_hit_way, s1_hit_state, s1_meta, s1_victim_meta) =
     if (usingDataScratchpad) {
-      val baseAddr = p(LookupByHartId)(_.dcache.flatMap(_.scratch.map(_.U)), io.hartid)
+      val baseAddr = p(LookupByHartId)(_.dcache.flatMap(_.scratch.map(_.U)), io_hartid)
       val inScratchpad = s1_paddr >= baseAddr && s1_paddr < baseAddr + nSets * cacheBlockBytes
       val hitState = Mux(inScratchpad, ClientMetadata.maximum, ClientMetadata.onReset)
       val dummyMeta = L1Metadata(UInt(0), ClientMetadata.onReset)
diff --git a/src/main/scala/rocket/Frontend.scala b/src/main/scala/rocket/Frontend.scala
index 9d9adb739bc..ae5653a65f1 100644
--- a/src/main/scala/rocket/Frontend.scala
+++ b/src/main/scala/rocket/Frontend.scala
@@ -59,15 +59,15 @@ class FrontendIO(implicit p: Parameters) extends CoreBundle()(p) {
   val perf = new FrontendPerfEvents().asInput
 }
 
-class Frontend(val icacheParams: ICacheParams, hartid: Int)(implicit p: Parameters) extends LazyModule {
+class Frontend(val icacheParams: ICacheParams, staticIdForMetadataUseOnly: Int)(implicit p: Parameters) extends LazyModule {
   lazy val module = new FrontendModule(this)
-  val icache = LazyModule(new ICache(icacheParams, hartid))
+  val icache = LazyModule(new ICache(icacheParams, staticIdForMetadataUseOnly))
   val masterNode = icache.masterNode
   val slaveNode = icache.slaveNode
+  val resetVectorSinkNode = BundleBridgeSink[UInt](Some(() => UInt(masterNode.edges.out.head.bundle.addressBits.W)))
 }
 
-class FrontendBundle(val outer: Frontend) extends CoreBundle()(outer.p)
-    with HasExternallyDrivenTileConstants {
+class FrontendBundle(val outer: Frontend) extends CoreBundle()(outer.p) {
   val cpu = new FrontendIO().flip
   val ptw = new TLBPTWIO()
   val errors = new ICacheErrors
@@ -78,6 +78,7 @@ class FrontendModule(outer: Frontend) extends LazyModuleImp(outer)
     with HasRocketCoreParameters
     with HasL1ICacheParameters {
   val io = IO(new FrontendBundle(outer))
+  val io_reset_vector = outer.resetVectorSinkNode.bundle
   implicit val edge = outer.masterNode.edges.out(0)
   val icache = outer.icache.module
   require(fetchWidth*coreInstBytes == outer.icacheParams.fetchBytes)
@@ -108,7 +109,7 @@ class FrontendModule(outer: Frontend) extends LazyModuleImp(outer)
   s1_valid := s0_valid
   val s1_pc = Reg(UInt(width=vaddrBitsExtended))
   val s1_speculative = Reg(Bool())
-  val s2_pc = RegInit(t = UInt(width = vaddrBitsExtended), alignPC(io.reset_vector))
+  val s2_pc = RegInit(t = UInt(width = vaddrBitsExtended), alignPC(io_reset_vector))
   val s2_btb_resp_valid = if (usingBTB) Reg(Bool()) else false.B
   val s2_btb_resp_bits = Reg(new BTBResp)
   val s2_btb_taken = s2_btb_resp_valid && s2_btb_resp_bits.taken
@@ -153,7 +154,6 @@ class FrontendModule(outer: Frontend) extends LazyModuleImp(outer)
   tlb.io.sfence := io.cpu.sfence
   tlb.io.kill := !s2_valid
 
-  icache.io.hartid := io.hartid
   icache.io.req.valid := s0_valid
   icache.io.req.bits.addr := io.cpu.npc
   icache.io.invalidate := io.cpu.flush_icache
@@ -349,9 +349,11 @@ class FrontendModule(outer: Frontend) extends LazyModuleImp(outer)
 /** Mix-ins for constructing tiles that have an ICache-based pipeline frontend */
 trait HasICacheFrontend extends CanHavePTW { this: BaseTile =>
   val module: HasICacheFrontendModule
-  val frontend = LazyModule(new Frontend(tileParams.icache.get, hartId))
+  val frontend = LazyModule(new Frontend(tileParams.icache.get, staticIdForMetadataUseOnly))
   tlMasterXbar.node := frontend.masterNode
   connectTLSlave(frontend.slaveNode, tileParams.core.fetchBytes)
+  frontend.icache.hartIdSinkNode := hartIdNode
+  frontend.resetVectorSinkNode := resetVectorNode
   nPTWPorts += 1
 
   // This should be a None in the case of not having an ITIM address, when we
diff --git a/src/main/scala/rocket/HellaCache.scala b/src/main/scala/rocket/HellaCache.scala
index 86737210fa9..57a7b44b356 100644
--- a/src/main/scala/rocket/HellaCache.scala
+++ b/src/main/scala/rocket/HellaCache.scala
@@ -175,17 +175,17 @@ class HellaCacheIO(implicit p: Parameters) extends CoreBundle()(p) {
 
 /** Base classes for Diplomatic TL2 HellaCaches */
 
-abstract class HellaCache(hartid: Int)(implicit p: Parameters) extends LazyModule
+abstract class HellaCache(staticIdForMetadataUseOnly: Int)(implicit p: Parameters) extends LazyModule
     with HasNonDiplomaticTileParameters {
   protected val cfg = tileParams.dcache.get
 
   protected def cacheClientParameters = cfg.scratch.map(x => Seq()).getOrElse(Seq(TLMasterParameters.v1(
-    name          = s"Core ${hartid} DCache",
+    name          = s"Core ${staticIdForMetadataUseOnly} DCache",
     sourceId      = IdRange(0, 1 max cfg.nMSHRs),
     supportsProbe = TransferSizes(cfg.blockBytes, cfg.blockBytes))))
 
   protected def mmioClientParameters = Seq(TLMasterParameters.v1(
-    name          = s"Core ${hartid} DCache MMIO",
+    name          = s"Core ${staticIdForMetadataUseOnly} DCache MMIO",
     sourceId      = IdRange(firstMMIO, firstMMIO + cfg.nMMIOs),
     requestFifo   = true))
 
@@ -196,6 +196,8 @@ abstract class HellaCache(hartid: Int)(implicit p: Parameters) extends LazyModul
     minLatency = 1,
     requestFields = tileParams.core.useVM.option(Seq()).getOrElse(Seq(AMBAProtField())))))
 
+  val hartIdSinkNode = BundleBridgeSink[UInt]()
+
   val module: HellaCacheModule
 
   def flushOnFenceI = cfg.scratch.isEmpty && !node.edges.out(0).manager.managers.forall(m => !m.supportsAcquireT || !m.executable || m.regionType >= RegionType.TRACKED || m.regionType <= RegionType.IDEMPOTENT)
@@ -208,7 +210,6 @@ abstract class HellaCache(hartid: Int)(implicit p: Parameters) extends LazyModul
 }
 
 class HellaCacheBundle(val outer: HellaCache)(implicit p: Parameters) extends CoreBundle()(p) {
-  val hartid = UInt(INPUT, hartIdLen)
   val cpu = (new HellaCacheIO).flip
   val ptw = new TLBPTWIO()
   val errors = new DCacheErrors
@@ -219,6 +220,7 @@ class HellaCacheModule(outer: HellaCache) extends LazyModuleImp(outer)
   implicit val edge = outer.node.edges.out(0)
   val (tl_out, _) = outer.node.out(0)
   val io = IO(new HellaCacheBundle(outer))
+  val io_hartid = outer.hartIdSinkNode.bundle
   dontTouch(io.cpu.resp) // Users like to monitor these fields even if the core ignores some signals
   dontTouch(io.cpu.s1_data)
 
@@ -237,9 +239,9 @@ case object BuildHellaCache extends Field[BaseTile => Parameters => HellaCache](
 object HellaCacheFactory {
   def apply(tile: BaseTile)(p: Parameters): HellaCache = {
     if (tile.tileParams.dcache.get.nMSHRs == 0)
-      new DCache(tile.hartId, tile.crossing)(p)
+      new DCache(tile.staticIdForMetadataUseOnly, tile.crossing)(p)
     else
-      new NonBlockingDCache(tile.hartId)(p)
+      new NonBlockingDCache(tile.staticIdForMetadataUseOnly)(p)
   }
 }
 
@@ -252,6 +254,7 @@ trait HasHellaCache { this: BaseTile =>
   lazy val dcache: HellaCache = LazyModule(p(BuildHellaCache)(this)(p))
 
   tlMasterXbar.node := dcache.node
+  dcache.hartIdSinkNode := hartIdNode
 }
 
 trait HasHellaCacheModule {
diff --git a/src/main/scala/rocket/ICache.scala b/src/main/scala/rocket/ICache.scala
index b3c53cd23c6..002ee177d00 100644
--- a/src/main/scala/rocket/ICache.scala
+++ b/src/main/scala/rocket/ICache.scala
@@ -52,13 +52,14 @@ class ICacheErrors(implicit p: Parameters) extends CoreBundle()(p)
   val bus = Valid(UInt(width = paddrBits))
 }
 
-class ICache(val icacheParams: ICacheParams, val hartId: Int)(implicit p: Parameters) extends LazyModule {
+class ICache(val icacheParams: ICacheParams, val staticIdForMetadataUseOnly: Int)(implicit p: Parameters) extends LazyModule {
   lazy val module = new ICacheModule(this)
+  val hartIdSinkNode = BundleBridgeSink[UInt]()
   val useVM = p(TileKey).core.useVM
   val masterNode = TLClientNode(Seq(TLMasterPortParameters.v1(
     clients = Seq(TLMasterParameters.v1(
       sourceId = IdRange(0, 1 + icacheParams.prefetch.toInt), // 0=refill, 1=hint
-      name = s"Core ${hartId} ICache")),
+      name = s"Core ${staticIdForMetadataUseOnly} ICache")),
     requestFields = useVM.option(Seq()).getOrElse(Seq(AMBAProtField())))))
 
   val size = icacheParams.nSets * icacheParams.nWays * icacheParams.blockBytes
@@ -109,7 +110,6 @@ class ICachePerfEvents extends Bundle {
 }
 
 class ICacheBundle(val outer: ICache) extends CoreBundle()(outer.p) {
-  val hartid = UInt(INPUT, hartIdLen)
   val req = Decoupled(new ICacheReq).flip
   val s1_paddr = UInt(INPUT, paddrBits) // delayed one cycle w.r.t. req
   val s2_vaddr = UInt(INPUT, vaddrBits) // delayed two cycles w.r.t. req
@@ -143,11 +143,12 @@ class ICacheModule(outer: ICache) extends LazyModuleImp(outer)
   require(!usingVM || outer.icacheParams.itimAddr.isEmpty || pgIdxBits >= untagBits,
     s"When VM and ITIM are enabled, I$$ set size must not exceed ${1<<(pgIdxBits-10)} KiB; got ${(outer.size/nWays)>>10} KiB")
 
+  val io_hartid = outer.hartIdSinkNode.bundle
   val scratchpadOn = RegInit(false.B)
   val scratchpadMax = tl_in.map(tl => Reg(UInt(width = log2Ceil(nSets * (nWays - 1)))))
   def lineInScratchpad(line: UInt) = scratchpadMax.map(scratchpadOn && line <= _).getOrElse(false.B)
   val scratchpadBase = outer.icacheParams.itimAddr.map { dummy =>
-    p(LookupByHartId)(_.icache.flatMap(_.itimAddr.map(_.U)), io.hartid)
+    p(LookupByHartId)(_.icache.flatMap(_.itimAddr.map(_.U)), io_hartid)
   }
   def addrMaybeInScratchpad(addr: UInt) = scratchpadBase.map(base => addr >= base && addr < base + outer.size).getOrElse(false.B)
   def addrInScratchpad(addr: UInt) = addrMaybeInScratchpad(addr) && lineInScratchpad(addr(untagBits+log2Ceil(nWays)-1, blockOffBits))
diff --git a/src/main/scala/rocket/NBDcache.scala b/src/main/scala/rocket/NBDcache.scala
index ced4f754ceb..6adce7cde1d 100644
--- a/src/main/scala/rocket/NBDcache.scala
+++ b/src/main/scala/rocket/NBDcache.scala
@@ -673,7 +673,7 @@ class DataArray(implicit p: Parameters) extends L1HellaCacheModule()(p) {
   io.write.ready := Bool(true)
 }
 
-class NonBlockingDCache(hartid: Int)(implicit p: Parameters) extends HellaCache(hartid)(p) {
+class NonBlockingDCache(staticIdForMetadataUseOnly: Int)(implicit p: Parameters) extends HellaCache(staticIdForMetadataUseOnly)(p) {
   override lazy val module = new NonBlockingDCacheModule(this)
   override def getOMSRAMs(): Seq[OMSRAM] = Nil // this is just a dummy value and that we need to eventually fix it
 }
diff --git a/src/main/scala/subsystem/Configs.scala b/src/main/scala/subsystem/Configs.scala
index 78ed31dcf18..20fd3dd455d 100644
--- a/src/main/scala/subsystem/Configs.scala
+++ b/src/main/scala/subsystem/Configs.scala
@@ -37,7 +37,8 @@ class BaseSubsystemConfig extends Config ((site, here, up) => {
     beatBytes = site(XLen)/8,
     blockBytes = site(CacheBlockBytes))
   // Additional device Parameters
-  case BootROMParams => BootROMParams(contentFileName = "./bootrom/bootrom.img")
+  case BootROMLocated(InSubsystem) => Some(BootROMParams(contentFileName = "./bootrom/bootrom.img"))
+  case SubsystemExternalResetVectorKey => false
   case DebugModuleKey => Some(DefaultDebugModuleParams(site(XLen)))
   case CLINTKey => Some(CLINTParams())
   case PLICKey => Some(PLICParams())
@@ -307,7 +308,7 @@ class WithFPUWithoutDivSqrt extends Config((site, here, up) => {
 })
 
 class WithBootROMFile(bootROMFile: String) extends Config((site, here, up) => {
-  case BootROMParams => up(BootROMParams, site).copy(contentFileName = bootROMFile)
+  case BootROMLocated(x) => up(BootROMLocated(x), site).map(_.copy(contentFileName = bootROMFile))
 })
 
 class WithSynchronousRocketTiles extends Config((site, here, up) => {
diff --git a/src/main/scala/subsystem/HasTiles.scala b/src/main/scala/subsystem/HasTiles.scala
index 22ba7eae6be..752cbc45a47 100644
--- a/src/main/scala/subsystem/HasTiles.scala
+++ b/src/main/scala/subsystem/HasTiles.scala
@@ -10,25 +10,48 @@ import freechips.rocketchip.devices.tilelink.{BasicBusBlocker, BasicBusBlockerPa
 import freechips.rocketchip.diplomacy._
 import freechips.rocketchip.diplomaticobjectmodel.logicaltree.{LogicalModuleTree}
 import freechips.rocketchip.interrupts._
-import freechips.rocketchip.tile.{BaseTile, LookupByHartIdImpl, TileParams, HasExternallyDrivenTileConstants, InstantiableTileParams, PriorityMuxHartIdFromSeq}
+import freechips.rocketchip.tile.{BaseTile, LookupByHartIdImpl, TileParams, InstantiableTileParams, MaxHartIdBits}
 import freechips.rocketchip.tilelink._
 import freechips.rocketchip.util._
 
 /** Entry point for Config-uring the presence of Tiles */
 case class TilesLocated(loc: HierarchicalLocation) extends Field[Seq[CanAttachTile]](Nil)
 
+/** Whether to add timing-closure registers along the path of the hart id
+  * as it propagates through the subsystem and into the tile.
+  *
+  * These are typically only desirable when a dynamically programmable prefix is being combined
+  * with the static hart id via [[freechips.rocketchip.subsystem.HasTiles.tileHartIdNexusNode]].
+  */
+case object InsertTimingClosureRegistersOnHartIds extends Field[Boolean](false)
+
+/** Whether per-tile hart ids are going to be driven as inputs into the subsystem,
+  * and if so, what their width should be.
+  */
+case object SubsystemExternalHartIdWidthKey extends Field[Option[Int]](None)
+
+/** Whether per-tile reset vectors are going to be driven as inputs into the subsystem.
+  *
+  * Unlike the hart ids, the reset vector width is determined by the sinks within the tiles,
+  * based on the size of the address map visible to the tiles.
+  */
+case object SubsystemExternalResetVectorKey extends Field[Boolean](true)
+
 /** An interface for describing the parameteization of how Tiles are connected to interconnects */
 trait TileCrossingParamsLike {
+  /** The type of clock crossing that should be inserted at the tile boundary. */
   val crossingType: ClockCrossingType
+  /** Parameters describing the contents and behavior of the point where the tile is attached as an interconnect master. */
   val master: TilePortParamsLike
+  /** Parameters describing the contents and behavior of the point where the tile is attached as an interconnect slave. */
   val slave: TilePortParamsLike
 }
 
 /** An interface for describing the parameterization of how a particular tile port is connected to an interconnect */
 trait TilePortParamsLike {
-  // the subnetwork location of the interconnect to which this tile port should be connected
+  /** The subnetwork location of the interconnect to which this tile port should be connected. */
   def where: TLBusWrapperLocation
-  // allows port-specific adapters to be injected into the interconnect side of the attachment point
+  /** Allows port-specific adapters to be injected into the interconnect side of the attachment point. */
   def injectNode(context: Attachable)(implicit p: Parameters): TLNode
 }
 
@@ -41,7 +64,7 @@ case class TileMasterPortParams(
   where: TLBusWrapperLocation = SBUS
 ) extends TilePortParamsLike {
   def injectNode(context: Attachable)(implicit p: Parameters): TLNode = {
-    (TLBuffer(buffers) :=* cork.map { u => TLCacheCork(unsafe = u) } .getOrElse { TLTempNode() })
+    (TLBuffer.chainNode(buffers) :=* cork.map { u => TLCacheCork(unsafe = u) } .getOrElse { TLTempNode() })
   }
 }
 
@@ -62,8 +85,8 @@ case class TileSlavePortParams(
       .map { bbbp =>
         val blocker = LazyModule(new BasicBusBlocker(bbbp))
         blockerBus.coupleTo("tile_slave_port_bus_blocker") { blocker.controlNode := TLFragmenter(blockerBus) := _ }
-        blocker.node :*= TLBuffer(buffers)
-      } .getOrElse { TLBuffer(buffers) }
+        blocker.node :*= TLBuffer.chainNode(buffers)
+      } .getOrElse { TLBuffer.chainNode(buffers) }
   }
 }
 
@@ -75,7 +98,7 @@ trait HasTileInterruptSources
   with CanHavePeripheryCLINT
   with HasPeripheryDebug
 { this: BaseSubsystem => // TODO ideally this bound would be softened to LazyModule
-  // meipNode is used to create a single bit subsystem input in Configs without a PLIC
+  /** meipNode is used to create a single bit subsystem input in Configs without a PLIC */
   val meipNode = p(PLICKey) match {
     case Some(_) => None
     case None    => Some(IntNexusNode(
@@ -86,7 +109,76 @@ trait HasTileInterruptSources
   }
 }
 
+/** These are sources of "constants" that are driven into the tile.
+  * 
+  * While they are not expected to change dyanmically while the tile is executing code,
+  * they may be either tied to a contant value or programmed during boot or reset.
+  * They need to be instantiated before tiles are attached within the subsystem containing them.
+  */
+trait HasTileInputConstants extends InstantiatesTiles { this: BaseSubsystem =>
+  /** tileHartIdNode is used to collect publishers and subscribers of hartids. */
+  val tileHartIdNode = BundleBridgeEphemeralNode[UInt]()
+
+  /** tileHartIdNexusNode is a BundleBridgeNexus that collects dynamic hart prefixes.
+    *
+    *   Each "prefix" input is actually the same full width as the outer hart id; the expected usage
+    *   is that each prefix source would set only some non-overlapping portion of the bits to non-zero values.
+    *   This node orReduces them, and further combines the reduction with the static ids assigned to each tile,
+    *   producing a unique, dynamic hart id for each tile.
+    *
+    *   If p(InsertTimingClosureRegistersOnHartIds) is set, the input and output values are registered.
+    *
+    *   The output values are [[dontTouch]]'d to prevent constant propagation from pulling the values into
+    *   the tiles if they are constant, which would ruin deduplication of tiles that are otherwise homogeneous.
+    */
+  val tileHartIdNexusNode = BundleBridgeNexus[UInt](
+    inputFn = BundleBridgeNexus.orReduction[UInt](registered = p(InsertTimingClosureRegistersOnHartIds)) _,
+    outputFn = (prefix: UInt, n: Int) =>  Seq.tabulate(n) { i =>
+      val y = dontTouch(prefix | hartIdList(i).U(p(MaxHartIdBits).W))
+      if (p(InsertTimingClosureRegistersOnHartIds)) BundleBridgeNexus.safeRegNext(y) else y
+    },
+    default = Some(() => 0.U(p(MaxHartIdBits).W)),
+    inputRequiresOutput = true // guard against this being driven but then ignored in tileHartIdIONodes below
+  )
+  // TODO: Replace the DebugModuleHartSelFuncs config key with logic to consume the dynamic hart IDs
+
+  /** tileResetVectorNode is used to collect publishers and subscribers of tile reset vector addresses. */
+  val tileResetVectorNode = BundleBridgeEphemeralNode[UInt]()
+
+  /** tileResetVectorNexusNode is a BundleBridgeNexus that accepts a single reset vector source, and broadcasts it to all tiles. */
+  val tileResetVectorNexusNode = BundleBroadcast[UInt](
+    inputRequiresOutput = true // guard against this being driven but ignored in tileResetVectorIONodes below
+  )
+
+  /** tileHartIdIONodes may generate subsystem IOs, one per tile, allowing the parent to assign unique hart ids.
+    *
+    *   Or, if such IOs are not configured to exist, tileHartIdNexusNode is used to supply an id to each tile.
+    */
+  val tileHartIdIONodes: Seq[BundleBridgeSource[UInt]] = p(SubsystemExternalHartIdWidthKey) match {
+    case Some(w) => Seq.fill(tiles.size) {
+      val hartIdSource = BundleBridgeSource(() => UInt(w.W))
+      tileHartIdNode := hartIdSource
+      hartIdSource
+    }
+    case None => { tileHartIdNode :*= tileHartIdNexusNode; Nil }
+  }
+
+  /** tileResetVectorIONodes may generate subsystem IOs, one per tile, allowing the parent to assign unique reset vectors.
+    *
+    *   Or, if such IOs are not configured to exist, tileResetVectorNexusNode is used to supply a single reset vector to every tile.
+    */
+  val tileResetVectorIONodes: Seq[BundleBridgeSource[UInt]] = p(SubsystemExternalResetVectorKey) match {
+    case true => Seq.fill(tiles.size) {
+      val resetVectorSource = BundleBridgeSource[UInt]()
+      tileResetVectorNode := resetVectorSource
+      resetVectorSource
+    }
+    case false => { tileResetVectorNode :*= tileResetVectorNexusNode; Nil }
+  }
+}
+
 /** These are sinks of notifications that are driven out from the tile.
+  *
   * They need to be instantiated before tiles are attached to the subsystem containing them.
   */
 trait HasTileNotificationSinks { this: LazyModule =>
@@ -103,33 +195,8 @@ trait HasTileNotificationSinks { this: LazyModule =>
   tileCeaseSinkNode := tileCeaseXbarNode
 }
 
-/** HasTiles adds a Config-urable sequence of tiles of any type
-  *   to the subsystem class into which it is mixed.
-  */
-trait HasTiles extends HasCoreMonitorBundles with DefaultTileContextType
-{ this: BaseSubsystem => // TODO: ideally this bound would be softened to Attachable
-  implicit val p: Parameters
-
-  // Actually instantiate all tiles, in order based on statically-assigned hartids
-  val tileAttachParams: Seq[CanAttachTile] = p(TilesLocated(location)).sortBy(_.tileParams.hartId)
-  val tiles: Seq[BaseTile] = tileAttachParams.map(_.instantiate(p))
-
-  // Helper functions for accessing certain parameters the are popular to refer to in subsystem code
-  val tileParams: Seq[TileParams] = tileAttachParams.map(_.tileParams)
-  val tileCrossingTypes = tileAttachParams.map(_.crossingParams.crossingType)
-  def nTiles: Int = tileAttachParams.size
-  def hartIdList: Seq[Int] = tileParams.map(_.hartId)
-  def localIntCounts: Seq[Int] = tileParams.map(_.core.nLocalInterrupts)
-
-  require(hartIdList.distinct.size == tiles.size, s"Every tile must be statically assigned a unique id, but got:\n${hartIdList}")
-
-  // connect all the tiles to interconnect attachment points made available in this subsystem context
-  tileAttachParams.zip(tiles).foreach { case (params, t) =>
-    params.connect(t.asInstanceOf[params.TileType], this.asInstanceOf[params.TileContextType])
-  }
-}
-
 /** Most tile types require only these traits in order for their standardized connect functions to apply.
+  *
   *    BaseTiles subtypes with different needs can extend this trait to provide themselves with
   *    additional external connection points.
   */
@@ -137,9 +204,11 @@ trait DefaultTileContextType
   extends Attachable
   with HasTileInterruptSources
   with HasTileNotificationSinks
+  with HasTileInputConstants
 { this: BaseSubsystem => } // TODO: ideally this bound would be softened to LazyModule
 
 /** Standardized interface by which parameterized tiles can be attached to contexts containing interconnect resources.
+  *
   *   Sub-classes of this trait can optionally override the individual connect functions in order to specialize
   *   their attachment behaviors, but most use cases should be be handled simply by changing the implementation
   *   of the injectNode functions in crossingParams.
@@ -151,21 +220,22 @@ trait CanAttachTile {
   def crossingParams: TileCrossingParamsLike
   def lookup: LookupByHartIdImpl
 
-  // narrow waist through which all tiles are intended to pass while being instantiated
+  /** Narrow waist through which all tiles are intended to pass while being instantiated. */
   def instantiate(implicit p: Parameters): TileType = {
     val tile = LazyModule(tileParams.instantiate(crossingParams, lookup))
     tile
   }
 
-  // a default set of connections that need to occur for most tile types
+  /** A default set of connections that need to occur for most tile types */
   def connect(tile: TileType, context: TileContextType): Unit = {
     connectMasterPorts(tile, context)
     connectSlavePorts(tile, context)
     connectInterrupts(tile, context)
+    connectInputConstants(tile, context)
     LogicalModuleTree.add(context.logicalTreeNode, tile.logicalTreeNode)
   }
 
-  // connect the port where the tile is the master to a TileLink interconnect
+  /** Connect the port where the tile is the master to a TileLink interconnect. */
   def connectMasterPorts(tile: TileType, context: Attachable): Unit = {
     implicit val p = context.p
     val dataBus = context.locateTLBusWrapper(crossingParams.master.where)
@@ -174,7 +244,7 @@ trait CanAttachTile {
     }
   }
 
-  // connect the port where the tile is the slave to a TileLink interconnect
+  /** Connect the port where the tile is the slave to a TileLink interconnect. */
   def connectSlavePorts(tile: TileType, context: Attachable): Unit = {
     implicit val p = context.p
     DisableMonitors { implicit p =>
@@ -185,7 +255,7 @@ trait CanAttachTile {
     }
   }
 
-  // connect the various interrupt and notification wires going to and from the tile
+  /** Connect the various interrupt and notification wires going to and from the tile. */
   def connectInterrupts(tile: TileType, context: TileContextType): Unit = {
     implicit val p = context.p
     // NOTE: The order of calls to := matters! They must match how interrupts
@@ -234,20 +304,56 @@ trait CanAttachTile {
     context.tileWFIXbarNode :=* tile.wfiNode
     context.tileCeaseXbarNode :=* tile.ceaseNode
   }
+
+  /** Connect the input to the tile that are assumed to be constant during normal operation. */
+  def connectInputConstants(tile: TileType, context: TileContextType): Unit = {
+    implicit val p = context.p
+    tile.hartIdNode := context.tileHartIdNode
+    tile.resetVectorNode := context.tileResetVectorNode
+  }
 }
 
-/** Provides some Chisel connectivity to certain tile IOs */
-trait HasTilesModuleImp extends LazyModuleImp with HasPeripheryDebugModuleImp {
-  val outer: HasTiles with HasTileInterruptSources
+/** InstantiatesTiles adds a Config-urable sequence of tiles of any type
+  *   to the subsystem class into which it is mixed.
+  */
+trait InstantiatesTiles { this: BaseSubsystem =>
+  /** Record the order in which to instantiate all tiles, based on statically-assigned ids.
+    *
+    * Note that these ids, which are often used as the tiles' default hartid input,
+    * may or may not be those actually reflected at runtime in e.g. the $mhartid CSR
+    */
+  val tileAttachParams: Seq[CanAttachTile] = p(TilesLocated(location)).sortBy(_.tileParams.hartId)
+
+  /** The actual list of instantiated tiles in this subsystem. */
+  val tiles: Seq[BaseTile] = tileAttachParams.map(_.instantiate(p))
+
+  // Helper functions for accessing certain parameters that are popular to refer to in subsystem code
+  val tileParams: Seq[TileParams] = tileAttachParams.map(_.tileParams)
+  val tileCrossingTypes = tileAttachParams.map(_.crossingParams.crossingType)
+  def nTiles: Int = tileAttachParams.size
+  def hartIdList: Seq[Int] = tileParams.map(_.hartId)
+  def localIntCounts: Seq[Int] = tileParams.map(_.core.nLocalInterrupts)
+
+  require(hartIdList.distinct.size == tiles.size, s"Every tile must be statically assigned a unique id, but got:\n${hartIdList}")
+}
+
+/** HasTiles instantiates and also connects a Config-urable sequence of tiles of any type to subsystem interconnect resources. */
+trait HasTiles extends InstantiatesTiles with HasCoreMonitorBundles with DefaultTileContextType
+{ this: BaseSubsystem => // TODO: ideally this bound would be softened to Attachable
+  implicit val p: Parameters
 
-  def resetVectorBits: Int = {
-    // Consider using the minimum over all widths, rather than enforcing homogeneity
-    val vectors = outer.tiles.map(_.module.constants.reset_vector)
-    require(vectors.tail.forall(_.getWidth == vectors.head.getWidth))
-    vectors.head.getWidth
+  // connect all the tiles to interconnect attachment points made available in this subsystem context
+  tileAttachParams.zip(tiles).foreach { case (params, t) =>
+    params.connect(t.asInstanceOf[params.TileType], this.asInstanceOf[params.TileContextType])
   }
+}
+
+/** Provides some Chisel connectivity to certain tile IOs */
+trait HasTilesModuleImp extends LazyModuleImp with HasPeripheryDebugModuleImp {
+  val outer: HasTiles with HasTileInterruptSources with HasTileInputConstants
 
-  val tile_inputs = outer.tiles.map(_.module.constants)
+  val reset_vector = outer.tileResetVectorIONodes.zipWithIndex.map { case (n, i) => n.makeIO(s"reset_vector_$i") }
+  val tile_hartids = outer.tileHartIdIONodes.zipWithIndex.map { case (n, i) => n.makeIO(s"tile_hartids_$i") }
 
   val meip = if(outer.meipNode.isDefined) Some(IO(Vec(outer.meipNode.get.out.size, Bool()).asInput)) else None
   meip.foreach { m =>
diff --git a/src/main/scala/subsystem/ResetVector.scala b/src/main/scala/subsystem/ResetVector.scala
deleted file mode 100644
index 56366d17582..00000000000
--- a/src/main/scala/subsystem/ResetVector.scala
+++ /dev/null
@@ -1,11 +0,0 @@
-// See LICENSE.SiFive for license details.
-
-package freechips.rocketchip.subsystem
-
-import Chisel._
-
-/** A single place for all tiles to find out the reset vector */
-trait HasResetVectorWire {
-  def resetVectorBits: Int
-  val global_reset_vector = Wire(UInt(width = resetVectorBits))
-}
diff --git a/src/main/scala/subsystem/RocketSubsystem.scala b/src/main/scala/subsystem/RocketSubsystem.scala
index e26ea2361c9..ebbd2237e87 100644
--- a/src/main/scala/subsystem/RocketSubsystem.scala
+++ b/src/main/scala/subsystem/RocketSubsystem.scala
@@ -12,8 +12,6 @@ import freechips.rocketchip.diplomaticobjectmodel.logicaltree._
 import freechips.rocketchip.diplomaticobjectmodel.model._
 import freechips.rocketchip.tile._
 
-case object HartPrefixKey extends Field[Boolean](false)
-
 case class RocketCrossingParams(
   crossingType: ClockCrossingType = SynchronousCrossing(),
   master: TileMasterPortParams = TileMasterPortParams(),
@@ -37,38 +35,9 @@ trait HasRocketTiles extends HasTiles { this: BaseSubsystem =>
   }).toList
 }
 
-// Field for specifying MaskROM addition to subsystem
-case object PeripheryMaskROMKey extends Field[Seq[MaskROMParams]](Nil)
-
-class RocketSubsystem(implicit p: Parameters) extends BaseSubsystem
-    with HasRocketTiles {
-        
-  // add Mask ROM devices
-  val maskROMs = p(PeripheryMaskROMKey).map { MaskROM.attach(_, cbus) }
-
-  val hartPrefixNode = if (p(HartPrefixKey)) {
-    Some(BundleBroadcast[UInt](registered = true))
-  } else {
-    None
-  }
-
-  val hartPrefixes = hartPrefixNode.map { hpn => Seq.fill(tiles.size) {
-   val hps = BundleBridgeSink[UInt]()
-   hps := hpn
-   hps
-  } }.getOrElse(Nil)
-
+class RocketSubsystem(implicit p: Parameters) extends BaseSubsystem with HasRocketTiles {
   override lazy val module = new RocketSubsystemModuleImp(this)
 }
 
 class RocketSubsystemModuleImp[+L <: RocketSubsystem](_outer: L) extends BaseSubsystemModuleImp(_outer)
-    with HasResetVectorWire
-    with HasTilesModuleImp {
-
-  for (i <- 0 until outer.tiles.size) {
-    val wire = tile_inputs(i)
-    val prefix = outer.hartPrefixes.lift(i).map(_.bundle).getOrElse(UInt(0))
-    wire.hartid := prefix | UInt(outer.hartIdList(i))
-    wire.reset_vector := global_reset_vector
-  }
-}
+    with HasTilesModuleImp
diff --git a/src/main/scala/system/ExampleRocketSystem.scala b/src/main/scala/system/ExampleRocketSystem.scala
index 9e06f95e453..64958ca19ec 100644
--- a/src/main/scala/system/ExampleRocketSystem.scala
+++ b/src/main/scala/system/ExampleRocketSystem.scala
@@ -16,12 +16,16 @@ class ExampleRocketSystem(implicit p: Parameters) extends RocketSubsystem
     with CanHaveMasterAXI4MemPort
     with CanHaveMasterAXI4MMIOPort
     with CanHaveSlaveAXI4Port
-    with HasPeripheryBootROM {
+{
+  // optionally add ROM devices
+  // Note that setting BootROMLocated will override the reset_vector for all tiles
+  val bootROM  = p(BootROMLocated(location)).map { BootROM.attach(_, this, CBUS) }
+  val maskROMs = p(MaskROMLocated(location)).map { MaskROM.attach(_, this, CBUS) }
+
   override lazy val module = new ExampleRocketSystemModuleImp(this)
 }
 
 class ExampleRocketSystemModuleImp[+L <: ExampleRocketSystem](_outer: L) extends RocketSubsystemModuleImp(_outer)
     with HasRTCModuleImp
     with HasExtInterruptsModuleImp
-    with HasPeripheryBootROMModuleImp
     with DontTouch
diff --git a/src/main/scala/tile/BaseTile.scala b/src/main/scala/tile/BaseTile.scala
index 5bb6b2582b4..874f7c849e5 100644
--- a/src/main/scala/tile/BaseTile.scala
+++ b/src/main/scala/tile/BaseTile.scala
@@ -17,8 +17,6 @@ import freechips.rocketchip.util._
 
 case object TileVisibilityNodeKey extends Field[TLEphemeralNode]
 case object TileKey extends Field[TileParams]
-case object ResetVectorBits extends Field[Int]
-case object MaxHartIdBits extends Field[Int]
 case object LookupByHartId extends Field[LookupByHartIdImpl]
 
 trait TileParams {
@@ -66,10 +64,20 @@ trait HasNonDiplomaticTileParameters {
     res
   }
   def asIdBits: Int = p(ASIdBits)
-  def maxPAddrBits: Int = xLen match { case 32 => 34; case 64 => 56 }
+  lazy val maxPAddrBits: Int = {
+    require(xLen == 32 || xLen == 64, s"Only XLENs of 32 or 64 are supported, but got $xLen")
+    xLen match { case 32 => 34; case 64 => 56 }
+  }
 
-  def hartId: Int = tileParams.hartId
-  def hartIdLen: Int = p(MaxHartIdBits)
+  /** Use staticIdForMetadataUseOnly to emit information during the build or identify a component to diplomacy.
+    *
+    *   Including it in a constructed Chisel circuit by converting it to a UInt will prevent
+    *   Chisel/FIRRTL from being able to deduplicate tiles that are otherwise homogeneous,
+    *   a property which is important for hierarchical place & route flows.
+    */
+  def staticIdForMetadataUseOnly: Int = tileParams.hartId
+  @deprecated("use hartIdSinkNode.bundle or staticIdForMetadataUseOnly", "rocket-chip 1.3")
+  def hartId: Int = staticIdForMetadataUseOnly
 
   def cacheBlockBytes = p(CacheBlockBytes)
   def lgCacheBlockBytes = log2Up(cacheBlockBytes)
@@ -132,7 +140,11 @@ trait HasNonDiplomaticTileParameters {
   */
 trait HasTileParameters extends HasNonDiplomaticTileParameters {
   protected def tlBundleParams = p(TileVisibilityNodeKey).edges.out.head.bundle
-  def paddrBits: Int = tlBundleParams.addressBits
+  lazy val paddrBits: Int = {
+    val bits = tlBundleParams.addressBits
+    require(bits <= maxPAddrBits, s"Requested $bits paddr bits, but since xLen is $xLen only $maxPAddrBits will fit")
+    bits
+  }
   def vaddrBits: Int =
     if (usingVM) {
       val v = maxSVAddrBits
@@ -147,8 +159,6 @@ trait HasTileParameters extends HasNonDiplomaticTileParameters {
   def ppnBits: Int = paddrBits - pgIdxBits
   def vpnBitsExtended: Int = vpnBits + (vaddrBits < xLen).toInt
   def vaddrBitsExtended: Int = vpnBitsExtended + pgIdxBits
-
-  def resetVectorLen: Int = paddrBits
 }
 
 /** Base class for all Tiles that use TileLink */
@@ -181,16 +191,30 @@ abstract class BaseTile private (val crossing: ClockCrossingType, q: Parameters)
   protected val tlSlaveXbar = LazyModule(new TLXbar)
   protected val intXbar = LazyModule(new IntXbar)
 
+  private val hartid = BundleBridgeIdentityNode[UInt]()
+  val hartIdNode: BundleBridgeNode[UInt] = BundleBroadcast[UInt](registered = p(InsertTimingClosureRegistersOnHartIds)) := hartid
+  val hartIdSinkNode = BundleBridgeSink[UInt]()
+  hartIdSinkNode := hartIdNode
+
+  private val reset_vector = BundleBridgeIdentityNode[UInt]()
+  val resetVectorNode: BundleBridgeNode[UInt] = BundleBroadcast[UInt]() := reset_vector
+  val resetVectorSinkNode = BundleBridgeSink[UInt](Some(() => UInt(visiblePhysAddrBits.W)))
+  resetVectorSinkNode := resetVectorNode
+
   // Node for legacy instruction trace from core
   val traceSourceNode = BundleBridgeSource(() => Vec(tileParams.core.retireWidth, new TracedInstruction()))
   val traceNode = BundleBroadcast[Vec[TracedInstruction]](Some("trace"))
   traceNode := traceSourceNode
 
   // Trace sideband signals into core
-  val traceAuxNode = BundleBridgeNexus[TraceAux]()
+  val traceAuxNode = BundleBridgeNexus[TraceAux](default = Some(() => {
+    val aux = Wire(new TraceAux)
+    aux.stall  := false.B
+    aux.enable := false.B
+    aux
+  }))
   val traceAuxSinkNode = BundleBridgeSink[TraceAux]()
-  val traceAuxDefaultNode = BundleBridgeSource(() => new TraceAux)
-  traceAuxSinkNode := traceAuxNode := traceAuxDefaultNode
+  traceAuxSinkNode := traceAuxNode
 
   // Node for instruction trace conforming to RISC-V Processor Trace spec V1.0
   val traceCoreSourceNode = BundleBridgeSource(() => new TraceCoreInterface(new TraceCoreParams()))
@@ -215,6 +239,7 @@ abstract class BaseTile private (val crossing: ClockCrossingType, q: Parameters)
 
   val visibilityNode = p(TileVisibilityNodeKey)
   protected def visibleManagers = visibilityNode.edges.out.flatMap(_.manager.managers)
+  protected def visiblePhysAddrBits = visibilityNode.edges.out.head.bundle.addressBits
   def unifyManagers: List[TLManagerParameters] = ManagerUnification(visibleManagers)
 
   // Find resource labels for all the outward caches
@@ -265,24 +290,4 @@ abstract class BaseTile private (val crossing: ClockCrossingType, q: Parameters)
   this.suggestName(tileParams.name)
 }
 
-abstract class BaseTileModuleImp[+L <: BaseTile](val outer: L) extends LazyModuleImp(outer) with HasTileParameters {
-
-  require(xLen == 32 || xLen == 64)
-  require(paddrBits <= maxPAddrBits, "asked for " + paddrBits + " paddr bits, but since xLen is " + xLen + ", only " + maxPAddrBits + " can fit")
-  require(resetVectorLen <= xLen)
-  require(resetVectorLen <= vaddrBitsExtended)
-  require (log2Up(hartId + 1) <= hartIdLen, s"p(MaxHartIdBits) of $hartIdLen is not enough for hartid $hartId")
-
-  outer.traceAuxDefaultNode.bundle.stall := false.B
-  outer.traceAuxDefaultNode.bundle.enable := false.B
-
-  val constants = IO(new TileInputConstants)
-}
-
-/** Some other non-tilelink but still standard inputs */
-trait HasExternallyDrivenTileConstants extends Bundle with HasTileParameters {
-  val hartid = UInt(INPUT, hartIdLen)
-  val reset_vector = UInt(INPUT, resetVectorLen)
-}
-
-class TileInputConstants(implicit val p: Parameters) extends Bundle with HasExternallyDrivenTileConstants
+abstract class BaseTileModuleImp[+L <: BaseTile](val outer: L) extends LazyModuleImp(outer) with HasTileParameters
diff --git a/src/main/scala/tile/Core.scala b/src/main/scala/tile/Core.scala
index 118dc9a72c9..114443ea3ea 100644
--- a/src/main/scala/tile/Core.scala
+++ b/src/main/scala/tile/Core.scala
@@ -9,6 +9,7 @@ import freechips.rocketchip.rocket._
 import freechips.rocketchip.util._
 
 case object XLen extends Field[Int]
+case object MaxHartIdBits extends Field[Int]
 
 // These parameters can be varied per-core
 trait CoreParams {
@@ -103,6 +104,14 @@ trait HasCoreParameters extends HasTileParameters {
     require(vMemDataBits >= eLen && vLen % vMemDataBits == 0, s"vMemDataBits ($vMemDataBits) must divide vLen ($vLen) and be no less than eLen ($eLen)")
   }
 
+  lazy val hartIdLen: Int = p(MaxHartIdBits)
+  lazy val resetVectorLen: Int = {
+    val externalLen = paddrBits
+    require(externalLen <= xLen, s"External reset vector length ($externalLen) must be <= XLEN ($xLen)")
+    require(externalLen <= vaddrBitsExtended, s"External reset vector length ($externalLen) must be <= virtual address bit width ($vaddrBitsExtended)")
+    externalLen
+  }
+
   // Print out log of committed instructions and their writeback values.
   // Requires post-processing due to out-of-order writebacks.
   val enableCommitLog = false
@@ -121,7 +130,9 @@ class CoreInterrupts(implicit p: Parameters) extends TileInterrupts()(p) {
 
 trait HasCoreIO extends HasTileParameters {
   implicit val p: Parameters
-  val io = new CoreBundle()(p) with HasExternallyDrivenTileConstants {
+  val io = new CoreBundle()(p) {
+    val hartid = UInt(hartIdLen).asInput
+    val reset_vector = UInt(resetVectorLen).asInput
     val interrupts = new CoreInterrupts().asInput
     val imem  = new FrontendIO
     val dmem = new HellaCacheIO
diff --git a/src/main/scala/tile/RocketTile.scala b/src/main/scala/tile/RocketTile.scala
index e5a713445df..c2b8b5f56b5 100644
--- a/src/main/scala/tile/RocketTile.scala
+++ b/src/main/scala/tile/RocketTile.scala
@@ -11,7 +11,7 @@ import freechips.rocketchip.diplomaticobjectmodel.logicaltree.{DCacheLogicalTree
 import freechips.rocketchip.interrupts._
 import freechips.rocketchip.tilelink._
 import freechips.rocketchip.rocket._
-import freechips.rocketchip.subsystem.{SubsystemResetSchemeKey, ResetSynchronous, HartPrefixKey, TileCrossingParamsLike}
+import freechips.rocketchip.subsystem.{SubsystemResetSchemeKey, ResetSynchronous, TileCrossingParamsLike}
 import freechips.rocketchip.util._
 
 case class RocketTileParams(
@@ -103,7 +103,7 @@ class RocketTile private(
   }
 
   ResourceBinding {
-    Resource(cpuDevice, "reg").bind(ResourceAddress(hartId))
+    Resource(cpuDevice, "reg").bind(ResourceAddress(staticIdForMetadataUseOnly))
   }
 
   override lazy val module = new RocketTileModuleImp(this)
@@ -154,16 +154,13 @@ class RocketTileModuleImp(outer: RocketTile) extends BaseTileModuleImp(outer)
     beu.module.io.errors.icache := outer.frontend.module.io.errors
   }
 
-  // Pass through various external constants and reports
+  // Pass through various external constants and reports that were bundle-bridged into the tile
   outer.traceSourceNode.bundle <> core.io.trace
   core.io.traceStall := outer.traceAuxSinkNode.bundle.stall
   outer.bpwatchSourceNode.bundle <> core.io.bpwatch
-  outer.frontend.module.io.reset_vector := constants.reset_vector
-
-  def regHart(x: UInt): UInt = if (p(HartPrefixKey)) RegNext(x) else x
-  core.io.hartid := regHart(constants.hartid)
-  outer.dcache.module.io.hartid := regHart(constants.hartid)
-  outer.frontend.module.io.hartid := regHart(constants.hartid)
+  core.io.hartid := outer.hartIdSinkNode.bundle
+  require(core.io.hartid.getWidth >= outer.hartIdSinkNode.bundle.getWidth,
+    s"core hartid wire (${core.io.hartid.getWidth}b) truncates external hartid wire (${outer.hartIdSinkNode.bundle.getWidth}b)")
 
   // Connect the core pipeline to other intra-tile modules
   outer.frontend.module.io.cpu <> core.io.imem
diff --git a/src/main/scala/util/package.scala b/src/main/scala/util/package.scala
index aabee6525e9..f6e2d610dec 100644
--- a/src/main/scala/util/package.scala
+++ b/src/main/scala/util/package.scala
@@ -69,6 +69,11 @@ package object util {
 
   implicit class DataToAugmentedData[T <: Data](val x: T) extends AnyVal {
     def holdUnless(enable: Bool): T = Mux(enable, x, RegEnable(x, enable))
+
+    def getElements: Seq[Element] = x match {
+      case e: Element => Seq(e)
+      case a: Aggregate => a.getElements.flatMap(_.getElements)
+    }
   }
 
   implicit class SeqMemToAugmentedSeqMem[T <: Data](val x: SeqMem[T]) extends AnyVal {