Support multi-run of binaries

common.mk

@@ -45,6 +45,9 @@ HELP_COMMANDS += \
 "   run-binary                  = run [./$(shell basename $(sim))] and log instructions to file" \
 "   run-binary-fast             = run [./$(shell basename $(sim))] and don't log instructions" \
 "   run-binary-debug            = run [./$(shell basename $(sim_debug))] and log instructions and waveform to files" \
+"   run-binaries                = run [./$(shell basename $(sim))] and log instructions to file" \
+"   run-binaries-fast           = run [./$(shell basename $(sim))] and don't log instructions" \
+"   run-binaries-debug          = run [./$(shell basename $(sim_debug))] and log instructions and waveform to files" \
 "   verilog                     = generate intermediate verilog files from chisel elaboration and firrtl passes" \
 "   firrtl                      = generate intermediate firrtl files from chisel elaboration" \
 "   run-tests                   = run all assembly and benchmark tests" \
@@ -287,49 +290,82 @@ verilog: $(sim_common_files)
 # helper rules to run simulations
 #########################################################################################
 .PHONY: run-binary run-binary-fast run-binary-debug run-fast
+	%.check-exists check-binary check-binaries
 
 check-binary:
 ifeq (,$(BINARY))
 	$(error BINARY variable is not set. Set it to the simulation binary)
 endif
-ifneq (none,$(BINARY))
-ifeq ("$(wildcard $(BINARY))","")
-	$(error BINARY=$(BINARY) not found)
-endif
+
+check-binaries:
+ifeq (,$(BINARIES))
+	$(error BINARIES variable is not set. Set it to the list of simulation binaries to run)
 endif
 
+%.check-exists:
+	if [ "$*" != "none" ] && [ ! -f "$*" ]; then printf "\n\nBinary $* not found\n\n"; exit 1; fi
+
 # allow you to override sim prereq
 ifeq (,$(BREAK_SIM_PREREQ))
 SIM_PREREQ = $(sim)
 SIM_DEBUG_PREREQ = $(sim_debug)
 endif
 
+# Function to generate the loadmem flag. First arg is the binary
+ifeq ($(LOADMEM),1)
+# If LOADMEM=1, assume BINARY is the loadmem elf
+get_loadmem_flag = +loadmem=$(1)
+else ifneq ($(LOADMEM),)
+# Otherwise, assume the variable points to an elf file
+get_loadmem_flag = +loadmem=$(LOADMEM)
+endif
+
+ifneq ($(LOADARCH),)
+get_loadarch_flag = +loadarch=$(subst mem.elf,loadarch,$(1))
+endif
+
+# get the output path base name for simulation outputs, First arg is the binary
+get_sim_out_name = $(output_dir)/$(call get_out_name,$(1))
+# sim flags that are common to run-binary/run-binary-fast/run-binary-debug
+get_common_sim_flags = $(SIM_FLAGS) $(EXTRA_SIM_FLAGS) $(SEED_FLAG) $(call get_loadmem_flag,$(1)) $(call get_loadarch_flag,$(1))
+
+.PHONY: %.run %.run.debug %.run.fast
+
 # run normal binary with hardware-logged insn dissassembly
-run-binary: $(SIM_PREREQ) check-binary | $(output_dir)
-	(set -o pipefail && $(NUMA_PREFIX) $(sim) $(PERMISSIVE_ON) $(SIM_FLAGS) $(EXTRA_SIM_FLAGS) $(SEED_FLAG) $(VERBOSE_FLAGS) $(PERMISSIVE_OFF) $(BINARY) </dev/null 2> >(spike-dasm > $(sim_out_name).out) | tee $(sim_out_name).log)
+run-binary: check-binary $(BINARY).run
+run-binaries: check-binaries $(addsuffix .run,$(BINARIES))
+
+%.run: %.check-exists $(SIM_PREREQ) | $(output_dir)
+	(set -o pipefail && $(NUMA_PREFIX) $(sim) $(PERMISSIVE_ON) $(call get_common_sim_flags,$*) $(VERBOSE_FLAGS) $(PERMISSIVE_OFF) $* </dev/null 2> >(spike-dasm > $(call get_sim_out_name,$*).out) | tee $(call get_sim_out_name,$*).log)
 
 # run simulator as fast as possible (no insn disassembly)
-run-binary-fast: $(SIM_PREREQ) check-binary | $(output_dir)
-	(set -o pipefail && $(NUMA_PREFIX) $(sim) $(PERMISSIVE_ON) $(SIM_FLAGS) $(EXTRA_SIM_FLAGS) $(SEED_FLAG) $(PERMISSIVE_OFF) $(BINARY) </dev/null | tee $(sim_out_name).log)
+run-binary-fast: check-binary $(BINARY).run.fast
+run-binaries-fast: check-binaries $(addsuffix .run.fast,$(BINARIES))
+
+%.run.fast: %.check-exists $(SIM_PREREQ) | $(output_dir)
+	(set -o pipefail && $(NUMA_PREFIX) $(sim) $(PERMISSIVE_ON) $(call get_common_sim_flags,$*) $(PERMISSIVE_OFF) $* </dev/null | tee $(call get_sim_out_name,$*).log)
 
 # run simulator with as much debug info as possible
-run-binary-debug: $(SIM_DEBUG_PREREQ) check-binary | $(output_dir)
-ifneq (none,$(BINARY))
-	riscv64-unknown-elf-objdump -D $(BINARY) > $(sim_out_name).dump
+run-binary-debug: check-binary $(BINARY).run.debug
+run-binaries-debug: check-binaries $(addsuffix .run.debug,$(BINARIES))
+
+%.run.debug: %.check-exists $(SIM_DEBUG_PREREQ) | $(output_dir)
+ifneq (none,$*)
+	riscv64-unknown-elf-objdump -D $* > $(call get_sim_out_name,$*).dump
 endif
-	(set -o pipefail && $(NUMA_PREFIX) $(sim_debug) $(PERMISSIVE_ON) $(SIM_FLAGS) $(EXTRA_SIM_FLAGS) $(SEED_FLAG) $(VERBOSE_FLAGS) $(WAVEFORM_FLAG) $(PERMISSIVE_OFF) $(BINARY) </dev/null 2> >(spike-dasm > $(sim_out_name).out) | tee $(sim_out_name).log)
+	(set -o pipefail && $(NUMA_PREFIX) $(sim_debug) $(PERMISSIVE_ON) $(call get_common_sim_flags,$*) $(VERBOSE_FLAGS) $(call get_waveform_flag,$(call get_sim_out_name,$*)) $(PERMISSIVE_OFF) $* </dev/null 2> >(spike-dasm > $(call get_sim_out_name,$*).out) | tee $(call get_sim_out_name,$*).log)
 
 run-fast: run-asm-tests-fast run-bmark-tests-fast
 
 #########################################################################################
 # helper rules to run simulator with fast loadmem
 # LEGACY - use LOADMEM=1 instead
 #########################################################################################
-run-binary-hex: run-binary
+run-binary-hex: $(BINARY).run
 run-binary-hex: override SIM_FLAGS += +loadmem=$(BINARY)
-run-binary-debug-hex: run-binary-debug
+run-binary-debug-hex: $(BINARY).run.debug
 run-binary-debug-hex: override SIM_FLAGS += +loadmem=$(BINARY)
-run-binary-fast-hex: run-binary-fast
+run-binary-fast-hex: $(BINARY).run.fast
 run-binary-fast-hex: override SIM_FLAGS += +loadmem=$(BINARY)
 
 #########################################################################################

sims/vcs/vcs.mk

@@ -5,9 +5,9 @@ HELP_SIMULATION_VARIABLES += \
 "   USE_VPD                = set to '1' to run VCS simulator emitting VPD instead of FSDB."
 
 ifndef USE_VPD
-WAVEFORM_FLAG=+fsdbfile=$(sim_out_name).fsdb
+get_waveform_flag=+fsdbfile=$(1).fsdb
 else
-WAVEFORM_FLAG=+vcdplusfile=$(sim_out_name).vpd
+get_waveform_flag=+vcdplusfile=$(1).vpd
 endif
 
 # If ntb_random_seed unspecified, vcs uses 1 as constant seed.

sims/verilator/Makefile

@@ -96,7 +96,7 @@ USE_FST ?= 0
 TRACING_OPTS := $(if $(filter $(USE_FST),0),\
 	                  --trace,--trace-fst --trace-threads 1)
 # TODO: consider renaming +vcdfile in TestDriver.v to +waveformfile (or similar)
-WAVEFORM_FLAG := +vcdfile=$(sim_out_name).$(if $(filter $(USE_FST),0),vcd,fst)
+get_waveform_flag = +vcdfile=$(1).$(if $(filter $(USE_FST),0),vcd,fst)
 
 #----------------------------------------------------------------------------------------
 # verilation configuration/optimization

sims/xcelium/xcelium.mk

@@ -1,5 +1,5 @@
 
-WAVEFORM_FLAG=+vcdfile=$(sim_out_name).vcd
+get_waveform_flag=+vcdfile=$(1).vcd
 
 # If ntb_random_seed unspecified, xcelium uses 1 as constant seed.
 # Set ntb_random_seed_automatic to actually get a random seed

variables.mk

@@ -25,6 +25,7 @@ HELP_PROJECT_VARIABLES = \
 
 HELP_SIMULATION_VARIABLES = \
 "   BINARY                 = riscv elf binary that the simulator will run when using the run-binary* targets" \
+"   BINARIES               = list of riscv elf binary that the simulator will run when using the run-binaries* targets" \
 "   LOADMEM                = riscv elf binary that should be loaded directly into simulated DRAM. LOADMEM=1 will load the BINARY elf" \
 "   LOADARCH               = path to a architectural checkpoint directory that should end in .loadarch/, for restoring from a checkpoint" \
 "   VERBOSE_FLAGS          = flags used when doing verbose simulation [$(VERBOSE_FLAGS)]" \
@@ -244,29 +245,21 @@ output_dir=$(sim_dir)/output/$(long_name)
 PERMISSIVE_ON=+permissive
 PERMISSIVE_OFF=+permissive-off
 BINARY ?=
+BINARIES ?=
 override SIM_FLAGS += +dramsim +dramsim_ini_dir=$(TESTCHIP_DIR)/src/main/resources/dramsim2_ini +max-cycles=$(timeout_cycles)
 VERBOSE_FLAGS ?= +verbose
-OUT_NAME ?= $(subst $() $(),_,$(notdir $(basename $(BINARY))))
+# get_out_name is a function, 1st argument is the binary
+get_out_name = $(subst $() $(),_,$(notdir $(basename $(1))))
 LOADMEM ?=
 LOADARCH ?=
 
 ifneq ($(LOADARCH),)
-override BINARY = $(LOADARCH)/mem.elf
-override OUT_NAME = $(shell basename $(LOADARCH))
+override BINARY = $(addsuffix /mem.elf,$(LOADARCH))
+override BINARIES = $(addsuffix /mem.elf,$(LOADARCH))
+override get_out_name = $(shell basename $(dir $(1)))
 override LOADMEM = 1
-override SIM_FLAGS += +loadarch=$(LOADARCH)/loadarch
 endif
 
-ifeq ($(LOADMEM),1)
-# If LOADMEM=1, assume BINARY is the loadmem elf
-override SIM_FLAGS += +loadmem=$(BINARY)
-else ifneq ($(LOADMEM),)
-# Otherwise, assume the variable points to an elf file
-override SIM_FLAGS += +loadmem=$(LOADMEM)
-endif
-
-sim_out_name = $(output_dir)/$(OUT_NAME)
-
 #########################################################################################
 # build output directory for compilation
 #########################################################################################

vlsi/power.mk

@@ -6,7 +6,7 @@ POWER_PAR_HIER_CONF = $(OBJ_DIR)/power-par-$(VLSI_TOP)-inputs.yml
 
 .PHONY: $(POWER_CONF) $(POWER_RTL_CONF) $(POWER_SYN_CONF) $(POWER_PAR_CONF) $(POWER_PAR_HIER_CONF)
 
-$(POWER_CONF): $(VLSI_RTL)
+$(POWER_CONF): $(VLSI_RTL) check-binary
 	mkdir -p $(dir $@)
 	echo "power.inputs:" > $@
 	echo "  top_module: $(VLSI_TOP)" >> $@
@@ -15,9 +15,9 @@ $(POWER_CONF): $(VLSI_RTL)
 ifneq ($(BINARY), )
 	echo "  waveforms: [" >> $@
 ifndef USE_VPD
-	echo "    '$(sim_out_name).fsdb'" >> $@
+	echo "    '$(call get_sim_out_name,$(BINARY)).fsdb'" >> $@
 else
-	echo "    '$(sim_out_name).vpd'" >> $@
+	echo "    '$(call get_sim_out_name,$(BINARY)).vpd'" >> $@
 endif
 	echo "  ]" >> $@
 endif

vlsi/sim.mk

@@ -4,7 +4,7 @@ SIM_TIMING_CONF = $(OBJ_DIR)/sim-timing-inputs.yml
 
 .PHONY: $(SIM_CONF) $(SIM_DEBUG_CONF) $(SIM_TIMING_CONF)
 
-$(SIM_CONF): $(sim_common_files)
+$(SIM_CONF): $(sim_common_files) check-binary
 	mkdir -p $(dir $@)
 	echo "sim.inputs:" > $@
 	echo "  top_module: $(VLSI_TOP)" >> $@
@@ -47,14 +47,14 @@ ifneq ($(BINARY), )
 endif
 	echo "  tb_dut: 'TestDriver.testHarness.$(VLSI_MODEL_DUT_NAME)'" >> $@
 
-$(SIM_DEBUG_CONF): $(sim_common_files)
+$(SIM_DEBUG_CONF): $(sim_common_files) check-binary
 	mkdir -p $(dir $@)
 	mkdir -p $(output_dir)
 	echo "sim.inputs:" > $@
 	echo "  defines: ['DEBUG']" >> $@
 	echo "  defines_meta: 'append'" >> $@
 	echo "  execution_flags:" >> $@
-	for x in $(VERBOSE_FLAGS) $(WAVEFORM_FLAG); do \
+	for x in $(VERBOSE_FLAGS) $(call get_waveform_flag,$(call get_sim_out_name,$(BINARY))); do \
 	  echo '    - "'$$x'"' >> $@; \
 	done
 	echo "  execution_flags_meta: 'append'" >> $@
@@ -65,9 +65,9 @@ ifndef USE_VPD
 	echo "  options:" >> $@
 	echo '    - "-kdb"' >> $@
 	echo "  options_meta: 'append'" >> $@
-	echo "sim.outputs.waveforms: ['$(sim_out_name).fsdb']" >> $@
+	echo "sim.outputs.waveforms: ['$(call get_sim_out_name,$(BINARY)).fsdb']" >> $@
 else
-	echo "sim.outputs.waveforms: ['$(sim_out_name).vpd']" >> $@
+	echo "sim.outputs.waveforms: ['$(call get_sim_out_name,$(BINARY)).vpd']" >> $@
 endif
 
 $(SIM_TIMING_CONF): $(sim_common_files)