.testing: Fix concurrency errors in tc4 rules

This patch fixes two issues in the preprocessing of tc4.

* ocean_hgrid.nc is marked as a dependency of gen_data
* Multiple ouputs are handled more safely in the Makefile

Expanding on the second point: We were directing one rule to produce two
output files, which resulted in the rule being run twice when invoked in
parallel (make -j).

This has been replaced with the recommended solution for handling
concurrent outputs: use one to generate both, and connect the second to
the first with a separate rule.

I have also generalized the `make` command in the .testing Makefile.

This should address (and hopefully fix) some intermittent errors in the
.testing build on Gaea.

.testing/Makefile

@@ -522,7 +522,7 @@ $(foreach c,$(CONFIGS),$(eval $(call CONFIG_DIM_RULE,$(c))))
 # NOTE: This only support tc4, but can be generalized over all tests.
 .PHONY: preproc
 preproc: tc4/Makefile
-	cd tc4 && make
+	cd tc4 && $(MAKE) LAUNCHER="$(MPIRUN)"
 
 tc4/Makefile: tc4/configure tc4/Makefile.in
 	cd $(@D) && ./configure || (cat config.log && false)

.testing/tc4/Makefile.in

@@ -4,22 +4,43 @@ FCFLAGS = @FCFLAGS@
 LDFLAGS = @LDFLAGS@
 LIBS = @LIBS@
 
-OUT = topog.nc ocean_hgrid.nc temp_salt_ic.nc sponge.nc
+LAUNCHER ?=
 
-all: $(OUT)
+OUT = ocean_hgrid.nc topog.nc temp_salt_ic.nc sponge.nc
 
-ocean_hgrid.nc topog.nc: gen_grid
-	./gen_grid
+# Since each program generates two outputs, we can only use one to track the
+# creation.  The second rule is used to indirectly re-invoke the first rule.
+#
+# Reference:
+# https://www.gnu.org/software/automake/manual/html_node/Multiple-Outputs.html
 
-temp_salt_ic.nc sponge.nc: gen_data
-	./gen_data
+# Program output
+all: ocean_hgrid.nc temp_salt_ic.nc
+
+ocean_hgrid.nc: gen_grid
+	$(LAUNCHER) ./gen_grid
+topog.nc: ocean_hgrid.nc
+	@test -f $@ || rm -f $^
+	@test -f $@ || $(MAKE) $^
+
+temp_salt_ic.nc: gen_data ocean_hgrid.nc
+	$(LAUNCHER) ./gen_data
+sponge.nc: temp_salt_ic.nc
+	@test -f $@ || rm -f $^
+	@test -f $@ || $(MAKE) $^
+
+
+# Programs
 
 gen_grid: gen_grid.F90
 	$(FC) $(FCFLAGS) $(LDFLAGS) -o $@ $^ $(LIBS)
 
 gen_data: gen_data.F90
 	$(FC) $(FCFLAGS) $(LDFLAGS) -o $@ $^ $(LIBS)
 
+
+# Support
+
 .PHONY: clean
 clean:
 	rm -rf $(OUT) gen_grid gen_data