From b0c6f8c7e2a4fc45602e2b4c24d8f702bfaa85e2 Mon Sep 17 00:00:00 2001
From: hlin730 <hlin730@users.noreply.github.com>
Date: Mon, 4 Dec 2023 14:58:45 -0600
Subject: [PATCH] Add difference imaging steps

Add difference imaging tasks in step4, step5, and step6.  Also modify step2bcde (move transformSourceTable, consolidateSourceTable, writeRecalibratedSourceTable from step4 to here) and step3b (add selectGoodSeeingVisits and templateGen).
---
 pipelines/LATISS/DRP.yaml | 55 ++++++++++++++++++++++++++++++++++++---
 1 file changed, 52 insertions(+), 3 deletions(-)

diff --git a/pipelines/LATISS/DRP.yaml b/pipelines/LATISS/DRP.yaml
index 4c1dedb5..603cccdd 100644
--- a/pipelines/LATISS/DRP.yaml
+++ b/pipelines/LATISS/DRP.yaml
@@ -135,7 +135,10 @@ subsets:
       - fgcmBuildFromIsolatedStars
       - fgcmFitCycle
       - fgcmOutputProducts
+      - transformSourceTable
+      - consolidateSourceTable
       - updateVisitSummary
+      - writeRecalibratedSourceTable
       - makeCcdVisitTable
       - makeVisitTable
     description: |
@@ -178,6 +181,8 @@ subsets:
   step3b:
     subset:
       - consolidateObjectTable
+      - selectGoodSeeingVisits
+      - templateGen
     description: |
       Tract-level tasks.
       Allowed data query constraints: skymap, tract.
@@ -212,9 +217,13 @@ subsets:
 
   step4:
     subset:
-      - writeRecalibratedSourceTable
-      - transformSourceTable
-      - consolidateSourceTable
+      - forcedPhotCcd
+      - forcedPhotDiffim
+      - getTemplate
+      - subtractImages
+      - detectAndMeasureDiaSources
+      - transformDiaSourceCat
+      - writeForcedSourceTable
     description: |
       Tasks that be run together, but only after the 'step1', 'step2', and
       'step3' subsets.
@@ -223,3 +232,43 @@ subsets:
       These detector-level tasks should not be run with 'tract' or 'patch' as
       part of the data ID expression if all reference catalogs are desired,
       and to ensure that consolidateSourceTable contains complete visits.
+
+  step5:
+    subset:
+      - drpAssociation
+      - drpDiaCalculation
+      - forcedPhotCcdOnDiaObjects
+      - forcedPhotDiffOnDiaObjects
+      - transformForcedSourceTable
+      - consolidateForcedSourceTable
+      - consolidateAssocDiaSourceTable
+      - consolidateFullDiaObjectTable
+      - writeForcedSourceOnDiaObjectTable
+      - transformForcedSourceOnDiaObjectTable
+      - consolidateForcedSourceOnDiaObjectTable
+    description: |
+      Tasks that can be run together, but only after the 'step1', 'step2',
+      'step3', and 'step4' subsets.
+
+      This step includes patch-level aggregation Tasks. These should be run
+      with explicit 'tract' constraints in the data query, otherwise quanta
+      will be created for jobs with only partial visit coverage.
+      'consolidateForcedSourceTable' is a tract-level task that aggregates
+      patches and should be rerun if any of the patches fail.
+
+  step6:
+    subset:
+      - consolidateDiaSourceTable
+    description: |
+      Tasks that can be run together, but only after the 'step1', 'step2',
+      'step3', 'step4', and 'step5' subsets.
+
+      This step includes visit-level aggregation tasks. Running without tract
+      or patch in the data query is recommended, otherwise the outputs of
+      consolidateDiaSourceTable will not contain complete visits.
+
+      This subset is separate from step4 to signal to operators to pause to
+      assess unexpected image differencing failures before these aggregation
+      steps. Otherwise, if run in the same quantum graph, aggregated data
+      products (e.g. diaObjects) would not be created if one or more of the
+      expected inputs is missing.