From b47955f8abc8d9d0d59941ddf2f97b226829701f Mon Sep 17 00:00:00 2001
From: Eli Rykoff <erykoff@stanford.edu>
Date: Mon, 11 Sep 2023 15:10:18 -0700
Subject: [PATCH] Add fgcmcal to LATISS pipeline, and split steps accordingly.

---
 pipelines/LATISS/DRP.yaml | 78 +++++++++++++++++++++++++++++++++++----
 1 file changed, 70 insertions(+), 8 deletions(-)

diff --git a/pipelines/LATISS/DRP.yaml b/pipelines/LATISS/DRP.yaml
index 3fc424e1..762ea38e 100644
--- a/pipelines/LATISS/DRP.yaml
+++ b/pipelines/LATISS/DRP.yaml
@@ -1,20 +1,25 @@
 description: DRP specialized for LATISS surveys
 instrument: lsst.obs.lsst.Latiss
 imports:
-  - location: $DRP_PIPE_DIR/pipelines/_ingredients/DRP-minimal-calibration.yaml
+  - location: $DRP_PIPE_DIR/pipelines/_ingredients/DRP-full.yaml
     include:
       - isr
       - characterizeImage
       - calibrate
-      - writeSourceTable
-      - transformSourceTable
-      - consolidateSourceTable
+      - writePreSourceTable
+      - transformPreSourceTable
+      - consolidatePreSourceTable
       - consolidateVisitSummary
       - isolatedStarAssociation
+      - fgcmBuildFromIsolatedStars
+      - fgcmFitCycle
+      - fgcmOutputProducts
       - finalizeCharacterization
       - updateVisitSummary
       - makeCcdVisitTable
       - makeVisitTable
+      - transformSourceTable
+      - writeRecalibratedSourceTable
       - makeWarp
       - assembleCoadd
       - detection
@@ -27,14 +32,26 @@ imports:
       - consolidateObjectTable
       - forcedPhotCoadd
 
+tasks:
+  fgcmFitCycle:
+    class: lsst.fgcmcal.fgcmFitCycle.FgcmFitCycleTask
+    config:
+      doMultipleCycles: true
+      multipleCyclesFinalCycleNumber: 6
+
+  fgcmOutputProducts:
+    class: lsst.fgcmcal.fgcmOutputProducts.FgcmOutputProductsTask
+    config:
+      connections.cycleNumber: 6
+
 subsets:
   step1:
     subset:
       - isr
       - characterizeImage
       - calibrate
-      - writeSourceTable
-      - transformSourceTable
+      - writePreSourceTable
+      - transformPreSourceTable
     description: |
       Per-detector tasks that can be run together to start the DRP pipeline.
       These should never be run with 'tract' or 'patch' as part of the data ID
@@ -44,15 +61,47 @@ subsets:
       this may be moot for a single-detector camera, we would still like to
       follow best practices envisioned for the full LSST survey).
 
-  step2:
+  step2a:
     subset:
-      - consolidateSourceTable
+      - consolidatePreSourceTable
       - consolidateVisitSummary
       - isolatedStarAssociation
+    description: |
+      Global tasks: This is a mix of visit-level and tract-level tasks that
+      must be run with a skymap data query constraint only (an instrument
+      constraint is fine, but generally unnecessary). For example, running
+      with 'tract' (and 'patch') constraints will select partial visits that
+      overlap that region.  A skymap constraint is necessary to prevent
+      isolatedStarAssociation from producing results for all skymaps in the
+      data repository.
+
+      Visit-level: consolidatePreSourceTable, consolidateVisitSummary
+      Tract-level: isolatedStarAssociation
+
+  step2bcde:
+    subset:
       - finalizeCharacterization
+      - fgcmBuildFromIsolatedStars
+      - fgcmFitCycle
+      - fgcmOutputProducts
       - updateVisitSummary
       - makeCcdVisitTable
       - makeVisitTable
+    description: |
+      Per-visit and per-collection tasks that can be run together after step2a
+      with no data query constraints other than instrument.
+
+      FGCM requires full visits and 'tract' and 'patch' constraints will
+      always select partial visits that overlap that region.
+
+      This includes FGCM because it's configured here to run in "global" mode,
+      which means one should not use a 'tract' expression to constrain it, and
+      if one _did_ run it with a tract constraint (which would be a common
+      occurrence if it was included in any later step).
+
+  step2:
+    subset:
+      - consolidateSourceTable
     description: |
       This is a mix of visit-level and tract-level tasks that must not be run
       with data query constraints other than instrument.  These are tasks that
@@ -93,3 +142,16 @@ subsets:
       'step3a' subsets (seperation into step3a and step3b is necessary for now
       as otherwise failues in upstream tasks will cause these to fail.  This is
       hoped to be a temporary limitation).
+
+  step4:
+    subset:
+      - writeRecalibratedSourceTable
+      - transformSourceTable
+      - consolidateSourceTable
+    description: |
+      Tasks that be run together, but only after the 'step1', 'step2', and
+      'step3' subsets.
+
+      These detector-level tasks should not be run with 'tract' or 'patch' as
+      part of the data ID expression if all reference catalogs are desired,
+      and to ensure that consolidateSourceTable contains complete visits.