Merge pull request #83 from lsst/tickets/DM-40555

DM-40555: Refactor LATISS photometric processing pipeline to add fgcmcal

pipelines/LATISS/DRP.yaml

@@ -1,20 +1,26 @@
 description: DRP specialized for LATISS surveys
 instrument: lsst.obs.lsst.Latiss
 imports:
-  - location: $DRP_PIPE_DIR/pipelines/_ingredients/DRP-minimal-calibration.yaml
+  - location: $DRP_PIPE_DIR/pipelines/_ingredients/DRP-full.yaml
     include:
       - isr
       - characterizeImage
       - calibrate
-      - writeSourceTable
-      - transformSourceTable
-      - consolidateSourceTable
+      - writePreSourceTable
+      - transformPreSourceTable
+      - consolidatePreSourceTable
       - consolidateVisitSummary
       - isolatedStarAssociation
+      - fgcmBuildFromIsolatedStars
+      - fgcmFitCycle
+      - fgcmOutputProducts
       - finalizeCharacterization
       - updateVisitSummary
       - makeCcdVisitTable
       - makeVisitTable
+      - transformSourceTable
+      - consolidateSourceTable
+      - writeRecalibratedSourceTable
       - makeWarp
       - assembleCoadd
       - detection
@@ -27,14 +33,33 @@ imports:
       - consolidateObjectTable
       - forcedPhotCoadd
 
+tasks:
+  fgcmFitCycle:
+    class: lsst.fgcmcal.fgcmFitCycle.FgcmFitCycleTask
+    config:
+      doMultipleCycles: true
+      multipleCyclesFinalCycleNumber: 6
+
+  fgcmOutputProducts:
+    class: lsst.fgcmcal.fgcmOutputProducts.FgcmOutputProductsTask
+    config:
+      connections.cycleNumber: 6
+
+  updateVisitSummary:
+    class: lsst.drp.tasks.update_visit_summary.UpdateVisitSummaryTask
+    config:
+      wcs_provider: "input_summary"
+      photo_calib_provider: "global"
+      background_provider: "input_summary"
+
 subsets:
   step1:
     subset:
       - isr
       - characterizeImage
       - calibrate
-      - writeSourceTable
-      - transformSourceTable
+      - writePreSourceTable
+      - transformPreSourceTable
     description: |
       Per-detector tasks that can be run together to start the DRP pipeline.
       These should never be run with 'tract' or 'patch' as part of the data ID
@@ -44,24 +69,43 @@ subsets:
       this may be moot for a single-detector camera, we would still like to
       follow best practices envisioned for the full LSST survey).
 
-  step2:
+  step2a:
     subset:
-      - consolidateSourceTable
+      - consolidatePreSourceTable
       - consolidateVisitSummary
       - isolatedStarAssociation
+    description: |
+      Global tasks: This is a mix of visit-level and tract-level tasks that
+      must be run with a skymap data query constraint only (an instrument
+      constraint is fine, but generally unnecessary). For example, running
+      with 'tract' (and 'patch') constraints will select partial visits that
+      overlap that region.  A skymap constraint is necessary to prevent
+      isolatedStarAssociation from producing results for all skymaps in the
+      data repository.
+
+      Visit-level: consolidatePreSourceTable, consolidateVisitSummary
+      Tract-level: isolatedStarAssociation
+
+  step2bcde:
+    subset:
       - finalizeCharacterization
+      - fgcmBuildFromIsolatedStars
+      - fgcmFitCycle
+      - fgcmOutputProducts
       - updateVisitSummary
       - makeCcdVisitTable
       - makeVisitTable
     description: |
-      This is a mix of visit-level and tract-level tasks that must not be run
-      with data query constraints other than instrument.  These are tasks that
-      can be run together, but only after the 'step1' subset.
+      Per-visit and per-collection tasks that can be run together after step2a
+      with no data query constraints other than instrument.
 
-      Visit-level: consolidateSourceTable, consolidateVisitSummary,
-                   finalizeCharacterization, updateVisitSummary,
-                   makeCcdVisitTable, makVisitTable
-      Tract-level: isolatedStarAssociation
+      FGCM requires full visits and 'tract' and 'patch' constraints will
+      always select partial visits that overlap that region.
+
+      This includes FGCM because it's configured here to run in "global" mode,
+      which means one should not use a 'tract' expression to constrain it, and
+      if one _did_ run it with a tract constraint (which would be a common
+      occurrence if it was included in any later step).
 
   step3a:
     subset:
@@ -93,3 +137,16 @@ subsets:
       'step3a' subsets (seperation into step3a and step3b is necessary for now
       as otherwise failues in upstream tasks will cause these to fail.  This is
       hoped to be a temporary limitation).
+
+  step4:
+    subset:
+      - writeRecalibratedSourceTable
+      - transformSourceTable
+      - consolidateSourceTable
+    description: |
+      Tasks that be run together, but only after the 'step1', 'step2', and
+      'step3' subsets.
+
+      These detector-level tasks should not be run with 'tract' or 'patch' as
+      part of the data ID expression if all reference catalogs are desired,
+      and to ensure that consolidateSourceTable contains complete visits.