DM-45087: Remove step8 from HSC/ImSim and move its analysis tasks upstream

bps/clustering/DRP-recalibrated.yaml

@@ -36,8 +36,8 @@ cluster:
     pipetasks: assembleCoadd,inject_coadd,templateGen,detection
     dimensions: tract,patch,band
 
-  sourceTable:
-    pipetasks: writeRecalibratedSourceTable,transformSourceTable
+  finalizeSourceTable:
+    pipetasks: finalizeCharacterization,updateVisitSummary,writeRecalibratedSourceTable,transformSourceTable,consolidateSourceTable
     dimensions: visit
     equalDimensions: visit:exposure
 

bps/resources/HSC/DRP-RC2.yaml

@@ -64,8 +64,6 @@ pipetask:
     requestMemory: 10000
   detection:
     requestMemory: 16384
-
-  # step8 tasks - sasquatch
   analyzeObjectTableCore:
     requestMemory: 16000
   analyzeMatchedVisitCore:

pipelines/HSC/DRP-Prod.yaml

@@ -24,6 +24,7 @@ subsets:
       - calibrate
       - writePreSourceTable
       - transformPreSourceTable
+      - analyzeAmpOffsetMetadata
     description: |
       Per-detector tasks that can be run together to start the DRP pipeline.
 
@@ -99,6 +100,7 @@ subsets:
       and CcdVisits.
   step3:
     subset:
+      # per-patch tasks
       - makeDirectWarp
       - makePsfMatchedWarp
       - selectDeepCoaddVisits
@@ -111,8 +113,18 @@ subsets:
       - forcedPhotCoadd
       - writeObjectTable
       - transformObjectTable
+      # per tract tasks
       - consolidateObjectTable
       - healSparsePropertyMaps
+      - analyzeObjectTableCore
+      - catalogMatchTract
+      - refCatObjectTract
+      - validateObjectTableCore
+      - analyzeMatchedVisitCore
+      - photometricCatalogMatch
+      - photometricRefCatObjectTract
+      - plotPropertyMapTract
+
     description: |
       tract-level tasks.  Allowed data query constraints: tract
 
@@ -134,30 +146,11 @@ subsets:
   step7:
     subset:
       - consolidateHealSparsePropertyMaps
+      - analyzeObjectTableSurveyCore
     description: |
       Tasks that should be run as the final step that require global inputs,
       and can be run after the 'step3' subset.
 
       This step has global aggregation tasks to run over all visits, detectors,
       tracts, etc.  This step should be run only with the instrument constraint
       in the data query.
-  step8:
-    subset:
-      - analyzeAmpOffsetMetadata
-      - analyzeObjectTableCore
-      - analyzeObjectTableSurveyCore
-      - catalogMatchTract
-      - refCatObjectTract
-      - validateObjectTableCore
-      - analyzeMatchedVisitCore
-      - photometricCatalogMatch
-      - photometricRefCatObjectTract
-      - plotPropertyMapTract
-    description: |
-      Core (most important), coadd-level plots/metrics from analysis_tools.
-
-      Must be run after 'step3' (writeObjectTable). Cannot be run with any
-      dataquery constraints if analyzeObjectTableSurveyCore is included.
-
-      Can be run together with analysis_drp subsets, e.g. pipeline.yaml
-      (hash/pound sign)step8,analysis_coadd_plots,imsim_analysis_coadd_plots

pipelines/HSC/DRP-RC2.yaml

@@ -92,14 +92,7 @@ tasks:
   objectTableCoreRefCatMatchWholeSkyPlot:
     class: lsst.analysis.tools.tasks.WholeSkyAnalysisTask
     config:
-      connections.inputName: "objectTable_tract_gaia_dr3_20230707_match_metricsTable"
-      connections.outputName: "objectTableCoreRefCatMatch_wholeSky"
-      atools.wholeSkyMetric: WholeSkyPlotTool
       atools.wholeSkyMetric.bands: ["g", "r", "i", "z", "y"]
-      atools.wholeSkyMetric.plotKeys: []
-      atools.wholeSkyMetric.keysWithBand: parameters.objectTableCoreRefCatMatchKeysWithBand
-      python: |
-        from lsst.analysis.tools.atools import *
 subsets:
   step1:
     subset:
@@ -108,6 +101,7 @@ subsets:
       - calibrate
       - writePreSourceTable
       - transformPreSourceTable
+      - analyzeAmpOffsetMetadata
     description: |
       Per-detector tasks that can be run together to start the DRP pipeline.
 
@@ -156,11 +150,17 @@ subsets:
       - fgcmFitCycle
       - fgcmOutputProducts
       - updateVisitSummary
+      - writeRecalibratedSourceTable
+      - transformSourceTable
+      - consolidateSourceTable
+      - catalogMatchVisit
+      - astrometricRefCatSourceVisit
       - makeCcdVisitTable
       - makeVisitTable
     description: |
-      Per-visit and per-collection tasks that can be run together after step2b
-      with no data query constraints other than instrument.
+      Per-detector, Per-visit, and per-collection tasks that can be run
+      together after step2b with no data query constraints other than
+      instrument.
 
       FGCM requires full visits and 'tract' and 'patch' constraints will
       always select partial visits that overlap that region.
@@ -172,6 +172,7 @@ subsets:
       wrong (partial-visit) inputs to its 'background' connection.
   step3:
     subset:
+      # per-patch Tasks
       - makeDirectWarp
       - makePsfMatchedWarp
       - selectDeepCoaddVisits
@@ -184,10 +185,20 @@ subsets:
       - forcedPhotCoadd
       - transformObjectTable
       - writeObjectTable
-      - consolidateObjectTable
       - healSparsePropertyMaps
       - selectGoodSeeingVisits
       - templateGen
+      # per-tract Tasks
+      - consolidateObjectTable
+      - analyzeMatchedVisitCore
+      - analyzeMatchedPreVisitCore
+      - analyzeObjectTableCore
+      - catalogMatchTract
+      - photometricCatalogMatch
+      - photometricRefCatObjectTract
+      - plotPropertyMapTract
+      - refCatObjectTract
+      - validateObjectTableCore
     description: |
       Tasks that can be run together, but only after the 'step1' and 'step2'
       subsets.
@@ -207,8 +218,6 @@ subsets:
       - detectAndMeasureDiaSources
       - transformDiaSourceCat
       - writeForcedSourceTable
-      - writeRecalibratedSourceTable
-      - transformSourceTable
     description: |
       Tasks that can be run together, but only after the 'step1', 'step2' and
       'step3' subsets
@@ -241,9 +250,6 @@ subsets:
   step6:
     subset:
       - consolidateDiaSourceTable
-      - consolidateSourceTable
-      - catalogMatchVisit
-      - astrometricRefCatSourceVisit
       - sourceObjectMatch
     description: |
       Tasks that can be run together, but only after the 'step1', 'step2',
@@ -262,40 +268,17 @@ subsets:
   step7:
     subset:
       - consolidateHealSparsePropertyMaps
+      - matchedVisitCoreWholeSkyPlot
+      - objectTableCoreWholeSkyPlot
+      - makeMetricTableMatchedVisitCore
+      - makeMetricTableObjectTableCore
+      - makeMetricTableObjectTableCoreRefCatMatch
+      - objectTableCoreRefCatMatchWholeSkyPlot
+      - analyzeObjectTableSurveyCore
     description: |
       Tasks that should be run as the final step that require global inputs,
       and can be run after the 'step3' subset.
 
       This step has global aggregation tasks to run over all visits, detectors,
       tracts, etc.  This step should be run only with the instrument constraint
       in the data query.
-  step8:
-    subset:
-      - analyzeAmpOffsetMetadata
-      - analyzeMatchedVisitCore
-      - analyzeMatchedPreVisitCore
-      - analyzeObjectTableCore
-      - analyzeObjectTableSurveyCore
-      - catalogMatchTract
-      - makeMetricTableMatchedVisitCore
-      - makeMetricTableObjectTableCore
-      - makeMetricTableObjectTableCoreRefCatMatch
-      - matchedVisitCoreWholeSkyPlot
-      - objectTableCoreWholeSkyPlot
-      - objectTableCoreRefCatMatchWholeSkyPlot
-      - photometricCatalogMatch
-      - photometricRefCatObjectTract
-      - plotPropertyMapTract
-      - refCatObjectTract
-      - validateObjectTableCore
-    description: |
-      Core (most important), coadd-level plots/metrics from analysis_tools.
-
-      Must be run after 'step3' (writeObjectTable). If the task
-      analyzeObjectTableSurveyCore is included, should be run with either no
-      data query constraints or with a tract constraint that *exactly* matches
-      that from 'step3'. Running with only a subset of tracts from step3 will
-      result in incomplete survey-level outputs.
-
-      Can be run together with other analysis_drp/analysis_tools subsets, e.g.
-      'pipeline.yaml#step8,analysis_coadd_plots,imsim_analysis_coadd_plots'

pipelines/HSC/DRP-RC2_subset.yaml

@@ -158,14 +158,7 @@ tasks:
   objectTableCoreRefCatMatchWholeSkyPlot:
     class: lsst.analysis.tools.tasks.WholeSkyAnalysisTask
     config:
-      connections.inputName: "objectTable_tract_gaia_dr3_20230707_match_metricsTable"
-      connections.outputName: "objectTableCoreRefCatMatch_wholeSky"
-      atools.wholeSkyMetric: WholeSkyPlotTool
       atools.wholeSkyMetric.bands: ["g", "r", "i", "z", "y"]
-      atools.wholeSkyMetric.plotKeys: []
-      atools.wholeSkyMetric.keysWithBand: parameters.objectTableCoreRefCatMatchKeysWithBand
-      python: |
-        from lsst.analysis.tools.atools import *
   psfPhotRepStar1:
     class: lsst.faro.measurement.TractMatchedMeasurementTask
     config:

pipelines/_ingredients/LSSTCam-imSim/DRP.yaml

@@ -125,6 +125,7 @@ subsets:
       - calibrate
       - writeSourceTable
       - transformSourceTable
+      - analyzeAmpOffsetMetadata
     description: |
       Per-detector tasks that can be run together to start the DRP pipeline.
 
@@ -144,6 +145,8 @@ subsets:
       - makeCcdVisitTable
       - makeVisitTable
       - updateVisitSummary
+      - catalogMatchVisit
+      - astrometricRefCatSourceVisit
     description: |
       Tasks that can be run together, but only after the 'step1'.
 
@@ -161,6 +164,7 @@ subsets:
       Full collection-level tasks include: makeCcdVisitTable, makeVisitTable
   step3:
     subset:
+      # Per-patch Tasks
       - makeDirectWarp
       - makePsfMatchedWarp
       - selectDeepCoaddVisits
@@ -173,12 +177,20 @@ subsets:
       - forcedPhotCoadd
       - transformObjectTable
       - writeObjectTable
-      - consolidateObjectTable
-      - healSparsePropertyMaps
       - selectGoodSeeingVisits
       - templateGen
+      # Per-tract Tasks
+      - consolidateObjectTable
+      - healSparsePropertyMaps
       - matchObjectToTruth
       - compareObjectToTruth
+      - analyzeMatchedVisitCore
+      - analyzeObjectTableCore
+      - catalogMatchTract
+      - diff_matched_analysis
+      - refCatObjectTract
+      - validateObjectTableCore
+
     description: |
       Tasks that can be run together, but only after the 'step1' and 'step2'
       subsets.
@@ -236,8 +248,6 @@ subsets:
   step6:
     subset:
       - consolidateDiaSourceTable
-      - catalogMatchVisit
-      - astrometricRefCatSourceVisit
       - sourceObjectMatch
     description: |
       Tasks that can be run together, but only after the 'step1', 'step2',
@@ -255,35 +265,14 @@ subsets:
   step7:
     subset:
       - consolidateHealSparsePropertyMaps
+      - analyzeObjectTableSurveyCore
     description: |
       Tasks that should be run as the final step that require global inputs,
       and can be run after the 'step3' subset.
 
       This step has global aggregation tasks to run over all visits, detectors,
       tracts, etc.  This step should be run only with the instrument constraint
       in the data query.
-  step8:
-    subset:
-      - analyzeAmpOffsetMetadata
-      - analyzeMatchedVisitCore
-      - analyzeObjectTableCore
-      - analyzeObjectTableSurveyCore
-      - catalogMatchTract
-      - diff_matched_analysis
-      - refCatObjectTract
-      - validateObjectTableCore
-
-    description: |
-      Core (most important), coadd-level plots/metrics from analysis_tools.
-
-      Must be run after 'step3' (writeObjectTable). Cannot be run with any
-      dataquery constraints if analyzeObjectTableSurveyCore is included.
-
-      Can be run together with analysis_drp subsets, e.g. pipeline.yaml
-      (hash/pound sign)step8,analysis_coadd_plots,imsim_analysis_coadd_plots
-
-      Includes extended-level diff_matched_analysis metrics, which are
-      considered core for DC2 as they can't be run on other datasets yet.
   faro_visit:
     subset:
       - nsrcMeasVisit

tests/test_pipelines.py

@@ -524,7 +524,7 @@ def test_lsstcam_imsim_drp_ci_imsim(self):
         butler = self.makeButler(writeable=True)
         tester = PipelineStepTester(
             os.path.join(PIPELINES_DIR, "LSSTCam-imSim", "DRP-ci_imsim.yaml"),
-            [f"#step{N}" for N in range(1, 9)],
+            [f"#step{N}" for N in range(1, 8)],
             [
                 ("cal_ref_cat_2_2", {"htm7"}, "SimpleCatalog", False),
             ],
@@ -537,7 +537,7 @@ def test_lsstcam_imsim_drp_test_med_1(self):
         butler = self.makeButler(writeable=True)
         tester = PipelineStepTester(
             os.path.join(PIPELINES_DIR, "LSSTCam-imSim", "DRP-test-med-1.yaml"),
-            [f"#step{N}" for N in range(1, 9)],
+            [f"#step{N}" for N in range(1, 8)],
             [
                 ("cal_ref_cat_2_2", {"htm7"}, "SimpleCatalog", False),
             ],