Merge pull request #266 from lsst/tickets/DM-45856

DM-45856: Add new calibration pipelines for IsrTaskLSST to _ingredients

pipelines/_ingredients/cpBfk.yaml

@@ -1,6 +1,8 @@
 description: cp_pipe brighter-fatter kernel calibration construction.
 tasks:
-  cpBfkSolve:
+  # TODO DM-46439: This can be renamed back to cpBfkSolve when repos
+  # are cleaned up for the previous dimensionality error.
+  cpBfkSolveX:
     class: lsst.cp.pipe.BrighterFatterKernelSolveTask
     config:
       connections.inputPtc: ptc

pipelines/_ingredients/cpBfkLSST.yaml

@@ -0,0 +1,52 @@
+description: cp_pipe brighter-fatter kernel calibration construction.
+tasks:
+  cpBfkIsr:
+    class: lsst.ip.isr.IsrTaskLSST
+    config:
+      connections.ccdExposure: "raw"
+      connections.outputExposure: "cpBfkIsrExp"
+      python: |
+        from lsst.cp.pipe import configureIsrTaskLSSTForCalibrations
+
+        configureIsrTaskLSSTForCalibrations(config)
+
+        config.doBootstrap = True
+        config.doCrosstalk = True
+        config.crosstalk.doQuadraticCrosstalkCorrection = False
+        config.doLinearize = True
+        # TODO DM-46426: Add cpCtiLSST pipeline so that this can be True.
+        config.doDeferredCharge = False
+        config.doDefect = True
+  cpBfkPtcExtract:
+    class: lsst.cp.pipe.ptc.PhotonTransferCurveExtractTask
+    config:
+      connections.inputExp: "cpBfkIsrExp"
+      connections.outputCovariances: "cpBfkPtcPartial"
+      maximumRangeCovariancesAstier: 15
+      numEdgeSuspect: 20
+      edgeMaskLevel: "AMP"
+      useEfdPhotodiodeData: true
+      auxiliaryHeaderKeys: ["TEMP6"]
+  cpBfkPtcSolve:
+    class: lsst.cp.pipe.ptc.PhotonTransferCurveSolveTask
+    config:
+      connections.inputCovariances: "cpBfkPtcPartial"
+      connections.outputPtcDataset: "bfkPtc"
+      ptcFitType: "FULLCOVARIANCE"
+      maximumRangeCovariancesAstier: 15
+      maximumRangeCovariancesAstierFullCovFit: 15
+  # TODO DM-46439: This can be renamed back to cpBfkSolve when repos
+  # are cleaned up for the previous dimensionality error.
+  cpBfkSolveX:
+    class: lsst.cp.pipe.BrighterFatterKernelSolveTask
+    config:
+      useBfkPtc: true
+      connections.inputPtc: bfkPtc
+      connections.outputBFK: bfk
+contracts:
+  - cpBfkIsr.doBootstrap == True
+  - cpBfkPtcExtract.connections.inputExp == cpBfkIsr.connections.outputExposure
+  - cpBfkPtcSolve.binSize == cpBfkPtcExtract.binSize
+  - cpBfkPtcSolve.maximumRangeCovariancesAstier == cpBfkPtcExtract.maximumRangeCovariancesAstier
+  - cpBfkPtcSolve.connections.inputCovariances == cpBfkPtcExtract.connections.outputCovariances
+  - cpBfkSolveX.connections.inputPtc == cpBfkPtcSolve.connections.outputPtcDataset

pipelines/_ingredients/cpBiasBootstrapLSST.yaml

@@ -0,0 +1,29 @@
+description: cp_pipe bootstrap bias calibration for LSST isr task
+tasks:
+  cpBiasBootstrapIsr:
+    class: lsst.ip.isr.IsrTaskLSST
+    config:
+      connections.ccdExposure: "raw"
+      connections.outputExposure: "cpBiasBootstrapIsrExp"
+      python: |
+        from lsst.cp.pipe import configureIsrTaskLSSTForCalibrations
+
+        configureIsrTaskLSSTForCalibrations(config)
+
+        config.doBootstrap = True
+        config.doCrosstalk = True
+        config.crosstalk.doQuadraticCrosstalkCorrection = False
+  cpBiasBootstrapCombine:
+    class: lsst.cp.pipe.CalibCombineTask
+    config:
+      connections.inputExpHandles: "cpBiasBootstrapIsrExp"
+      connections.outputData: "biasBootstrap"
+      calibrationType: "bias"
+      exposureScaling: "Unity"
+      mask: ["DETECTED", "INTRP"]
+contracts:
+  - cpBiasBootstrapIsr.doBias == False
+  - cpBiasBootstrapIsr.doBootstrap == True
+  - cpBiasBootstrapCombine.calibrationType == "bias"
+  - cpBiasBootstrapCombine.exposureScaling == "Unity"
+  - cpBiasBootstrapIsr.connections.outputExposure == cpBiasBootstrapCombine.connections.inputExpHandles

pipelines/_ingredients/cpBiasLSST.yaml

@@ -0,0 +1,60 @@
+description: cp_pipe bias calibration for LSST isr task
+tasks:
+  cpBiasIsr:
+    class: lsst.ip.isr.IsrTaskLSST
+    config:
+      connections.ccdExposure: "raw"
+      connections.outputExposure: "cpBiasIsrExp"
+      python: |
+        from lsst.cp.pipe import configureIsrTaskLSSTForCalibrations
+
+        configureIsrTaskLSSTForCalibrations(config)
+
+        config.doCrosstalk = True
+        config.crosstalk.doQuadraticCrosstalkCorrection = True
+        config.doApplyGains = True
+        config.doLinearize = True
+  cpBiasCombine:
+    class: lsst.cp.pipe.CalibCombineTask
+    config:
+      connections.inputExpHandles: "cpBiasIsrExp"
+      connections.outputData: "bias"
+      calibrationType: "bias"
+      exposureScaling: "Unity"
+      mask: ["DETECTED", "INTRP"]
+  cpBiasBin8:
+    class: lsst.pipe.tasks.visualizeVisit.VisualizeBinCalibTask
+    config:
+      connections.inputExp: "bias"
+      connections.outputExp: "biasBin8"
+      binning: 8
+  cpBiasBin64:
+    class: lsst.pipe.tasks.visualizeVisit.VisualizeBinCalibTask
+    config:
+      connections.inputExp: "bias"
+      connections.outputExp: "biasBin64"
+      binning: 64
+  cpBiasMosaic8:
+    class: lsst.pipe.tasks.visualizeVisit.VisualizeMosaicCalibTask
+    config:
+      connections.inputExps: "biasBin8"
+      connections.outputData: "biasMosaic8"
+      binning: 8
+  cpBiasMosaic64:
+    class: lsst.pipe.tasks.visualizeVisit.VisualizeMosaicCalibTask
+    config:
+      connections.inputExps: "biasBin64"
+      connections.outputData: "biasMosaic64"
+      binning: 64
+contracts:
+  - cpBiasIsr.doBias == False
+  - cpBiasIsr.doBootstrap == False
+  - cpBiasCombine.calibrationType == "bias"
+  - cpBiasCombine.exposureScaling == "Unity"
+  - cpBiasBin8.binning == cpBiasMosaic8.binning
+  - cpBiasBin64.binning == cpBiasMosaic64.binning
+  - cpBiasCombine.connections.inputExpHandles == cpBiasIsr.connections.outputExposure
+  - cpBiasBin8.connections.inputExp == cpBiasCombine.connections.outputData
+  - cpBiasBin64.connections.inputExp == cpBiasCombine.connections.outputData
+  - cpBiasMosaic8.connections.inputExps == cpBiasBin8.connections.outputExp
+  - cpBiasMosaic64.connections.inputExps == cpBiasBin64.connections.outputExp

pipelines/_ingredients/cpDarkBootstrapLSST.yaml

@@ -0,0 +1,31 @@
+description: cp_pipe bootstrap dark calibration for LSST isr task
+tasks:
+  cpDarkBootstrapIsr:
+    class: lsst.ip.isr.IsrTaskLSST
+    config:
+      connections.ccdExposure: "raw"
+      connections.outputExposure: "cpDarkBootstrapIsrExp"
+      connections.bias: "biasBootstrap"
+      python: |
+        from lsst.cp.pipe import configureIsrTaskLSSTForCalibrations
+
+        configureIsrTaskLSSTForCalibrations(config)
+
+        config.doBootstrap = True
+        config.doCrosstalk = True
+        config.crosstalk.doQuadraticCrosstalkCorrection = False
+        doBias = True
+  cpDarkBootstrapCombine:
+    class: lsst.cp.pipe.CalibCombineTask
+    config:
+      connections.inputExpHandles: "cpDarkBootstrapIsrExp"
+      connections.outputData: "darkBootstrap"
+      calibrationType: "dark"
+      exposureScaling: "DarkTime"
+      mask: ["DETECTED", "INTRP"]
+contracts:
+  - cpDarkBootstrapIsr.doDark == False
+  - cpDarkBootstrapIsr.doBootstrap == True
+  - cpDarkBootstrapCombine.calibrationType == "dark"
+  - cpDarkBootstrapCombine.exposureScaling == "DarkTime"
+  - cpDarkBootstrapIsr.connections.outputExposure == cpDarkBootstrapCombine.connections.inputExpHandles

pipelines/_ingredients/cpDarkLSST.yaml

@@ -0,0 +1,69 @@
+description: cp_pipe dark calibration for LSST isr task
+tasks:
+  cpDarkIsr:
+    class: lsst.ip.isr.IsrTaskLSST
+    config:
+      connections.ccdExposure: "raw"
+      connections.outputExposure: "cpDarkIsrExp"
+      python: |
+        from lsst.cp.pipe import configureIsrTaskLSSTForCalibrations
+
+        configureIsrTaskLSSTForCalibrations(config)
+
+        config.doCrosstalk = True
+        config.crosstalk.doQuadraticCrosstalkCorrection = True
+        config.doApplyGains = True
+        config.doLinearize = True
+        config.doBias = True
+        # Defects are flagged but not interpolated for CR detection.
+        config.doDefect = True
+  cpDark:
+    class: lsst.cp.pipe.CpDarkTask
+    config:
+      connections.inputExp: 'cpDarkIsrExp'
+      connections.outputExp: 'cpDarkRemoveCRIsrExp'
+  cpDarkCombine:
+    class: lsst.cp.pipe.CalibCombineTask
+    config:
+      connections.inputExpHandles: "cpDarkRemoveCRIsrExp"
+      connections.outputData: "dark"
+      calibrationType: "dark"
+      exposureScaling: "DarkTime"
+      python: config.mask.append("CR")
+  cpDarkBin8:
+    class: lsst.pipe.tasks.visualizeVisit.VisualizeBinCalibTask
+    config:
+      connections.inputExp: "dark"
+      connections.outputExp: "darkBin8"
+      binning: 8
+  cpDarkBin64:
+    class: lsst.pipe.tasks.visualizeVisit.VisualizeBinCalibTask
+    config:
+      connections.inputExp: "dark"
+      connections.outputExp: "darkBin64"
+      binning: 64
+  cpDarkMosaic8:
+    class: lsst.pipe.tasks.visualizeVisit.VisualizeMosaicCalibTask
+    config:
+      connections.inputExps: "darkBin8"
+      connections.outputData: "darkMosaic8"
+      binning: 8
+  cpDarkMosaic64:
+    class: lsst.pipe.tasks.visualizeVisit.VisualizeMosaicCalibTask
+    config:
+      connections.inputExps: "darkBin64"
+      connections.outputData: "darkMosaic64"
+      binning: 64
+contracts:
+  - cpDarkIsr.doDark == False
+  - cpDarkIsr.doBootstrap == False
+  - cpDarkCombine.calibrationType == "dark"
+  - cpDarkCombine.exposureScaling == "DarkTime"
+  - cpDarkBin8.binning == cpDarkMosaic8.binning
+  - cpDarkBin64.binning == cpDarkMosaic64.binning
+  - cpDark.connections.inputExp == cpDarkIsr.connections.outputExposure
+  - cpDarkCombine.connections.inputExpHandles == cpDark.connections.outputExp
+  - cpDarkBin8.connections.inputExp == cpDarkCombine.connections.outputData
+  - cpDarkBin64.connections.inputExp == cpDarkCombine.connections.outputData
+  - cpDarkMosaic8.connections.inputExps == cpDarkBin8.connections.outputExp
+  - cpDarkMosaic64.connections.inputExps == cpDarkBin64.connections.outputExp

pipelines/_ingredients/cpDefectsLSST.yaml

@@ -0,0 +1,29 @@
+description: cp_pipe defect calibration construction from combined images for LSST isr.
+tasks:
+  cpMeasureBiasDefects:
+    class: lsst.cp.pipe.MeasureDefectsCombinedTask
+    config:
+      connections.inputExp: "biasBootstrap"
+      connections.outputDefects: "cpDefectsFromBiasBootstrap"
+      thresholdType: "VALUE"
+  cpMeasureDarkDefects:
+    class: lsst.cp.pipe.MeasureDefectsCombinedTask
+    config:
+      connections.inputExp: "darkBootstrap"
+      connections.outputDefects: "cpDefectsFromDarkBootstrap"
+  cpMeasureFlatDefects:
+    class: lsst.cp.pipe.MeasureDefectsCombinedWithFilterTask
+    config:
+      connections.inputExp: "flatBootstrap"
+      connections.outputDefects: "cpDefectsFromFlatBootstrap"
+  cpMergeDefects:
+    class: lsst.cp.pipe.MergeDefectsCombinedTask
+    config:
+      combinationMode: OR
+      connections.inputBiasDefects: "cpDefectsFromBiasBootstrap"
+      connections.inputDarkDefects: "cpDefectsFromDarkBootstrap"
+      connections.inputFlatDefects: "cpDefectsFromFlatBootstrap"
+contracts:
+  - cpMergeDefects.connections.inputBiasDefects == cpMeasureBiasDefects.connections.outputDefects
+  - cpMergeDefects.connections.inputDarkDefects == cpMeasureDarkDefects.connections.outputDefects
+  - cpMergeDefects.connections.inputFlatDefects == cpMeasureFlatDefects.connections.outputDefects

pipelines/_ingredients/cpFilterScanLSST.yaml

@@ -0,0 +1,32 @@
+description: cp_pipe pipeline to measure filter scans.
+tasks:
+  cpFilterScanIsr:
+    class: lsst.ip.isr.IsrTaskLSST
+    config:
+      connections.ccdExposure: "raw"
+      connections.outputExposure: "cpFilterScanIsrExp"
+      python: |
+        from lsst.cp.pipe import configureIsrTaskLSSTForCalibrations
+
+        configureIsrTaskLSSTForCalibrations(config)
+
+        config.doCrosstalk = True
+        config.crosstalk.doQuadraticCrosstalkCorrection = True
+        config.doApplyGains = True
+        config.doLinearize = True
+        # TODO DM-46426: Add cpCtiLSST pipeline so that this can be True.
+        config.doDeferredCharge = False
+        config.doBias = True
+        config.doDark = True
+        config.doBrighterFatter = True
+        # TODO DM-46356: Check if this should be True or False.
+        config.doFlat = False
+        config.doInterpolate = True
+  cpFilterScanMerge:
+    class: lsst.cp.pipe.CpFilterScanTask
+    config:
+      connections.inputExpHandles: "cpFilterScanIsrExp"
+      # This is still an intermediate product, although it"s the end of this pipeline.
+      connections.outputData: "cpFilterScan"
+contracts:
+  - cpFilterScanMerge.connections.inputExpHandles == cpFilterScanIsr.connections.outputExposure

pipelines/_ingredients/cpFlatBootstrapLSST.yaml

@@ -0,0 +1,47 @@
+description: cp_pipe bootstrap flat calibration for LSST isr task
+tasks:
+  cpFlatBootstrapIsr:
+    class: lsst.ip.isr.IsrTaskLSST
+    config:
+      connections.ccdExposure: "raw"
+      connections.outputExposure: "cpFlatBootstrapIsrExp"
+      connections.bias: "biasBootstrap"
+      connections.dark: "darkBootstrap"
+      python: |
+        from lsst.cp.pipe import configureIsrTaskLSSTForCalibrations
+
+        configureIsrTaskLSSTForCalibrations(config)
+
+        config.doBootstrap = True
+        config.doCrosstalk = True
+        config.crosstalk.doQuadraticCrosstalkCorrection = False
+        config.doBias = True
+        config.doDark = True
+  cpFlatBootstrapMeasure:
+    class: lsst.cp.pipe.CpFlatMeasureTask
+    config:
+      connections.inputExp: "cpFlatBootstrapIsrExp"
+      connections.outputStats: "cpFlatBootstrapStatistics"
+  cpFlatBootstrapNormalize:
+    class: lsst.cp.pipe.CpFlatNormalizationTask
+    config:
+      connections.inputMDs: "cpFlatBootstrapStatistics"
+      connections.outputScales: "cpFlatBootstrapNormalizeScales"
+  cpFlatBootstrapCombine:
+    class: lsst.cp.pipe.CalibCombineByFilterTask
+    config:
+      connections.inputExpHandles: "cpFlatBootstrapIsrExp"
+      connections.inputScales: "cpFlatBootstrapNormalizeScales"
+      connections.outputData: "flatBootstrap"
+      calibrationType: "flat"
+      exposureScaling: InputList
+      scalingLevel: AMP
+contracts:
+  - cpFlatBootstrapIsr.doFlat == False
+  - cpFlatBootstrapIsr.doBootstrap == True
+  - cpFlatBootstrapMeasure.connections.inputExp == cpFlatBootstrapIsr.connections.outputExposure
+  - cpFlatBootstrapNormalize.connections.inputMDs == cpFlatBootstrapMeasure.connections.outputStats
+  - cpFlatBootstrapCombine.calibrationType == "flat"
+  - cpFlatBootstrapCombine.exposureScaling == "InputList"
+  - cpFlatBootstrapCombine.connections.inputExpHandles == cpFlatBootstrapIsr.connections.outputExposure
+  - cpFlatBootstrapCombine.connections.inputScales == cpFlatBootstrapNormalize.connections.outputScales

pipelines/_ingredients/cpFlatBootstrapSingleChipLSST.yaml

@@ -0,0 +1,32 @@
+description: cp_pipe bootstrap flat calibration for LSST isr task
+tasks:
+  cpFlatBootstrapIsr:
+    class: lsst.ip.isr.IsrTaskLSST
+    config:
+      connections.ccdExposure: "raw"
+      connections.outputExposure: "cpFlatBootstrapIsrExp"
+      connections.bias: "biasBootstrap"
+      connections.dark: "darkBootstrap"
+      python: |
+        from lsst.cp.pipe import configureIsrTaskLSSTForCalibrations
+
+        configureIsrTaskLSSTForCalibrations(config)
+
+        config.doBootstrap = True
+        config.doCrosstalk = True
+        config.crosstalk.doQuadraticCrosstalkCorrection = False
+        config.doBias = True
+        config.doDark = True
+  cpFlatBootstrapCombine:
+    class: lsst.cp.pipe.CalibCombineByFilterTask
+    config:
+      connections.inputExpHandles: "cpFlatBootstrapIsrExp"
+      connections.outputData: "flatBootstrap"
+      calibrationType: "flat"
+      exposureScaling: MeanStats
+contracts:
+  - cpFlatBootstrapIsr.doFlat == False
+  - cpFlatBootstrapIsr.doBootstrap == True
+  - cpFlatBootstrapCombine.calibrationType == "flat"
+  - cpFlatBootstrapCombine.connections.inputExpHandles == cpFlatBootstrapIsr.connections.outputExposure
+