diff --git a/pipelines/_ingredients/cpBfk.yaml b/pipelines/_ingredients/cpBfk.yaml index e5e745bc..b3eda87f 100644 --- a/pipelines/_ingredients/cpBfk.yaml +++ b/pipelines/_ingredients/cpBfk.yaml @@ -1,6 +1,8 @@ description: cp_pipe brighter-fatter kernel calibration construction. tasks: - cpBfkSolve: + # TODO DM-46439: This can be renamed back to cpBfkSolve when repos + # are cleaned up for the previous dimensionality error. + cpBfkSolveX: class: lsst.cp.pipe.BrighterFatterKernelSolveTask config: connections.inputPtc: ptc diff --git a/pipelines/_ingredients/cpBfkLSST.yaml b/pipelines/_ingredients/cpBfkLSST.yaml new file mode 100644 index 00000000..5ff83669 --- /dev/null +++ b/pipelines/_ingredients/cpBfkLSST.yaml @@ -0,0 +1,52 @@ +description: cp_pipe brighter-fatter kernel calibration construction. +tasks: + cpBfkIsr: + class: lsst.ip.isr.IsrTaskLSST + config: + connections.ccdExposure: "raw" + connections.outputExposure: "cpBfkIsrExp" + python: | + from lsst.cp.pipe import configureIsrTaskLSSTForCalibrations + + configureIsrTaskLSSTForCalibrations(config) + + config.doBootstrap = True + config.doCrosstalk = True + config.crosstalk.doQuadraticCrosstalkCorrection = False + config.doLinearize = True + # TODO DM-46426: Add cpCtiLSST pipeline so that this can be True. + config.doDeferredCharge = False + config.doDefect = True + cpBfkPtcExtract: + class: lsst.cp.pipe.ptc.PhotonTransferCurveExtractTask + config: + connections.inputExp: "cpBfkIsrExp" + connections.outputCovariances: "cpBfkPtcPartial" + maximumRangeCovariancesAstier: 15 + numEdgeSuspect: 20 + edgeMaskLevel: "AMP" + useEfdPhotodiodeData: true + auxiliaryHeaderKeys: ["TEMP6"] + cpBfkPtcSolve: + class: lsst.cp.pipe.ptc.PhotonTransferCurveSolveTask + config: + connections.inputCovariances: "cpBfkPtcPartial" + connections.outputPtcDataset: "bfkPtc" + ptcFitType: "FULLCOVARIANCE" + maximumRangeCovariancesAstier: 15 + maximumRangeCovariancesAstierFullCovFit: 15 + # TODO DM-46439: This can be renamed back to cpBfkSolve when repos + # are cleaned up for the previous dimensionality error. + cpBfkSolveX: + class: lsst.cp.pipe.BrighterFatterKernelSolveTask + config: + useBfkPtc: true + connections.inputPtc: bfkPtc + connections.outputBFK: bfk +contracts: + - cpBfkIsr.doBootstrap == True + - cpBfkPtcExtract.connections.inputExp == cpBfkIsr.connections.outputExposure + - cpBfkPtcSolve.binSize == cpBfkPtcExtract.binSize + - cpBfkPtcSolve.maximumRangeCovariancesAstier == cpBfkPtcExtract.maximumRangeCovariancesAstier + - cpBfkPtcSolve.connections.inputCovariances == cpBfkPtcExtract.connections.outputCovariances + - cpBfkSolveX.connections.inputPtc == cpBfkPtcSolve.connections.outputPtcDataset diff --git a/pipelines/_ingredients/cpBiasBootstrapLSST.yaml b/pipelines/_ingredients/cpBiasBootstrapLSST.yaml new file mode 100644 index 00000000..f4414bc5 --- /dev/null +++ b/pipelines/_ingredients/cpBiasBootstrapLSST.yaml @@ -0,0 +1,29 @@ +description: cp_pipe bootstrap bias calibration for LSST isr task +tasks: + cpBiasBootstrapIsr: + class: lsst.ip.isr.IsrTaskLSST + config: + connections.ccdExposure: "raw" + connections.outputExposure: "cpBiasBootstrapIsrExp" + python: | + from lsst.cp.pipe import configureIsrTaskLSSTForCalibrations + + configureIsrTaskLSSTForCalibrations(config) + + config.doBootstrap = True + config.doCrosstalk = True + config.crosstalk.doQuadraticCrosstalkCorrection = False + cpBiasBootstrapCombine: + class: lsst.cp.pipe.CalibCombineTask + config: + connections.inputExpHandles: "cpBiasBootstrapIsrExp" + connections.outputData: "biasBootstrap" + calibrationType: "bias" + exposureScaling: "Unity" + mask: ["DETECTED", "INTRP"] +contracts: + - cpBiasBootstrapIsr.doBias == False + - cpBiasBootstrapIsr.doBootstrap == True + - cpBiasBootstrapCombine.calibrationType == "bias" + - cpBiasBootstrapCombine.exposureScaling == "Unity" + - cpBiasBootstrapIsr.connections.outputExposure == cpBiasBootstrapCombine.connections.inputExpHandles diff --git a/pipelines/_ingredients/cpBiasLSST.yaml b/pipelines/_ingredients/cpBiasLSST.yaml new file mode 100644 index 00000000..b65b1dd2 --- /dev/null +++ b/pipelines/_ingredients/cpBiasLSST.yaml @@ -0,0 +1,60 @@ +description: cp_pipe bias calibration for LSST isr task +tasks: + cpBiasIsr: + class: lsst.ip.isr.IsrTaskLSST + config: + connections.ccdExposure: "raw" + connections.outputExposure: "cpBiasIsrExp" + python: | + from lsst.cp.pipe import configureIsrTaskLSSTForCalibrations + + configureIsrTaskLSSTForCalibrations(config) + + config.doCrosstalk = True + config.crosstalk.doQuadraticCrosstalkCorrection = True + config.doApplyGains = True + config.doLinearize = True + cpBiasCombine: + class: lsst.cp.pipe.CalibCombineTask + config: + connections.inputExpHandles: "cpBiasIsrExp" + connections.outputData: "bias" + calibrationType: "bias" + exposureScaling: "Unity" + mask: ["DETECTED", "INTRP"] + cpBiasBin8: + class: lsst.pipe.tasks.visualizeVisit.VisualizeBinCalibTask + config: + connections.inputExp: "bias" + connections.outputExp: "biasBin8" + binning: 8 + cpBiasBin64: + class: lsst.pipe.tasks.visualizeVisit.VisualizeBinCalibTask + config: + connections.inputExp: "bias" + connections.outputExp: "biasBin64" + binning: 64 + cpBiasMosaic8: + class: lsst.pipe.tasks.visualizeVisit.VisualizeMosaicCalibTask + config: + connections.inputExps: "biasBin8" + connections.outputData: "biasMosaic8" + binning: 8 + cpBiasMosaic64: + class: lsst.pipe.tasks.visualizeVisit.VisualizeMosaicCalibTask + config: + connections.inputExps: "biasBin64" + connections.outputData: "biasMosaic64" + binning: 64 +contracts: + - cpBiasIsr.doBias == False + - cpBiasIsr.doBootstrap == False + - cpBiasCombine.calibrationType == "bias" + - cpBiasCombine.exposureScaling == "Unity" + - cpBiasBin8.binning == cpBiasMosaic8.binning + - cpBiasBin64.binning == cpBiasMosaic64.binning + - cpBiasCombine.connections.inputExpHandles == cpBiasIsr.connections.outputExposure + - cpBiasBin8.connections.inputExp == cpBiasCombine.connections.outputData + - cpBiasBin64.connections.inputExp == cpBiasCombine.connections.outputData + - cpBiasMosaic8.connections.inputExps == cpBiasBin8.connections.outputExp + - cpBiasMosaic64.connections.inputExps == cpBiasBin64.connections.outputExp diff --git a/pipelines/_ingredients/cpDarkBootstrapLSST.yaml b/pipelines/_ingredients/cpDarkBootstrapLSST.yaml new file mode 100644 index 00000000..7bd832f5 --- /dev/null +++ b/pipelines/_ingredients/cpDarkBootstrapLSST.yaml @@ -0,0 +1,31 @@ +description: cp_pipe bootstrap dark calibration for LSST isr task +tasks: + cpDarkBootstrapIsr: + class: lsst.ip.isr.IsrTaskLSST + config: + connections.ccdExposure: "raw" + connections.outputExposure: "cpDarkBootstrapIsrExp" + connections.bias: "biasBootstrap" + python: | + from lsst.cp.pipe import configureIsrTaskLSSTForCalibrations + + configureIsrTaskLSSTForCalibrations(config) + + config.doBootstrap = True + config.doCrosstalk = True + config.crosstalk.doQuadraticCrosstalkCorrection = False + doBias = True + cpDarkBootstrapCombine: + class: lsst.cp.pipe.CalibCombineTask + config: + connections.inputExpHandles: "cpDarkBootstrapIsrExp" + connections.outputData: "darkBootstrap" + calibrationType: "dark" + exposureScaling: "DarkTime" + mask: ["DETECTED", "INTRP"] +contracts: + - cpDarkBootstrapIsr.doDark == False + - cpDarkBootstrapIsr.doBootstrap == True + - cpDarkBootstrapCombine.calibrationType == "dark" + - cpDarkBootstrapCombine.exposureScaling == "DarkTime" + - cpDarkBootstrapIsr.connections.outputExposure == cpDarkBootstrapCombine.connections.inputExpHandles diff --git a/pipelines/_ingredients/cpDarkLSST.yaml b/pipelines/_ingredients/cpDarkLSST.yaml new file mode 100644 index 00000000..b81f0f08 --- /dev/null +++ b/pipelines/_ingredients/cpDarkLSST.yaml @@ -0,0 +1,69 @@ +description: cp_pipe dark calibration for LSST isr task +tasks: + cpDarkIsr: + class: lsst.ip.isr.IsrTaskLSST + config: + connections.ccdExposure: "raw" + connections.outputExposure: "cpDarkIsrExp" + python: | + from lsst.cp.pipe import configureIsrTaskLSSTForCalibrations + + configureIsrTaskLSSTForCalibrations(config) + + config.doCrosstalk = True + config.crosstalk.doQuadraticCrosstalkCorrection = True + config.doApplyGains = True + config.doLinearize = True + config.doBias = True + # Defects are flagged but not interpolated for CR detection. + config.doDefect = True + cpDark: + class: lsst.cp.pipe.CpDarkTask + config: + connections.inputExp: 'cpDarkIsrExp' + connections.outputExp: 'cpDarkRemoveCRIsrExp' + cpDarkCombine: + class: lsst.cp.pipe.CalibCombineTask + config: + connections.inputExpHandles: "cpDarkRemoveCRIsrExp" + connections.outputData: "dark" + calibrationType: "dark" + exposureScaling: "DarkTime" + python: config.mask.append("CR") + cpDarkBin8: + class: lsst.pipe.tasks.visualizeVisit.VisualizeBinCalibTask + config: + connections.inputExp: "dark" + connections.outputExp: "darkBin8" + binning: 8 + cpDarkBin64: + class: lsst.pipe.tasks.visualizeVisit.VisualizeBinCalibTask + config: + connections.inputExp: "dark" + connections.outputExp: "darkBin64" + binning: 64 + cpDarkMosaic8: + class: lsst.pipe.tasks.visualizeVisit.VisualizeMosaicCalibTask + config: + connections.inputExps: "darkBin8" + connections.outputData: "darkMosaic8" + binning: 8 + cpDarkMosaic64: + class: lsst.pipe.tasks.visualizeVisit.VisualizeMosaicCalibTask + config: + connections.inputExps: "darkBin64" + connections.outputData: "darkMosaic64" + binning: 64 +contracts: + - cpDarkIsr.doDark == False + - cpDarkIsr.doBootstrap == False + - cpDarkCombine.calibrationType == "dark" + - cpDarkCombine.exposureScaling == "DarkTime" + - cpDarkBin8.binning == cpDarkMosaic8.binning + - cpDarkBin64.binning == cpDarkMosaic64.binning + - cpDark.connections.inputExp == cpDarkIsr.connections.outputExposure + - cpDarkCombine.connections.inputExpHandles == cpDark.connections.outputExp + - cpDarkBin8.connections.inputExp == cpDarkCombine.connections.outputData + - cpDarkBin64.connections.inputExp == cpDarkCombine.connections.outputData + - cpDarkMosaic8.connections.inputExps == cpDarkBin8.connections.outputExp + - cpDarkMosaic64.connections.inputExps == cpDarkBin64.connections.outputExp diff --git a/pipelines/_ingredients/cpDefectsLSST.yaml b/pipelines/_ingredients/cpDefectsLSST.yaml new file mode 100644 index 00000000..3904bfe9 --- /dev/null +++ b/pipelines/_ingredients/cpDefectsLSST.yaml @@ -0,0 +1,29 @@ +description: cp_pipe defect calibration construction from combined images for LSST isr. +tasks: + cpMeasureBiasDefects: + class: lsst.cp.pipe.MeasureDefectsCombinedTask + config: + connections.inputExp: "biasBootstrap" + connections.outputDefects: "cpDefectsFromBiasBootstrap" + thresholdType: "VALUE" + cpMeasureDarkDefects: + class: lsst.cp.pipe.MeasureDefectsCombinedTask + config: + connections.inputExp: "darkBootstrap" + connections.outputDefects: "cpDefectsFromDarkBootstrap" + cpMeasureFlatDefects: + class: lsst.cp.pipe.MeasureDefectsCombinedWithFilterTask + config: + connections.inputExp: "flatBootstrap" + connections.outputDefects: "cpDefectsFromFlatBootstrap" + cpMergeDefects: + class: lsst.cp.pipe.MergeDefectsCombinedTask + config: + combinationMode: OR + connections.inputBiasDefects: "cpDefectsFromBiasBootstrap" + connections.inputDarkDefects: "cpDefectsFromDarkBootstrap" + connections.inputFlatDefects: "cpDefectsFromFlatBootstrap" +contracts: + - cpMergeDefects.connections.inputBiasDefects == cpMeasureBiasDefects.connections.outputDefects + - cpMergeDefects.connections.inputDarkDefects == cpMeasureDarkDefects.connections.outputDefects + - cpMergeDefects.connections.inputFlatDefects == cpMeasureFlatDefects.connections.outputDefects diff --git a/pipelines/_ingredients/cpFilterScanLSST.yaml b/pipelines/_ingredients/cpFilterScanLSST.yaml new file mode 100644 index 00000000..7616429b --- /dev/null +++ b/pipelines/_ingredients/cpFilterScanLSST.yaml @@ -0,0 +1,32 @@ +description: cp_pipe pipeline to measure filter scans. +tasks: + cpFilterScanIsr: + class: lsst.ip.isr.IsrTaskLSST + config: + connections.ccdExposure: "raw" + connections.outputExposure: "cpFilterScanIsrExp" + python: | + from lsst.cp.pipe import configureIsrTaskLSSTForCalibrations + + configureIsrTaskLSSTForCalibrations(config) + + config.doCrosstalk = True + config.crosstalk.doQuadraticCrosstalkCorrection = True + config.doApplyGains = True + config.doLinearize = True + # TODO DM-46426: Add cpCtiLSST pipeline so that this can be True. + config.doDeferredCharge = False + config.doBias = True + config.doDark = True + config.doBrighterFatter = True + # TODO DM-46356: Check if this should be True or False. + config.doFlat = False + config.doInterpolate = True + cpFilterScanMerge: + class: lsst.cp.pipe.CpFilterScanTask + config: + connections.inputExpHandles: "cpFilterScanIsrExp" + # This is still an intermediate product, although it"s the end of this pipeline. + connections.outputData: "cpFilterScan" +contracts: + - cpFilterScanMerge.connections.inputExpHandles == cpFilterScanIsr.connections.outputExposure diff --git a/pipelines/_ingredients/cpFlatBootstrapLSST.yaml b/pipelines/_ingredients/cpFlatBootstrapLSST.yaml new file mode 100644 index 00000000..efd22410 --- /dev/null +++ b/pipelines/_ingredients/cpFlatBootstrapLSST.yaml @@ -0,0 +1,47 @@ +description: cp_pipe bootstrap flat calibration for LSST isr task +tasks: + cpFlatBootstrapIsr: + class: lsst.ip.isr.IsrTaskLSST + config: + connections.ccdExposure: "raw" + connections.outputExposure: "cpFlatBootstrapIsrExp" + connections.bias: "biasBootstrap" + connections.dark: "darkBootstrap" + python: | + from lsst.cp.pipe import configureIsrTaskLSSTForCalibrations + + configureIsrTaskLSSTForCalibrations(config) + + config.doBootstrap = True + config.doCrosstalk = True + config.crosstalk.doQuadraticCrosstalkCorrection = False + config.doBias = True + config.doDark = True + cpFlatBootstrapMeasure: + class: lsst.cp.pipe.CpFlatMeasureTask + config: + connections.inputExp: "cpFlatBootstrapIsrExp" + connections.outputStats: "cpFlatBootstrapStatistics" + cpFlatBootstrapNormalize: + class: lsst.cp.pipe.CpFlatNormalizationTask + config: + connections.inputMDs: "cpFlatBootstrapStatistics" + connections.outputScales: "cpFlatBootstrapNormalizeScales" + cpFlatBootstrapCombine: + class: lsst.cp.pipe.CalibCombineByFilterTask + config: + connections.inputExpHandles: "cpFlatBootstrapIsrExp" + connections.inputScales: "cpFlatBootstrapNormalizeScales" + connections.outputData: "flatBootstrap" + calibrationType: "flat" + exposureScaling: InputList + scalingLevel: AMP +contracts: + - cpFlatBootstrapIsr.doFlat == False + - cpFlatBootstrapIsr.doBootstrap == True + - cpFlatBootstrapMeasure.connections.inputExp == cpFlatBootstrapIsr.connections.outputExposure + - cpFlatBootstrapNormalize.connections.inputMDs == cpFlatBootstrapMeasure.connections.outputStats + - cpFlatBootstrapCombine.calibrationType == "flat" + - cpFlatBootstrapCombine.exposureScaling == "InputList" + - cpFlatBootstrapCombine.connections.inputExpHandles == cpFlatBootstrapIsr.connections.outputExposure + - cpFlatBootstrapCombine.connections.inputScales == cpFlatBootstrapNormalize.connections.outputScales diff --git a/pipelines/_ingredients/cpFlatBootstrapSingleChipLSST.yaml b/pipelines/_ingredients/cpFlatBootstrapSingleChipLSST.yaml new file mode 100644 index 00000000..17244b28 --- /dev/null +++ b/pipelines/_ingredients/cpFlatBootstrapSingleChipLSST.yaml @@ -0,0 +1,32 @@ +description: cp_pipe bootstrap flat calibration for LSST isr task +tasks: + cpFlatBootstrapIsr: + class: lsst.ip.isr.IsrTaskLSST + config: + connections.ccdExposure: "raw" + connections.outputExposure: "cpFlatBootstrapIsrExp" + connections.bias: "biasBootstrap" + connections.dark: "darkBootstrap" + python: | + from lsst.cp.pipe import configureIsrTaskLSSTForCalibrations + + configureIsrTaskLSSTForCalibrations(config) + + config.doBootstrap = True + config.doCrosstalk = True + config.crosstalk.doQuadraticCrosstalkCorrection = False + config.doBias = True + config.doDark = True + cpFlatBootstrapCombine: + class: lsst.cp.pipe.CalibCombineByFilterTask + config: + connections.inputExpHandles: "cpFlatBootstrapIsrExp" + connections.outputData: "flatBootstrap" + calibrationType: "flat" + exposureScaling: MeanStats +contracts: + - cpFlatBootstrapIsr.doFlat == False + - cpFlatBootstrapIsr.doBootstrap == True + - cpFlatBootstrapCombine.calibrationType == "flat" + - cpFlatBootstrapCombine.connections.inputExpHandles == cpFlatBootstrapIsr.connections.outputExposure + diff --git a/pipelines/_ingredients/cpFlatLSST.yaml b/pipelines/_ingredients/cpFlatLSST.yaml new file mode 100644 index 00000000..430e5b72 --- /dev/null +++ b/pipelines/_ingredients/cpFlatLSST.yaml @@ -0,0 +1,79 @@ +description: cp_pipe flat calibration for LSST isr task +tasks: + cpFlatIsr: + class: lsst.ip.isr.IsrTaskLSST + config: + connections.ccdExposure: "raw" + connections.outputExposure: "cpFlatIsrExp" + python: | + from lsst.cp.pipe import configureIsrTaskLSSTForCalibrations + + configureIsrTaskLSSTForCalibrations(config) + + config.doCrosstalk = True + config.crosstalk.doQuadraticCrosstalkCorrection = True + config.doApplyGains = True + config.doLinearize = True + # TODO DM-46426: Add cpCtiLSST pipeline so that this can be True. + config.doDeferredCharge = False + config.doBias = True + config.doDark = True + config.doBrighterFatter = True + cpFlatMeasure: + class: lsst.cp.pipe.CpFlatMeasureTask + config: + connections.inputExp: "cpFlatIsrExp" + connections.outputStats: "cpFlatStatistics" + cpFlatNormalize: + class: lsst.cp.pipe.CpFlatNormalizationTask + config: + connections.inputMDs: "cpFlatStatistics" + connections.outputScales: "cpFlatNormalizeScales" + cpFlatCombine: + class: lsst.cp.pipe.CalibCombineByFilterTask + config: + connections.inputExpHandles: "cpFlatIsrExp" + connections.inputScales: "cpFlatNormalizeScales" + connections.outputData: "flat" + calibrationType: "flat" + exposureScaling: "InputList" + scalingLevel: "DETECTOR" + cpFlatBin8: + class: lsst.pipe.tasks.visualizeVisit.VisualizeBinCalibFilterTask + config: + connections.inputExp: 'flat' + connections.outputExp: 'flatBin8' + binning: 8 + cpFlatBin64: + class: lsst.pipe.tasks.visualizeVisit.VisualizeBinCalibFilterTask + config: + connections.inputExp: 'flat' + connections.outputExp: 'flatBin64' + binning: 64 + cpFlatMosaic8: + class: lsst.pipe.tasks.visualizeVisit.VisualizeMosaicCalibFilterTask + config: + connections.inputExps: 'flatBin8' + connections.outputData: 'flatMosaic8' + binning: 8 + cpFlatMosaic64: + class: lsst.pipe.tasks.visualizeVisit.VisualizeMosaicCalibFilterTask + config: + connections.inputExps: 'flatBin64' + connections.outputData: 'flatMosaic64' + binning: 64 +contracts: + - cpFlatIsr.doFlat == False + - cpFlatIsr.doBootstrap == False + - cpFlatCombine.calibrationType == "flat" + - cpFlatCombine.exposureScaling == "InputList" + - cpFlatMeasure.connections.inputExp == cpFlatIsr.connections.outputExposure + - cpFlatNormalize.connections.inputMDs == cpFlatMeasure.connections.outputStats + - cpFlatCombine.connections.inputExpHandles == cpFlatIsr.connections.outputExposure + - cpFlatCombine.connections.inputScales == cpFlatNormalize.connections.outputScales + - cpFlatBin8.binning == cpFlatMosaic8.binning + - cpFlatBin64.binning == cpFlatMosaic64.binning + - cpFlatBin8.connections.inputExp == cpFlatCombine.calibrationType + - cpFlatBin64.connections.inputExp == cpFlatCombine.calibrationType + - cpFlatMosaic8.connections.inputExps == cpFlatBin8.connections.outputExp + - cpFlatMosaic64.connections.inputExps == cpFlatBin64.connections.outputExp diff --git a/pipelines/_ingredients/cpFlatSingleChipLSST.yaml b/pipelines/_ingredients/cpFlatSingleChipLSST.yaml new file mode 100644 index 00000000..9b169fe8 --- /dev/null +++ b/pipelines/_ingredients/cpFlatSingleChipLSST.yaml @@ -0,0 +1,63 @@ +description: cp_pipe flat calibration for LSST isr task +tasks: + cpFlatIsr: + class: lsst.ip.isr.IsrTaskLSST + config: + connections.ccdExposure: "raw" + connections.outputExposure: "cpFlatIsrExp" + python: | + from lsst.cp.pipe import configureIsrTaskLSSTForCalibrations + + configureIsrTaskLSSTForCalibrations(config) + + config.doCrosstalk = True + config.crosstalk.doQuadraticCrosstalkCorrection = True + config.doApplyGains = True + config.doLinearize = True + # TODO DM-46426: Add cpCtiLSST pipeline so that this can be True. + config.doDeferredCharge = False + config.doBias = True + config.doDark = True + config.doBrighterFatter = True + cpFlatCombine: + class: lsst.cp.pipe.CalibCombineByFilterTask + config: + connections.inputExpHandles: "cpFlatIsrExp" + connections.outputData: "flat" + calibrationType: "flat" + exposureScaling: MeanStats + cpFlatBin8: + class: lsst.pipe.tasks.visualizeVisit.VisualizeBinCalibFilterTask + config: + connections.inputExp: 'flat' + connections.outputExp: 'flatBin8' + binning: 8 + cpFlatBin64: + class: lsst.pipe.tasks.visualizeVisit.VisualizeBinCalibFilterTask + config: + connections.inputExp: 'flat' + connections.outputExp: 'flatBin64' + binning: 64 + cpFlatMosaic8: + class: lsst.pipe.tasks.visualizeVisit.VisualizeMosaicCalibFilterTask + config: + connections.inputExps: 'flatBin8' + connections.outputData: 'flatMosaic8' + binning: 8 + cpFlatMosaic64: + class: lsst.pipe.tasks.visualizeVisit.VisualizeMosaicCalibFilterTask + config: + connections.inputExps: 'flatBin64' + connections.outputData: 'flatMosaic64' + binning: 64 +contracts: + - cpFlatIsr.doFlat == False + - cpFlatIsr.doBootstrap == False + - cpFlatCombine.calibrationType == "flat" + - cpFlatCombine.connections.inputExpHandles == cpFlatIsr.connections.outputExposure + - cpFlatBin8.binning == cpFlatMosaic8.binning + - cpFlatBin64.binning == cpFlatMosaic64.binning + - cpFlatBin8.connections.inputExp == cpFlatCombine.calibrationType + - cpFlatBin64.connections.inputExp == cpFlatCombine.calibrationType + - cpFlatMosaic8.connections.inputExps == cpFlatBin8.connections.outputExp + - cpFlatMosaic64.connections.inputExps == cpFlatBin64.connections.outputExp diff --git a/pipelines/_ingredients/cpLinearizerLSST.yaml b/pipelines/_ingredients/cpLinearizerLSST.yaml new file mode 100644 index 00000000..e231a5ea --- /dev/null +++ b/pipelines/_ingredients/cpLinearizerLSST.yaml @@ -0,0 +1,52 @@ +description: cp_pipe linearizer calibration construction. +tasks: + cpLinearizerIsr: + class: lsst.ip.isr.IsrTaskLSST + config: + connections.ccdExposure: "raw" + connections.outputExposure: "cpLinearizerIsrExp" + python: | + from lsst.cp.pipe import configureIsrTaskLSSTForCalibrations + + configureIsrTaskLSSTForCalibrations(config) + + config.doBootstrap = True + config.doCrosstalk = True + config.crosstalk.doQuadraticCrosstalkCorrection = False + config.doDefect = True + cpLinearizerPtcExtract: + class: lsst.cp.pipe.ptc.PhotonTransferCurveExtractTask + config: + connections.inputExp: "cpLinearizerIsrExp" + connections.outputCovariances: "cpLinearizerPtcPartial" + maximumRangeCovariancesAstier: 1 + numEdgeSuspect: 20 + edgeMaskLevel: "AMP" + useEfdPhotodiodeData: true + auxiliaryHeaderKeys: ["TEMP6"] + cpLinearizerPtcSolve: + class: lsst.cp.pipe.ptc.PhotonTransferCurveSolveTask + config: + connections.inputCovariances: "cpLinearizerPtcPartial" + connections.outputPtcDataset: "linearizerPtc" + # This is just used as a placeholder. + ptcFitType: "EXPAPPROXIMATION" + maximumRangeCovariancesAstier: 1 + maximumRangeCovariancesAstierFullCovFit: 1 + cpLinearizerSolve: + class: lsst.cp.pipe.LinearitySolveTask + config: + useLinearizerPtc: true + connections.inputLinearizerPtc: "linearizerPtc" + connections.outputLinearizer: "linearizer" + linearityType: "Spline" + splineKnots: 10 + trimmedState: false + usePhotodiode: true +contracts: + - cpLinearizerIsr.doBootstrap == True + - cpLinearizerPtcExtract.connections.inputExp == cpLinearizerIsr.connections.outputExposure + - cpLinearizerPtcSolve.binSize == cpLinearizerPtcExtract.binSize + - cpLinearizerPtcSolve.maximumRangeCovariancesAstier == cpLinearizerPtcExtract.maximumRangeCovariancesAstier + - cpLinearizerPtcSolve.connections.inputCovariances == cpLinearizerPtcExtract.connections.outputCovariances + - cpLinearizerSolve.connections.inputLinearizerPtc == cpLinearizerPtcSolve.connections.outputPtcDataset diff --git a/pipelines/_ingredients/cpPtcLSST.yaml b/pipelines/_ingredients/cpPtcLSST.yaml new file mode 100644 index 00000000..0b102c03 --- /dev/null +++ b/pipelines/_ingredients/cpPtcLSST.yaml @@ -0,0 +1,55 @@ +description: cp_pipe ptc calibration construction. +tasks: + cpPtcIsr: + class: lsst.ip.isr.IsrTaskLSST + config: + connections.ccdExposure: "raw" + connections.outputExposure: "cpPtcIsrExp" + python: | + from lsst.cp.pipe import configureIsrTaskLSSTForCalibrations + + configureIsrTaskLSSTForCalibrations(config) + + config.doBootstrap = True + config.doCrosstalk = True + config.crosstalk.doQuadraticCrosstalkCorrection = False + config.doLinearize = True + config.doDefect = True + config.doAmpOffset = True + config.ampOffset.ampEdgeMaxOffset = 100000.0 + config.ampOffset.ampEdgeInset = 10 + config.ampOffset.doBackground = False + config.ampOffset.doDetection = False + config.ampOffset.doApplyAmpOffset = False + cpPtcExtract: + class: lsst.cp.pipe.ptc.PhotonTransferCurveExtractTask + config: + connections.inputExp: "cpPtcIsrExp" + connections.outputCovariances: "cpPtcPartial" + maximumRangeCovariancesAstier: 8 + numEdgeSuspect: 20 + edgeMaskLevel: "AMP" + useEfdPhotodiodeData: true + auxiliaryHeaderKeys: ["TEMP6"] + cpPtcSolve: + class: lsst.cp.pipe.ptc.PhotonTransferCurveSolveTask + config: + connections.inputCovariances: "cpPtcPartial" + connections.outputPtcDataset: "ptc" + ptcFitType: "FULLCOVARIANCE" + maximumRangeCovariancesAstier: 8 + doAmpOffsetGainRatioFixup: true +subsets: + # TODO DM-46448: This subset will no longer be necessary to run when the + # verification task can use the assembled PTC and the gainList therein. + cpPtcGainFromFlatPairs: + subset: + - cpPtcIsr + - cpPtcExtract + description: Gain estimation from pairs of flats at the same exposure time. +contracts: + - cpPtcIsr.doBootstrap == True + - cpPtcExtract.connections.inputExp == cpPtcIsr.connections.outputExposure + - cpPtcSolve.binSize == cpPtcExtract.binSize + - cpPtcSolve.maximumRangeCovariancesAstier == cpPtcExtract.maximumRangeCovariancesAstier + - cpPtcSolve.connections.inputCovariances == cpPtcExtract.connections.outputCovariances diff --git a/pipelines/_ingredients/cpSkyLSST.yaml b/pipelines/_ingredients/cpSkyLSST.yaml new file mode 100644 index 00000000..d9a0b687 --- /dev/null +++ b/pipelines/_ingredients/cpSkyLSST.yaml @@ -0,0 +1,28 @@ +description: Sky frame generation pipeline definition for LSST isr task. +tasks: + cpSkyIsr: + class: lsst.ip.isr.IsrTaskLSST + config: + connections.outputExposure: "cpSkyIsrExp" + doCrosstalk: true + crosstalk.doQuadraticCrosstalkCorrection: true + # TODO DM-46426: Add cpCtiLSST pipeline so that this can be True. + doDeferredCharge: false + cpSkyImage: + class: lsst.cp.pipe.CpSkyImageTask + config: + connections.inputExp: "cpSkyIsrExp" + cpSkyScaleMeasure: + class: lsst.cp.pipe.CpSkyScaleMeasureTask + cpSkySubtractBackground: + class: lsst.cp.pipe.CpSkySubtractBackgroundTask + cpSkyCombine: + class: lsst.cp.pipe.CpSkyCombineTask +contracts: + - cpSkyImage.connections.inputExp == cpSkyIsr.connections.outputExposure + - cpSkyScaleMeasure.connections.inputBkgs == cpSkyImage.connections.maskedBkg + - cpSkySubtractBackground.connections.inputExp == cpSkyImage.connections.maskedExp + - cpSkySubtractBackground.connections.inputBkg == cpSkyScaleMeasure.connections.outputBkg + - cpSkySubtractBackground.connections.inputScale == cpSkyScaleMeasure.connections.outputScale + - cpSkyCombine.connections.inputBkgs == cpSkySubtractBackground.connections.outputBkg + - cpSkyCombine.connections.inputExpHandles == cpSkyImage.connections.maskedExp diff --git a/python/lsst/cp/pipe/__init__.py b/python/lsst/cp/pipe/__init__.py index f6bfa42a..0c6cc45e 100644 --- a/python/lsst/cp/pipe/__init__.py +++ b/python/lsst/cp/pipe/__init__.py @@ -20,6 +20,7 @@ # along with this program. If not, see . # +from ._configure import * from .cpBfk import * from .cpCombine import * from .cpCrosstalk import * diff --git a/python/lsst/cp/pipe/_configure.py b/python/lsst/cp/pipe/_configure.py new file mode 100644 index 00000000..d60da4f8 --- /dev/null +++ b/python/lsst/cp/pipe/_configure.py @@ -0,0 +1,39 @@ +__all__ = ["configureIsrTaskLSSTForCalibrations"] + + +def configureIsrTaskLSSTForCalibrations(config): + """Apply configuration overrides for a baseline of building calibrations. + + Parameters + ---------- + config : `lsst.ip.isr.IsrTaskLSSTConfig` + Configuration object to override. + """ + # These are defined in application/run order. + config.doBootstrap = False + config.doDiffNonLinearCorrection = False + config.doCorrectGains = False + config.doSaturation = False + config.doApplyGains = False + config.doCrosstalk = False + config.crosstalk.doQuadraticCrosstalkCorrection = False + config.doLinearize = False + config.doDeferredCharge = False + config.doAssembleCcd = True + config.expectWcs = False + config.doBias = False + config.doDark = False + config.doDefect = False + config.doNanMasking = True + config.doWidenSaturationTrails = False + config.doBrighterFatter = False + config.doVariance = True + config.maskNegativeVariance = False + config.doFlat = False + config.doSaveInterpPixels = False + config.doSetBadRegions = False + config.doInterpolate = False + config.doAmpOffset = False + config.doStandardStatistics = True + config.doCalculateStatistics = True + config.doBinnedExposures = False diff --git a/python/lsst/cp/pipe/cpBfk.py b/python/lsst/cp/pipe/cpBfk.py index 955f9afd..547fc729 100644 --- a/python/lsst/cp/pipe/cpBfk.py +++ b/python/lsst/cp/pipe/cpBfk.py @@ -33,10 +33,11 @@ from lsst.ip.isr import (BrighterFatterKernel) from .utils import (funcPolynomial, irlsFit, extractCalibDate) +from .cpLinearitySolve import ptcLookup class BrighterFatterKernelSolveConnections(pipeBase.PipelineTaskConnections, - dimensions=("instrument", "exposure", "detector")): + dimensions=("instrument", "detector")): dummy = cT.Input( name="raw", doc="Dummy exposure.", @@ -58,6 +59,14 @@ class BrighterFatterKernelSolveConnections(pipeBase.PipelineTaskConnections, storageClass="PhotonTransferCurveDataset", dimensions=("instrument", "detector"), isCalibration=True, + lookupFunction=ptcLookup, + ) + inputBfkPtc = cT.Input( + name="bfkPtc", + doc="Input BFK PTC dataset.", + storageClass="PhotonTransferCurveDataset", + dimensions=("instrument", "detector"), + isCalibration=True, ) outputBFK = cT.Output( @@ -68,6 +77,13 @@ class BrighterFatterKernelSolveConnections(pipeBase.PipelineTaskConnections, isCalibration=True, ) + def __init__(self, *, config=None): + if config.useBfkPtc: + del self.inputPtc + del self.dummy + else: + del self.inputBfkPtc + class BrighterFatterKernelSolveConfig(pipeBase.PipelineTaskConfig, pipelineConnections=BrighterFatterKernelSolveConnections): @@ -150,6 +166,11 @@ class BrighterFatterKernelSolveConfig(pipeBase.PipelineTaskConfig, doc="Slope of the correlation model for radii larger than correlationModelRadius", default=-1.35, ) + useBfkPtc = pexConfig.Field( + dtype=bool, + doc="Use a BFK ptc in a single pipeline?", + default=False, + ) class BrighterFatterKernelSolveTask(pipeBase.PipelineTask): @@ -174,7 +195,14 @@ def runQuantum(self, butlerQC, inputRefs, outputRefs): inputs = butlerQC.get(inputRefs) # Use the dimensions to set calib/provenance information. - inputs['inputDims'] = dict(inputRefs.inputPtc.dataId.required) + + if self.config.useBfkPtc: + inputs["inputDims"] = dict(inputRefs.inputBfkPtc.dataId.required) + inputs["inputPtc"] = inputs["inputBfkPtc"] + inputs["dummy"] = [] + del inputs["inputBfkPtc"] + else: + inputs["inputDims"] = dict(inputRefs.inputPtc.dataId.required) # Add calibration provenance info to header. kwargs = dict() @@ -226,9 +254,6 @@ def run(self, inputPtc, dummy, camera, inputDims): Resulting Brighter-Fatter Kernel (`lsst.ip.isr.BrighterFatterKernel`). """ - if len(dummy) == 0: - self.log.warning("No dummy exposure found.") - detector = camera[inputDims['detector']] detName = detector.getName() diff --git a/tests/test_pipelines.py b/tests/test_pipelines.py index 6ab7ca37..f8dba239 100644 --- a/tests/test_pipelines.py +++ b/tests/test_pipelines.py @@ -73,6 +73,9 @@ def _get_pipelines(self, exclude=[]): "cpPlotPtc.yaml", "cpPtc.yaml", "cpSky.yaml", + "cpBiasBootstrap.yaml", + "cpDarkBootstrap.yaml", + "cpFlatBootstrap.yaml", } for ex in exclude: @@ -97,10 +100,14 @@ def _check_pipeline(self, pipeline_file): def test_ingredients(self): """Check that all pipelines in pipelines/_ingredients are tested.""" glob_str = os.path.join(self.pipeline_path, "_ingredients", "*.yaml") + # The *LSST.yaml pipelines are imported by LATISS/LSSTComCam/LSSTCam + # and are not to be tested on their own. ingredients = set( - [os.path.basename(pipeline) for pipeline in glob.glob(glob_str)] + [os.path.basename(pipeline) for pipeline in glob.glob(glob_str) if "LSST.yaml" not in pipeline] ) - expected = self._get_pipelines() + # The *Bootstrap* pipelines are used by LATISS/LSSTComCam/LSSTCam + # but are renamed on import. + expected = set([pipeline for pipeline in self._get_pipelines() if "Bootstrap" not in pipeline]) self.assertEqual(ingredients, expected) def test_cameras(self): @@ -125,14 +132,24 @@ def test_cameras(self): @unittest.skipIf(not has_obs_lsst, reason="Cannot test LATISS pipelines without obs_lsst") def test_latiss_pipelines(self): - for pipeline in self._get_pipelines(exclude=["cpMonochromatorScan.yaml"]): + for pipeline in self._get_pipelines(exclude=[ + "cpMonochromatorScan.yaml", + # TODO DM-46356: Remove following from exclusion list. + "cpBiasBootstrap.yaml", + "cpDarkBootstrap.yaml", + "cpFlatBootstrap.yaml", + ]): self._check_pipeline(os.path.join(self.pipeline_path, "LATISS", pipeline)) @unittest.skipIf(not has_obs_lsst, reason="Cannot test LSSTCam pipelines without obs_lsst") def test_lsstcam_pipelines(self): for pipeline in self._get_pipelines(exclude=[ "cpFilterScan.yaml", - "cpMonochromatorScan.yaml" + "cpMonochromatorScan.yaml", + # TODO DM-46358: Remove following from exclusion list. + "cpBiasBootstrap.yaml", + "cpDarkBootstrap.yaml", + "cpFlatBootstrap.yaml", ]): self._check_pipeline(os.path.join(self.pipeline_path, "LSSTCam", pipeline)) @@ -141,7 +158,10 @@ def test_lsstcam_imsim_pipelines(self): for pipeline in self._get_pipelines(exclude=[ "cpDarkForDefects.yaml", "cpFilterScan.yaml", - "cpMonochromatorScan.yaml" + "cpMonochromatorScan.yaml", + "cpBiasBootstrap.yaml", + "cpDarkBootstrap.yaml", + "cpFlatBootstrap.yaml", ]): self._check_pipeline(os.path.join(self.pipeline_path, "LSSTCam-imSim", pipeline)) @@ -149,7 +169,11 @@ def test_lsstcam_imsim_pipelines(self): def test_lsstcomcam_pipelines(self): for pipeline in self._get_pipelines(exclude=[ "cpFilterScan.yaml", - "cpMonochromatorScan.yaml" + "cpMonochromatorScan.yaml", + # TODO DM-46357: Remove following from exclusion list. + "cpBiasBootstrap.yaml", + "cpDarkBootstrap.yaml", + "cpFlatBootstrap.yaml", ]): self._check_pipeline(os.path.join(self.pipeline_path, "LSSTComCam", pipeline)) @@ -158,7 +182,11 @@ def test_lsstcomcamsim_pipelines(self): for pipeline in self._get_pipelines(exclude=[ "cpDarkForDefects.yaml", "cpFilterScan.yaml", - "cpMonochromatorScan.yaml" + "cpMonochromatorScan.yaml", + # TODO DM-46357: Remove following from exclusion list. + "cpBiasBootstrap.yaml", + "cpDarkBootstrap.yaml", + "cpFlatBootstrap.yaml", ]): self._check_pipeline(os.path.join(self.pipeline_path, "LSSTComCamSim", pipeline)) @@ -166,7 +194,10 @@ def test_lsstcomcamsim_pipelines(self): def test_lsst_ts8_pipelines(self): for pipeline in self._get_pipelines(exclude=[ "cpFilterScan.yaml", - "cpMonochromatorScan.yaml" + "cpMonochromatorScan.yaml", + "cpBiasBootstrap.yaml", + "cpDarkBootstrap.yaml", + "cpFlatBootstrap.yaml", ]): self._check_pipeline(os.path.join(self.pipeline_path, "LSST-TS8", pipeline)) @@ -175,7 +206,10 @@ def test_decam_pipelines(self): for pipeline in self._get_pipelines(exclude=[ "cpDarkForDefects.yaml", "cpFilterScan.yaml", - "cpMonochromatorScan.yaml" + "cpMonochromatorScan.yaml", + "cpBiasBootstrap.yaml", + "cpDarkBootstrap.yaml", + "cpFlatBootstrap.yaml", ]): self._check_pipeline(os.path.join(self.pipeline_path, "DECam", pipeline)) @@ -184,7 +218,10 @@ def test_hsc_pipelines(self): for pipeline in self._get_pipelines(exclude=[ "cpDarkForDefects.yaml", "cpFilterScan.yaml", - "cpMonochromatorScan.yaml" + "cpMonochromatorScan.yaml", + "cpBiasBootstrap.yaml", + "cpDarkBootstrap.yaml", + "cpFlatBootstrap.yaml", ]): self._check_pipeline(os.path.join(self.pipeline_path, "HSC", pipeline))