From 97d644d58143e697d58630bc4322a8f54f6c2153 Mon Sep 17 00:00:00 2001 From: Arun Kannawadi Date: Thu, 30 May 2024 19:16:33 -0400 Subject: [PATCH 1/7] Replace makeWarp in OR3 pipeline --- pipelines/LSSTComCamSim/DRP.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pipelines/LSSTComCamSim/DRP.yaml b/pipelines/LSSTComCamSim/DRP.yaml index 0ee4f1fe..445b0fff 100644 --- a/pipelines/LSSTComCamSim/DRP.yaml +++ b/pipelines/LSSTComCamSim/DRP.yaml @@ -79,7 +79,8 @@ subsets: and CcdVisits. step3: subset: - - makeWarp + - makeDirectWarp + - makePsfMatchedWarp - selectDeepCoaddVisits - assembleCoadd - detection From 52f18dace4947375443d282d7bf1bb6f5504d7bc Mon Sep 17 00:00:00 2001 From: Arun Kannawadi Date: Thu, 30 May 2024 19:17:30 -0400 Subject: [PATCH 2/7] Replace makeWarp in LATISS pipeline --- pipelines/LATISS/DRP.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pipelines/LATISS/DRP.yaml b/pipelines/LATISS/DRP.yaml index d259ee52..1560786d 100644 --- a/pipelines/LATISS/DRP.yaml +++ b/pipelines/LATISS/DRP.yaml @@ -168,7 +168,8 @@ subsets: step3a: subset: - - makeWarp + - makeDirectWarp + - makePsfMatchedWarp - selectDeepCoaddVisits - assembleCoadd - detection From afd364f7805d9827c47add5681ca59a0b61b61fb Mon Sep 17 00:00:00 2001 From: Arun Kannawadi Date: Thu, 30 May 2024 19:28:24 -0400 Subject: [PATCH 3/7] Replace makeWarp in DRP-minimal-calibration pipeline --- .../_ingredients/DRP-minimal-calibration.yaml | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/pipelines/_ingredients/DRP-minimal-calibration.yaml b/pipelines/_ingredients/DRP-minimal-calibration.yaml index 7021bc74..b7c382dc 100644 --- a/pipelines/_ingredients/DRP-minimal-calibration.yaml +++ b/pipelines/_ingredients/DRP-minimal-calibration.yaml @@ -18,10 +18,8 @@ tasks: updateVisitSummary: lsst.drp.tasks.update_visit_summary.UpdateVisitSummaryTask makeCcdVisitTable: lsst.pipe.tasks.postprocess.MakeCcdVisitTableTask makeVisitTable: lsst.pipe.tasks.postprocess.MakeVisitTableTask - makeWarp: - class: lsst.pipe.tasks.makeWarp.MakeWarpTask - config: - makePsfMatched: true + makeDirectWarp: lsst.pipe.tasks.make_direct_warp.MakeDirectWarpTask + makePsfMatchedWarp: lsst.pipe.tasks.make_psf_matched_warp.MakePsfMatchedWarpTask selectDeepCoaddVisits: class: lsst.pipe.tasks.selectImages.BestSeeingSelectVisitsTask config: @@ -188,7 +186,8 @@ subsets: subset. coaddition: subset: - - makeWarp + - makeDirectWarp + - makePsfMatchedWarp - assembleCoadd - healSparsePropertyMaps description: > @@ -248,8 +247,7 @@ contracts: - drpAssociation.connections.diaObjectTable == drpDiaCalculation.connections.diaObjectTable - forcedPhotDiffim.connections.refCat == forcedPhotCcd.connections.refCat - consolidateHealSparsePropertyMaps.property_maps == healSparsePropertyMaps.property_maps - - "'calib_psf_candidate' not in measure.propagateFlags.source_flags if makeWarp.useVisitSummaryPsf else True" - - "'calib_psf_used' not in measure.propagateFlags.source_flags if makeWarp.useVisitSummaryPsf else True" - - "'calib_psf_reserved' not in measure.propagateFlags.source_flags if makeWarp.useVisitSummaryPsf else True" - - "makeWarp.warpAndPsfMatch.warp.cacheSize == assembleCoadd.coaddPsf.cacheSize" - - "makeWarp.matchingKernelSize == assembleCoadd.matchingKernelSize" # TODO: Remove this in DM-44688. + - "'calib_psf_candidate' not in measure.propagateFlags.source_flags if makeDirectWarp.useVisitSummaryPsf else True" + - "'calib_psf_used' not in measure.propagateFlags.source_flags if makeDirectWarp.useVisitSummaryPsf else True" + - "'calib_psf_reserved' not in measure.propagateFlags.source_flags if makeDirectWarp.useVisitSummaryPsf else True" + - "makeDirectWarp.warper.cacheSize == assembleCoadd.coaddPsf.cacheSize" From 6fbb557ab5eddec3ac77388f50f1110be40e3a09 Mon Sep 17 00:00:00 2001 From: Arun Kannawadi Date: Thu, 30 May 2024 19:28:45 -0400 Subject: [PATCH 4/7] Replace makeWarp in other pipelines --- pipelines/HSC/DRP-Prod.yaml | 3 ++- pipelines/HSC/DRP-RC2.yaml | 3 ++- pipelines/HSC/DRP-RC2_subset.yaml | 3 ++- pipelines/_ingredients/LSSTCam-imSim/DRP.yaml | 3 ++- 4 files changed, 8 insertions(+), 4 deletions(-) diff --git a/pipelines/HSC/DRP-Prod.yaml b/pipelines/HSC/DRP-Prod.yaml index a121d04a..a06c987a 100644 --- a/pipelines/HSC/DRP-Prod.yaml +++ b/pipelines/HSC/DRP-Prod.yaml @@ -99,7 +99,8 @@ subsets: and CcdVisits. step3: subset: - - makeWarp + - makeDirectWarp + - makePsfMatchedWarp - selectDeepCoaddVisits - assembleCoadd - detection diff --git a/pipelines/HSC/DRP-RC2.yaml b/pipelines/HSC/DRP-RC2.yaml index 9b9b5ee0..6cf66774 100644 --- a/pipelines/HSC/DRP-RC2.yaml +++ b/pipelines/HSC/DRP-RC2.yaml @@ -178,7 +178,8 @@ subsets: wrong (partial-visit) inputs to its 'background' connection. step3: subset: - - makeWarp + - makeDirectWarp + - makePsfMatchedWarp - selectDeepCoaddVisits - assembleCoadd - detection diff --git a/pipelines/HSC/DRP-RC2_subset.yaml b/pipelines/HSC/DRP-RC2_subset.yaml index cee43830..b41f40b8 100644 --- a/pipelines/HSC/DRP-RC2_subset.yaml +++ b/pipelines/HSC/DRP-RC2_subset.yaml @@ -361,7 +361,8 @@ subsets: Per-visit tasks. These should never be run with tract constraints nightlyStep3: subset: - - makeWarp + - makeDirectWarp + - makePsfMatchedWarp - selectDeepCoaddVisits - assembleCoadd - detection diff --git a/pipelines/_ingredients/LSSTCam-imSim/DRP.yaml b/pipelines/_ingredients/LSSTCam-imSim/DRP.yaml index 82e29e60..6b0e884e 100644 --- a/pipelines/_ingredients/LSSTCam-imSim/DRP.yaml +++ b/pipelines/_ingredients/LSSTCam-imSim/DRP.yaml @@ -161,7 +161,8 @@ subsets: Full collection-level tasks include: makeCcdVisitTable, makeVisitTable step3: subset: - - makeWarp + - makeDirectWarp + - makePsfMatchedWarp - selectDeepCoaddVisits - assembleCoadd - detection From 10d8d6470d4c6779b078637260c02a7315b82165 Mon Sep 17 00:00:00 2001 From: Arun Kannawadi Date: Thu, 30 May 2024 19:29:02 -0400 Subject: [PATCH 5/7] Replace makeWarp in Merian pipeline --- pipelines/_ingredients/DECam/DRP.yaml | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/pipelines/_ingredients/DECam/DRP.yaml b/pipelines/_ingredients/DECam/DRP.yaml index 6a7bd98b..d9b502c6 100644 --- a/pipelines/_ingredients/DECam/DRP.yaml +++ b/pipelines/_ingredients/DECam/DRP.yaml @@ -46,21 +46,24 @@ tasks: background_provider: "input_summary" connections.photoCalibName: "jointcal" connections.skyWcsName: "jointcal" - makeWarp: - class: lsst.pipe.tasks.makeWarp.MakeWarpTask + makeDirectWarp: + class: lsst.pipe.tasks.make_direct_warp.MakeDirectWarpTask config: # These configs match obs_subaru, to facilitate 1:1 comparisons between DECam and HSC - matchingKernelSize: 29 - modelPsf.defaultFwhm: 7.7 - warpAndPsfMatch.warp.warpingKernelName: lanczos5 + warper.warpingKernelName: lanczos5 coaddPsf.warpingKernelName: lanczos5 - # Legacy DECam config to write out warps, even if they are empty - doWriteEmptyWarps: true + doSelectPreWarp: true python: | - config.warpAndPsfMatch.psfMatch.kernel['AL'].kernelSize = 29 - config.warpAndPsfMatch.psfMatch.kernel['AL'].alardSigGauss = [1.0, 2.0, 4.5] from lsst.pipe.tasks.selectImages import PsfWcsSelectImagesTask config.select.retarget(PsfWcsSelectImagesTask) + makePsfMatchedWarp: + class: lsst.pipe.tasks.make_psf_matched_warp.MakePsfMatchedWarpTask + config: + # These configs match obs_subaru, to facilitate 1:1 comparisons between DECam and HSC + modelPsf.defaultFwhm: 7.7 + python: | + config.psfMatch.kernel['AL'].kernelSize = 29 + config.psfMatch.kernel['AL'].alardSigGauss = [1.0, 2.0, 4.5] templateGen: class: lsst.drp.tasks.assemble_coadd.CompareWarpAssembleCoaddTask config: @@ -199,7 +202,8 @@ subsets: subset: - selectGoodSeeingVisits - selectDeepCoaddVisits - - makeWarp + - makeDirectWarp + - makePsfMatchedWarp - templateGen - deblend - detection From d9a61c271862556ebfa3aa1acc6df60e51c67063 Mon Sep 17 00:00:00 2001 From: Arun Kannawadi Date: Thu, 30 May 2024 20:03:04 -0400 Subject: [PATCH 6/7] Request memory for individual warp tasks --- bps/resources/LSSTCam-imSim/DRP-DP0.2.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/bps/resources/LSSTCam-imSim/DRP-DP0.2.yaml b/bps/resources/LSSTCam-imSim/DRP-DP0.2.yaml index d6e0d393..0bbe62e7 100644 --- a/bps/resources/LSSTCam-imSim/DRP-DP0.2.yaml +++ b/bps/resources/LSSTCam-imSim/DRP-DP0.2.yaml @@ -10,7 +10,9 @@ # pipetask: # DRP tasks - makeWarp: + makeDirectWarp: + requestMemory: 8192 + makePsfMatchedWarp: requestMemory: 8192 assembleCoadd: requestMemory: 16384 From ed9b019125665d48f1bd895715d0d2ebddf53d6b Mon Sep 17 00:00:00 2001 From: Arun Kannawadi Date: Wed, 17 Jul 2024 11:11:08 -0400 Subject: [PATCH 7/7] Add contracts between warping tasks makeDirectWarp and makePsfMatchedWarp. --- pipelines/_ingredients/DRP-minimal-calibration.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pipelines/_ingredients/DRP-minimal-calibration.yaml b/pipelines/_ingredients/DRP-minimal-calibration.yaml index b7c382dc..d63642ed 100644 --- a/pipelines/_ingredients/DRP-minimal-calibration.yaml +++ b/pipelines/_ingredients/DRP-minimal-calibration.yaml @@ -251,3 +251,5 @@ contracts: - "'calib_psf_used' not in measure.propagateFlags.source_flags if makeDirectWarp.useVisitSummaryPsf else True" - "'calib_psf_reserved' not in measure.propagateFlags.source_flags if makeDirectWarp.useVisitSummaryPsf else True" - "makeDirectWarp.warper.cacheSize == assembleCoadd.coaddPsf.cacheSize" + - "makeDirectWarp.border >= makePsfMatchedWarp.psfMatch.kernel.active.sizeCellX" + - "makeDirectWarp.border >= makePsfMatchedWarp.psfMatch.kernel.active.sizeCellY"