From d65edad732144b5126a3ead9d02cbb17d31e186e Mon Sep 17 00:00:00 2001 From: Jakob T McCann Date: Thu, 14 Sep 2023 09:35:07 -0700 Subject: [PATCH 1/4] Propogate injected source mask planes --- python/lsst/ip/diffim/subtractImages.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/lsst/ip/diffim/subtractImages.py b/python/lsst/ip/diffim/subtractImages.py index 75717ee4..90abc437 100644 --- a/python/lsst/ip/diffim/subtractImages.py +++ b/python/lsst/ip/diffim/subtractImages.py @@ -200,7 +200,7 @@ class AlardLuptonSubtractBaseConfig(lsst.pex.config.Config): ) preserveTemplateMask = lsst.pex.config.ListField( dtype=str, - default=("NO_DATA", "BAD", "SAT"), + default=("NO_DATA", "BAD", "SAT", "INJECTED", "INJECTED_CORE"), doc="Mask planes from the template to propagate to the image difference." ) From 9a805245704876f64d5b7635171530e110a7a8ab Mon Sep 17 00:00:00 2001 From: Clare Saunders Date: Fri, 25 Aug 2023 13:39:12 -0700 Subject: [PATCH 2/4] Add coordinate error fields to diaSrc --- python/lsst/ip/diffim/detectAndMeasure.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/python/lsst/ip/diffim/detectAndMeasure.py b/python/lsst/ip/diffim/detectAndMeasure.py index d4b06118..819efa99 100644 --- a/python/lsst/ip/diffim/detectAndMeasure.py +++ b/python/lsst/ip/diffim/detectAndMeasure.py @@ -178,6 +178,8 @@ class DetectAndMeasureTask(lsst.pipe.base.PipelineTask): def __init__(self, **kwargs): super().__init__(**kwargs) self.schema = afwTable.SourceTable.makeMinimalSchema() + # Add coordinate error fields: + afwTable.CoordKey.addErrorFields(self.schema) self.algMetadata = dafBase.PropertyList() self.makeSubtask("detection", schema=self.schema) From e8a5b81093ec443e6fa1a93ae8a2db96b2950218 Mon Sep 17 00:00:00 2001 From: Ian Sullivan Date: Tue, 11 Jul 2023 17:28:47 -0700 Subject: [PATCH 3/4] Replace masked image pixels with median values before convolution Only use with preconvolution and the Score image used for detection for now. --- python/lsst/ip/diffim/subtractImages.py | 40 ++++++++++++++++++++++--- 1 file changed, 36 insertions(+), 4 deletions(-) diff --git a/python/lsst/ip/diffim/subtractImages.py b/python/lsst/ip/diffim/subtractImages.py index 90abc437..5b464bac 100644 --- a/python/lsst/ip/diffim/subtractImages.py +++ b/python/lsst/ip/diffim/subtractImages.py @@ -612,7 +612,10 @@ def _validateExposures(template, science): def _convolveExposure(exposure, kernel, convolutionControl, bbox=None, psf=None, - photoCalib=None): + photoCalib=None, + interpolateBadMaskPlanes=False, + badMaskPlanes=None, + ): """Convolve an exposure with the given kernel. Parameters @@ -640,8 +643,11 @@ def _convolveExposure(exposure, kernel, convolutionControl, convolvedExposure.setPsf(psf) if photoCalib is not None: convolvedExposure.setPhotoCalib(photoCalib) - convolvedImage = lsst.afw.image.MaskedImageF(exposure.getBBox()) - lsst.afw.math.convolve(convolvedImage, exposure.maskedImage, kernel, convolutionControl) + if interpolateBadMaskPlanes and badMaskPlanes is not None: + _interpolateImage(convolvedExposure.maskedImage, + badMaskPlanes) + convolvedImage = lsst.afw.image.MaskedImageF(convolvedExposure.getBBox()) + lsst.afw.math.convolve(convolvedImage, convolvedExposure.maskedImage, kernel, convolutionControl) convolvedExposure.setMaskedImage(convolvedImage) if bbox is None: return convolvedExposure @@ -849,7 +855,9 @@ def run(self, template, science, sources, finalizedPsfApCorrCatalog=None, visitS # TODO: DM-37212 we need to mirror the kernel in order to get correct cross correlation scienceKernel = science.psf.getKernel() - matchedScience = self._convolveExposure(science, scienceKernel, self.convolutionControl) + matchedScience = self._convolveExposure(science, scienceKernel, self.convolutionControl, + badMaskPlanes=self.config.badMaskPlanes, + interpolateBadMaskPlanes=True) selectSources = self._sourceSelector(sources, matchedScience.mask) subtractResults = self.runPreconvolve(template, science, matchedScience, selectSources, scienceKernel) @@ -910,6 +918,8 @@ def runPreconvolve(self, template, science, matchedScience, selectSources, preCo self.convolutionControl, bbox=bbox, psf=science.psf, + badMaskPlanes=self.config.badMaskPlanes, + interpolateBadMaskPlanes=True, photoCalib=science.photoCalib) score = _subtractImages(matchedScience, matchedTemplate, backgroundModel=(kernelResult.backgroundModel @@ -1025,3 +1035,25 @@ def _shapeTest(exp1, exp2, fwhmExposureBuffer, fwhmExposureGrid): xTest = shape1[0] <= shape2[0] yTest = shape1[1] <= shape2[1] return xTest | yTest + + +def _interpolateImage(maskedImage, badMaskPlanes, fallbackValue=None): + """Replace masked image pixels with interpolated values. + + Parameters + ---------- + maskedImage : `lsst.afw.image.MaskedImage` + Image on which to perform interpolation. + badMaskPlanes : `list` of `str` + List of mask planes to interpolate over. + fallbackValue : `float`, optional + Value to set when interpolation fails. + """ + image = maskedImage.image.array + badPixels = (maskedImage.mask.array & maskedImage.mask.getPlaneBitMask(badMaskPlanes)) > 0 + image[badPixels] = np.nan + if fallbackValue is None: + fallbackValue = np.nanmedian(image) + # For this initial implementation, skip the interpolation and just fill with + # the median value. + image[badPixels] = fallbackValue From 2e5f4c33b7c2300ad6f11c846ba2b61855eb539b Mon Sep 17 00:00:00 2001 From: Ian Sullivan Date: Mon, 18 Sep 2023 15:23:13 -0700 Subject: [PATCH 4/4] Add subtractTask metadata --- python/lsst/ip/diffim/subtractImages.py | 25 +++++++++++++++++-------- 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/python/lsst/ip/diffim/subtractImages.py b/python/lsst/ip/diffim/subtractImages.py index 5b464bac..dabdaced 100644 --- a/python/lsst/ip/diffim/subtractImages.py +++ b/python/lsst/ip/diffim/subtractImages.py @@ -372,6 +372,8 @@ def run(self, template, science, sources, finalizedPsfApCorrCatalog=None, ) self.log.info("Science PSF FWHM: %f pixels", sciencePsfSize) self.log.info("Template PSF FWHM: %f pixels", templatePsfSize) + self.metadata.add("sciencePsfSize", sciencePsfSize) + self.metadata.add("templatePsfSize", templatePsfSize) selectSources = self._sourceSelector(sources, science.mask) if self.config.mode == "auto": @@ -397,8 +399,10 @@ def run(self, template, science, sources, finalizedPsfApCorrCatalog=None, raise RuntimeError("Cannot handle AlardLuptonSubtract mode: %s", self.config.mode) if convolveTemplate: + self.metadata.add("convolvedExposure", "Template") subtractResults = self.runConvolveTemplate(template, science, selectSources) else: + self.metadata.add("convolvedExposure", "Science") subtractResults = self.runConvolveScience(template, science, selectSources) return subtractResults @@ -608,13 +612,11 @@ def _validateExposures(template, science): assert templateBBox.contains(scienceBBox),\ "Template bbox does not contain all of the science image." - @staticmethod - def _convolveExposure(exposure, kernel, convolutionControl, + def _convolveExposure(self, exposure, kernel, convolutionControl, bbox=None, psf=None, photoCalib=None, interpolateBadMaskPlanes=False, - badMaskPlanes=None, ): """Convolve an exposure with the given kernel. @@ -643,9 +645,10 @@ def _convolveExposure(exposure, kernel, convolutionControl, convolvedExposure.setPsf(psf) if photoCalib is not None: convolvedExposure.setPhotoCalib(photoCalib) - if interpolateBadMaskPlanes and badMaskPlanes is not None: - _interpolateImage(convolvedExposure.maskedImage, - badMaskPlanes) + if interpolateBadMaskPlanes and self.config.badMaskPlanes is not None: + nInterp = _interpolateImage(convolvedExposure.maskedImage, + self.config.badMaskPlanes) + self.metadata.add("nInterpolated", nInterp) convolvedImage = lsst.afw.image.MaskedImageF(convolvedExposure.getBBox()) lsst.afw.math.convolve(convolvedImage, convolvedExposure.maskedImage, kernel, convolutionControl) convolvedExposure.setMaskedImage(convolvedImage) @@ -694,6 +697,7 @@ def _sourceSelector(self, sources, mask): "%i selected but %i needed for the calculation.", len(selectSources), self.config.makeKernel.nStarPerCell) raise RuntimeError("Cannot compute PSF matching kernel: too few sources selected.") + self.metadata.add("nPsfSources", len(selectSources)) return selectSources.copy(deep=True) @@ -856,9 +860,9 @@ def run(self, template, science, sources, finalizedPsfApCorrCatalog=None, visitS # TODO: DM-37212 we need to mirror the kernel in order to get correct cross correlation scienceKernel = science.psf.getKernel() matchedScience = self._convolveExposure(science, scienceKernel, self.convolutionControl, - badMaskPlanes=self.config.badMaskPlanes, interpolateBadMaskPlanes=True) selectSources = self._sourceSelector(sources, matchedScience.mask) + self.metadata.add("convolvedExposure", "Preconvolution") subtractResults = self.runPreconvolve(template, science, matchedScience, selectSources, scienceKernel) @@ -918,7 +922,6 @@ def runPreconvolve(self, template, science, matchedScience, selectSources, preCo self.convolutionControl, bbox=bbox, psf=science.psf, - badMaskPlanes=self.config.badMaskPlanes, interpolateBadMaskPlanes=True, photoCalib=science.photoCalib) score = _subtractImages(matchedScience, matchedTemplate, @@ -1048,6 +1051,11 @@ def _interpolateImage(maskedImage, badMaskPlanes, fallbackValue=None): List of mask planes to interpolate over. fallbackValue : `float`, optional Value to set when interpolation fails. + + Returns + ------- + result: `float` + The number of masked pixels that were replaced. """ image = maskedImage.image.array badPixels = (maskedImage.mask.array & maskedImage.mask.getPlaneBitMask(badMaskPlanes)) > 0 @@ -1057,3 +1065,4 @@ def _interpolateImage(maskedImage, badMaskPlanes, fallbackValue=None): # For this initial implementation, skip the interpolation and just fill with # the median value. image[badPixels] = fallbackValue + return np.sum(badPixels)