Skip to content

Commit 76175f0

Browse files
laldorotylaldoroty
committed
update to be compatible with snappl changes
1 parent 76de905 commit 76175f0

File tree

4 files changed

+129
-115
lines changed

4 files changed

+129
-115
lines changed

‎phrosty/imagesubtraction.py‎

Lines changed: 8 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
__all__ = [ 'sky_subtract', 'stampmaker' ]
22

33
# IMPORTS Standard:
4+
from astropy.io import fits
45
import numpy as np
56
import pathlib
67
import random
@@ -63,9 +64,10 @@ def sky_subtract( img, temp_dir=None ):
6364
if isinstance( origimg, snappl.image.FITSImageOnDisk ):
6465
img = origimg.uncompressed_version( include=['data'] )
6566
else:
66-
img = snappl.image.FITSImageOnDisk( path=tmpimpath )
67+
# Can take out header arg when snappl issue #77 is resolved.
68+
img = snappl.image.FITSImage( path=tmpimpath, header=fits.header.Header() )
6769
img.data = origimg.data
68-
img.save_data()
70+
img.save_data( which='data' )
6971

7072
SNLogger.debug( "Calling SEx_SkySubtract.SSS..." )
7173
( _SKYDIP, _SKYPEAK, _PixA_skysub,
@@ -78,10 +80,12 @@ def sky_subtract( img, temp_dir=None ):
7880
DETECT_THRESH=1.5, DETECT_MINAREA=5,
7981
DETECT_MAXAREA=0,
8082
VERBOSE_LEVEL=2, MDIR=None)
83+
84+
8185
SNLogger.debug( "...back from SEx_SkySubtract.SSS" )
8286

83-
subim = snappl.image.FITSImageOnDisk( path=tmpsubpath )
84-
detmaskim = snappl.image.FITSImageOnDisk( path=tmpdetmaskpath )
87+
subim = snappl.image.FITSImage( path=tmpsubpath )
88+
detmaskim = snappl.image.FITSImage( path=tmpdetmaskpath )
8589
skyrms = np.median( PixA_skyrms )
8690
return subim, detmaskim, skyrms
8791

‎phrosty/pipeline.py‎

Lines changed: 78 additions & 57 deletions
Original file line numberDiff line numberDiff line change
@@ -125,15 +125,10 @@ def save_sky_subtract_info( self, info ):
125125
for phrosty.imagesubtraction.sky_subtract().
126126
"""
127127

128-
try:
129-
SNLogger.debug( f"Saving sky_subtract info for path {info[0]}" )
130-
self.skysub_img = info[0]
131-
self.detmask_img = info[1]
132-
self.skyrms = info[2]
133-
134-
except:
135-
if self.skysub_img is None or self.detmask_img is None or self.skyrms is None:
136-
pipeline.failures['skysub'].append(f'{self.image.band} {self.image.pointing} {self.image.sca}')
128+
SNLogger.debug( f"Saving sky_subtract info for path {info[0]}" )
129+
self.skysub_img = info[0]
130+
self.detmask_img = info[1]
131+
self.skyrms = info[2]
137132

138133
def get_psf( self, ra, dec ):
139134
"""Get the at the right spot on the image.
@@ -177,8 +172,7 @@ def get_psf( self, ra, dec ):
177172
return None
178173

179174
def keep_psf_data( self, psf_data ):
180-
"""Save PSF data to attribute. If self.get_psf() failed,
181-
then the image is recorded as a failure.
175+
"""Save PSF data to attribute.
182176
183177
Parameters
184178
----------
@@ -189,9 +183,6 @@ def keep_psf_data( self, psf_data ):
189183

190184
self.psf_data = psf_data
191185

192-
if self.psf_data is None:
193-
pipeline.failures['get_psf'].append(f'{self.image.band} {self.image.pointing} {self.image.sca}')
194-
195186
def free( self ):
196187
"""Try to free memory. More might be done here."""
197188
self.image.free()
@@ -283,8 +274,17 @@ def __init__( self, diaobj, imgcol, band,
283274
self.template_images = [ PipelineImage( i, self ) for i in template_images ]
284275

285276
# All of our failures.
286-
self.failures = {'skysub': [],
287-
'get_psf': [],
277+
# LA NOTE: I want to add keys for when this fails in sky subtraction and PSF retrieval
278+
# as well. But that is slightly more involved because those things happen in
279+
# PipelineImage, not Pipeline. So, for now, anything that fails at 'make_lightcurve'
280+
# may have failed at earlier steps first. Also, source extractor doesn't fail on
281+
# an image full of NaNs for some reason. It just reports 0 sources. I would expect
282+
# that it should fail, here...
283+
self.failures = {'align_and_preconvolve': [],
284+
'find_decorrelation': [],
285+
'subtract': [],
286+
'variance': [],
287+
'apply_decorrelation': [],
288288
'make_lightcurve': [],
289289
'make_stamps': []}
290290

@@ -344,9 +344,7 @@ def sky_sub_all_images( self ):
344344
all_imgs.extend( self.template_images )
345345

346346
def log_error( img, x ):
347-
348347
SNLogger.error( f"Sky subtraction failure on {img.image.path}: {x}" )
349-
self.failures['skysub'].append( f'{img.image.band} {img.image.pointing} {img.image.sca}' )
350348

351349
if self.nprocs > 1:
352350
with Pool( self.nprocs ) as pool:
@@ -373,7 +371,6 @@ def get_psfs( self ):
373371

374372
def log_error( img, x ):
375373
SNLogger.error( f"get_psf failure on {img.image.path}: {x}" )
376-
self.failures['get_psf'].append( f'{img.image.band} {img.image.pointing} {img.image.sca}' )
377374

378375
if self.nprocs > 1:
379376
with Pool( self.nprocs ) as pool:
@@ -861,56 +858,80 @@ def log_fits_write_error( savepath, x ):
861858
for sci_image in self.science_images:
862859
SNLogger.info( f"Processing {sci_image.image.name} minus {templ_image.image.name}" )
863860
sfftifier = None
861+
fail_info = {'science': f'{sci_image.image.band} {sci_image.image.pointing} {sci_image.image.sca}',
862+
'template': f'{templ_image.image.band} {templ_image.image.pointing} {templ_image.image.sca}'
863+
}
864+
i_failed = False
864865

865866
if 'align_and_preconvolve' in steps:
866867
SNLogger.info( "...align_and_preconvolve" )
867868
with nvtx.annotate( "align_and_pre_convolve", color=0x8888ff ):
868-
sfftifier = self.align_and_pre_convolve( templ_image, sci_image )
869+
try:
870+
sfftifier = self.align_and_pre_convolve( templ_image, sci_image )
871+
except:
872+
i_failed = True
873+
self.failures['align_and_preconvolve'].append(fail_info)
869874

870-
if 'subtract' in steps:
875+
if 'subtract' in steps and not i_failed:
871876
SNLogger.info( "...subtract" )
872877
with nvtx.annotate( "subtraction", color=0x44ccff ):
873-
sfftifier.sfft_subtraction()
878+
try:
879+
sfftifier.sfft_subtraction()
880+
except:
881+
i_failed = True
882+
self.failures['subtract'].append(fail_info)
874883

875-
if 'find_decorrelation' in steps:
884+
if 'find_decorrelation' in steps and not i_failed:
876885
SNLogger.info( "...find_decorrelation" )
877886
with nvtx.annotate( "find_decor", color=0xcc44ff ):
878-
sfftifier.find_decorrelation()
887+
try:
888+
sfftifier.find_decorrelation()
889+
except:
890+
i_failed = True
891+
self.failures['find_decorrelation'].append(fail_info)
879892

880893
SNLogger.info( "...generate variance image" )
881894
with nvtx.annotate( "variance", color=0x44ccff ):
882-
diff_var = sfftifier.create_variance_image()
895+
try:
896+
diff_var = sfftifier.create_variance_image()
897+
mess = f"{sci_image.image.name}-{templ_image.image.name}"
898+
diff_var_path = self.dia_out_dir / f"diff_var_{mess}"
899+
except:
900+
i_failed = True
901+
self.failures['variance'].append(fail_info)
902+
903+
if 'apply_decorrelation' in steps and not i_failed:
904+
try:
883905
mess = f"{sci_image.image.name}-{templ_image.image.name}"
884-
diff_var_path = self.dia_out_dir / f"diff_var_{mess}"
885-
886-
if 'apply_decorrelation' in steps:
887-
mess = f"{sci_image.image.name}-{templ_image.image.name}"
888-
decorr_psf_path = self.dia_out_dir / f"decorr_psf_{mess}"
889-
decorr_zptimg_path = self.dia_out_dir / f"decorr_zptimg_{mess}"
890-
decorr_diff_path = self.dia_out_dir / f"decorr_diff_{mess}"
891-
892-
images = [ sfftifier.PixA_DIFF_GPU, diff_var,
893-
sfftifier.PixA_Ctarget_GPU, sfftifier.PSF_target_GPU ]
894-
savepaths = [ decorr_diff_path, diff_var_path,
895-
decorr_zptimg_path, decorr_psf_path ]
896-
headers = [ sfftifier.hdr_target, sfftifier.hdr_target,
897-
sfftifier.hdr_target, None ]
898-
899-
for img, savepath, hdr in zip( images, savepaths, headers ):
900-
with nvtx.annotate( "apply_decor", color=0xccccff ):
901-
SNLogger.info( f"...apply_decor to {savepath}" )
902-
decorimg = sfftifier.apply_decorrelation( img )
903-
with nvtx.annotate( "submit writefits", color=0xff8888 ):
904-
SNLogger.info( f"...writefits {savepath}" )
905-
fits_writer_pool.apply_async( self.write_fits_file,
906-
( cp.asnumpy( decorimg ).T, hdr, savepath ), {},
907-
error_callback=partial(log_fits_write_error, savepath) )
908-
sci_image.decorr_psf_path[ templ_image.image.name ] = decorr_psf_path
909-
sci_image.decorr_zptimg_path[ templ_image.image.name ] = decorr_zptimg_path
910-
sci_image.decorr_diff_path[ templ_image.image.name ] = decorr_diff_path
911-
sci_image.diff_var_path[ templ_image.image.name ] = diff_var_path
912-
913-
if self.keep_intermediate:
906+
decorr_psf_path = self.dia_out_dir / f"decorr_psf_{mess}"
907+
decorr_zptimg_path = self.dia_out_dir / f"decorr_zptimg_{mess}"
908+
decorr_diff_path = self.dia_out_dir / f"decorr_diff_{mess}"
909+
910+
images = [ sfftifier.PixA_DIFF_GPU, diff_var,
911+
sfftifier.PixA_Ctarget_GPU, sfftifier.PSF_target_GPU ]
912+
savepaths = [ decorr_diff_path, diff_var_path,
913+
decorr_zptimg_path, decorr_psf_path ]
914+
headers = [ sfftifier.hdr_target, sfftifier.hdr_target,
915+
sfftifier.hdr_target, None ]
916+
917+
for img, savepath, hdr in zip( images, savepaths, headers ):
918+
with nvtx.annotate( "apply_decor", color=0xccccff ):
919+
SNLogger.info( f"...apply_decor to {savepath}" )
920+
decorimg = sfftifier.apply_decorrelation( img )
921+
with nvtx.annotate( "submit writefits", color=0xff8888 ):
922+
SNLogger.info( f"...writefits {savepath}" )
923+
fits_writer_pool.apply_async( self.write_fits_file,
924+
( cp.asnumpy( decorimg ).T, hdr, savepath ), {},
925+
error_callback=partial(log_fits_write_error, savepath) )
926+
sci_image.decorr_psf_path[ templ_image.image.name ] = decorr_psf_path
927+
sci_image.decorr_zptimg_path[ templ_image.image.name ] = decorr_zptimg_path
928+
sci_image.decorr_diff_path[ templ_image.image.name ] = decorr_diff_path
929+
sci_image.diff_var_path[ templ_image.image.name ] = diff_var_path
930+
except:
931+
i_failed = True
932+
self.failures['apply_decorrelation'].append(fail_info)
933+
934+
if self.keep_intermediate and not i_failed:
914935
# Each key is the file prefix addition.
915936
# Each list has [descriptive filetype, image file name, data, header].
916937

@@ -978,7 +999,7 @@ def log_fits_write_error( savepath, x ):
978999
fits_writer_pool.join()
9791000
SNLogger.info( "...FITS writer processes done." )
9801001

981-
if 'make_stamps' in steps:
1002+
if 'make_stamps' in steps and not i_failed:
9821003
SNLogger.info( "Starting to make stamps..." )
9831004
with nvtx.annotate( "make stamps", color=0xff8888 ):
9841005

@@ -1026,7 +1047,7 @@ def log_stamp_err( sci_image, templ_image, x ):
10261047
SNLogger.info( f"After make_stamps, memory usage = {tracemalloc.get_traced_memory()[1]/(1024**2):.2f} MB" )
10271048

10281049
lightcurve_path = None
1029-
if 'make_lightcurve' in steps:
1050+
if 'make_lightcurve' in steps and not i_failed:
10301051
with nvtx.annotate( "make_lightcurve", color=0xff8888 ):
10311052
lightcurve_path = self.make_lightcurve()
10321053

‎phrosty/tests/conftest.py‎

Lines changed: 5 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -177,20 +177,17 @@ def two_ou2024_science_images( ou2024_image_collection ):
177177
return img1, img2
178178

179179
@pytest.fixture
180-
def nan_image( ou2024_image_collection ):
181-
ou_img = ou2024_image_collection.get_image( pointing=35198, sca=2, band='Y106' )
182-
ou_header = ou_img.get_fits_header()
183-
180+
def nan_image():
184181
nan_arr = np.empty((4088,4088))
185182
nan_arr[:] = np.nan
186183

187184
# NOTE: path='/dev/null' because at this time, snappl requires a path to instantiate
188185
# a FITSImage object.
189186
nan_img = FITSImageStdHeaders( path='/dev/null',
190-
header=ou_header,
191-
data=nan_arr,
192-
noise=nan_arr,
193-
flags=nan_arr )
187+
data=nan_arr
188+
)
194189
nan_img.band = 'Y106'
190+
nan_img.pointing = -1 # So it's not real and doesn't interpret.
191+
nan_img.sca = 1
195192

196193
return nan_img

‎phrosty/tests/test_pipeline.py‎

Lines changed: 38 additions & 46 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@
44
import pytest
55

66
from phrosty.pipeline import Pipeline
7-
from snappl.image import FITSImageOnDisk
7+
from snappl.image import FITSImageStdHeaders
88

99
# TODO : separate tests for PipelineImage, for all the functions in#
1010
# PipelineImage and Pipeline. Right now we just have this regression
@@ -77,59 +77,51 @@ def test_pipeline_run( object_for_tests, ou2024_image_collection,
7777

7878
@pytest.mark.skipif( os.getenv("SKIP_GPU_TESTS", 0 ), reason="SKIP_GPU_TESTS is set" )
7979
def test_pipeline_failures( object_for_tests, ou2024_image_collection,
80-
one_ou2024_template_image, two_ou2024_science_images,
81-
nan_image ):
80+
one_ou2024_template_image, two_ou2024_science_images):
8281

8382
nprocss = [1, 3]
8483
nwrites = [1, 3]
8584

86-
for i in nprocss:
87-
for j in nwrites:
88-
pip = Pipeline( object_for_tests, ou2024_image_collection, 'Y106',
89-
science_images=two_ou2024_science_images,
90-
template_images=[one_ou2024_template_image],
91-
nprocs=i, nwrite=j )
85+
# for i in nprocss:
86+
# for j in nwrites:
87+
# pip = Pipeline( object_for_tests, ou2024_image_collection, 'Y106',
88+
# science_images=two_ou2024_science_images,
89+
# template_images=[one_ou2024_template_image],
90+
# nprocs=i, nwrite=j )
9291

93-
lctv = pip()
92+
# lctv = pip()
9493

95-
# First, check the images as-is. Make sure there are no failures.
96-
for key in pip.failures:
97-
assert len(pip.failures[key]) == 0
98-
99-
# Below is commented out because I can't verify that the tests are working
100-
# because of NERSC spin issues. What it's trying to do is verify that for
101-
# an array full of NaNs, the thing fails in the sky subtraction portion of
102-
# the code, and self.failures['skysub'] gets the image identifier appended
103-
# to it.
94+
# # First, check the images as-is. Make sure there are no failures.
95+
# for key in pip.failures:
96+
# assert len(pip.failures[key]) == 0
10497

105-
# This is messy and file-savey because right now, snappl requires paths to do things.
106-
# Or maybe it's sky subtract. Either way. Future improvements necessary.
10798
# try:
108-
# nanpaths = [
109-
# pathlib.Path( '/phrosty_temp/test_nan_img.fits' ),
110-
# pathlib.Path( '/phrosty_temp/test_nan_noise.fits' ),
111-
# pathlib.Path( 'phrosty_temp/test_nan_flags.fits' )
112-
# ]
113-
114-
# nan_image.save(
115-
# path=nanpaths[0],
116-
# noisepath=nanpaths[1],
117-
# flagspath=nanpaths[2]
118-
# )
119-
120-
# opened_nan_img = FITSImageOnDisk(nanpaths[0])
121-
122-
# print(opened_nan_img)
123-
# print(opened_nan_img.get_data())
124-
125-
# new_test_imgs = [opened_nan_image, two_ou2024_science_images[1]]
126-
127-
# pytest.set_trace()
128-
# pip = Pipeline( object_for_tests, ou2024_image_collection, 'Y106',
129-
# science_images=new_test_imgs,
130-
# template_images=[one_ou2024_template_image],
131-
# nprocs=1, nwrite=1 )
132-
# lctv = pip()
99+
nan_image = FITSImageStdHeaders( path='/phrosty_temp/test_nan_img',
100+
data=np.full(one_ou2024_template_image.image_shape, np.nan),
101+
flags=np.zeros(one_ou2024_template_image.image_shape),
102+
std_imagenames=True
103+
)
104+
105+
nan_image._wcs = one_ou2024_template_image.get_wcs()
106+
nan_image.noise = nan_image.data
107+
nan_image.band = 'Y106'
108+
nan_image.pointing = -1 # Give it a fake pointing on purpose
109+
nan_image.sca = 1
110+
nan_image.save( which='data', overwrite=True )
111+
112+
# pytest.set_trace()
113+
114+
new_test_imgs = [nan_image, two_ou2024_science_images[1]]
115+
116+
pip = Pipeline( object_for_tests, ou2024_image_collection, 'Y106',
117+
science_images=new_test_imgs,
118+
template_images=[one_ou2024_template_image],
119+
nprocs=1, nwrite=1 )
120+
lctv = pip()
121+
122+
for key in pip.failures:
123+
print(key)
124+
print(len(pip.failures[key]))
133125

134126
# finally:
135127
# for path in nanpaths:

0 commit comments

Comments
 (0)