diff --git a/changes/97.feature.rst b/changes/97.feature.rst new file mode 100644 index 0000000..fefce3c --- /dev/null +++ b/changes/97.feature.rst @@ -0,0 +1 @@ +Move nuke_temp_dir kwarg out of Pipeline function and into the phrosty_config.yaml file. Also, make it work instead of leaving it as an empty parameter. Also, rename nuke_temp_dir to remove_temp_dir. diff --git a/examples/perlmutter/phrosty_config.yaml b/examples/perlmutter/phrosty_config.yaml index dc4bacb..a4db085 100644 --- a/examples/perlmutter/phrosty_config.yaml +++ b/examples/perlmutter/phrosty_config.yaml @@ -7,6 +7,7 @@ photometry: image_type: ou2024fits force_sky_subtract: true keep_intermediate: true + remove_temp_dir: false mem_trace: false paths: diff --git a/phrosty/pipeline.py b/phrosty/pipeline.py index 2fdfeb4..7f65779 100644 --- a/phrosty/pipeline.py +++ b/phrosty/pipeline.py @@ -9,6 +9,7 @@ import nvtx import pathlib import re +import shutil import tracemalloc # Imports ASTRO @@ -180,7 +181,7 @@ def free( self ): class Pipeline: def __init__( self, object_id, ra, dec, band, science_images, template_images, nprocs=1, nwrite=5, - nuke_temp_dir=False, verbose=False ): + verbose=False ): """Create the a pipeline object. @@ -215,6 +216,7 @@ def __init__( self, object_id, ra, dec, band, science_images, template_images, n self.image_base_dir = pathlib.Path( self.config.value( 'photometry.phrosty.paths.image_base_dir' ) ) self.dia_out_dir = pathlib.Path( self.config.value( 'photometry.phrosty.paths.dia_out_dir' ) ) self.scratch_dir = pathlib.Path( self.config.value( 'photometry.phrosty.paths.scratch_dir' ) ) + self.temp_dir = pathlib.Path( self.config.value( 'photometry.phrosty.paths.temp_dir' ) ) self.ltcv_dir = pathlib.Path( self.config.value( 'photometry.phrosty.paths.ltcv_dir' ) ) self.object_id = object_id @@ -227,11 +229,9 @@ def __init__( self, object_id, ra, dec, band, science_images, template_images, n for ppsmb in template_images if ppsmb[4] == self.band ] ) self.nprocs = nprocs self.nwrite = nwrite - self.nuke_temp_dir = nuke_temp_dir - if self.nuke_temp_dir: - SNLogger.warning( "nuke_temp_dir not implemented" ) self.keep_intermediate = self.config.value( 'photometry.phrosty.keep_intermediate' ) + self.remove_temp_dir = self.config.value( 'photometry.phrosty.remove_temp_dir' ) self.mem_trace = self.config.value( 'photometry.phrosty.mem_trace' ) @@ -593,12 +593,22 @@ def make_lightcurve( self ): def write_fits_file( self, data, header, savepath ): fits.writeto( savepath, data, header=header, overwrite=True ) + def clear_contents( self, directory ): + for f in directory.iterdir(): + try: + if f.is_dir(): + shutil.rmtree( f ) + else: + f.unlink() + + except Exception as e: + print( f'Oops! Deleting {f} from {directory} did not work.\nReason: {e}' ) + def __call__( self, through_step=None ): if self.mem_trace: tracemalloc.start() tracemalloc.reset_peak() - if through_step is None: through_step = 'make_lightcurve' @@ -826,6 +836,9 @@ def log_fits_write_error( savepath, x ): SNLogger.info( f"After make_lightcurve, memory usage = \ {tracemalloc.get_traced_memory()[1]/(1024**2):.2f} MB" ) + if self.remove_temp_dir: + self.clear_contents( self.temp_dir ) + # ====================================================================== @@ -874,7 +887,7 @@ def main(): imlist.append( ( pathlib.Path(img), int(point), int(sca), float(mjd), band ) ) pipeline = Pipeline( args.oid, args.ra, args.dec, args.band, science_images, template_images, - nprocs=args.nprocs, nwrite=args.nwrite, nuke_temp_dir=False, verbose=args.verbose ) + nprocs=args.nprocs, nwrite=args.nwrite, verbose=args.verbose ) pipeline( args.through_step )