Skip to content

Commit 5d81366

Browse files
authored
First step of refactoring to use snappl (#72)
1 parent 03f7cd9 commit 5d81366

12 files changed

+373
-340
lines changed

changes/72.feature.rst

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
Starting on a refactor to use snpit_utils and snappl

examples/perlmutter/20172782_2sci1templ.sh

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,6 @@ echo
1717
podman-hpc run --gpu \
1818
--mount type=bind,source=$PWD,target=/home \
1919
--mount type=bind,source=$PWD/phrosty,target=/phrosty \
20-
--mount type=bind,source=$PWD/sfft,target=/sfft \
2120
--mount type=bind,source=$PWD/sn_info_dir,target=/sn_info_dir \
2221
--mount type=bind,source=$PWD/dia_out_dir,target=/dia_out_dir \
2322
--mount type=bind,source=$PWD/lc_out_dir,target=/lc_out_dir \
@@ -34,13 +33,12 @@ podman-hpc run --gpu \
3433
--env VECLIB_MAXIMUM_THREADS=1 \
3534
--env SIMS_DIR=/sims_dir \
3635
--env SNANA_PQ_DIR=/snana_pq_dir \
37-
--env SN_INFO_DIR=sn_info_dir \
38-
--env DIA_OUT_DIR=dia_out_dir \
3936
--env TERM=xterm \
4037
-w /home \
4138
-it \
4239
registry.nersc.gov/m4385/rknop/roman-snpit-env:cuda-dev \
4340
python phrosty/phrosty/pipeline.py \
41+
-c phrosty_config.yaml \
4442
--oid 20172782 \
4543
-r 7.551093401915147 \
4644
-d -44.80718106491529 \

examples/perlmutter/20172782_instances_science.csv

Lines changed: 54 additions & 54 deletions
Large diffs are not rendered by default.
Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,2 +1,3 @@
1-
RomanTDS/images/simple_model/R062/35083/Roman_TDS_simple_model_R062_35083_8.fits.gz 35083 8 62455.174
2-
RomanTDS/images/simple_model/R062/41205/Roman_TDS_simple_model_R062_41205_12.fits.gz 41205 12 62535.053
1+
path pointing sca mjd band
2+
RomanTDS/images/simple_model/R062/35083/Roman_TDS_simple_model_R062_35083_8.fits.gz 35083 8 62455.174 R062
3+
RomanTDS/images/simple_model/R062/41205/Roman_TDS_simple_model_R062_41205_12.fits.gz 41205 12 62535.053 R062
Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1,2 @@
1-
RomanTDS/images/simple_model/R062/6/Roman_TDS_simple_model_R062_6_17.fits.gz 6 17 62000.04011
1+
path pointing sca mjd band
2+
RomanTDS/images/simple_model/R062/6/Roman_TDS_simple_model_R062_6_17.fits.gz 6 17 62000.04011 R062
Lines changed: 11 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,11 @@
1-
path pointing sca mjd
2-
/global/cfs/cdirs/lsst/shared/external/roman-desc-sims/Roman_data/RomanTDS/images/simple_model/R062/6/Roman_TDS_simple_model_R062_6_17.fits.gz 6 17 62000.04011
3-
/global/cfs/cdirs/lsst/shared/external/roman-desc-sims/Roman_data/RomanTDS/images/simple_model/R062/1928/Roman_TDS_simple_model_R062_1928_10.fits.gz 1928 10 62025.0294
4-
/global/cfs/cdirs/lsst/shared/external/roman-desc-sims/Roman_data/RomanTDS/images/simple_model/R062/2314/Roman_TDS_simple_model_R062_2314_6.fits.gz 2314 6 62030.0321
5-
/global/cfs/cdirs/lsst/shared/external/roman-desc-sims/Roman_data/RomanTDS/images/simple_model/R062/2319/Roman_TDS_simple_model_R062_2319_8.fits.gz 2319 8 62030.0481
6-
/global/cfs/cdirs/lsst/shared/external/roman-desc-sims/Roman_data/RomanTDS/images/simple_model/R062/2709/Roman_TDS_simple_model_R062_2709_7.fits.gz 2709 7 62035.0642
7-
/global/cfs/cdirs/lsst/shared/external/roman-desc-sims/Roman_data/RomanTDS/images/simple_model/R062/3089/Roman_TDS_simple_model_R062_3089_17.fits.gz 3089 17 62040.0481
8-
/global/cfs/cdirs/lsst/shared/external/roman-desc-sims/Roman_data/RomanTDS/images/simple_model/R062/3094/Roman_TDS_simple_model_R062_3094_10.fits.gz 3094 10 62040.0642
9-
/global/cfs/cdirs/lsst/shared/external/roman-desc-sims/Roman_data/RomanTDS/images/simple_model/R062/4264/Roman_TDS_simple_model_R062_4264_7.fits.gz 4264 7 62055.1123
10-
/global/cfs/cdirs/lsst/shared/external/roman-desc-sims/Roman_data/RomanTDS/images/simple_model/R062/5044/Roman_TDS_simple_model_R062_5044_9.fits.gz 5044 9 62065.1444
11-
/global/cfs/cdirs/lsst/shared/external/roman-desc-sims/Roman_data/RomanTDS/images/simple_model/R062/5429/Roman_TDS_simple_model_R062_5429_4.fits.gz 5429 4 62070.1444
1+
path pointing sca mjd band
2+
/global/cfs/cdirs/lsst/shared/external/roman-desc-sims/Roman_data/RomanTDS/images/simple_model/R062/6/Roman_TDS_simple_model_R062_6_17.fits.gz 6 17 62000.04011 R062
3+
/global/cfs/cdirs/lsst/shared/external/roman-desc-sims/Roman_data/RomanTDS/images/simple_model/R062/1928/Roman_TDS_simple_model_R062_1928_10.fits.gz 1928 10 62025.0294 R062
4+
/global/cfs/cdirs/lsst/shared/external/roman-desc-sims/Roman_data/RomanTDS/images/simple_model/R062/2314/Roman_TDS_simple_model_R062_2314_6.fits.gz 2314 6 62030.0321 R062
5+
/global/cfs/cdirs/lsst/shared/external/roman-desc-sims/Roman_data/RomanTDS/images/simple_model/R062/2319/Roman_TDS_simple_model_R062_2319_8.fits.gz 2319 8 62030.0481 R062
6+
/global/cfs/cdirs/lsst/shared/external/roman-desc-sims/Roman_data/RomanTDS/images/simple_model/R062/2709/Roman_TDS_simple_model_R062_2709_7.fits.gz 2709 7 62035.0642 R062
7+
/global/cfs/cdirs/lsst/shared/external/roman-desc-sims/Roman_data/RomanTDS/images/simple_model/R062/3089/Roman_TDS_simple_model_R062_3089_17.fits.gz 3089 17 62040.0481 R062
8+
/global/cfs/cdirs/lsst/shared/external/roman-desc-sims/Roman_data/RomanTDS/images/simple_model/R062/3094/Roman_TDS_simple_model_R062_3094_10.fits.gz 3094 10 62040.0642 R062
9+
/global/cfs/cdirs/lsst/shared/external/roman-desc-sims/Roman_data/RomanTDS/images/simple_model/R062/4264/Roman_TDS_simple_model_R062_4264_7.fits.gz 4264 7 62055.1123 R062
10+
/global/cfs/cdirs/lsst/shared/external/roman-desc-sims/Roman_data/RomanTDS/images/simple_model/R062/5044/Roman_TDS_simple_model_R062_5044_9.fits.gz 5044 9 62065.1444 R062
11+
/global/cfs/cdirs/lsst/shared/external/roman-desc-sims/Roman_data/RomanTDS/images/simple_model/R062/5429/Roman_TDS_simple_model_R062_5429_4.fits.gz 5429 4 62070.1444 R062

examples/perlmutter/README.md

Lines changed: 11 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -48,20 +48,6 @@ Work in one of two places. You make yourself a subdirectory underneath `/pscrat
4848

4949
I'm going to call the place you've picked to work your "parent" directory.
5050

51-
### Get sfft
52-
53-
In your parent directory, do
54-
```
55-
git clone https://github.com/Roman-Supernova-PIT/sfft.git
56-
cd sfft
57-
git checkout fixes_20241022
58-
cd ..
59-
```
60-
61-
This will pull down the Roman PITs version of the SFFT archive, and check out the specific branch that (as of this writing) we require. Below, you will make this directory available inside the podman container you'll use to run *phrosty*. (Note: you may prefer to clone `[email protected]:Roman-Supernova-PIT/sfft.git` instead of `https:...`. If you use github with ssh keys, then you already probably know that you want to do this. If not, don't worry about it.)
62-
63-
In the future, we will merge the changes we need back to the `master` branch of sfft, but for now, you need the `fixes_20241022` branch.
64-
6551
### Get phrosty
6652

6753
```
@@ -71,7 +57,7 @@ git checkout main
7157
cd ..
7258
```
7359

74-
This will pull the actual phrosty code, including the pipeline you'll run. This README file you're reading is within that repository (in `examples/perlmutter/README.md`). The `git checkout main` is probably redundant, becuase it's likely to check that branch out by default. (Again, you may prefer to clone `[email protected]:Roman-Supernova-PIT/phrosty.git`, but you will already know if you want to do that.)
60+
This will pull the actual phrosty code, including the pipeline you'll run. This README file you're reading is within that repository (in `examples/perlmutter/README.md`). The `git checkout main` is probably redundant, becuase it's likely to check that branch out by default. (Again, you may prefer to clone `[email protected]:Roman-Supernova-PIT/phrosty.git`, but you will already know if you want to do that.) The last `cd ..` puts you back in your parent directory.
7561

7662

7763
### Locate existing directories
@@ -91,13 +77,14 @@ You need to make the following directories. (They don't have to have exactly th
9177
* `lc_out_dir`
9278

9379
In addition, create a directory `phrosty_temp` somewhere underneath `$SCRATCH`, e.g.:
94-
```mkdir $SCRATCH/phrosty_temp```. (The further examples below will assume that this is where you made it.)
80+
```mkdir $SCRATCH/phrosty_temp```
81+
(The further examples below will assume that this is where you made it.)
9582

9683
### Populate your sn_info_dir
9784

9885
This is where you put information that *phrosty* needs to find information about the OpenUniverse sims, and about any supernova from that sim you want to run it on. The first file you need is `tds.yaml`; copy that file from this directory (i.e. `phrosty/examples/perlmutter/tds.yaml`) into your `sn_info_dir`; when in the `sn_info_dir`, run:)
9986
```
100-
cp -p ../phrosty/examples/perlmutter/tds.yaml ./
87+
cp -p phrosty/examples/perlmutter/tds.yaml ./
10188
```
10289
(This file is a modified version of the standard OpenUniverse sims file `/global/cfs/cdirs/lsst/shared/external/roman-desc-sims/Roman_data/RomanTDS`, fixing some paths for our own purposes.)
10390

@@ -122,7 +109,7 @@ If you look at these `.csv` files, there are four pieces of information on each
122109
* The pointing of the image
123110
* The SCA on which the supernova is present for this pointing
124111
* The MJD of the pointing
125-
112+
* The band (filter) of the exposure
126113

127114
## Running interactively
128115

@@ -134,13 +121,13 @@ salloc -t 04:00:00 -A m4385 --constraint=gpu -q interactive
134121
```
135122
after a minute or so, that should log you into one of the nodes with a session that will last 4 hours. (This is overkill; if you know it won't be that long, shorten the time after the `-t` flag.) You can verify that you're on a compute node by running `nvidia-smi`; you should see four different GPUs listed each with either 40MB or 80GB of memory, but no GPU processes running.
136123

137-
cd into your "parent" directory.
124+
cd into your "parent" directory (if you're not there already).
138125

139-
Copy the file `interactive_podman.sh` from `phrosty/examples/perlmutter` to your parent directory:
126+
Copy the files `interactive_podman.sh` and `phrosty_config.yaml` from `phrosty/examples/perlmutter` to your parent directory:
140127
```
141-
cp -p phrosty/examples/perlmutter/interactive_podman.sh ./
128+
cp -p phrosty/examples/perlmutter/interactive_podman.sh phrosty_config.yaml ./
142129
```
143-
Look at this file. (There's no need to edit it; this is so you can see what's going on.) You'll see number of `--mount` parameters. Each of these takes a directory on the host machine (the `source`) and maps it to a directory inside the podman container (the `target`). For example, you will see your phrosty checkout goes to `/phrosty` inside the container. In addition, a bunch of environment variables are set so that *phrosty* will be able to find all of these directories inside the container.
130+
Look at the `.sh` file. (There's no need to edit it; this is so you can see what's going on.) You'll see number of `--mount` parameters. Each of these takes a directory on the host machine (the `source`) and maps it to a directory inside the podman container (the `target`). For example, you will see your phrosty checkout goes to `/phrosty` inside the container. In addition, a bunch of environment variables are set so that *phrosty* will be able to find all of these directories inside the container.
144131

145132
Now do
146133
```
@@ -163,6 +150,7 @@ to see how it works, and to see what the various parameters you can specify are.
163150
Run this on your example lightcurve with:
164151
```
165152
python phrosty/phrosty/pipeline.py \
153+
-c phrosty_config.yaml \
166154
--oid 20172782 \
167155
-r 7.551093401915147 \
168156
-d -44.80718106491529 \
@@ -195,6 +183,7 @@ nsys profile \
195183
--python-sampling=true \
196184
--trace=cuda,nvtx,cublas,cusparse,cudnn,cudla,cusolver,opengl,openacc,openmp,osrt,mpi,nvvideo,vulkan,python-gil \
197185
python phrosty/phrosty/pipeline.py \
186+
-c phrosty_config.yaml \
198187
--oid 20172782 \
199188
-r 7.551093401915147 \
200189
-d -44.80718106491529 \

examples/perlmutter/interactive_podman.sh

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,6 @@
33
podman-hpc run --gpu \
44
--mount type=bind,source=$PWD,target=/home \
55
--mount type=bind,source=$PWD/phrosty,target=/phrosty \
6-
--mount type=bind,source=$PWD/sfft,target=/sfft \
76
--mount type=bind,source=$PWD/sn_info_dir,target=/sn_info_dir \
87
--mount type=bind,source=$PWD/dia_out_dir,target=/dia_out_dir \
98
--mount type=bind,source=$PWD/lc_out_dir,target=/lc_out_dir \
@@ -20,8 +19,6 @@ podman-hpc run --gpu \
2019
--env VECLIB_MAXIMUM_THREADS=1 \
2120
--env SIMS_DIR=/sims_dir \
2221
--env SNANA_PQ_DIR=/snana_pq_dir \
23-
--env SN_INFO_DIR=/sn_info_dir \
24-
--env DIA_OUT_DIR=/dia_out_dir \
2522
--env TERM=xterm \
2623
-it \
2724
registry.nersc.gov/m4385/rknop/roman-snpit-env:cuda-dev \
Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
ou24psf:
2+
config_file: /sn_info_dir/tds.yaml
3+
4+
photometry:
5+
phrosty:
6+
7+
image_type: ou2024fits
8+
force_sky_subtract: true
9+
10+
paths:
11+
temp_dir: /phrosty_temp
12+
sn_info_dir: /sn_info_dir
13+
dia_out_dir: /dia_out_dir
14+
image_base_dir: /sims_dir
15+
snana_pq_dir: /snana_pq_dir
16+
snid_lc_dir: /snid_lc_dir
17+
ltcv_dir: /lc_out_dir
18+
19+
psf:
20+
type: ou24PSF

phrosty/imagesubtraction.py

Lines changed: 17 additions & 55 deletions
Original file line numberDiff line numberDiff line change
@@ -19,28 +19,28 @@
1919
from astropy.wcs import WCS
2020

2121
# IMPORTS SFFT:
22-
from roman_imsim.utils import roman_utils
2322
from sfft.utils.SExSkySubtract import SEx_SkySubtract
2423
from sfft.utils.StampGenerator import Stamp_Generator
2524
from sfft.utils.CudaResampling import Cuda_Resampling
2625

2726
# IMPORTS internal
2827
from snpit_utils.logger import SNLogger
28+
from snpit_utils.config import Config
2929

3030
"""
3131
This module was written with significant contributions from
3232
Dr. Lei Hu (https://github.com/thomasvrussell/), and relies on his
3333
SFFT image subtraction package (https://github.com/thomasvrussell/sfft).
3434
"""
3535

36-
output_files_rootdir = os.getenv('DIA_OUT_DIR', None)
37-
assert output_files_rootdir is not None, 'You need to set DIA_OUT_DIR as an environment variable.'
36+
# output_files_rootdir = os.getenv('DIA_OUT_DIR', None)
37+
# assert output_files_rootdir is not None, 'You need to set DIA_OUT_DIR as an environment variable.'
3838

3939

40-
def check_and_mkdir(dirname):
41-
"""Utility function for checking if a directory exists, and if not, makes that directory."""
42-
if not os.path.exists(dirname):
43-
os.mkdir(dirname)
40+
# def check_and_mkdir(dirname):
41+
# """Utility function for checking if a directory exists, and if not, makes that directory."""
42+
# if not os.path.exists(dirname):
43+
# os.mkdir(dirname)
4444

4545

4646
def gz_and_ext(in_path, out_path):
@@ -111,6 +111,7 @@ def sky_subtract( inpath, skysubpath, detmaskpath, temp_dir=pathlib.Path("/tmp")
111111
decompressed_path = inpath
112112

113113

114+
SNLogger.debug( "Calling SEx_SkySubtract.SSS..." )
114115
( SKYDIP, SKYPEAK, PixA_skysub,
115116
PixA_sky, PixA_skyrms ) = SEx_SkySubtract.SSS(FITS_obj=decompressed_path,
116117
FITS_skysub=skysubpath,
@@ -121,6 +122,7 @@ def sky_subtract( inpath, skysubpath, detmaskpath, temp_dir=pathlib.Path("/tmp")
121122
DETECT_THRESH=1.5, DETECT_MINAREA=5,
122123
DETECT_MAXAREA=0,
123124
VERBOSE_LEVEL=2, MDIR=None)
125+
SNLogger.debug( "...back from SEx_SkySubtract.SSS" )
124126

125127
return np.median( PixA_skyrms )
126128

@@ -209,50 +211,6 @@ def run_resample(FITS_obj, FITS_targ, FITS_resamp):
209211
# return rotate_angle
210212

211213

212-
def get_imsim_psf(image_path, ra, dec, band, pointing, sca, size=201, config_yaml_file=None,
213-
psf_path=None, force=False, **kwargs):
214-
215-
"""Retrieve the PSF from roman_imsim/galsim.
216-
217-
Transform the WCS so that CRPIX and CRVAL are centered on the image
218-
instead of at the corner.
219-
220-
kwargs match getPSF_Image args, listed here with their defaults:
221-
pupil_bin=8,
222-
sed=None,
223-
oversampling_factor=1,
224-
include_photonOps=False,
225-
n_phot=1e6
226-
227-
force parameter does not currently do anything.
228-
229-
"""
230-
231-
if psf_path is None:
232-
raise ValueError( "psf_path can't be None" )
233-
234-
# Get WCS of the image you need the PSF for.
235-
with fits.open( image_path ) as hdu:
236-
wcs = WCS(hdu[0].header)
237-
coord = SkyCoord(ra=ra*u.deg, dec=dec*u.deg)
238-
x, y = wcs.world_to_pixel(coord)
239-
240-
# Get PSF at specified ra, dec.
241-
assert config_yaml_file is not None, "config_yaml_file is a required argument"
242-
config_path = config_yaml_file
243-
config = roman_utils(config_path, pointing, sca)
244-
#### TEMP TEST
245-
#### Want to see if this is the source of non-reproducible results
246-
# For the future: if we adapt the snpit_utils config system,
247-
# we could set a config variable that is the seed, and if
248-
# it's not None, set the seed here. For now, we aceept
249-
# the variance in our tests.
250-
# config.rng = galsim.BaseDeviate(12345)
251-
#### END OF TEMP TEST
252-
psf = config.getPSF_Image(size, x, y, **kwargs)
253-
psf.write( str(psf_path) )
254-
255-
256214
# def rotate_psf(ra,dec,psf,target,savename=None,force=False,verbose=False):
257215
# """
258216
# 2. Rotate PSF model to match reference WCS.
@@ -359,13 +317,16 @@ def stampmaker(ra, dec, shape, imgpath, savedir=None, savename=None):
359317
"""
360318

361319
if savedir is None:
362-
savedir = os.path.join(output_files_rootdir, 'stamps')
363-
check_and_mkdir(savedir)
320+
cfg = Config.get()
321+
savedir = pathlib.Path( cfg.value( 'photometry.phrosty.paths.dia_out_dir' ) ) / "stamps"
322+
else:
323+
savedir = pathlib.Path( savedir )
324+
savedir.mkdir( parents=True, exist_ok=True )
364325

365326
if savename is None:
366327
savename = f'stamp_{os.path.basename(imgpath)}.fits'
367328

368-
savepath = os.path.join(savedir, savename)
329+
savepath = savedir / savename
369330

370331
coord = SkyCoord(ra=ra*u.deg, dec=dec*u.deg)
371332
with fits.open(imgpath) as hdu:
@@ -375,8 +336,9 @@ def stampmaker(ra, dec, shape, imgpath, savedir=None, savename=None):
375336
x, y = skycoord_to_pixel(coord, wcs)
376337
pxradec = np.array([[x, y]])
377338

339+
# TODO : if Stamp_Generator.SG can take a Path in FITS_StpLst, remove the str()
378340
Stamp_Generator.SG(FITS_obj=imgpath, EXTINDEX=hdun, COORD=pxradec, COORD_TYPE='IMAGE',
379-
STAMP_IMGSIZE=shape, FILL_VALUE=np.nan, FITS_StpLst=savepath)
341+
STAMP_IMGSIZE=shape, FILL_VALUE=np.nan, FITS_StpLst=str(savepath))
380342

381343
return savepath
382344

0 commit comments

Comments
 (0)