Skip to content

Commit 45eb10c

Browse files
smsimonsmsimon
authored
Merge pull request #8 from CMB-S4/updated_noise
updated detector noise
2 parents fa9d7cf + c2732f0 commit 45eb10c

15 files changed

+524
-4735
lines changed

‎reference_tool/atm.2.slrm

+119
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,119 @@
1+
#!/bin/bash
2+
#SBATCH --partition=regular
3+
#SBATCH --qos=premium
4+
#SBATCH --time=24:00:00
5+
#SBATCH --nodes=64
6+
#SBATCH --job-name=cache_atm
7+
#SBATCH --licenses=SCRATCH
8+
#SBATCH --constraint=knl
9+
#SBATCH --core-spec=4
10+
#SBATCH --account=mp107
11+
12+
ulimit -c unlimited
13+
export MALLOC_MMAP_THRESHOLD_=131072
14+
export PYTHONSTARTUP=""
15+
export PYTHONNOUSERSITE=1
16+
export HOME=$SCRATCH
17+
export OMP_NUM_THREADS=4
18+
export OMP_PLACES=threads
19+
export OMP_PROC_BIND=spread
20+
21+
let nnode_tot=64
22+
let nnode=1
23+
let njob=$nnode_tot/$nnode
24+
let ntask_node=64/$OMP_NUM_THREADS
25+
let ntask=$nnode*$ntask_node
26+
let ncore=4*$OMP_NUM_THREADS
27+
# Make sure nnode is divisible by nnode_group
28+
let nnode_group=$nnode
29+
let groupsize=nnode_group*ntask_node
30+
31+
echo "Running with"
32+
echo " nnode = ${nnode}"
33+
echo " OMP_NUM_THREADS = ${OMP_NUM_THREADS}"
34+
echo " ntask_node = ${ntask_node}"
35+
echo " ntask = ${ntask}"
36+
echo " nnode_group = ${nnode_group}"
37+
echo " groupsize = ${groupsize}"
38+
39+
export PYTHONSTARTUP=""
40+
export PYTHONNOUSERSITE=1
41+
42+
logdir=atm_logs
43+
mkdir -p $logdir
44+
45+
#for site in chile pole; do
46+
for site in chile; do
47+
case $site in
48+
chile)
49+
weather="weather_Atacama.fits"
50+
;;
51+
pole)
52+
weather="weather_South_Pole.fits"
53+
;;
54+
*) echo "Unknown telescope: $tel"; exit;;
55+
esac
56+
for tel in sat lat; do
57+
case $tel in
58+
sat)
59+
tube=ST16
60+
band=LFS1
61+
nside=512
62+
fpradius=20.5
63+
hardware=hardware_SAT_LFS.toml.gz
64+
;;
65+
lat)
66+
tube=LT17
67+
band=LFL1
68+
nside=4096
69+
fpradius=4.3
70+
hardware=hardware_LAT_LFL.toml.gz
71+
;;
72+
*) echo "Unknown telescope: $tel"; exit;;
73+
esac
74+
75+
fnames=scan_strategy/${site}_${tel}/split_schedules/*
76+
# Random wait time to reduce clashes
77+
sleep $((RANDOM % 15))
78+
for schedule in $fnames; do
79+
let nrunning=`ps | grep srun | wc -l`-1
80+
while [[ $nrunning -ge $njob ]]; do
81+
echo "All slots are full"
82+
sleep 60
83+
let nrunning=`ps | grep srun | wc -l`-1
84+
done
85+
echo "There are $nrunning running jobs"
86+
rootname=`basename $schedule .txt`
87+
rootname=${site}_${rootname}
88+
logfile=$logdir/${rootname}.log
89+
if [[ ! -e $logfile ]]; then
90+
echo "Writing $logfile"
91+
srun -N $nnode -n $ntask -c $ncore --cpu_bind=cores \
92+
toast_s4_sim.py @general.par \
93+
--group-size $groupsize \
94+
--MC-count 8 \
95+
--bands $band \
96+
--focalplane-radius $fpradius \
97+
--hardware $hardware \
98+
--no-maps \
99+
--sample-rate 10 \
100+
--scan-accel 1 \
101+
--scan-rate 1 \
102+
--schedule $schedule \
103+
--simulate-atmosphere \
104+
--site $site \
105+
--thinfp 1 \
106+
--tubes $tube \
107+
--weather $weather \
108+
>& ${logfile} &
109+
sleep 1
110+
else
111+
echo "$logfile exists"
112+
fi
113+
done
114+
done
115+
done
116+
117+
echo "Waiting for all jobs to finish"
118+
119+
wait

‎reference_tool/general.par

+3
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,6 @@
1+
--gainscrambler
2+
--gain-sigma
3+
0.01
14
--sort-schedule
25
--single-precision-pointing
36
--flush

‎reference_tool/generate_slurm.py

+61-34
Original file line numberDiff line numberDiff line change
@@ -7,14 +7,16 @@
77
import os
88
import sys
99

10-
1110
"""
12-
--bands BANDS Comma-separated list of bands: ULFL1 (20 GHz, LAT),
13-
LFL1 (27 GHz LAT), LFL2 (39 GHz, LAT), LFS1 (30 GHz,
11+
--bands BANDS Comma-separated list of bands: ULFPL1 (20 GHz, Pole
12+
LAT), LFL1 (27 GHz LAT), LFL2 (39 GHz, LAT), LFPL1 (27
13+
GHz Pole LAT), LFPL2 (39 GHz, Pole LAT), LFS1 (30 GHz,
1414
SAT), LFS2 (40 GHz, SAT), MFL1 (93 GHz, LAT), MFL2
15-
(145 GHz, LAT), MFLS1 (85 GHz, SAT), MFLS2 (145.1 GHz,
15+
(145 GHz, LAT), MFPL1 (93 GHz, Pole LAT), MFPL2 (145
16+
GHz, Pole LAT), MFLS1 (85 GHz, SAT), MFLS2 (145.1 GHz,
1617
SAT), MFHS1 (95 GHz, SAT), MFHS2 (155.1 GHz, SAT),
17-
HFL1(225 GHz, LAT), HFL2 (278 GHz, LAT), HFS1 (220
18+
HFL1(225 GHz, LAT), HFL2 (278 GHz, LAT), HFPL1 (225
19+
GHz, Pole LAT), HFPL2 (278 GHz, Pole LAT), HFS1 (220
1820
GHz, SAT), HFS2 (270 GHz, SAT).Length of list must
1921
equal --tubes
2022
--tubes TUBES Comma-separated list of optics tubes: LT0 (HFL), LT1
@@ -26,19 +28,20 @@
2628
(MFL), LT25 (MFL), LT26 (MFL), LT27 (MFL), LT28 (MFL),
2729
LT29 (MFL), LT30 (MFL), LT31 (MFL), LT32 (MFL), LT33
2830
(MFL), LT34 (MFL), LT35 (MFL), LT36 (LFL), LT37 (LFL),
29-
LT38 (HFL), LT39 (HFL), LT40 (HFL), LT41 (HFL), LT42
30-
(MFL), LT43 (MFL), LT44 (MFL), LT45 (MFL), LT46 (MFL),
31-
LT47 (MFL), LT48 (MFL), LT49 (MFL), LT50 (MFL), LT51
32-
(MFL), LT52 (MFL), LT53 (MFL), LT54 (LFL), LT55 (LFL),
33-
LT56 (ULFL), ST0 (MFLS), ST1 (MFLS), ST2 (MFLS), ST3
34-
(MFLS), ST4 (MFLS), ST5 (MFLS), ST6 (MFHS), ST7
35-
(MFHS), ST8 (MFHS), ST9 (MFHS), ST10 (MFHS), ST11
36-
(MFHS), ST12 (HFS), ST13 (HFS), ST14 (HFS), ST15 (HFS),
37-
ST16 (LFS), ST17 (LFS).Length of list must equal
38-
--bands
31+
LT38 (HFPL), LT39 (HFPL), LT40 (HFPL), LT41 (HFPL),
32+
LT42 (MFPL), LT43 (MFPL), LT44 (MFPL), LT45 (MFPL),
33+
LT46 (MFPL), LT47 (MFPL), LT48 (MFPL), LT49 (MFPL),
34+
LT50 (MFPL), LT51 (MFPL), LT52 (MFPL), LT53 (MFPL),
35+
LT54 (LFPL), LT55 (LFPL), LT56 (ULFPL), ST0 (MFLS),
36+
ST1 (MFLS), ST2 (MFLS), ST3 (MFLS), ST4 (MFLS), ST5
37+
(MFLS), ST6 (MFHS), ST7 (MFHS), ST8 (MFHS), ST9
38+
(MFHS), ST10 (MFHS), ST11 (MFHS), ST12 (HFS),ST13
39+
(HFS), ST14 (HFS), ST15 (HFS), ST16 (LFS), ST17
40+
(LFS).Length of list must equal --bands
41+
3942
"""
4043

41-
input_map_dir = "/global/cscratch1/sd/zonca/cmbs4/map_based_simulations/202002_foregrounds_extragalactic_cmb_tophat"
44+
input_map_dir = "/global/cscratch1/sd/zonca/cmbs4/map_based_simulations/202006_foregrounds_extragalactic_cmb_tophat"
4245

4346
flavors = (
4447
"noise",
@@ -51,10 +54,13 @@
5154

5255
telescopes = {
5356
"LAT": {
54-
"LT0": ["HFL1", "HFL2"], # 225 & 278 GHz
55-
"LT5": ["MFL1", "MFL2"], # 93 & 145 GHz
56-
"LT17": ["LFL1", "LFL2"], # 27 & 39 GHz
57-
"LT56": ["ULFL1"], # 20 GHz
57+
"LT0": ["HFL1", "HFL2"], # 225 & 278 GHz
58+
"LT5": ["MFL1", "MFL2"], # 93 & 145 GHz
59+
"LT17": ["LFL1", "LFL2"], # 27 & 39 GHz
60+
"LT38": ["HFPL1", "HFPL2"], # Pole 225 & 278 GHz
61+
"LT42": ["MFPL1", "MFPL2"], # Pole 93 & 145 GHz
62+
"LT54": ["LFPL1", "LFPL2"], # Pole 27 & 39 GHz
63+
"LT56": ["ULFPL1"], # Pole 20 GHz
5864
},
5965
"SAT": {
6066
"ST0": ["MFLS1", "MFLS2"], # 85 & 145.1 GHz - SAT0 - FOV 14.5 deg
@@ -134,17 +140,21 @@
134140

135141
for tube, bands in tubes.items():
136142
for band in bands:
143+
if telescope == "LAT" and site == "pole" and "P" not in band:
144+
continue
145+
if telescope == "LAT" and site == "chile" and "P" in band and band != "ULFPL1":
146+
continue
137147
thinfp_temp = thinfp
138148
if band.startswith("ULF") or band.startswith("LF"):
139149
thinfp_temp = 1
140150
elif band.startswith("HFS"):
141151
thinfp_temp = 8
142152
hardware = "hardware_{}_{}.toml.gz".format(telescope, band[:-1])
143153
for flavor in flavors:
144-
rootname = "{}_{}_{}_{}".format(site, flavor, telescope, band)
154+
rootname = "{}_{}_{}_{}".format(site, flavor, telescope, band.replace("P", ""))
145155
os.makedirs("slurm", exist_ok=True)
146156
os.makedirs("logs", exist_ok=True)
147-
157+
148158
params = {
149159
"bands": band,
150160
"tubes": tube,
@@ -161,7 +171,10 @@
161171
"madam-prefix": rootname,
162172
"thinfp": thinfp_temp,
163173
"hardware": hardware,
174+
"out" : "out-hwp",
164175
}
176+
if hwprpm:
177+
params["hwp-rpm"] = hwprpm
165178
if cosecant_scan:
166179
params["scan-cosecant-modulate"] = None
167180
if poly_order is not None:
@@ -177,7 +190,10 @@
177190
params["wcov"] = None
178191
params["wcov-inv"] = None
179192
params["MC-count"] = 8
180-
walltime = "02:00:00"
193+
if telescope == "LAT":
194+
walltime = "02:00:00"
195+
else:
196+
walltime = "08:00:00"
181197
elif flavor == "atmosphere":
182198
params["simulate-atmosphere"] = None
183199
params["no-hits"] = None
@@ -199,27 +215,38 @@
199215
signal_name = {
200216
"cmb-unlensed" : "cmb_unlensed_solardipole_nest",
201217
"cmb-lensing" : "cmb_lensing_signal",
202-
"cmb-tensors" : "cmb_tensor_nest",
218+
#"cmb-tensors" : "cmb_tensor_nest",
219+
"cmb-tensors" : "cmb_tensor",
203220
"foreground" : "combined_foregrounds",
204221
}[flavor]
205222
num = "0000"
206-
params["input-map"] = os.path.join(
207-
input_map_dir,
208-
str(nside),
209-
signal_name,
210-
num,
211-
"cmbs4_{}_uKCMB_{}-{}_nside{}_{}.fits".format(
212-
signal_name, telescope, band, nside, num
223+
if flavor == "cmb-tensors":
224+
# cmb_tensor/4096/cmbs4_cmb_tensor_uKCMB_LAT-HFL1_nside4096_0000.fits
225+
params["input-map"] = os.path.join(
226+
"cmb_tensor",
227+
str(nside),
228+
"cmbs4_cmb_tensor_uKCMB_{}-{}_nside{}_{}.fits".format(
229+
telescope, band.replace("P", ""), nside, num
230+
)
213231
)
214-
)
215-
walltime = "00:30:00"
232+
else:
233+
params["input-map"] = os.path.join(
234+
input_map_dir,
235+
str(nside),
236+
signal_name,
237+
num,
238+
"cmbs4_{}_uKCMB_{}-{}_nside{}_{}.fits".format(
239+
signal_name, telescope, band.replace("P", ""), nside, num
240+
)
241+
)
242+
walltime = "02:00:00"
216243
else:
217244
raise RuntimeError(
218245
"Unknown simulation flavor: '{}'".format(flavor)
219246
)
220247

221248
params.update(madampars)
222-
249+
223250
fname_slurm = os.path.join("slurm", "{}.slrm".format(rootname))
224251
with open(fname_slurm, "w") as slurm:
225252
for line in [

0 commit comments

Comments
 (0)