Skip to content

Commit b82532f

Browse files
author
Joe Hamman
authored
Merge pull request #50 from NCAR/develop
Update master for 0.4 tag
2 parents d96026d + 1fcb993 commit b82532f

38 files changed

+2800
-1272
lines changed

.gitignore

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -35,4 +35,5 @@ test_regression
3535
test_calendar
3636
test_qm
3737
test_config
38-
downscale
38+
test_random
39+
gard

.travis.yml

Lines changed: 20 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -1,19 +1,27 @@
1-
sudo: false
1+
sudo: false # use container-based build
22
language: fortran
3-
compiler:
4-
- gfortran
5-
os:
6-
- linux
3+
notifications:
4+
email: false
5+
6+
compiler: gfortran-6
7+
os: linux
8+
env: TESTID='gard_linux'
79
addons:
810
apt:
9-
source:
11+
sources:
1012
- ubuntu-toolchain-r-test
1113
packages:
12-
- gfortran
13-
- libnetcdf-dev
1414
- liblapack-dev
15+
- libnetcdf-dev
16+
- gfortran-6
17+
before_install:
18+
- source ci/gard_install_utils
19+
- gard_before_install
20+
install:
21+
- gard_install
1522
script:
16-
- sed -i "s|NCDF_PATH = /usr/local|NCDF_PATH = /usr|" src/makefile
17-
- sed -i "s|LAPACK_PATH = /usr/local|LAPACK_PATH = /usr|" src/makefile
18-
- make -C src -j4 test
19-
- make -C src -j4
23+
- gard_script
24+
after_success:
25+
- gard_after_success
26+
after_failure:
27+
- gard_after_failure

ci/gard_install_utils

Lines changed: 115 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,115 @@
1+
#!/usr/bin/env bash
2+
3+
set -e
4+
set -x
5+
6+
export CC=/usr/bin/gcc-6
7+
export FC=/usr/bin/gfortran-6
8+
9+
if [ -z "$WORKDIR" ]; then
10+
export WORKDIR=$HOME/workdir
11+
mkdir -p $WORKDIR
12+
fi
13+
14+
if [ -z "$INSTALLDIR" ]; then
15+
export INSTALLDIR=$HOME/installdir
16+
mkdir -p $INSTALLDIR
17+
fi
18+
19+
function install_szip {
20+
echo install_szip
21+
cd $WORKDIR
22+
wget --no-check-certificate -q http://www.hdfgroup.org/ftp/lib-external/szip/2.1/src/szip-2.1.tar.gz
23+
tar -xzf szip-2.1.tar.gz
24+
cd szip-2.1
25+
./configure --prefix=$INSTALLDIR &> config.log
26+
make &> make.log
27+
make install
28+
export CPPFLAGS="$CPPFLAGS -I${INSTALLDIR}/include"
29+
export LDFLAGS="$LDFLAGS -L${INSTALLDIR}/lib"
30+
}
31+
32+
function install_hdf5 {
33+
echo install_hdf5
34+
cd $WORKDIR
35+
wget --no-check-certificate -q http://www.hdfgroup.org/ftp/HDF5/current/src/hdf5-1.10.0-patch1.tar.gz
36+
tar -xzf hdf5-1.10.0-patch1.tar.gz
37+
cd hdf5-1.10.0-patch1
38+
./configure --prefix=$INSTALLDIR &> config.log
39+
make &> make.log
40+
make install
41+
export LIBDIR=${INSTALLDIR}/lib
42+
}
43+
44+
function install_netcdf_c {
45+
echo install_netcdf_c
46+
cd $WORKDIR
47+
wget --no-check-certificate -q ftp://ftp.unidata.ucar.edu/pub/netcdf/netcdf-4.4.1.tar.gz
48+
tar -xzf netcdf-4.4.1.tar.gz
49+
cd netcdf-4.4.1
50+
./configure --prefix=$INSTALLDIR &> config.log
51+
make &> make.log
52+
make install
53+
export LD_LIBRARY_PATH=${INSTALLDIR}/lib
54+
}
55+
56+
function install_netcdf_fortran {
57+
echo install_netcdf_fortran
58+
cd $WORKDIR
59+
wget --no-check-certificate -q ftp://ftp.unidata.ucar.edu/pub/netcdf/netcdf-fortran-4.4.4.tar.gz
60+
tar -xzf netcdf-fortran-4.4.4.tar.gz
61+
cd netcdf-fortran-4.4.4
62+
./configure --prefix=$INSTALLDIR &> config.log
63+
make &> make.log
64+
make install
65+
}
66+
67+
function gard_before_install {
68+
echo gard_before_install
69+
# Install szip (used by hdf5)
70+
install_szip
71+
# Install HDF5
72+
install_hdf5
73+
# Install NetCDF-C
74+
install_netcdf_c
75+
# Install NetCDF fortran
76+
install_netcdf_fortran
77+
}
78+
79+
function gard_install {
80+
echo gard_install
81+
cd ${TRAVIS_BUILD_DIR}
82+
sed -i "s|NCDF_PATH = /usr/local|NCDF_PATH = ${INSTALLDIR}|" src/makefile
83+
sed -i "s|LAPACK_PATH = /usr/local|LAPACK_PATH = /usr|" src/makefile
84+
make -C src clean; make -C src -j4 test
85+
make -C src clean; make -C src -j4 MODE=debugslow
86+
make -C src clean; make -C src -j4 MODE=debug
87+
make -C src clean; make -C src -j4 MODE=debugompslow
88+
make -C src clean; make -C src -j4 MODE=debugomp
89+
make -C src clean; make -C src -j4 MODE=profile
90+
make -C src clean; make -C src -j4 MODE=fast
91+
make -C src clean; make -C src -j4
92+
echo "GARD install succeeded"
93+
}
94+
95+
function gard_script {
96+
cd ./src
97+
./gard --version
98+
./gard -h
99+
./test_calendar
100+
# ./test_random
101+
./test_regression
102+
# ./test_config ../run/downscale_options.txt
103+
cd ../
104+
echo "GARD script succeeded"
105+
}
106+
107+
function gard_after_success {
108+
echo gard_after_success
109+
echo "GARD build succeeded"
110+
}
111+
112+
function gard_after_failure {
113+
echo gard_after_failure
114+
echo "GARD build failed"
115+
}

docs/config.md

Lines changed: 47 additions & 34 deletions
Large diffs are not rendered by default.

docs/running.md

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,8 @@ GARD has the following dependencies:
1313
1. LAPACK — Linear Algebra PACKage.
1414
1. netCDF4 - Network Common Data Form.
1515

16+
*Note: GARD allocates memory to the stack. Users should set the "The maximum stack size." to "unlimited" prior to building/running GARD. `ulimit -s unlimited`*
17+
1618
## Building GARD
1719

1820
GARD is built using a standard `makefile`. From the command line, simply run the following command:
@@ -43,3 +45,10 @@ After building GARD, it is run on the command line following this syntax:
4345
Use the following to generate a list of e.g. GEFS precipitation files for input.
4446

4547
ls -1 gefs/2010/*/apcp_sfc_*_mean.nc | sed 's/*//g;s/$/"/g;s/^/"/g'>gefs_pr_file.txt
48+
49+
## Common Errors
50+
51+
1. Segmentation Fault
52+
- GARD allocates memory to the stack. Users should set the "The maximum stack size" to "unlimited" prior to building/running GARD. `ulimit -s unlimited`
53+
2. Random errors (e.g. debug not staying set at False)
54+
- Make sure all filenames in the namelist are in quotations.

run/downscale_options.txt

Lines changed: 50 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@
44
! observation_file = "downscale_options.nml"
55
! prediction_file = "downscale_options.nml"
66

7-
output_file = "log_reg_output/"
7+
output_file = "output/gard_out_" ! prefix for output files
88

99
start_date = "2000-01-01 00:00:00" ! start time for the output data (only needs to exist in the prediction dataset)
1010
end_date = "2000-01-31 23:00:00" ! end time for the output data
@@ -15,23 +15,35 @@
1515
start_transform = "1990-01-01 00:00:00" ! start time for any transformations (e.g. quantile mapping) (must exist in both the training and the prediction datasets)
1616
end_transform = "1999-01-01 23:00:00" ! end time for the transformation period
1717

18+
start_post = "1990-01-01 00:00:00" ! start time for any post processing transformations (e.g. quantile mapping) (must exist in the prediction dataset)
19+
end_post = "1999-01-01 23:00:00" ! end time for the post-proc transformation period
20+
21+
1822
! model types
19-
pure_regression = False ! set to true (and others to false) to compute a single regression (no analogs) between training variables and observations to be applied to all predictor data
20-
pure_analog = False ! set to true (and others to false) to use the selected analogs (no regression) to predict the output
21-
analog_regression = True ! set to true (and others to false) to perform a regression only on the selected analog days when predicting the output
23+
pure_regression = False ! set to true (and others to false) to compute a single regression (no analogs) between training variables and observations to be applied to all predictor data
24+
pure_analog = False ! set to true (and others to false) to use the selected analogs (no regression) to predict the output
25+
analog_regression = True ! set to true (and others to false) to perform a regression only on the selected analog days when predicting the output
26+
27+
! Non-model type
28+
pass_through = False ! set to true to simply pass a predictor variable through instead of computing regressions or analogs
29+
pass_through_var = 1 ! Option to select which predictor variable is passed through
2230

2331
! analog selection parameters
24-
n_analogs = 200 ! set the number of analogs to find for each time step
25-
! n_log_analogs = 20 ! set to the number of analogs to use for the logistic_from_analog_exceedance calculation if using something other than n_analogs
26-
! analog_threshold = 0.25 ! set to use a threshold distance when selecting analogs instead of a fixed number (in units of standard deviations averaged across input variables)
32+
n_analogs = 200 ! set the number of analogs to find for each time step
33+
! n_log_analogs = 20 ! set to the number of analogs to use for the logistic_from_analog_exceedance calculation if using something other than n_analogs
34+
! analog_threshold = 0.25 ! set to use a threshold distance when selecting analogs instead of a fixed number (in units of standard deviations averaged across input variables)
2735

2836
! model options
2937
sample_analog = False ! when using pure_analog this will sample randomly from the selection of analogs instead of using the mean
3038
logistic_from_analog_exceedance = False ! when true the logistic probability is calculated from the number of analogs that exceed the threshold instead of from a logistic regression
3139
logistic_threshold = 0 ! when set, the probability of exceeding this value will be calculated and the standard output will be trained only for the data that exceed this value
3240
weight_analogs = True ! when calculating analog_mean, analog_error, or logistic_from_analog, weight each analog by it's inverse square distance from the current input
3341

42+
! output post processing transformation
43+
post_correction_transform = 0 ! post processing to apply (e.g. quantile mapping to entire input obs period) (one entry for every output variable)
44+
3445
debug = False ! prints more output at runtime and outputs files including the coefficients used in each analog regression (or analog values) as well as the predictor data
46+
interactive = False ! Print downscaling status as a percentage on the command line
3547
/
3648

3749
! define the input atmospheric model data to be used when training the model
@@ -45,6 +57,13 @@
4557
lon_name = "XLONG" ! name of a variable in the input data that contains the longitude of each grid cell (can be a 1D or 2D variable)
4658
time_name = "XTIME" ! name of a variable in the input data that contains the time data (must be a number ideally with a units attribute such as "days since YYYY-MM-DD hh:mm:ss")
4759
nfiles = 21 ! the number of files to be read
60+
selected_level = -1 ! If the input data have 4 dimensions (e.g. one dimension is elevation) this is the index that will be used
61+
! this assumes that the z dimension is the 2nd (time, z, y, x) in the NetCDF file
62+
! if -1, the first level will be used.
63+
64+
! Controls normalization applied to training data.
65+
! 0 = don't normalize, 1 = normalize based on the training data mean and standard deviation
66+
normalization_method = 1
4867

4968
! the following are arrays with one element for each input variable
5069
input_transformations = 0, 0, 3, 3 ! transformations to apply to each input variable (0=none, 1=qm?, 2=log, 3=cuberoot, 4=fifthroot)
@@ -55,6 +74,13 @@
5574

5675
! calendar to be used when interpreting the time variable, "gregorian", "standard", "noleap", "365-day","360-day" all work
5776
calendar = "gregorian"
77+
calendar_start_year = 1900 ! set this to the year the time data use as time 0, will be read from the NetCDF units attribute if possible
78+
timezone_offset = 0 ! offset (in hours) to add to the time data to permit a better match with the local obs. data
79+
80+
! the following options are primarily for forecasting
81+
! GEFS data have n time-steps per file, corresponding to the n-forecast lead times
82+
selected_time = -1 ! if set, only this time step will be read from each input file
83+
time_indices = -1, -1, -1 ! this can be a list of time steps to read and average over instead
5884
/
5985

6086
! Define the input atmospheric model data to be used when applying the model for future predictions
@@ -72,13 +98,23 @@
7298

7399
input_transformations = 0, 0, 3, 3
74100

75-
! this is the main difference between training and prediction data, permits an additional transformation to e.g. quantile map (transformation=1) each variable to match the training data
101+
! Here is the main difference between training and prediction data
102+
! This permits an additional transformation to e.g. quantile map (transformation=1) each variable to match the training data
103+
! this is primarily for climate simulations
76104
transformations = 0, 0, 0, 0
105+
! Also, if normalization_method = 2, the means and standard deviations from the training data will be used to normalize the predictors
106+
! this is primarily for forecasting applications. 1=Normalize based on prediction data, 0=no normalization applied
107+
normalization_method = 1
77108

78109
var_names = "T2", "PSFC", "PREC_ACC_NC", "PREC_ACC_C"
79110
file_list = "filelists/erai_files_200x.txt", "filelists/erai_files_200x.txt", "filelists/erai_files_200x.txt", "filelists/erai_files_200x.txt"
80111

81112
calendar = "gregorian"
113+
calendar_start_year = 1900
114+
timezone_offset = 0
115+
116+
selected_time = -1
117+
time_indices = -1, -1, -1
82118
/
83119

84120
! Define the input observation data to be used when training the model
@@ -99,4 +135,10 @@
99135
file_list = "filelists/obs_files_complete.txt"
100136

101137
calendar = "gregorian"
138+
calendar_start_year = 1900
139+
140+
! specify a variable to use to find which grid cells should be masked
141+
mask_variable = 1
142+
! specify a value to use to in that variable to define masked gridcells
143+
mask_value = 1e20
102144
/

0 commit comments

Comments
 (0)