forked from NOAA-EMC/global-workflow
-
Notifications
You must be signed in to change notification settings - Fork 0
/
Copy pathexglobal_fetch.py
executable file
·39 lines (26 loc) · 1.14 KB
/
exglobal_fetch.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
#!/usr/bin/env python3
import os
from pygfs.task.fetch import Fetch
from wxflow import AttrDict, Logger, cast_strdict_as_dtypedict, logit
# initialize root logger
logger = Logger(level=os.environ.get("LOGGING_LEVEL", "DEBUG"), colored_log=True)
@logit(logger)
def main():
config = cast_strdict_as_dtypedict(os.environ)
# Instantiate the Fetch object
fetch = Fetch(config)
# Pull out all the configuration keys needed to run the fetch step
keys = ['current_cycle', 'previous_cycle', 'RUN', 'PDY', 'PARMgfs', 'PSLOT', 'ROTDIR',
'FETCH_YAML_TMPL', 'FETCHDIR', 'ntiles', 'DATAROOT', 'waveGRD']
fetch_dict = AttrDict()
for key in keys:
fetch_dict[key] = fetch.task_config.get(key)
if fetch_dict[key] is None:
print(f"Warning: key ({key}) not found in task_config!")
# Determine which archives to retrieve from HPSS
# Read the input YAML file to get the list of tarballs on tape
fetchdir_set = fetch.configure(fetch_dict)
# Pull the data from tape or locally and store the specified destination
fetch.execute_pull_data(fetchdir_set)
if __name__ == '__main__':
main()