Add option for reading wind from restart for wmesmf and a few other t…

…hings (NOAA-EMC#109)

* superscript to registered trademark in README

* Branch esmf_hybrid: Adding hybrid compile options for OMP threading in esmf build for NEMS app.

* Branch esmf_hybrid: fixing makefile bug for completing make process in esmf area.

* updating w3_make for esmf for hybrid compile option (NOAA-EMC#78)

updating esmf makefile for option to only build library
updating switch to remove FLD2 which is only used for 2way coupling

* spaces to tab in makefile for esmf

* updating comp/link for theia issue for hybrid compilation

* Adding to skip files in matrix.comp Github Issue NOAA-EMC#87

* Removing T400 until parameters can be confirmed and updated

* removing a temp manual file and adding to git ignore list

* Adding hera for NEMS coupled application builds

* more hera updates for NEMS

* adding wind restarts for esmf issues

* updates for proper indexing of tw* and bug fixes
plus debug statements to be removed after more testing

* adding implicit none to wmesmf

* removing w3tide computed gotos - update from Mickael

* remove computed goto

* bugfix for w3iors

* adding WXNwrst because WXN/WYN are not allocated yet

* removing debug write statements

* adding wrst to manual

* adding WRST to w3_new

* when WRST is allocated a second time it was zeroing out field
so a variable was added to make sure it was only allocated once

.gitignore

@@ -33,6 +33,7 @@ manual/ww3_systrk.tex
 manual/ww3_uprstr.tex
 manual/ww3_grib.tex
 manual/ww3_gint.tex
+manual/ww3_trnc.tex
 manual/gx_outf.tex
 manual/gx_outp.tex
 manual/*.log

README.md

@@ -1,6 +1,6 @@
 # The WAVEWATCH III Framework
 
-WAVEWATCH III ® is a community wave modeling framework that includes the 
+WAVEWATCH III<sup>&reg;</sup>  is a community wave modeling framework that includes the 
 latest scientific advancements in the field of wind-wave modeling and dynamics.
 
 ## General Features
@@ -11,11 +11,11 @@ for shallow-water (surf zone) applications, as well as wetting and drying of
 grid points. Propagation of a wave spectrum can be solved using regular 
 (rectilinear or curvilinear) and unstructured (triangular) grids. See 
 [About WW3](https://github.com/NOAA-EMC/WW3/wiki/About-WW3) for a 
-detailed description of WAVEWATCH III &reg;.
+detailed description of WAVEWATCH III<sup>&reg;</sup> .
 
 ## Installation
 
-The WAVEWATCH III framework package has two parts that need to be combined so 
+The WAVEWATCH III<sup>&reg;</sup>  framework package has two parts that need to be combined so 
 all runs smoothly: the GitHub repo itself, and a binary data file bundle that 
 needs to be obtained from our ftp site. Steps to successfully acquire and install 
 the framework are outlined in our [Quick Start](https://github.com/NOAA-EMC/WW3/wiki/Quick-Start)

manual/app/nuopc.tex

@@ -25,6 +25,10 @@ \subsection{~Building and Installing the NUOPC Cap} \label{sec:nuopcbuild}
 this makefile will subsequently call {\code w3\_make}. As part of this process a nuopc.mk makefile
 fragment will also be created, which tells NUOPC/ESMF where the \ws\ library is located. 
 
+Note there is a new switch {\code WRST} which will add 10 m wind to the restart file and use that wind field 
+at the initial time step of the wave model. This can be used in situations where the coupled atmospheric model 
+does not have 10 m wind speeds at initialization.
+
 \vssub
 \subsection{~Import/Export Fields in the NUOPC Cap} \label{sec:nuopcfields}
 \vssub

manual/impl/switch.tex

@@ -365,6 +365,7 @@ \subsubsection{~Optional switches} \label{sub:opt_switch}
            NetCDF-3 files. Selecting both TRKNC and NC4 will generate 
            NetCDF-4 files.}
 \sit{uost}{Enable the unresolved obstacles source term.}
+\sit{wrst}{Save wind in restart and use in first time step in wmesmf.}
 \sit{xw0 }{Swell diffusion only in \uq\ scheme.}
 \sit{xw1 }{Id. wave growth diffusion only.}
 \end{slist}

model/bin/comp.theia

@@ -106,7 +106,7 @@
   # open mpi implementation
   if [ "$omp_mod" = 'yes' ]
   then
-    opt="$opt -openmp"
+    opt="$opt -qopenmp"
   fi
 
   # oasis coupler include dir

model/bin/link.theia

@@ -109,7 +109,7 @@
   # open mpi implementation
   if [ "$omp_mod" = 'yes' ]
   then
-    opt="$opt -openmp"
+    opt="$opt -qopenmp"
   fi
 
   # oasis coupler archive

model/bin/make_makefile.sh

@@ -99,7 +99,7 @@
               dstress s_ice s_is reflection s_xx \
               wind windx wcor rwind curr currx mgwind mgprop mggse \
               subsec tdyn dss0 pdif tide refrx ig rotag arctic nnt mprf \
-              cou oasis agcm ogcm igcm trknc setup pdlib memck uost
+              cou oasis agcm ogcm igcm trknc setup pdlib memck uost rstwind
   do
     case $type in
 #sort:mach:
@@ -257,6 +257,13 @@
                ID='wind vs. current definition'
                TS='RWND'
                OK='RWND' ;;
+
+#sort:rstwind:
+      rstwind  ) TY='upto1'
+               ID='wind in restart for wmesmf'
+               TS='WRST'
+               OK='WRST' ;;
+
 #sort:curr:
       curr   ) TY='one'
                ID='current interpolation in time'

model/bin/w3_make

@@ -709,8 +709,14 @@ EOF
       if [ -n "`echo $prog | grep esmf 2>/dev/null`" ]
       then
         mkfile=$main_dir/nuopc.mk
-        mod_dir=$main_dir/mod_MPI
-        obj_dir=$main_dir/obj_MPI
+        if [ -n "`grep OMP $switch_file`" ]
+        then
+          mod_dir=$main_dir/mod_HYB
+          obj_dir=$main_dir/obj_HYB
+        else 
+          mod_dir=$main_dir/mod_MPI
+          obj_dir=$main_dir/obj_MPI
+        fi 
         rm -f $mkfile
         touch $mkfile
         echo "#-----------------------------------------------"      >> $mkfile

model/bin/w3_new

@@ -320,6 +320,9 @@
                                    touch w3gdatmd.ftn
                                    touch w3iogrmd.ftn
                                    touch ww3_grid.ftn ;;
+     'rstwind') cd $main_dir/ftn ; touch wmesmfmd.ftn
+                                   touch w3idatmd.ftn
+                                   touch w3iorsmd.ftn ;;
      'curr'   ) cd $main_dir/ftn ; touch w3updtmd.ftn
                                    touch ww3_prnc.ftn
                                    touch ww3_prep.ftn ;; 

model/esmf/Makefile

@@ -89,6 +89,9 @@ ww3_nems: env setup gout switch
 	$(WW3_BINDIR)/w3_make ww3_multi_esmf
 	$(WW3_BINDIR)/w3_make ww3_multi
 
+ww3_nemslibonly: env setup switch
+	$(WW3_BINDIR)/w3_make ww3_multi_esmf
+
 ww3_multi_esmf: esmApp.o $(DEP_LINK_OBJS)
 	$(ESMF_F90LINKER) $(ESMF_F90LINKOPTS) -o $(EXE) $^ \
 		$(ESMF_F90LINKPATHS) $(ESMF_F90LINKRPATHS) $(ESMF_F90ESMFLINKLIBS)
@@ -157,6 +160,8 @@ switch:
 gout: 
 	@echo "$(SWITCHES)" > $(WW3_BINDIR)/tempswitch
 	@sed -e "s/DIST/SHRD/g"\
+	     -e "s/OMPG/ /g"\
+	     -e "s/OMPH/ /g"\
 	     -e "s/MPIT/ /g"\
              -e "s/MPI/ /g"\
              -e "s/PDLIB/ /g"\
@@ -172,6 +177,8 @@ gout:
 	$(WW3_BINDIR)/w3_make ww3_ounp
 	@echo "$(SWITCHES)" > $(WW3_BINDIR)/tempswitch
 	@sed -e "s/DIST/SHRD/g"\
+	     -e "s/OMPG/ /g"\
+	     -e "s/OMPH/ /g"\
              -e "s/MPIT/ /g"\
              -e "s/MPI/ /g"\
              -e "s/PDLIB/ /g"\

model/esmf/switch

@@ -5,12 +5,12 @@ DIST
 MPI
 SCRIP
 SCRIPNC
+WRST
 NC4
 PR3
 UQ
 FLX0
 SEED
-FLD2
 ST4
 STAB0
 NL1

model/ftn/w3idatmd.ftn

@@ -149,6 +149,7 @@
         INTEGER               :: TFN(2,-7:8), TC0(2), TW0(2),     &
                                  TDN(2), TG0(2)
         REAL                  :: GA0, GD0, GAN, GDN
+!/WRST        REAL, POINTER         :: WXNwrst(:,:),WYNwrst(:,:)
         REAL, POINTER         :: WX0(:,:), WY0(:,:), DT0(:,:),        &
                                  WXN(:,:), WYN(:,:), DTN(:,:),        &
                                  CX0(:,:), CY0(:,:), CXN(:,:),        &
@@ -159,6 +160,7 @@
 !/TIDE REAL, POINTER         ::  CXTIDE(:,:,:,:), CYTIDE(:,:,:,:),    &
 !/TIDE                           WLTIDE(:,:,:,:)
         LOGICAL               :: IINIT
+!/WRST        LOGICAL               :: WRSTIINIT=.FALSE.
 ! note that if size of INFLAGS1 is changed, then TFLAGS in wminitmd.ftn
 !    also must be resized.
         LOGICAL               :: INFLAGS1(-7:12), FLAGSC(-7:12),      &
@@ -179,6 +181,7 @@
       REAL, POINTER           :: GA0, GD0, GAN, GDN
       REAL, POINTER           :: WX0(:,:), WY0(:,:), DT0(:,:),        &
                                  WXN(:,:), WYN(:,:), DTN(:,:),        &
+!/WRST                                 WXNwrst(:,:),WYNwrst(:,:),           &
                                  CX0(:,:), CY0(:,:), CXN(:,:),        &
                                  CYN(:,:), WLEV(:,:), ICEI(:,:),      &
                                  BERGI(:,:), MUDT(:,:), MUDV(:,:),    &
@@ -527,6 +530,13 @@
 !/TIDE          CHECK_ALLOC_STATUS ( ISTAT )
 !/TIDE        END IF
 !
+
+!/WRST      IF(.NOT.(INPUTS(IMOD)%WRSTIINIT)) THEN 
+!/WRST        ALLOCATE (   INPUTS(IMOD)%WXNwrst(NX,NY) ,              &
+!/WRST                     INPUTS(IMOD)%WYNwrst(NX,NY) , STAT=ISTAT )
+!/WRST        INPUTS(IMOD)%WRSTIINIT=.TRUE.
+!/WRST      ENDIF
+
       IF ( FLWIND ) THEN
 !/SMC       IF( FSWND ) THEN
 !/SMC          ALLOCATE ( INPUTS(IMOD)%WX0(NSEA,1) ,              &
@@ -795,6 +805,9 @@
 !/TIDE              CYTIDE => INPUTS(IMOD)%CYTIDE
 !/TIDE            END IF
 !
+!/WRST          WXNwrst    => INPUTS(IMOD)%WXNwrst
+!/WRST          WYNwrst    => INPUTS(IMOD)%WYNwrst
+
           IF ( FLWIND  ) THEN
               WX0    => INPUTS(IMOD)%WX0
               WY0    => INPUTS(IMOD)%WY0

model/ftn/w3iorsmd.ftn

@@ -234,6 +234,8 @@
                           GNAME, FILEXT, GTYPE, UNGTYPE
       USE W3TRIAMD, ONLY: SETUGIOBP
       USE W3WDATMD
+!/WRST      USE W3IDATMD, ONLY: WXN, WYN, W3SETI
+!/WRST      USE W3IDATMD, ONLY: WXNwrst, WYNwrst 
       USE W3ODATMD, ONLY: NDSE, NDST, IAPROC, NAPROC, NAPERR, NAPRST, &
                           IFILE => IFILE4, FNMPRE, NTPROC, IOSTYP
 !/MPI      USE W3ODATMD, ONLY: NRQRS, NBLKRS, RSBLKS, IRQRS, IRQRSS, VAAUX
@@ -270,7 +272,7 @@
       INTEGER                 :: IGRD, I, J, LRECL, NSIZE, IERR,      &
                                  NSEAT, MSPEC, TTIME(2), ISEA, JSEA,  &
                                  NREC, NPART, IPART, IX, IY, IXL, IP, &
-                                 NPRTX2
+                                 NPRTX2, NPRTY2, IYL
       INTEGER, ALLOCATABLE    :: MAPTMP(:,:)
 !/S      INTEGER, SAVE           :: IENT = 0
 !/MPI      INTEGER                 :: IERR_MPI, IH, IB, ISEA0, ISEAN, &
@@ -279,6 +281,7 @@
 !/MPI      INTEGER, ALLOCATABLE    :: STAT1(:,:), STAT2(:,:)
 !/MPI      REAL, ALLOCATABLE       :: VGBUFF(:), VLBUFF(:)
       REAL(KIND=LRB), ALLOCATABLE :: WRITEBUFF(:)
+
       LOGICAL                 :: WRITE, IOSFLG
       CHARACTER(LEN=4)        :: TYPE
       CHARACTER(LEN=10)       :: VERTST
@@ -313,6 +316,7 @@
       CALL W3SETO ( IGRD, NDSE, NDST )
       CALL W3SETG ( IGRD, NDSE, NDST )
       CALL W3SETW ( IGRD, NDSE, NDST )
+!/WRST      CALL W3SETI ( IGRD, NDSE, NDST )
 !
       IF (INXOUT.NE.'READ' .AND. INXOUT.NE.'HOT'  .AND.               &
           INXOUT.NE.'COLD' .AND. INXOUT.NE.'WIND' .AND.               &
@@ -684,6 +688,7 @@
       NREC   = NSEA + 3
       NPART  = 1 + (NSEA-1)/NSIZE
       NPRTX2 = 1 + (NX-1)/NSIZE
+      NPRTY2 = 1 + (NY-1)/NSIZE
 !
 !/DEBUGIO        WRITE(740+IAPROC,*)  'W3IORS, step 8'
 !/DEBUGIO        FLUSH(740+IAPROC)
@@ -701,8 +706,8 @@
                   RPOS  = 1_8 + LRECL*(NREC-1_8)
                   WRITEBUFF(:) = 0.
                   WRITE (NDSR,POS=RPOS,ERR=803,IOSTAT=IERR) WRITEBUFF
-                  WRITE (NDSR,POS=RPOS,ERR=803,IOSTAT=IERR) TLEV, TICE
-
+                  WRITE (NDSR,POS=RPOS,ERR=803,IOSTAT=IERR)           & 
+                          TLEV, TICE                                  
                   DO IPART=1,NPART
                     NREC  = NREC + 1
                     RPOS  = 1_8 + LRECL*(NREC-1_8)
@@ -718,7 +723,27 @@
                     WRITE (NDSR,POS=RPOS,ERR=803,IOSTAT=IERR)         &
                           (ICE(ISEA),ISEA=1+(IPART-1)*NSIZE,          &
                                           MIN(NSEA,IPART*NSIZE))
-                    END DO
+                  END DO
+!/WRST                  DO IX=1, NX
+!/WRST                    DO IPART=1,NPRTY2
+!/WRST                      NREC  = NREC + 1
+!/WRST                      RPOS  = 1_8 + LRECL*(NREC-1_8)
+!/WRST                      WRITE (NDSR,POS=RPOS,ERR=803,IOSTAT=IERR) WRITEBUFF
+!/WRST                      WRITE (NDSR,POS=RPOS,ERR=803,IOSTAT=IERR)       &
+!/WRST                          (WXN(IX,IYL),IYL=1+(IPART-1)*NSIZE,         &
+!/WRST                                         MIN(NY,IPART*NSIZE))
+!/WRST                    END DO        
+!/WRST                  END DO
+!/WRST                  DO IX=1, NX
+!/WRST                    DO IPART=1,NPRTY2
+!/WRST                      NREC  = NREC + 1
+!/WRST                      RPOS  = 1_8 + LRECL*(NREC-1_8)
+!/WRST                      WRITE (NDSR,POS=RPOS,ERR=803,IOSTAT=IERR) WRITEBUFF
+!/WRST                      WRITE (NDSR,POS=RPOS,ERR=803,IOSTAT=IERR)       &
+!/WRST                          (WYN(IX,IYL),IYL=1+(IPART-1)*NSIZE,         &
+!/WRST                                         MIN(NY,IPART*NSIZE))
+!/WRST                    END DO        
+!/WRST                  END DO
                   ALLOCATE ( MAPTMP(NY,NX) )
                   MAPTMP = MAPSTA + 8*MAPST2
                   DO IY=1, NY
@@ -777,7 +802,8 @@
         ELSE
           IF (TYPE.EQ.'FULL') THEN
               RPOS = 1_8 + LRECL*(NREC-1_8)
-              READ (NDSR,POS=RPOS,ERR=802,IOSTAT=IERR) TLEV, TICE
+              READ (NDSR,POS=RPOS,ERR=802,IOSTAT=IERR)                &
+                      TLEV, TICE
 !/DEBUGINIT         WRITE(740+IAPROC,*) 'Before reading WLV'
               DO IPART=1,NPART
                 NREC  = NREC + 1
@@ -793,7 +819,25 @@
                 READ (NDSR,POS=RPOS,ERR=802,IOSTAT=IERR)              &
                       (ICE(ISEA),ISEA=1+(IPART-1)*NSIZE,              &
                                       MIN(NSEA,IPART*NSIZE))
-                END DO
+              END DO
+!/WRST              DO IX=1, NX
+!/WRST               DO IPART=1,NPRTY2
+!/WRST                NREC  = NREC + 1
+!/WRST                RPOS = 1_8 + LRECL*(NREC-1_8)
+!/WRST                READ (NDSR,POS=RPOS,ERR=802,IOSTAT=IERR)              &
+!/WRST                      (WXNwrst(IX,IYL),IYL=1+(IPART-1)*NSIZE,         &
+!/WRST                                      MIN(NY,IPART*NSIZE))
+!/WRST               END DO
+!/WRST              END DO
+!/WRST              DO IX=1, NX
+!/WRST               DO IPART=1,NPRTY2
+!/WRST                NREC  = NREC + 1
+!/WRST                RPOS = 1_8 + LRECL*(NREC-1_8)
+!/WRST                READ (NDSR,POS=RPOS,ERR=802,IOSTAT=IERR)              &
+!/WRST                      (WYNwrst(IX,IYL),IYL=1+(IPART-1)*NSIZE,         &
+!/WRST                                      MIN(NY,IPART*NSIZE))
+!/WRST               END DO
+!/WRST              END DO
               ALLOCATE ( MAPTMP(NY,NX) )
 !/DEBUGINIT         WRITE(740+IAPROC,*) 'Before reading MAPTMP'
               DO IY=1, NY
@@ -859,6 +903,8 @@
               TIC1(2) =  0
               TIC5(1) = -1
               TIC5(2) =  0
+!/WRST              WXNwrst =  0. 
+!/WRST              WYNwrst =  0. 
               WLV     =  0.
               ICE     =  0.
               ASF     =  1.

model/ftn/w3profsmd.ftn

@@ -2030,7 +2030,29 @@ END MODULE W3PROFSMD
 !     where to go
 !
       if (ipar(1).gt.0) then
-         goto (10, 20, 40, 50, 60, 70, 80, 90, 100, 110) ipar(10)
+         !!goto (10, 20, 40, 50, 60, 70, 80, 90, 100, 110) ipar(10)     
+        SELECT CASE (ipar(10))
+          CASE (1)
+            GOTO 10
+          CASE (2)
+            GOTO 20
+          CASE (3)
+            GOTO 40
+          CASE (4)
+            GOTO 50
+          CASE (5)
+            GOTO 60
+          CASE (6)
+            GOTO 70
+          CASE (7)
+            GOTO 80
+          CASE (8)
+            GOTO 90
+          CASE (9)
+            GOTO 100
+          CASE (10)
+            GOTO 110
+        END SELECT
       else if (ipar(1).lt.0) then
          goto 900
       endif