#!/bin/csh -f
# +-======-+ 
#  Copyright (c) 2003-2007 United States Government as represented by 
#  the Admistrator of the National Aeronautics and Space Administration.  
#  All Rights Reserved.
#  
#  THIS OPEN  SOURCE  AGREEMENT  ("AGREEMENT") DEFINES  THE  RIGHTS  OF USE,
#  REPRODUCTION,  DISTRIBUTION,  MODIFICATION AND REDISTRIBUTION OF CERTAIN 
#  COMPUTER SOFTWARE ORIGINALLY RELEASED BY THE UNITED STATES GOVERNMENT AS 
#  REPRESENTED BY THE GOVERNMENT AGENCY LISTED BELOW ("GOVERNMENT AGENCY").  
#  THE UNITED STATES GOVERNMENT, AS REPRESENTED BY GOVERNMENT AGENCY, IS AN 
#  INTENDED  THIRD-PARTY  BENEFICIARY  OF  ALL  SUBSEQUENT DISTRIBUTIONS OR 
#  REDISTRIBUTIONS  OF THE  SUBJECT  SOFTWARE.  ANYONE WHO USES, REPRODUCES, 
#  DISTRIBUTES, MODIFIES  OR REDISTRIBUTES THE SUBJECT SOFTWARE, AS DEFINED 
#  HEREIN, OR ANY PART THEREOF,  IS,  BY THAT ACTION, ACCEPTING IN FULL THE 
#  RESPONSIBILITIES AND OBLIGATIONS CONTAINED IN THIS AGREEMENT.
#  
#  Government Agency: National Aeronautics and Space Administration
#  Government Agency Original Software Designation: GSC-15354-1
#  Government Agency Original Software Title:  GEOS-5 GCM Modeling Software
#  User Registration Requested.  Please Visit http://opensource.gsfc.nasa.gov
#  Government Agency Point of Contact for Original Software:  
#  			Dale Hithon, SRA Assistant, (301) 286-2691
#  
# +-======-+ 

#######################################################################
#                     Batch Parameters for Run Job
#######################################################################

#PBS -l walltime=@RUN_T
#@RUN_FP
#PBS -N @RUN_FN
#@RUN_Q
#@BATCH_GROUP
#PBS -o FCSTOUT

#######################################################################
#                  System Environment Variables
#######################################################################

umask 022

limit stacksize unlimited

@SETENVS

#######################################################################
#           Architecture Specific Environment Variables
#######################################################################

setenv ARCH `uname`

setenv SITE             @SITE
setenv GEOSBIN          @GEOSBIN 
setenv GEOSUTIL         @GEOSUTIL
setenv RUN_CMD         "@RUN_CMD"

source $GEOSBIN/g5_modules
setenv LD_LIBRARY_PATH ${LD_LIBRARY_PATH}:${BASEDIR}/${ARCH}/lib

#######################################################################
#             Experiment Specific Environment Variables
#######################################################################

setenv    EXPID   @EXPID
setenv    EXPDIR  @EXPDIR
setenv    HOMDIR  @HOMDIR
setenv    G6DEV   /discover/nobackup/projects/gmao/g6dev/ltakacs

#######################################################################
#                  Set Forecast Run Parameters
#######################################################################

set date = `echo &DATE 000000`

set HEARTBEAT = &HEARTBEAT
set   AGCM_IM = &AGCM_IM
set   AGCM_JM = &AGCM_JM
set        NX = &NX
set        NY = &NY
set     NFMAX = &NFMAX
set       ANA = &ANA

set  FSEGMENT = &FSEGMENT

#######################################################################
#                 Create Forecast Sub-Directory
#######################################################################

     @  NFCST  = 1
while( $NFCST <= $NFMAX )

set date0 = $date[1]
set time0 = $date[2]
set hour  = `echo $time0 | cut -c1-2`
set fdate = ${date0}_${hour}z

setenv   FCSTDIR  $EXPDIR/forecasts/$fdate
setenv    SCRDIR  $FCSTDIR/scratch
if (! -e $SCRDIR ) mkdir -p $SCRDIR
cd       $SCRDIR

/bin/rm -rf *

set date  = `$GEOSUTIL/post/tick $date0 $time0 -10800`
set nymd0 = $date[1]
set nhms0 = $date[2]
set hour  = `echo $nhms0 | cut -c1-2`

##################################################################
######      Create STRIP Utility for File Modifications
######           (Removes formatted BLANK spaces)
##################################################################

set      FILE = strip
/bin/rm $FILE
cat << EOF > $FILE
#!/bin/ksh
/bin/mv \$1 \$1.tmp
touch   \$1
while read line
do
echo \$line >> \$1
done < \$1.tmp
exit
EOF
chmod +x $FILE

#######################################################################
#                 Set Model Resolution Parameters
#######################################################################

set  AGCM_LM = `grep  AGCM_LM: $HOMDIR/AGCM.rc | cut -d':' -f2`
set  OGCM_IM = `grep  OGCM_IM: $HOMDIR/AGCM.rc | cut -d':' -f2`
set  OGCM_JM = `grep  OGCM_JM: $HOMDIR/AGCM.rc | cut -d':' -f2`

# Set ATMOS and OCEAN Horizontal Resolution Tags
# ----------------------------------------------
set AGCM_IM_Tag = `echo $AGCM_IM | awk '{printf "%4.4i", $1}'`
set AGCM_JM_Tag = `echo $AGCM_JM | awk '{printf "%4.4i", $1}'`
set OGCM_IM_Tag = `echo $OGCM_IM | awk '{printf "%4.4i", $1}'`
set OGCM_JM_Tag = `echo $OGCM_JM | awk '{printf "%4.4i", $1}'`

>>>FVLATLON<<<set ATMOStag = DC${AGCM_IM_Tag}xPC${AGCM_JM_Tag}
>>>FVCUBED<<<set ATMOStag = CF${AGCM_IM_Tag}x6C
>>>DATAOCEAN<<<set OCEANtag = DE${OGCM_IM_Tag}xPE${OGCM_JM_Tag}
>>>COUPLED<<<set OCEANtag = TM${OGCM_IM_Tag}xTM${OGCM_JM_Tag}

#######################################################################
#            Copy and Modify RC Files from Home Directory
#######################################################################

/bin/ln -sf $EXPDIR/RC/* .
/bin/cp -f  $HOMDIR/*.rc .

echo $nymd0 $nhms0 > cap_restart

/bin/cp  CAP.rc  CAP.rc.orig

# Set LAYOUT based on gcm_forecast.setup
# --------------------------------------
./strip AGCM.rc
set oldstring =  `cat AGCM.rc | grep NX:`
set newstring =  "NX: $NX"
/bin/mv AGCM.rc AGCM.tmp
cat AGCM.tmp | sed -e "s?$oldstring?$newstring?g" > AGCM.rc

./strip AGCM.rc
set oldstring =  `cat AGCM.rc | grep NY:`
set newstring =  "NY: $NY"
/bin/mv AGCM.rc AGCM.tmp
cat AGCM.tmp | sed -e "s?$oldstring?$newstring?g" > AGCM.rc

# Turn Off Intermediate Restarts
# ------------------------------
./strip AGCM.rc
set oldstring =  `cat AGCM.rc | grep RECORD_FREQUENCY:`
set newstring =  "#RECORD_FREQUENCY:"
/bin/mv -f AGCM.rc AGCM.tmp
cat AGCM.tmp | sed -e "s?$oldstring?$newstring?g" > AGCM.rc

./strip AGCM.rc
set oldstring =  `cat AGCM.rc | grep RECORD_REF_DATE:`
set newstring =  "#RECORD_REF_DATE:"
/bin/mv -f AGCM.rc AGCM.tmp
cat AGCM.tmp | sed -e "s?$oldstring?$newstring?g" > AGCM.rc

./strip AGCM.rc
set oldstring =  `cat AGCM.rc | grep RECORD_REF_TIME:`
set newstring =  "#RECORD_REF_TIME:"
/bin/mv -f AGCM.rc AGCM.tmp
cat AGCM.tmp | sed -e "s?$oldstring?$newstring?g" > AGCM.rc

# Initialize REPLAY SHUTOFF Option for Forecasts
# ----------------------------------------------
set oldstring =  `cat AGCM.rc | grep "REPLAY_SHUTOFF"`
set newstring =  "REPLAY_SHUTOFF: 21600"
/bin/mv -f AGCM.rc AGCM.tmp
cat AGCM.tmp | sed -e "s?$oldstring?$newstring?g" > AGCM.rc

#######################################################################
#                 Create Forecast History RC Files
#######################################################################

set         FILE = HISTORY.rc
/bin/rm -f $FILE
cat << _EOF_ > $FILE

EXPID:  G5$ANA
EXPDSC: G5${ANA}_based_on_${EXPID}

COLLECTIONS: 'geosgcm_fcst'
             'geosgcm_surf'
             ::

  geosgcm_fcst.template:  '%y4%m2%d2_%h2%n2z.nc4',
  geosgcm_fcst.archive:   '%c/Y%y4',
  geosgcm_fcst.format:    'CFIO',
  geosgcm_fcst.frequency:  060000,
  geosgcm_fcst.duration:   000000,
  geosgcm_fcst.resolution: 1152 721,
  geosgcm_fcst.vscale:     100.0,
  geosgcm_fcst.vunit:     'hPa',
  geosgcm_fcst.vvars:     'log(PLE)' , 'DYN'          ,
  geosgcm_fcst.levels:     1000 975 950 925 900 850 800 750 700 600 500 400 300 250 200 150 100,
  geosgcm_fcst.fields:    'PHIS'     , 'AGCM'         ,
                          'SLP'      , 'DYN'          ,
                          'U'        , 'DYN'          ,
                          'V'        , 'DYN'          ,
                          'T'        , 'DYN'          ,
                          'PS'       , 'DYN'          ,
                          'ZLE'      , 'DYN'          , 'H'   ,
                          'OMEGA'    , 'DYN'          ,
                          'Q'        , 'MOIST'        , 'QV'  ,
                          'RH2'      , 'MOIST'        , 'RH'  ,
                          ::

  geosgcm_surf.template:  '%y4%m2%d2_%h2%n2z.nc4',
  geosgcm_surf.archive:   '%c/Y%y4',
  geosgcm_surf.format:    'CFIO',
  geosgcm_surf.mode:      'time-averaged',
  geosgcm_surf.frequency:  030000,
  geosgcm_surf.duration:   000000,
  geosgcm_surf.resolution: 1152 721,
  geosgcm_surf.fields:    'PHIS'     , 'AGCM'         ,
                          'SLP'      , 'DYN'          ,
                          'PS'       , 'DYN'          ,
                          'DMDTDYN'  , 'DYN'          ,
                          'TQV'      , 'AGCM'         ,
                          'TQL'      , 'AGCM'         ,
                          'TQI'      , 'AGCM'         ,
                          'TPREC'    , 'MOIST'        ,
                          'EVAP'     , 'SURFACE'      ,
                          'TS'       , 'SURFACE'      ,
                          ::

_EOF_

#######################################################################
#              Create HISTORY Collection Directories
#######################################################################

set collections = ''
foreach line ("`cat HISTORY.rc`")
   set firstword  = `echo $line | awk '{print $1}'`
   set firstchar  = `echo $firstword | cut -c1`
   set secondword = `echo $line | awk '{print $2}'`

   if ( $firstword == "::" ) goto done

   if ( $firstchar != "#" ) then
      set collection  = `echo $firstword | sed -e "s/'//g"`
      set collections = `echo $collections $collection`
      if ( $secondword == :: ) goto done
   endif

   if ( $firstword == COLLECTIONS: ) then
      set collections = `echo $secondword | sed -e "s/'//g"`
   endif
end

done:

#######################################################################
#                        Link Boundary Datasets
#######################################################################

setenv BCSDIR   @BCSDIR
setenv SSTDIR   @SSTDIR
setenv CHMDIR   @CHMDIR
setenv BCSTAG   ${ATMOStag}_${OCEANtag}
setenv DATELINE DC

set             FILE = linkbcs
/bin/rm -f     $FILE
cat << _EOF_ > $FILE
#!/bin/csh -f

/bin/mkdir -p            ExtData
/bin/ln    -sf $CHMDIR/* ExtData

>>>DATAOCEAN<<</bin/ln -sf $BCSDIR/$BCSTAG/${BCSTAG}-Pfafstetter.til  tile.data
>>>DATAOCEAN<<<if(     -e  $BCSDIR/$BCSTAG/${BCSTAG}-Pfafstetter.TIL) then
>>>DATAOCEAN<<</bin/ln -sf $BCSDIR/$BCSTAG/${BCSTAG}-Pfafstetter.TIL  tile.bin
>>>DATAOCEAN<<<endif

/bin/ln -sf $BCSDIR/Shared/pchem.species.CMIP-5.1870-2097.z_91x72.nc4 species.data
/bin/ln -sf $BCSDIR/Shared/*bin .

/bin/ln -sf $BCSDIR/$BCSTAG/visdf_@RES_DATELINE.dat visdf.dat
/bin/ln -sf $BCSDIR/$BCSTAG/nirdf_@RES_DATELINE.dat nirdf.dat
/bin/ln -sf $BCSDIR/$BCSTAG/vegdyn_@RES_DATELINE.dat vegdyn.data
/bin/ln -sf $BCSDIR/$BCSTAG/lai_clim_@RES_DATELINE.data lai.data
/bin/ln -sf $BCSDIR/$BCSTAG/green_clim_@RES_DATELINE.data green.data
/bin/ln -sf $BCSDIR/$BCSTAG/topo_DYN_ave_@RES_DATELINE.data topo_dynave.data
/bin/ln -sf $BCSDIR/$BCSTAG/topo_GWD_var_@RES_DATELINE.data topo_gwdvar.data
/bin/ln -sf $BCSDIR/$BCSTAG/topo_TRB_var_@RES_DATELINE.data topo_trbvar.data

>>>FVCUBED<<<if(     -e  $BCSDIR/$BCSTAG/Gnomonic_$BCSTAG.dat ) then
>>>FVCUBED<<</bin/ln -sf $BCSDIR/$BCSTAG/Gnomonic_$BCSTAG.dat .
>>>FVCUBED<<<endif

_EOF_

>>>DATAOCEAN<<<echo "/bin/ln -sf $SSTDIR"'/@SSTFILE   sst.data' >> $FILE
>>>DATAOCEAN<<<echo "/bin/ln -sf $SSTDIR"'/@ICEFILE fraci.data' >> $FILE
>>>DATAOCEAN<<<echo "/bin/ln -sf $SSTDIR"'/@KPARFILE SEAWIFS_KPAR_mon_clim.data' >> $FILE

chmod +x linkbcs
/bin/cp  linkbcs $EXPDIR

#######################################################################
#          Get C2L History weights/index file for Cubed-Sphere
#######################################################################

set C_NPX = `echo $AGCM_IM | awk '{printf "%5.5i", $1}'`
set C_NPY = `echo $AGCM_JM | awk '{printf "%5.5i", $1}'`
set H_NPX = `echo @HIST_IM | awk '{printf "%5.5i", $1}'`
set H_NPY = `echo @HIST_JM | awk '{printf "%5.5i", $1}'`

set c2l_file = "${C_NPX}x${C_NPY}_c2l_${H_NPX}x${H_NPY}.bin"

if (-e $BCSDIR/@BCSRES/${c2l_file}) /bin/ln -s $BCSDIR/@BCSRES/${c2l_file} .

##############################################################################
#                    Get Executable and RESTARTS 
#    Note: MERRA Restarts must be Pre-Fetched, Regridded, and Locally Stored
##############################################################################

/bin/cp $EXPDIR/GEOSgcm.x .

set rst_types = `cat AGCM.rc | grep "RESTART_FILE"    | cut -d ":" -f1 | cut -d "_" -f1-2`
set chk_types = `cat AGCM.rc | grep "CHECKPOINT_FILE" | cut -d ":" -f1 | cut -d "_" -f1-2`
set rst_files = `cat AGCM.rc | grep "RESTART_FILE"    | cut -d ":" -f2`
set chk_files = `cat AGCM.rc | grep "CHECKPOINT_FILE" | cut -d ":" -f2`

# Remove possible bootstrap parameter (+/-)
# -----------------------------------------
set dummy = `echo $rst_files`
set rst_files = ''
foreach rst ( $dummy )
  set length  = `echo $rst | awk '{print length($0)}'`
  set    bit  = `echo $rst | cut -c1`
  if(  "$bit" == "+" | \
       "$bit" == "-" ) set rst = `echo $rst | cut -c2-$length`
  set rst_files = `echo $rst_files $rst`
end

# Copy Restarts to Scratch Directory
# ----------------------------------
set year  = `echo $nymd0 | cut -c1-4`
set month = `echo $nymd0 | cut -c5-6`
/bin/ln -s $EXPDIR/restarts/*${nymd0}_${hour}*z.*  .
$GEOSBIN/stripname ${EXPID}.
$GEOSBIN/stripname .e${nymd0}_${hour}\*

#######################################################################
#                 Create Binary Tile File from ASCII
#######################################################################

# Link Boundary Conditions for Appropriate Date
# ---------------------------------------------
set year = `echo $date0 | cut -b1-4`
setenv YEAR $year
./linkbcs

if (! -e tile.bin) then
$GEOSBIN/binarytile.x tile.data tile.bin
endif

##################################################################
######
######          Perform REPLAY Forecast of Model Run
######
##################################################################

/bin/cp CAP.rc.orig CAP.rc
./strip CAP.rc
set oldstring =  `cat CAP.rc | grep JOB_SGMT:`
set newstring =  "JOB_SGMT: $FSEGMENT 030000"
/bin/mv CAP.rc CAP.tmp
cat CAP.tmp | sed -e "s?$oldstring?$newstring?g" > CAP.rc

set oldstring =  `cat CAP.rc | grep HEARTBEAT_DT:`
set newstring =  "HEARTBEAT_DT: $HEARTBEAT"
/bin/mv CAP.rc CAP.tmp
cat CAP.tmp | sed -e "s?$oldstring?$newstring?g" > CAP.rc

       @  NPES = $NX * $NY
$RUN_CMD $NPES ./GEOSgcm.x
 
#######################################################################
#               Move HISTORY Files to Holding Directory
#######################################################################

foreach collection ( $collections )
  /bin/mv `/bin/ls -1 *.${collection}.*` $FCSTDIR
end

#######################################################################
#                               Create STATS
#######################################################################

set nymd   =  $date0
set yyyymm = `echo $nymd | cut -b1-6`
mkdir -p $EXPDIR/forecasts/G5${ANA}.stats.$yyyymm
cd       $EXPDIR/forecasts/G5${ANA}.stats.$yyyymm

set year0      = `echo $nymd | cut -b1-4`
set month0     = `echo $nymd | cut -b5-6`
set fcst_files = `/bin/ls -1 $EXPDIR/forecasts/${nymd}_00z/G5${ANA}*geosgcm_fcst*nc4`

set anal_files = ''
@ n = 1
while ($n <= 6)

set year  = `echo $nymd | cut -b1-4`
set month = `echo $nymd | cut -b5-6`


# G5REPLAY Verification
# ---------------------
set next = `/bin/ls -1 $EXPDIR/holding/geosgcm_prog/${year}*/$EXPID.geosgcm_prog.${nymd}*nc4`

set anal_files = `echo $anal_files $next`
set date = `$GEOSUTIL/post/tick $nymd 0 86400`
set nymd = $date[1]
@ n = $n + 1
end

$GEOSUTIL/post/stats.x -fcst $fcst_files \
                       -ana  $anal_files \
                       -cli $SHARE/gmao_ops/verification/stats/merrasc.197901-200812.clim_00z.144x91.data.nc \
                            $SHARE/gmao_ops/verification/stats/merrasc.197901-200812.clim_06z.144x91.data.nc \
                            $SHARE/gmao_ops/verification/stats/merrasc.197901-200812.clim_12z.144x91.data.nc \
                            $SHARE/gmao_ops/verification/stats/merrasc.197901-200812.clim_18z.144x91.data.nc \
                       -tag $EXPID -nfreq 060000 -rc $GEOSUTIL/post/stats.rc

#######################################################################
#                         Update Forecast Counter
#######################################################################

 @ NFCST  = $NFCST + 1
set date  = `$GEOSUTIL/post/tick $date0 $time0 86400`
end