#!/bin/csh -f
# +-======-+ 
#  Copyright (c) 2003-2007 United States Government as represented by 
#  the Admistrator of the National Aeronautics and Space Administration.  
#  All Rights Reserved.
#  
#  THIS OPEN  SOURCE  AGREEMENT  ("AGREEMENT") DEFINES  THE  RIGHTS  OF USE,
#  REPRODUCTION,  DISTRIBUTION,  MODIFICATION AND REDISTRIBUTION OF CERTAIN 
#  COMPUTER SOFTWARE ORIGINALLY RELEASED BY THE UNITED STATES GOVERNMENT AS 
#  REPRESENTED BY THE GOVERNMENT AGENCY LISTED BELOW ("GOVERNMENT AGENCY").  
#  THE UNITED STATES GOVERNMENT, AS REPRESENTED BY GOVERNMENT AGENCY, IS AN 
#  INTENDED  THIRD-PARTY  BENEFICIARY  OF  ALL  SUBSEQUENT DISTRIBUTIONS OR 
#  REDISTRIBUTIONS  OF THE  SUBJECT  SOFTWARE.  ANYONE WHO USES, REPRODUCES, 
#  DISTRIBUTES, MODIFIES  OR REDISTRIBUTES THE SUBJECT SOFTWARE, AS DEFINED 
#  HEREIN, OR ANY PART THEREOF,  IS,  BY THAT ACTION, ACCEPTING IN FULL THE 
#  RESPONSIBILITIES AND OBLIGATIONS CONTAINED IN THIS AGREEMENT.
#  
#  Government Agency: National Aeronautics and Space Administration
#  Government Agency Original Software Designation: GSC-15354-1
#  Government Agency Original Software Title:  GEOS-5 GCM Modeling Software
#  User Registration Requested.  Please Visit http://opensource.gsfc.nasa.gov
#  Government Agency Point of Contact for Original Software:  
#  			Dale Hithon, SRA Assistant, (301) 286-2691
#  
# +-======-+ 

#######################################################################
#                     Batch Parameters for Run Job
#######################################################################

#PBS -l walltime=@RUN_T
#PBS -l @RUN_P
#PBS -N @RUN_N
#PBS -q @RUN_Q
#PBS -W group_list=@GROUP

#######################################################################
#                  System Environment Variables
#######################################################################

umask 022

limit stacksize unlimited

@SETENVS

#######################################################################
#           Architecture Specific Environment Variables
#######################################################################

setenv ARCH `uname`

setenv SITE             @SITE
setenv GEOSBIN          @GEOSBIN 
setenv RUN_CMD         "@RUN_CMD"

source $GEOSBIN/g5_modules
setenv LD_LIBRARY_PATH ${LD_LIBRARY_PATH}:${BASEDIR}/${ARCH}/lib

#######################################################################
#             Experiment Specific Environment Variables
#######################################################################

setenv    EXPID   @EXPID
setenv    EXPDIR  @EXPDIR
setenv    HOMDIR  @HOMDIR
setenv    SCRDIR  $EXPDIR/scratch

#######################################################################
#                 Create Experiment Sub-Directories
#######################################################################

if (! -e $EXPDIR/restarts   ) mkdir -p $EXPDIR/restarts
if (! -e $EXPDIR/holding    ) mkdir -p $EXPDIR/holding
if (! -e $EXPDIR/archive    ) mkdir -p $EXPDIR/archive
if (! -e $EXPDIR/post       ) mkdir -p $EXPDIR/post
if (! -e $EXPDIR/plot       ) mkdir -p $EXPDIR/plot
if (! -e $SCRDIR            ) mkdir -p $SCRDIR

#######################################################################
#                   Set Experiment Run Parameters
#######################################################################

set       NX = `grep       NX: $HOMDIR/AGCM.rc | cut -d':' -f2`
set       NY = `grep       NY: $HOMDIR/AGCM.rc | cut -d':' -f2`
set  AGCM_IM = `grep  AGCM_IM: $HOMDIR/AGCM.rc | cut -d':' -f2`
set  AGCM_JM = `grep  AGCM_JM: $HOMDIR/AGCM.rc | cut -d':' -f2`
set  AGCM_LM = `grep  AGCM_LM: $HOMDIR/AGCM.rc | cut -d':' -f2`
set END_DATE = `grep END_DATE:  $HOMDIR/CAP.rc | cut -d':' -f2`
set NUM_SGMT = `grep NUM_SGMT:  $HOMDIR/CAP.rc | cut -d':' -f2`

#######################################################################
#   Move to Scratch Directory and Copy RC Files from Home Directory
#######################################################################

cd $SCRDIR
/bin/rm -rf *
                             /bin/cp -f $HOMDIR/*.rc .
if(! -e $EXPDIR/HISTORY.rc ) /bin/cp HISTORY.rc $EXPDIR   ## For Plots

#######################################################################
#              Create HISTORY Collection Directories
#######################################################################

set HISTORY = `cat HISTORY.rc`
@ n = 1
while( $n <= $#HISTORY )
   if( $HISTORY[$n] == COLLECTIONS: )  then
      set collections = ''
      @ m = $n + 1
      while( $HISTORY[$m]   != "::" )
             set collection  = `echo $HISTORY[$m] | sed -e "s/'//g"`
             set collections = `echo $collections $collection`
      @ m = $m + 1
      end
   @ n = $#HISTORY + 1
   endif
@ n = $n + 1
end

foreach collection ( $collections )
  if (! -e $EXPDIR/$collection )         mkdir $EXPDIR/$collection
  if (! -e $EXPDIR/holding/$collection ) mkdir $EXPDIR/holding/$collection
end

#######################################################################
#                        Link Boundary Datasets
#######################################################################

setenv BCSDIR   @BCSDIR
setenv SSTDIR   @SSTDIR
setenv CHMDIR   @CHMDIR
setenv DATELINE DC

set             FILE = linkbcs
/bin/rm -f     $FILE
cat << _EOF_ > $FILE

/bin/mkdir -p          ExtData
/bin/ln    -sf $CHMDIR ExtData

/bin/ln -sf $SSTDIR/dataoceanfile_MERRA_sst_1971-current.360x180.LE                                       sst.data
/bin/ln -sf $SSTDIR/dataoceanfile_MERRA_fraci_1971-current.360x180.LE                                   fraci.data
/bin/ln -sf $SSTDIR/SEAWIFS_KPAR_mon_clim.360x180                                       SEAWIFS_KPAR_mon_clim.data

/bin/ln -sf $BCSDIR/Shared/pchem.species.CMIP-5.1870-2005.z_91x72                                     species.data

/bin/ln -sf $BCSDIR/${AGCM_IM}x${AGCM_JM}/visdf_${AGCM_IM}x${AGCM_JM}_${DATELINE}.dat                   visdf.dat
/bin/ln -sf $BCSDIR/${AGCM_IM}x${AGCM_JM}/nirdf_${AGCM_IM}x${AGCM_JM}_${DATELINE}.dat                   nirdf.dat
/bin/ln -sf $BCSDIR/${AGCM_IM}x${AGCM_JM}/vegdyn_${AGCM_IM}x${AGCM_JM}_${DATELINE}.dat                 vegdyn.data
/bin/ln -sf $BCSDIR/${AGCM_IM}x${AGCM_JM}/FV_${AGCM_IM}x${AGCM_JM}_${DATELINE}_360x180_DE.til            tile.data
/bin/ln -sf $BCSDIR/${AGCM_IM}x${AGCM_JM}/lai_green_clim_${AGCM_IM}x${AGCM_JM}_${DATELINE}.data        laigrn.data
/bin/ln -sf $BCSDIR/${AGCM_IM}x${AGCM_JM}/topo_DYN_ave_${AGCM_IM}x${AGCM_JM}_${DATELINE}.data     topo_dynave.data
/bin/ln -sf $BCSDIR/${AGCM_IM}x${AGCM_JM}/topo_GWD_var_${AGCM_IM}x${AGCM_JM}_${DATELINE}.data     topo_gwdvar.data
/bin/ln -sf $BCSDIR/${AGCM_IM}x${AGCM_JM}/topo_TRB_var_${AGCM_IM}x${AGCM_JM}_${DATELINE}.data     topo_trbvar.data

_EOF_

chmod +x linkbcs
/bin/cp  linkbcs $EXPDIR
       ./linkbcs

$GEOSBIN/binarytile.x tile.data tile.bin

#######################################################################
#             Get Executable, RC files, and RESTARTS 
#######################################################################

/bin/cp     $EXPDIR/GEOSgcm.x .
/bin/ln -sf $EXPDIR/RC/* .
/bin/cp     $EXPDIR/cap_restart .

set rst_types = `cat AGCM.rc | grep "RESTART_FILE"    | cut -d ":" -f1 | cut -d "_" -f1-2`
set chk_types = `cat AGCM.rc | grep "CHECKPOINT_FILE" | cut -d ":" -f1 | cut -d "_" -f1-2`
set rst_files = `cat AGCM.rc | grep "RESTART_FILE"    | cut -d ":" -f2`
set chk_files = `cat AGCM.rc | grep "CHECKPOINT_FILE" | cut -d ":" -f2`

# Remove possible bootstrap parameter (-)
# ---------------------------------------
set dummy = `echo $rst_files`
set rst_files = ''
foreach rst ( $dummy )
  set length  = `echo $rst | awk '{print length($0)}'`
  set    bit  = `echo $rst | cut -c1`
  if(  "$bit" == "-" ) set rst = `echo $rst | cut -c2-$length`
  set rst_files = `echo $rst_files $rst`
end

# Copy Restarts to Scratch Directory
# ----------------------------------
foreach rst ( $rst_files )
  if(-e $EXPDIR/$rst ) /bin/cp $EXPDIR/$rst .
end

# Copy and Tar Initial Restarts to Restarts Directory
# ---------------------------------------------------
set edate = e`cat cap_restart | cut -c1-8`_`cat cap_restart | cut -c10-11`z
set numrs = `/bin/ls -1 ${EXPDIR}/restarts/*${edate}* | wc -l`
if($numrs == 0) then
   foreach rst ( $rst_files )
      if( -e $rst & ! -e ${EXPDIR}/restarts/${rst}.${edate} ) then
            /bin/cp $rst ${EXPDIR}/restarts/${rst}.${edate}
      endif
   end
   cd $EXPDIR/restarts
      tar cf  restarts.${edate}.tar *.${edate}
     /bin/rm -f `ls -1  *.${edate}`
   cd $SCRDIR
endif

##################################################################
######
######         Perform multiple iterations of Model Run
######
##################################################################

@ counter    = 1
while ( $counter <= ${NUM_SGMT} )

       @  NPES = $NX * $NY
$RUN_CMD $NPES ./GEOSgcm.x
set rc =  $status
echo       Status = $rc
 
#######################################################################
#      Rename Checkpoints => Restarts for Next Segment and Archive
#        Note: cap_restart contains the current NYMD and NHMS
#######################################################################

set edate  = e`cat cap_restart | cut -c1-8`_`cat cap_restart | cut -c10-11`z

set numrst = `echo $rst_types | wc -w`
set numchk = `echo $chk_types | wc -w`

@ n = 1
@ z = $numrst + 1
while ( $n <= $numchk )
   if ( -e $chk_files[$n] ) then
       @ m = 1
       while ( $m <= $numrst )
       if(    $chk_types[$n] == $rst_types[$m] || \
            \#$chk_types[$n] == $rst_types[$m]    ) then
           /bin/mv $chk_files[$n] $rst_files[$m]
           /bin/cp $rst_files[$m] ${EXPDIR}/restarts/${rst_files[$m]}.${edate}
           @ m = $numrst + 999
       else
           @ m = $m + 1
       endif
       end
       if( $m == $z ) then
           echo "Warning!!  Could not find CHECKPOINT/RESTART match for:  " $chk_types[$n]
           exit
       endif
   endif
@ n = $n + 1
end

cd $EXPDIR/restarts
   tar cf  restarts.${edate}.tar *.${edate}
  /bin/rm -f             `ls -1  *.${edate}`
cd $SCRDIR

#######################################################################
#               Move HISTORY Files to Holding Directory
#######################################################################

foreach    collection ( $collections )
  /bin/mv `ls -1 *.${collection}.*` $EXPDIR/holding/$collection
end

#######################################################################
#                      Perform Post Processing
#######################################################################

cd   $EXPDIR/post

/bin/rm -f sedfile
cat >      sedfile << EOF
s/@POST_O/gcm_post.${edate}/g
EOF
sed -f sedfile gcm_post.j > gcm_post.jtmp

qsub gcm_post.jtmp
/bin/rm -f sedfile gcm_post.jtmp
cd   $SCRDIR

#######################################################################
#                         Update Iteration Counter
#######################################################################

set enddate = `echo  $END_DATE | cut -c1-8`
set capdate = `cat cap_restart | cut -c1-8`

if ( $capdate < $enddate ) then
@ counter = $counter    + 1
else
@ counter = ${NUM_SGMT} + 1
endif

end

#######################################################################
#                              Re-Submit Job
#######################################################################

foreach rst ( $rst_files )
   /bin/rm -f $EXPDIR/$rst  
end
   /bin/rm -f $EXPDIR/cap_restart

foreach rst ( $rst_files )
  /bin/cp $rst $EXPDIR/$rst  
end
/bin/cp cap_restart $EXPDIR/cap_restart

if ( $rc == 0 ) then
      cd   $HOMDIR
      if ( $capdate < $enddate ) qsub $HOMDIR/gcm_run.j
endif