#!/bin/csh -f
# +-======-+ 
#  Copyright (c) 2003-2007 United States Government as represented by 
#  the Admistrator of the National Aeronautics and Space Administration.  
#  All Rights Reserved.
#  
#  THIS OPEN  SOURCE  AGREEMENT  ("AGREEMENT") DEFINES  THE  RIGHTS  OF USE,
#  REPRODUCTION,  DISTRIBUTION,  MODIFICATION AND REDISTRIBUTION OF CERTAIN 
#  COMPUTER SOFTWARE ORIGINALLY RELEASED BY THE UNITED STATES GOVERNMENT AS 
#  REPRESENTED BY THE GOVERNMENT AGENCY LISTED BELOW ("GOVERNMENT AGENCY").  
#  THE UNITED STATES GOVERNMENT, AS REPRESENTED BY GOVERNMENT AGENCY, IS AN 
#  INTENDED  THIRD-PARTY  BENEFICIARY  OF  ALL  SUBSEQUENT DISTRIBUTIONS OR 
#  REDISTRIBUTIONS  OF THE  SUBJECT  SOFTWARE.  ANYONE WHO USES, REPRODUCES, 
#  DISTRIBUTES, MODIFIES  OR REDISTRIBUTES THE SUBJECT SOFTWARE, AS DEFINED 
#  HEREIN, OR ANY PART THEREOF,  IS,  BY THAT ACTION, ACCEPTING IN FULL THE 
#  RESPONSIBILITIES AND OBLIGATIONS CONTAINED IN THIS AGREEMENT.
#  
#  Government Agency: National Aeronautics and Space Administration
#  Government Agency Original Software Designation: GSC-15354-1
#  Government Agency Original Software Title:  GEOS-5 GCM Modeling Software
#  User Registration Requested.  Please Visit http://opensource.gsfc.nasa.gov
#  Government Agency Point of Contact for Original Software:  
#  			Dale Hithon, SRA Assistant, (301) 286-2691
#  
# +-======-+ 

#######################################################################
#                     Batch Parameters for Run Job
#######################################################################

#PBS -l walltime=@RUN_T
#PBS -l @RUN_P
#PBS -N @RUN_N
#PBS -q @RUN_Q
#PBS -W group_list=@GROUP

#######################################################################
#                  System Environment Variables
#######################################################################

umask 022

limit stacksize unlimited

@SETENVS

#######################################################################
#           Architecture Specific Environment Variables
#######################################################################

setenv ARCH `uname`

setenv SITE             @SITE
setenv GEOSBIN          @GEOSBIN 
setenv RUN_CMD         "@RUN_CMD"

source $GEOSBIN/g5_modules
setenv LD_LIBRARY_PATH ${LD_LIBRARY_PATH}:${BASEDIR}/${ARCH}/lib

#######################################################################
#             Experiment Specific Environment Variables
#######################################################################

setenv    EXPID   @EXPID
setenv    EXPDIR  @EXPDIR
setenv    HOMDIR  @HOMDIR
setenv    SCRDIR  $EXPDIR/scratch

#######################################################################
#                 Create Experiment Sub-Directories
#######################################################################

if (! -e $EXPDIR/restarts   ) mkdir -p $EXPDIR/restarts
if (! -e $EXPDIR/holding    ) mkdir -p $EXPDIR/holding
if (! -e $EXPDIR/archive    ) mkdir -p $EXPDIR/archive
if (! -e $EXPDIR/post       ) mkdir -p $EXPDIR/post
if (! -e $EXPDIR/plot       ) mkdir -p $EXPDIR/plot
if (! -e $SCRDIR            ) mkdir -p $SCRDIR

#######################################################################
#                   Set Experiment Run Parameters
#######################################################################

set       NX = `grep       NX: $HOMDIR/AGCM.rc | cut -d':' -f2`
set       NY = `grep       NY: $HOMDIR/AGCM.rc | cut -d':' -f2`
set  AGCM_IM = `grep  AGCM_IM: $HOMDIR/AGCM.rc | cut -d':' -f2`
set  AGCM_JM = `grep  AGCM_JM: $HOMDIR/AGCM.rc | cut -d':' -f2`
set  AGCM_LM = `grep  AGCM_LM: $HOMDIR/AGCM.rc | cut -d':' -f2`
set  OGCM_IM = `grep  OGCM_IM: $HOMDIR/AGCM.rc | cut -d':' -f2`
set  OGCM_JM = `grep  OGCM_JM: $HOMDIR/AGCM.rc | cut -d':' -f2`
set END_DATE = `grep END_DATE:  $HOMDIR/CAP.rc | cut -d':' -f2`
set NUM_SGMT = `grep NUM_SGMT:  $HOMDIR/CAP.rc | cut -d':' -f2`

# Check for Over-Specification of CPU Resources
# ---------------------------------------------
  if ($?PBS_NODEFILE) then
     set  NCPUS = `qstat -f $PBS_JOBID | grep -i Resource_List.ncpus | cut -d'=' -f2`
     @    NPES  = $NX * $NY
        if( $NPES > $NCPUS ) then
             echo "CPU Resources are Over-Specified"
             echo "--------------------------------"
             echo "Allotted NCPUs: $NCPUS"
             echo "Specified  NX : $NX"
             echo "Specified  NY : $NY"
             exit
        endif
     endif
  endif

#######################################################################
#   Move to Scratch Directory and Copy RC Files from Home Directory
#######################################################################

cd $SCRDIR
/bin/rm -rf *
                             /bin/ln -sf $EXPDIR/RC/* .
                             /bin/cp     $EXPDIR/cap_restart .
                             /bin/cp -f  $HOMDIR/*.rc .

#######################################################################
#              Create HISTORY Collection Directories
#######################################################################

set collections = ''
foreach line ("`cat HISTORY.rc`")
   set firstword  = `echo $line | awk '{print $1}'`
   set firstchar  = `echo $firstword | cut -c1`
   set secondword = `echo $line | awk '{print $2}'`

   if ( $firstword == "::" ) goto done

   if ( $firstchar != "#" ) then
      set collection  = `echo $firstword | sed -e "s/'//g"`
      set collections = `echo $collections $collection`
      if ( $secondword == :: ) goto done
   endif

   if ( $firstword == COLLECTIONS: ) then
      set collections = `echo $secondword | sed -e "s/'//g"`
   endif
end

done:
   foreach collection ( $collections )
      if (! -e $EXPDIR/$collection )         mkdir $EXPDIR/$collection
      if (! -e $EXPDIR/holding/$collection ) mkdir $EXPDIR/holding/$collection
   end

#######################################################################
#                        Link Boundary Datasets
#######################################################################

setenv BCSDIR   @BCSDIR
setenv SSTDIR   @SSTDIR
setenv CHMDIR   @CHMDIR
setenv DATELINE DC

>>>COUPLED<<<setenv GRIDDIR  @COUPLEDIR/a${AGCM_IM}x${AGCM_JM}_o${OGCM_IM}x${OGCM_JM}

set             FILE = linkbcs
/bin/rm -f     $FILE
cat << _EOF_ > $FILE
#!/bin/csh -f

>>>COUPLED<<</bin/mkdir -p RESTART
/bin/mkdir -p            ExtData
/bin/ln    -sf $CHMDIR/* ExtData

>>>COUPLED<<</bin/ln -s $GRIDDIR/SEAWIFS_KPAR_mon_clim.${OGCM_IM}x${OGCM_JM}               SEAWIFS_KPAR_mon_clim.data
>>>COUPLED<<</bin/ln -s $GRIDDIR/DC0${AGCM_IM}xPC00${AGCM_JM}_TM0${OGCM_IM}xTM0${OGCM_JM}-Pfafstetter.til   tile.data
>>>COUPLED<<</bin/ln -s $GRIDDIR/DC0${AGCM_IM}xPC00${AGCM_JM}_TM0${OGCM_IM}xTM0${OGCM_JM}-Pfafstetter.TRN   runoff.bin
>>>COUPLED<<</bin/ln -s $GRIDDIR/tripolar_${OGCM_IM}x${OGCM_JM}.ascii .
>>>COUPLED<<</bin/ln -s $GRIDDIR/vgrid50.ascii .

>>>DATAOCEAN<<</bin/ln -sf $SSTDIR/dataoceanfile_MERRA_sst_1971-current.360x180.LE sst.data
>>>DATAOCEAN<<</bin/ln -sf $SSTDIR/dataoceanfile_MERRA_fraci_1971-current.360x180.LE fraci.data
>>>DATAOCEAN<<</bin/ln -sf $SSTDIR/SEAWIFS_KPAR_mon_clim.360x180 SEAWIFS_KPAR_mon_clim.data
>>>DATAOCEAN<<</bin/ln -sf $BCSDIR/@BCSRES/@TILEDATA tile.data

/bin/ln -sf $BCSDIR/Shared/pchem.species.CMIP-5.1870-2097.z_91x72.nc4 species.data

/bin/ln -sf $BCSDIR/@BCSRES/visdf_@RES_DATELINE.dat visdf.dat
/bin/ln -sf $BCSDIR/@BCSRES/nirdf_@RES_DATELINE.dat nirdf.dat
/bin/ln -sf $BCSDIR/@BCSRES/vegdyn_@RES_DATELINE.dat vegdyn.data
>>>FVLATLON<<</bin/ln -sf $BCSDIR/@BCSRES/lai_green_clim_@RES_DATELINE.data laigrn.data
>>>FVCUBED<<</bin/ln -sf $BCSDIR/@BCSRES/lai_clim_@RES_DATELINE.data lai.data
>>>FVCUBED<<</bin/ln -sf $BCSDIR/@BCSRES/green_clim_@RES_DATELINE.data green.data
/bin/ln -sf $BCSDIR/@BCSRES/topo_DYN_ave_@RES_DATELINE.data topo_dynave.data
/bin/ln -sf $BCSDIR/@BCSRES/topo_GWD_var_@RES_DATELINE.data topo_gwdvar.data
/bin/ln -sf $BCSDIR/@BCSRES/topo_TRB_var_@RES_DATELINE.data topo_trbvar.data

if( -e $BCSDIR/@BCSRES/Gnomonic_@BCSRES.dat ) then
    /bin/ln -sf $BCSDIR/@BCSRES/Gnomonic_@BCSRES.dat .
endif

>>>COUPLED<<</bin/cp -R $GRIDDIR/INPUT .
>>>COUPLED<<</bin/cp $HOMDIR/input.nml .
>>>COUPLED<<</bin/cp $HOMDIR/*_table .
>>>COUPLED<<</bin/ln -sf $GRIDDIR/cice/kmt_cice.bin .
>>>COUPLED<<</bin/ln -sf $GRIDDIR/cice/grid_cice.bin .

_EOF_

chmod +x linkbcs
/bin/cp  linkbcs $EXPDIR
       ./linkbcs

#######################################################################
#                 Create Binary Tile File from ASCII
#######################################################################

$GEOSBIN/binarytile.x tile.data tile.bin

#######################################################################
#          Get C2L History weights/index file for Cubed-Sphere
#######################################################################

set C_NPX = `echo $AGCM_IM | awk '{printf "%5.5i", $1}'`
set C_NPY = `echo $AGCM_JM | awk '{printf "%5.5i", $1}'`
set H_NPX = `echo @HIST_IM | awk '{printf "%5.5i", $1}'`
set H_NPY = `echo @HIST_JM | awk '{printf "%5.5i", $1}'`

set c2l_file = "${C_NPX}x${C_NPY}_c2l_${H_NPX}x${H_NPY}.bin"

if (-e $BCSDIR/@BCSRES/${c2l_file}) /bin/ln -s $BCSDIR/@BCSRES/${c2l_file} .

#######################################################################
#                    Get Executable and RESTARTS 
#######################################################################

/bin/cp $EXPDIR/GEOSgcm.x .

set rst_types = `cat AGCM.rc | grep "RESTART_FILE"    | cut -d ":" -f1 | cut -d "_" -f1-2`
set chk_types = `cat AGCM.rc | grep "CHECKPOINT_FILE" | cut -d ":" -f1 | cut -d "_" -f1-2`
set rst_files = `cat AGCM.rc | grep "RESTART_FILE"    | cut -d ":" -f2`
set chk_files = `cat AGCM.rc | grep "CHECKPOINT_FILE" | cut -d ":" -f2`

# Remove possible bootstrap parameter (-)
# ---------------------------------------
set dummy = `echo $rst_files`
set rst_files = ''
foreach rst ( $dummy )
  set length  = `echo $rst | awk '{print length($0)}'`
  set    bit  = `echo $rst | cut -c1`
  if(  "$bit" == "-" ) set rst = `echo $rst | cut -c2-$length`
  set rst_files = `echo $rst_files $rst`
end

# Copy Restarts to Scratch Directory
# ----------------------------------
foreach rst ( $rst_files )
  if(-e $EXPDIR/$rst ) /bin/cp $EXPDIR/$rst .
end
>>>COUPLED<<</bin/cp $EXPDIR/RESTART/* INPUT

# Copy and Tar Initial Restarts to Restarts Directory
# ---------------------------------------------------
set edate = e`cat cap_restart | cut -c1-8`_`cat cap_restart | cut -c10-11`z
set numrs = `/bin/ls -1 ${EXPDIR}/restarts/*${edate}* | wc -l`
if($numrs == 0) then
   foreach rst ( $rst_files )
      if( -e $rst & ! -e ${EXPDIR}/restarts/${rst}.${edate} ) then
            /bin/cp $rst ${EXPDIR}/restarts/${rst}.${edate}
      endif
   end
>>>COUPLED<<<   /bin/cp $EXPDIR/RESTART ${EXPDIR}/restarts/RESTART.${edate}
   cd $EXPDIR/restarts
      tar cf  restarts.${edate}.tar *.${edate}
     /bin/rm -rf `ls -1  *.${edate}`
   cd $SCRDIR
endif

##################################################################
######
######         Perform multiple iterations of Model Run
######
##################################################################

@ counter    = 1
while ( $counter <= ${NUM_SGMT} )

       @  NPES = $NX * $NY
$RUN_CMD $NPES ./GEOSgcm.x
set rc =  $status
echo       Status = $rc
 
#######################################################################
#      Rename Checkpoints => Restarts for Next Segment and Archive
#        Note: cap_restart contains the current NYMD and NHMS
#######################################################################

set edate  = e`cat cap_restart | cut -c1-8`_`cat cap_restart | cut -c10-11`z

set numrst = `echo $rst_types | wc -w`
set numchk = `echo $chk_types | wc -w`

@ n = 1
@ z = $numrst + 1
while ( $n <= $numchk )
   if ( -e $chk_files[$n] ) then
       @ m = 1
       while ( $m <= $numrst )
       if(    $chk_types[$n] == $rst_types[$m] || \
            \#$chk_types[$n] == $rst_types[$m]    ) then
           /bin/mv $chk_files[$n] $rst_files[$m]
           /bin/cp $rst_files[$m] ${EXPDIR}/restarts/${rst_files[$m]}.${edate}
           @ m = $numrst + 999
       else
           @ m = $m + 1
       endif
       end
       if( $m == $z ) then
           echo "Warning!!  Could not find CHECKPOINT/RESTART match for:  " $chk_types[$n]
           exit
       endif
   endif
@ n = $n + 1
end

>>>COUPLED<<</bin/cp RESTART ${EXPDIR}/restarts/RESTART.${edate}
>>>COUPLED<<</bin/cp RESTART/* INPUT
cd $EXPDIR/restarts
   tar cf  restarts.${edate}.tar *.${edate}
  /bin/rm -rf            `ls -1  *.${edate}`
cd $SCRDIR

#######################################################################
#               Move HISTORY Files to Holding Directory
#######################################################################

foreach collection ( $collections )
   /bin/mv `ls -1 *.${collection}.*` $EXPDIR/holding/$collection
end

>>>COUPLED<<<# MOM-Specific Output Files
>>>COUPLED<<<# -------------------------
>>>COUPLED<<< set dsets="ocean_month"
>>>COUPLED<<< foreach dset ( $dsets )
>>>COUPLED<<< set num = `/bin/ls -1 $dset.nc | wc -l`
>>>COUPLED<<< if($num != 0) then
>>>COUPLED<<<    if(! -e $EXPDIR/MOM_Output) mkdir -p $EXPDIR/MOM_Output
>>>COUPLED<<<    /bin/mv $SCRDIR/$dset.nc $EXPDIR/MOM_Output/$dset.${edate}.nc
>>>COUPLED<<< endif
>>>COUPLED<<< end
>>>COUPLED<<<
#######################################################################
#                      Perform Post Processing
#######################################################################

cd   $EXPDIR/post

/bin/rm -f sedfile
cat >      sedfile << EOF
s/@POST_O/gcm_post.${edate}/g
EOF
sed -f sedfile gcm_post.j > gcm_post.jtmp
chmod 755 gcm_post.jtmp

if( ( $SITE    == "NAS" ) & \
    ( $AGCM_IM == "45"    | \
      $AGCM_IM == "90"    | \
      $AGCM_IM == "144"   | \
      $AGCM_IM == "288" ) )    then
       nohup ssh -t bridge1  "$EXPDIR/post/gcm_post.jtmp > $EXPDIR/post/gcm_post.${edate} ; \
                   /bin/rm -f $EXPDIR/post/gcm_post.jtmp &"
else
       qsub      gcm_post.jtmp
      /bin/rm -f gcm_post.jtmp
endif

/bin/rm -f sedfile
cd   $SCRDIR

#######################################################################
#                         Update Iteration Counter
#######################################################################

set enddate = `echo  $END_DATE | cut -c1-8`
set capdate = `cat cap_restart | cut -c1-8`

if ( $capdate < $enddate ) then
@ counter = $counter    + 1
else
@ counter = ${NUM_SGMT} + 1
endif

end

#######################################################################
#                              Re-Submit Job
#######################################################################

foreach rst ( $rst_files )
   /bin/rm -f $EXPDIR/$rst  
end
   /bin/rm -f $EXPDIR/cap_restart

foreach rst ( $rst_files )
  /bin/cp $rst $EXPDIR/$rst  
end
/bin/cp cap_restart $EXPDIR/cap_restart

>>>COUPLED<<</bin/cp -f RESTART $EXPDIR

if ( $rc == 0 ) then
      cd   $HOMDIR
      if ( $capdate < $enddate ) qsub $HOMDIR/gcm_run.j
endif