#!/bin/csh # +-======-+ # Copyright (c) 2003-2007 United States Government as represented by # the Admistrator of the National Aeronautics and Space Administration. # All Rights Reserved. # # THIS OPEN SOURCE AGREEMENT ("AGREEMENT") DEFINES THE RIGHTS OF USE, # REPRODUCTION, DISTRIBUTION, MODIFICATION AND REDISTRIBUTION OF CERTAIN # COMPUTER SOFTWARE ORIGINALLY RELEASED BY THE UNITED STATES GOVERNMENT AS # REPRESENTED BY THE GOVERNMENT AGENCY LISTED BELOW ("GOVERNMENT AGENCY"). # THE UNITED STATES GOVERNMENT, AS REPRESENTED BY GOVERNMENT AGENCY, IS AN # INTENDED THIRD-PARTY BENEFICIARY OF ALL SUBSEQUENT DISTRIBUTIONS OR # REDISTRIBUTIONS OF THE SUBJECT SOFTWARE. ANYONE WHO USES, REPRODUCES, # DISTRIBUTES, MODIFIES OR REDISTRIBUTES THE SUBJECT SOFTWARE, AS DEFINED # HEREIN, OR ANY PART THEREOF, IS, BY THAT ACTION, ACCEPTING IN FULL THE # RESPONSIBILITIES AND OBLIGATIONS CONTAINED IN THIS AGREEMENT. # # Government Agency: National Aeronautics and Space Administration # Government Agency Original Software Designation: GSC-15354-1 # Government Agency Original Software Title: GEOS-5 GCM Modeling Software # User Registration Requested. Please Visit http://opensource.gsfc.nasa.gov # Government Agency Point of Contact for Original Software: # Dale Hithon, SRA Assistant, (301) 286-2691 # # +-======-+ #BSUB -W 8:00 #BSUB -q gmao_hi #BSUB -n 16 #BSUB -o std.out%J #BSUB -e std.err%J #BSUB -J ocn_trans_1 #BSUB -P g0609 ####################################################################### # Batch Parameters for Altix ####################################################################### ##PBS -l walltime=8:00:00 ##PBS -l ncpus=32 ##PBS -V ##PBS -N geos_coupled ##PBS -q geos ####################################################################### # System Environment Variables ####################################################################### umask 022 limit stacksize unlimited setenv ARCH `uname` setenv MPI_TYPE_MAX 65536 setenv DSM_DISTRIBUTE setenv MPI_DSM_VERBOSE setenv MPI_DSM_DISTRIBUTE setenv MPI_USE_XPMEM setenv MPI_BUFFER_MAX 2000 setenv MPI_MSGS_MAX 1048576 if( $ARCH == 'OSF1' ) then # setenv BASEDIR /share/ESMA/baselibs/v1_8r1p setenv BASEDIR $HOME//baselibs/v2_2 setenv LD_LIBRARY_PATH ${LD_LIBRARY_PATH}:${BASEDIR}/${ARCH}/lib # setenv RUN_CMD "prun -n" setenv RUN_CMD "totalview prun -a -n" setenv BCSDIR /share/ltakacs/geos5/bcs endif if( $ARCH == 'IRIX64' ) then setenv BASEDIR /share/ESMA/baselibs/v1_8r1p setenv LD_LIBRARY64_PATH ${LD_LIBRARY64_PATH}:${BASEDIR/${ARCH}/lib setenv AGCM_N_THREADS_PER_PROCESS 1 setenv RUN_CMD "mpirun -np" setenv BCSDIR /share/ltakacs/geos5/bcs endif if ($ARCH == 'Linux' ) then setenv RUN_CMD "mpirun -np" # setenv RUN_CMD "totalview mpirun -a -np" setenv BCSDIR /u/ltakacs/geos5/bcs endif ####################################################################### # Set Experiment Directories ####################################################################### #setenv APPLDIR $HOME/esma/Coupled/GEOSGCM/src/Application/ setenv APPLDIR $HOME/esma/GEOSogcm-Poseidon/GEOSGCM/src/Application/ setenv EXPID exp001tran1 setenv MASTOR /nsippscr/trayanov/GEOS5.0/coupled setenv EXPDIR $MASTOR/$EXPID setenv WRKDIR $EXPDIR/wrkdir setenv GRIDDIR /nsippscr/jjacob/CGEOS/grid if (! -e $EXPDIR/listings ) mkdir $EXPDIR/listings if (! -e $EXPDIR/restarts ) mkdir $EXPDIR/restarts #if (! -e $MASTOR/do_monthly ) mkdir $MASTOR/do_monthly if (! -e $MASTOR/history ) mkdir $MASTOR/history if (! -e $WRKDIR ) mkdir -p $WRKDIR cd $WRKDIR /bin/rm -f * ####################################################################### # Set Model Run Parameters ####################################################################### set NX = 4 set NY = 4 set IM = 360 set JM = 180 set LM = 72 set DATELINE = DE set OCEAN_IM = 576 set OCEAN_JM = 540 set OCEAN_LM = 34 #set DT = 1800 set DT = 1200 set BEG_DATE = '19910301 000000' set END_DATE = '20050302 210000' set JOB_SGMT = '00000001 000000' set EXPDSC = "Coupled_GEOS5_and_Poseidon" set ANALYSIS = 0 ####################################################################### # Create CAP Resource File ####################################################################### set FILE = CAP.rc /bin/rm $FILE cat << _EOF_ > $FILE NX: $NX NY: $NY IM: $IM JM: $JM LM: $LM APP_IM: $IM APP_JM: $JM APP_LM: $LM OCEAN_IM: $OCEAN_IM OCEAN_JM: $OCEAN_JM OCEAN_LM: $OCEAN_LM CAP_GC1_config_file: GCM.rc CAP_GC2_config_file: HISTORY.rc BEG_DATE: $BEG_DATE END_DATE: $END_DATE JOB_SGMT: $JOB_SGMT RUN_DT: $DT DATELINE: $DATELINE GRIDNAME: PE${IM}x${JM}-DE _EOF_ ####################################################################### # Create History List for Output ####################################################################### set FILE = HISTORY.rc /bin/rm $FILE cat << _EOF_ > $FILE EXPID: $EXPID EXPDSC: $EXPDSC list(1)%filename: 'geosgcm_ocean', list(1)%frequency: 480000, list(1)%duration: 0, list(1)%fields: 'T' , 'OCEAN', 'S' , 'OCEAN', 'H' , 'OCEAN', 'U' , 'OCEAN', 'V' , 'OCEAN', 'Ue' , 'OCEAN', 'Ve' , 'OCEAN', 'TW1' , 'OCEAN', 'Taux' , 'OCEAN' , 'Tauy' , 'OCEAN' , 'MixedLayerDepth' , 'OCEAN' , 'BuoyancyFlux' , 'OCEAN' , 'U3' , 'OCEAN' , 'TURB' , 'OBIO' , :: _EOF_ ####################################################################### # Create GCM Resource File ####################################################################### set FILE = GCM.rc /bin/rm $FILE cat << _EOF_ > $FILE IM: $IM JM: $JM LM: $LM NX: $NX NY: $NY APP_IM: $IM APP_JM: $JM APP_LM: $LM OCEAN_IM: $OCEAN_IM OCEAN_JM: $OCEAN_JM OCEAN_LM: $OCEAN_LM DATELINE: $DATELINE GRIDNAME: PE${IM}x${JM}-DE RUN_DT: $DT DATA_SIT_FILE: sstsi.data DATA_FRT_FILE: fraci.data SSS_FILE: LEV_SSS_clim_GEOSfmt.ieee SOLAR_FILE: NCEP_NSWRS_clim_GEOSfmt.ieee TAUX_FILE: NCEP_UFLX_clim_GEOSfmt.ieee TAUY_FILE: NCEP_VFLX_clim_GEOSfmt.ieee PME_FILE: NCEP_WATER_clim_GEOSfmt.ieee KPAR_FILE: SEAWIFS_KPAR_mon_clim.ieee Q_FILE: COADS_QNET_clim_GEOSfmt.ieee SSS_FILE: LEV_SSS_clim_GEOSfmt.ieee SST_FILE: dataoceanfile_reynolds_v2_sst_clim.360x180 SST_RELAXTIME: 86400 SSS_RELAXTIME: 86400 SALTWATER_INTERNAL_RESTART_FILE: dataatm_internal_restart SALTWATER_INTERNAL_RESTART_TYPE: binary SALTWATER_INTERNAL_CHECKPOINT_FILE: dataatm_internal_checkpoint SALTWATER_INTERNAL_CHECKPOINT_TYPE: binary RUN_DT: $DT ##RUN_DT: 21600 DATA_SEA_FILE: ocean.data DATA_SEAICE_FILE: seaice.data # section for Poseidon OCEAN_GRID_FILE: gridfileesma.ZDF OCEAN_POSEIDON_RESTART_FILE: poseidon_restart OCEAN_POSEIDON_RESTART_TYPE: zdf OCEAN_POSEIDON_CHECKPOINT_FILE: poseidon_checkpoint OCEAN_POSEIDON_CHECKPOINT_TYPE: zdf OCEAN_POSEIDON_NAMELIST_FILE: layer_control #HistoryDiagnostics: 'Nu' 'Vmix' 'Ap' 'As' 'Aa' 'Af' 'ATend' 'AgradP' 'ACor' 'AFx' 'AFilt' 'AVmix' #History_Writing_Seconds: 3600 #History_Sampling_Seconds: 3600 # GEOS internal state restart/checkpoint for Poseidon_GridComp #OCEAN_INTERNAL_RESTART_FILE: ocean_internal_restart #OCEAN_INTERNAL_RESTART_TYPE: binary OCEAN_INTERNAL_CHECKPOINT_FILE: ocean_internal_checkpoint OCEAN_INTERNAL_CHECKPOINT_TYPE: binary #TREATMENT OF NON-LINEAR AND CORIOLIS TERMS: OCEAN_PV_SCHEME: 2 # What type of PV advection operator: # 0: Default # 1: Sadourny # 2: Arakawa-Lamb vector invariant # 4: Suarez-Takacs 4th order # 5: B-grid flux form OCEAN_TIMESTEP: 1200. # ocean timestep OCEAN_EXTERNAL_MODE: 1 # 1=Ext mode, 0=reduced gravity OCEAN_EXTERNAL_SUBSTEPS: 48 # Number of external steps per internal OCEAN_BETA_H: 0.53000 # Forward-backward coefficient OCEAN_MASS_DAMPING_TIME: 28800. # External mode damping time OCEAN_MASS_ADVECTION_ORDER: 3 # Order of advection operator OCEAN_TRACER_ADVECTION_ORDER: 3 # Order of advection operator OCEAN_REMAP_INTERVAL: 7200.0 # interval between vert remapping # FILTERING INFORMATION: OCEAN_H_FILTER_FREQUENCY: 14400 # How often to apply filter (seconds) OCEAN_H_FILTER_ORDER: 8 # Filter order OCEAN_H_FILTER_TIMESCALE: 86400.0 # timescale of filter (seconds) OCEAN_T_FILTER_FREQUENCY: 14400 # How often to apply filter (seconds) OCEAN_T_FILTER_ORDER: 8 # Filter order OCEAN_T_FILTER_TIMESCALE: 86400.0 # timescale of filter (seconds) OCEAN_U_FILTER_FREQUENCY: 14400 # How often to apply filter (seconds) OCEAN_U_FILTER_ORDER: 8 # Filter order OCEAN_U_FILTER_TIMESCALE: 432000.0 # timescale of filter (seconds) # HORIZONTAL DIFFUSION INFORMATION: OCEAN_AKH: 0.0 # Diffusivity of layer interfaces OCEAN_AKT: 500. # Diffusivity of heat, salt # Momentum: OCEAN_USE_LAPLACIAN_VISCOSITY: 0 # 1=TRUE, 0=FALSE OCEAN_AKU: 0.0 # Background Laplacian viscosity OCEAN_SMAGORINSKY_LAP_CONST: 0.00 # Coefficient for Laplacian Smag effect OCEAN_KH_VEL_SCALE: 0.00 # Velocity scale for Laplacian OCEAN_USE_BIHARMONIC_VISCOSITY: 1 # 1=TRUE, 0=FALSE OCEAN_AAU: 0.00 # Background Biharmonic viscosity OCEAN_SMAGORINSKY_BI_CONST: 0.010 # Coefficient for Biharmonic Smag effect OCEAN_AH_VEL_SCALE: 0.00 # Velocity scale for Biharmonic OCEAN_USE_NOSLIP: 0 # 0=FREESLIP 1=NOSLIP OCEAN_VISCOSITY_TIMELIMIT: 1800. # time scale to set upper bound on friction # Vertical Mixing Parameters: OCEAN_VMIX_INTERVAL: 7200 # seconds between vertical mixing calculation # HEAT: OCEAN_KAPPA_T_TYPE: 3 # 1=step 2=PetersGreggToole, 3=Pacanowski-Philander OCEAN_KAPPA_T_0: 5.00E-03 # Kappa at Ri=0.0 for PP, PGT curves Units:m^2/s OCEAN_KAPPA_T_MAX: 1.50E-03 # clipping limit on Kappa OCEAN_KAPPA_T_MIN: 5.00E-05 # Minimum Kappa OCEAN_KAPPA_T_CONVECT: 1.00E+03 # Kappa to use for convection OCEAN_KAPPA_T_CRITICAL_RI: 0.25 # Richardson number criterion for step scheme OCEAN_KAPPA_T_PGT1: 0.0 # coefficient for Peters, Gregg Toole OCEAN_KAPPA_T_POWER: 3 # Exponent for 1/Ri part of Pac Phil #SALT: OCEAN_KAPPA_S_TYPE: 3 OCEAN_KAPPA_S_0: 5.00E-03 OCEAN_KAPPA_S_MAX: 1.50E-03 OCEAN_KAPPA_S_MIN: 5.00E-05 OCEAN_KAPPA_S_CONVECT: 1.00E+03 OCEAN_KAPPA_S_KRAUS: 0.00E+00 OCEAN_KAPPA_S_CRITICAL_RI: 0.25 OCEAN_KAPPA_S_PGT1: 0.0 OCEAN_KAPPA_S_POWER: 3 #MOMENTUM: OCEAN_NU_TYPE: 3 OCEAN_NU_0: 5.00E-03 OCEAN_NU_MAX: 2.50E-03 OCEAN_NU_MIN: 1.00E-04 OCEAN_NU_CONVECT: 1.00E+03 OCEAN_NU_KRAUS: 0.0 OCEAN_NU_CRITICAL_RI: 0.25 OCEAN_NU_PGT1: 0.0 OCEAN_NU_POWER: 3 # If external mode: what is bottom drag law: OCEAN_BOTTOM_DRAG_CD: 1.00E-03 # Bottom drag CD OCEAN_BOTTOM_BL_THICKNESS: 1.0 # Thickness for bottom drag law (m) OCEAN_EXTRA_BOTTOM_SPEED: 0.01 # Extra speed to add to bottom layer speed # to compute drag coefficient (m/2) #MIXED LAYER INFORMATION: OCEAN_DO_MIXED_LAYER: 1 # 1 if TRUE, 0 = FALSE OCEAN_ML_INTERVAL: 86400 # Seconds between mixed layer computation OCEAN_ML_NLAYERS: 0 # How many layers follow the mixed layer OCEAN_ML_DISSIPATION: 1.00E-09 # background disispation in mixed layer OCEAN_ML_U3COEFF: 1.2500 # Coefficient on U*^3 OCEAN_ML_BACKU2: 0.0 # background level of turbulence in mixed layer OCEAN_ML_SHEAR_EFF: 0.2000 # efficiency of release of mean kinetic energy OCEAN_ML_QNEG_DISS: 0.2000 # how much negative buoyancy flux is dissipated OCEAN_BUFFER_LAYERS: 0 # number of buffer layers below mixed layer ML_PENDEPTH: 15.0 _EOF_ ####################################################################### # Create fvcore_layout Resource File ####################################################################### set FILE = fvcore_layout.rc /bin/rm $FILE cat << _EOF_ > $FILE xy_yz_decomp: $NX $NY $NY $NX im: $IM jm: $JM km: $LM dt: $DT nsplit: 0 ntotq: 2 _EOF_ ####################################################################### # Link Boundary Datasets ####################################################################### #bin/ln -s /atmos/pegion/sst/ESMF/dataoceanfile_1990-now.360x180 ocean.data #/bin/ln -s /atmos/pegion/sst/ESMF/reynolds_v2_weekly_dataoceanfile.576x540 ocean.data /bin/ln -s /atmos/pegion/sst/ESMF/dataoceanfile_reynolds_v2_weekly.${OCEAN_IM}x${OCEAN_JM} ocean.data /bin/ln -s /atmos/pegion/sst/ESMF/single_variable/${OCEAN_IM}x${OCEAN_JM}/dataoceanfile_reynolds_v2_weekly_sstsi.${OCEAN_IM}x${OCEAN_JM} sstsi.data /bin/ln -s /atmos/pegion/sst/ESMF/single_variable/${OCEAN_IM}x${OCEAN_JM}/dataoceanfile_reynolds_v2_weekly_fraci.${OCEAN_IM}x${OCEAN_JM} fraci.data cp ocean.data seaice.data /bin/ln -s /atmos/pegion/sst/ESMF/single_variable/360x180/dataoceanfile_reynolds_v2_sst_clim.360x180 . /bin/ln -s $HOME/PE_${IM}x${JM}_${DATELINE}_${OCEAN_IM}x${OCEAN_JM}_DE.til tile.data /bin/ln -s tile.data TILING_FILE /bin/ln -s $GRIDDIR/gridfileesma.ZDF . ####################################################################### # Get RESTARTS and Executable ####################################################################### set rst_types = `cat GCM.rc |grep -v "#" | grep "RESTART_FILE" | cut -d ":" -f1 | cut -d "_" -f1-2` set chk_types = `cat GCM.rc |grep -v "#" | grep "CHECKPOINT_FILE" | cut -d ":" -f1 | cut -d "_" -f1-2` set rst_files = `cat GCM.rc | grep -v "#"| grep "RESTART_FILE" | cut -d ":" -f2` set chk_files = `cat GCM.rc | grep -v "#"| grep "CHECKPOINT_FILE" | cut -d ":" -f2` foreach rst ( $rst_files ) if( -e $EXPDIR/$rst ) /bin/cp $EXPDIR/$rst . & end wait /bin/cp $EXPDIR/layer_control . /bin/cp $EXPDIR/cap_restart . set epdate = e`cat cap_restart | cut -c1-8`_`cat cap_restart | cut -c10-11`z /bin/cp /ocean/bcs/576X540/*.ieee . /bin/cp $EXPDIR/*.ieee . ################################################################## ###### ###### Perform multiple iterations of PRUN ###### ################################################################## @ iterations = 1 @ counter = 1 while ( $counter <= $iterations ) ####################################################################### # Run the Model ####################################################################### @ NPES = $NX * $NY $RUN_CMD $NPES $APPLDIR/GEOSgcm.x set rc = $status echo Status = $rc # ALT: for now we do not resubmit exit ####################################################################### # Rename Checkpoints => Restarts for Next Segment ####################################################################### foreach rst ( $rst_files ) /bin/rm $rst & end wait set numrst = `echo $rst_types | wc -w` set numchk = `echo $chk_types | wc -w` @ n = 1 @ z = $numrst + 1 while ( $n <= $numchk ) @ m = 1 while ( $m <= $numrst ) if( $chk_types[$n] == $rst_types[$m] || \ \#$chk_types[$n] == $rst_types[$m] ) then /bin/mv $chk_files[$n] $rst_files[$m] & @ m = $numrst + 999 else @ m = $m + 1 endif end if( $m == $z ) then echo "Warning!! Could not find CHECKPOINT/RESTART match for: " $chk_types[$n] exit endif @ n = $n + 1 end wait #exit ####################################################################### # Archive History Output and Restarts # Note: cap_restart contains the current NYMD and NHMS ####################################################################### set edate = e`cat cap_restart | cut -c1-8`_`cat cap_restart | cut -c10-11`z # Archive RESTARTS # ---------------- foreach rst ( $rst_files ) if( -e $rst ) then cp $rst $EXPDIR/restarts/${rst}.${edate} cp $rst $EXPDIR/$rst endif end cp cap_restart $EXPDIR/cap_restart cp cap_restart $EXPDIR/restarts/cap_restart.${edate} # Archive HISTORY mv geosgcm_ocean $EXPDIR/history/geosgcm_ocean.$edate cp $EXPID.geosgcm_ocean.ctl $EXPDIR/history # Resubmit the script (no counter) foreach rst ( $rst_files ) if(! -e $EXPDIR/restarts/${rst}.${edate} ) exit if(! -e $EXPDIR/${rst} ) exit end wait if(! -e $EXPDIR/restarts/cap_restart.${edate} ) exit if(! -e $EXPDIR/cap_restart) exit if(! -e $EXPDIR/history/geosgcm_ocean.${edate} ) exit if(! -e $EXPDIR/history/$EXPID.geosgcm_ocean.ctl) exit #if( $edate == '20050301') then # echo 'Series completed' # exit #endif if( $edate == $epdate ) then echo 'Work stopped' exit endif echo $epdate 'Updated ' echo $edate 'Created' #cd /u1/wly0m/gmao/exp001 #bsub < run_script_ocn2_1201 exit if( $ANALYSIS != 0 ) then /bin/mv $EXPDIR/restarts/moist_internal_restart.$edate $EXPDIR/restarts/moist_internal_restart.bkg.$edate /bin/mv $EXPDIR/restarts/fvcore_internal_restart.$edate $EXPDIR/restarts/fvcore_internal_restart.bkg.$edate endif ####################################################################### # Check for Monthly Processing ####################################################################### if( $num != 0 ) then #ssh halem3 /home1/ltakacs/aries/diag_eta2prs.script $MASTOR/do_monthly \ # $EXPDIR \ # /gmao/model/$LOGNAME & set FILE = /home1/$LOGNAME/diag_eta2prs.${EXPID}_${edate}.batch /bin/rm $FILE cat << _EOF_ > $FILE #!/bin/csh #PBS -l walltime=12:00:00 #PBS -l mem=2gb #PBS -l ncpus=4 #PBS -o /scr1/jimpf/$LOGNAME/diag_eta2prs.${EXPID}_${edate}.out #PBS -j oe #PBS -W group_list=g2546 # /home1/ltakacs/aries/diag_eta2prs.script $MASTOR/do_monthly $EXPDIR /scr2/jimpf/$LOGNAME exit _EOF_ ssh halem-cron << _EOF at now + 1 min <