#!/bin/csh
#
#  last update = 2 February 2009
#
#  This is a C-shell script to execute GAMESS, by typing
#       rungms JOB VERNO NCPUS >& JOB.log &
#  JOB    is the name of the 'JOB.inp' file to be executed,
#  VERNO  is the number of the executable you chose at 'lked' time,
#  NCPUS  is the number of processors to be used.
#
#  Unfortunately execution is harder to standardize than compiling,
#  so you have to do a bit more than name your machine type here:
#    a) choose the target for execution from: 
#
#       Select "sockets", or perhaps one of the following special cases:
#           compaq-sc, cray-t3e, cray-x1, cray-xt3, ibm64-sp, sgi64
#
#       Select "sockets" if your compile time target was any of these:
#           axp64, cray-pvp, cray-xd1, fuji-pp, hpux32, hpux64,
#           ibm32, ibm64, linux32, linux64, linux-ia64, mac32, macG5,
#           necsx, sgi32, sun32, sun64
#       as all of these systems use TCP/IP sockets.  Do not name your
#       machine, instead, type "sockets".
#
#       Select "mpi" only if you have deliberately chosen to do the hard
#       work to use MPI for some very special reason.  Your hard work
#       is not yet over, as the mpirun below is unlikely to be correct
#       for your system.
#
#           Search on the words typed in capital letters just below
#           in order to find the right place to select each one:
#    b) choose a directory SCR where large temporary files can reside.
#       This should be the fastest possible disk access, very spacious,
#       and almost certainly a local disk.
#       Translation: do not put these files on a slow network file system!
#    c) change the the VERNO default to the version number you chose when
#       running "lked" as the VERNO default, and maybe NCPUS' default.
#    d) perhaps change the location of the PUNCH, TRAJECT, MAKEFP, and
#       RESTART files below.  These, along with the standard output, should
#       probably be routed to a central file server (using a network file
#       system to place these in user's normal storage is quite OK).
#    e) make sure that the ERICFMT file name and MCPPATH pathname point to
#       your file server's GAMESS tree, so that all runs can find them.
#       Again, a network file system is quite OK for these two.
#    f) customize the execution section for your target below,
#       each has its own list of further requirements.
#    g) it is unwise to have every user take a copy of this script, as you
#       can *NEVER* update all the copies later on.  Instead, it is better
#       to ask other users to create an alias to point to a common script,
#       such as this in their C-shell .login file,
#             alias gms '/u1/mike/gamess/rungms'
#    h) it is entirely possible to make 'rungms' run in a batch queue,
#       be it PBS, DQS, et cetera.  This is so installation dependent
#       that we leave it to up to you, although we give examples.
#       See ~/gamess/misc, where there are two examples using this
#       file as the backend: one for Sun Grid Engine (sge-gms), and
#       one for LoadLeveler "ll-gms".
#
set TARGET=sockets
set SCR=./
set GMSPATH=./
# location where the .dat and .trj files go
set USERSCR=./
#
set JOB=$1      # name of the input file xxx.inp, give only the xxx part
set VERNO=$2    # revision number of the executable created by 'lked' step
set NCPUS=$3    # number of compute processes to be run

# provide defaults if last two arguments are not given to this script
if (null$VERNO == null) set VERNO=00
if (null$NCPUS == null) set NCPUS=1
#
#  ---- the top third of the script is input and other file assignments ----
#
echo ----- GAMESS execution script -----
set master=`hostname`
echo This job is running on host $master
echo under operating system `uname` at `date`
echo "Available scratch disk space (Kbyte units) at beginning of the job is"
df -k $SCR

#        this added as experiment, February 2007
#        its intent is to detect large arrays allocated off the stack
limit stacksize 8192

#  Grab a copy of the input file.
#  In the case of EXAMnn jobs, this file might be in the "tests" subdirectory.
if ($JOB:r.inp == $JOB) set JOB=$JOB:r      # strip off possible .inp
if (-e $JOB.inp) then
   set echo
   cp  $JOB.inp  $SCR/$JOB.F05
   unset echo
else
   if (-e tests/$JOB.inp) then
      set echo
      cp  tests/$JOB.inp  $SCR/$JOB.F05
      unset echo
   else
      echo "Input file $JOB.inp does not exist."
      echo "This job expected the input file to be in directory `pwd`"
      echo "Please fix your file name problem, and resubmit."
      exit 4
   endif
endif

#  file assignments.
#
#  All binary files should be put on a node's local disk ($SCR directory), 
#  for the highest speed access possible.  These .Fxx files are typically
#  not saved for the next run, but they may be big and/or I/O intensive.
#  
#  It is convenient to write ASCII output files (PUNCH, RESTART, TRAJECT,
#  and MAKEFP) to the user's permanent disk, on your file server.  They
#  are small, written only by the master process, and are useful outputs
#  for further runs.
#
#  Some data files may be read by a run, each is read only once, so
#  that storage of one (1) copy on your file server is appropriate.
#  a. The ERICFMT file is the Fm(t) data for ERI computations and is provided
#     with GAMESS.  It is essential for correct functioning of 2e- integrals.
#  b. The MCPPATH is just a path name, the program appends the specific
#     file names inside that path before opening.  The model core potentials
#     and basis set files come as part of GAMESS's normal distribution tree.
#  c. The EXTBAS file is user-supplied basis sets.
#  d. see NEO plug-in code's documentation regarding the NUCBAS file.
#         Note that you must edit a+b, but will probably skip c+d.
#
set echo
#                        ASCII input files (see explanation above)
setenv ERICFMT $GMSPATH/ericfmt.dat
setenv MCPPATH $GMSPATH/mcpdata
setenv  EXTBAS /dev/null
setenv  NUCBAS /dev/null
#
setenv  MAKEFP $USERSCR/$JOB.efp
setenv   GAMMA $USERSCR/$JOB.gamma
setenv TRAJECT $USERSCR/$JOB.trj
setenv RESTART $USERSCR/$JOB.rst
setenv   INPUT $SCR/$JOB.F05
setenv   PUNCH $USERSCR/$JOB.dat
setenv  AOINTS $SCR/$JOB.F08
setenv  MOINTS $SCR/$JOB.F09
setenv DICTNRY $SCR/$JOB.F10
setenv DRTFILE $SCR/$JOB.F11
setenv CIVECTR $SCR/$JOB.F12
setenv CASINTS $SCR/$JOB.F13
setenv  CIINTS $SCR/$JOB.F14
setenv  WORK15 $SCR/$JOB.F15
setenv  WORK16 $SCR/$JOB.F16
setenv CSFSAVE $SCR/$JOB.F17
setenv FOCKDER $SCR/$JOB.F18
setenv  WORK19 $SCR/$JOB.F19
setenv  DASORT $SCR/$JOB.F20
setenv DFTINTS $SCR/$JOB.F21
setenv DFTGRID $SCR/$JOB.F22
setenv  JKFILE $SCR/$JOB.F23
setenv  ORDINT $SCR/$JOB.F24
setenv  EFPIND $SCR/$JOB.F25
setenv PCMDATA $SCR/$JOB.F26                                     
setenv PCMINTS $SCR/$JOB.F27
setenv SVPWRK1 $SCR/$JOB.F26
setenv SVPWRK2 $SCR/$JOB.F27
setenv  MLTPL  $SCR/$JOB.F28
setenv  MLTPLT $SCR/$JOB.F29
setenv  DAFL30 $SCR/$JOB.F30
setenv  SOINTX $SCR/$JOB.F31
setenv  SOINTY $SCR/$JOB.F32
setenv  SOINTZ $SCR/$JOB.F33
setenv  SORESC $SCR/$JOB.F34
#   35 is used by RESTART, see above
setenv GCILIST $SCR/$JOB.F37
setenv HESSIAN $SCR/$JOB.F38
setenv QMMMTEI $SCR/$JOB.F39
setenv SOCCDAT $SCR/$JOB.F40
setenv  AABB41 $SCR/$JOB.F41
setenv  BBAA42 $SCR/$JOB.F42
setenv  BBBB43 $SCR/$JOB.F43
setenv  MCQD50 $SCR/$JOB.F50
setenv  MCQD51 $SCR/$JOB.F51
setenv  MCQD52 $SCR/$JOB.F52
setenv  MCQD53 $SCR/$JOB.F53
setenv  MCQD54 $SCR/$JOB.F54
setenv  MCQD55 $SCR/$JOB.F55
setenv  MCQD56 $SCR/$JOB.F56
setenv  MCQD57 $SCR/$JOB.F57
setenv  MCQD58 $SCR/$JOB.F58
setenv  MCQD59 $SCR/$JOB.F59
setenv  MCQD60 $SCR/$JOB.F60
setenv  MCQD61 $SCR/$JOB.F61
setenv  MCQD62 $SCR/$JOB.F62
setenv  MCQD63 $SCR/$JOB.F63
setenv  MCQD64 $SCR/$JOB.F64
setenv NMRINT1 $SCR/$JOB.F61
setenv NMRINT2 $SCR/$JOB.F62
setenv NMRINT3 $SCR/$JOB.F63
setenv NMRINT4 $SCR/$JOB.F64
setenv NMRINT5 $SCR/$JOB.F65
setenv NMRINT6 $SCR/$JOB.F66
setenv DCPHFH2 $SCR/$JOB.F67
setenv DCPHF21 $SCR/$JOB.F68
setenv ELNUINT $SCR/$JOB.F67
setenv NUNUINT $SCR/$JOB.F68
setenv   GVVPT $SCR/$JOB.F69
setenv NUMOIN  $SCR/$JOB.F69
setenv NUMOCAS $SCR/$JOB.F70
setenv NUELMO  $SCR/$JOB.F71
setenv NUELCAS $SCR/$JOB.F72

#    Next files are for GMCQDPT
setenv GMCREF $SCR/$JOB.F70
setenv GMCO2R $SCR/$JOB.F71
setenv GMCROC $SCR/$JOB.F72
setenv GMCOOC $SCR/$JOB.F73
setenv GMCCC0 $SCR/$JOB.F74
setenv GMCHMA $SCR/$JOB.F75
setenv GMCEI1 $SCR/$JOB.F76
setenv GMCEI2 $SCR/$JOB.F77
setenv GMCEOB $SCR/$JOB.F78
setenv GMCEDT $SCR/$JOB.F79
setenv GMCERF $SCR/$JOB.F80
setenv GMCHCR $SCR/$JOB.F81
setenv GMCGJK $SCR/$JOB.F82
setenv GMCGAI $SCR/$JOB.F83
setenv GMCGEO $SCR/$JOB.F84
setenv GMCTE1 $SCR/$JOB.F85
setenv GMCTE2 $SCR/$JOB.F86
setenv GMCHEF $SCR/$JOB.F87
setenv GMCMOL $SCR/$JOB.F88
setenv GMCMOS $SCR/$JOB.F89
setenv GMCWGT $SCR/$JOB.F90
setenv GMCRM2 $SCR/$JOB.F91
setenv GMCRM1 $SCR/$JOB.F92
setenv GMCR00 $SCR/$JOB.F93
setenv GMCRP1 $SCR/$JOB.F94
setenv GMCRP2 $SCR/$JOB.F95
setenv GMCVEF $SCR/$JOB.F96
setenv GMCDIN $SCR/$JOB.F97
setenv GMC2SZ $SCR/$JOB.F98
setenv GMCCCS $SCR/$JOB.F99

#    Next files are used only during closed shell coupled cluster runs.
#    Display the numerous definitions iff they are going to be used.
unset echo
set cctyp=`grep -i CCTYP= $SCR/$JOB.F05 | wc -l`
if ($cctyp > 0) set echo
setenv  CCREST $SCR/$JOB.F70
setenv  CCDIIS $SCR/$JOB.F71
setenv  CCINTS $SCR/$JOB.F72
setenv CCT1AMP $SCR/$JOB.F73
setenv CCT2AMP $SCR/$JOB.F74
setenv CCT3AMP $SCR/$JOB.F75
setenv    CCVM $SCR/$JOB.F76
setenv    CCVE $SCR/$JOB.F77
setenv CCQUADS $SCR/$JOB.F78
setenv QUADSVO $SCR/$JOB.F79
setenv EOMSTAR $SCR/$JOB.F80
setenv EOMVEC1 $SCR/$JOB.F81
setenv EOMVEC2 $SCR/$JOB.F82
setenv  EOMHC1 $SCR/$JOB.F83
setenv  EOMHC2 $SCR/$JOB.F84
setenv EOMHHHH $SCR/$JOB.F85
setenv EOMPPPP $SCR/$JOB.F86
setenv EOMRAMP $SCR/$JOB.F87
setenv EOMRTMP $SCR/$JOB.F88
setenv EOMDG12 $SCR/$JOB.F89
setenv    MMPP $SCR/$JOB.F90
setenv   MMHPP $SCR/$JOB.F91
setenv MMCIVEC $SCR/$JOB.F92
setenv MMCIVC1 $SCR/$JOB.F93
setenv MMCIITR $SCR/$JOB.F94
setenv  EOMVL1 $SCR/$JOB.F95
setenv  EOMVL2 $SCR/$JOB.F96
setenv EOMLVEC $SCR/$JOB.F97
setenv  EOMHL1 $SCR/$JOB.F98
setenv  EOMHL2 $SCR/$JOB.F99
setenv  CCVVVV $SCR/$JOB.F80
#
#    Next files are used only during open shell coupled cluster runs.
#
setenv AMPROCC $SCR/$JOB.F70
setenv ITOPNCC $SCR/$JOB.F71
setenv FOCKMTX $SCR/$JOB.F72
setenv  LAMB23 $SCR/$JOB.F73
setenv   VHHAA $SCR/$JOB.F74
setenv   VHHBB $SCR/$JOB.F75
setenv   VHHAB $SCR/$JOB.F76
setenv    VMAA $SCR/$JOB.F77
setenv    VMBB $SCR/$JOB.F78
setenv    VMAB $SCR/$JOB.F79
setenv    VMBA $SCR/$JOB.F80
setenv  VHPRAA $SCR/$JOB.F81
setenv  VHPRBB $SCR/$JOB.F82
setenv  VHPRAB $SCR/$JOB.F83
setenv  VHPLAA $SCR/$JOB.F84
setenv  VHPLBB $SCR/$JOB.F85
setenv  VHPLAB $SCR/$JOB.F86
setenv  VHPLBA $SCR/$JOB.F87
setenv    VEAA $SCR/$JOB.F88
setenv    VEBB $SCR/$JOB.F89
setenv    VEAB $SCR/$JOB.F90
setenv    VEBA $SCR/$JOB.F91
setenv   VPPPP $SCR/$JOB.F92
setenv INTERM1 $SCR/$JOB.F93
setenv INTERM2 $SCR/$JOB.F94
setenv INTERM3 $SCR/$JOB.F95
#
#    Next files are used only during elongation method runs.
#    Display the numerous definitions iff they are going to be used.
unset echo
set elgtyp = `grep -i NELONG= $SCR/$JOB.F05 | wc -l`
if ($elgtyp > 0) then
    set ELGNAME=$4
    if (null$4 == null) set ELGNAME=ELGFILE
    set echo
    setenv AOINTS   $SCR/$ELGNAME.F08
    setenv ELGDOS   ~$USER/scr/$JOB.ldos
    setenv ELGDAT   $SCR/$ELGNAME.F71
    setenv ELGPAR   $SCR/$ELGNAME.F72
    setenv ELGCUT   $SCR/$ELGNAME.F74
    setenv ELGVEC   $SCR/$ELGNAME.F75
    setenv EGINTA   $SCR/$ELGNAME.F77
    setenv EGINTB   $SCR/$ELGNAME.F78
    setenv EGTDHF   $SCR/$ELGNAME.F79
    setenv EGTEST   $SCR/$ELGNAME.F80
    unset echo
endif
#
#    Next files are used only during extended TDHF package runs.
#    Display the numerous definitions iff they are going to be used.
unset echo
set txtyp=`grep -i RUNTYP=TDHFX $SCR/$JOB.F05 | wc -l`
if ($txtyp > 0) set echo
setenv  OLI201 $SCR/$JOB.F201
setenv  OLI202 $SCR/$JOB.F202
setenv  OLI203 $SCR/$JOB.F203
setenv  OLI204 $SCR/$JOB.F204
setenv  OLI205 $SCR/$JOB.F205
setenv  OLI206 $SCR/$JOB.F206
setenv  OLI207 $SCR/$JOB.F207
setenv  OLI208 $SCR/$JOB.F208
setenv  OLI209 $SCR/$JOB.F209
setenv  OLI210 $SCR/$JOB.F210
setenv  OLI211 $SCR/$JOB.F211
setenv  OLI212 $SCR/$JOB.F212
setenv  OLI213 $SCR/$JOB.F213
setenv  OLI214 $SCR/$JOB.F214
setenv  OLI215 $SCR/$JOB.F215
setenv  OLI216 $SCR/$JOB.F216
setenv  OLI217 $SCR/$JOB.F217
setenv  OLI218 $SCR/$JOB.F218
setenv  OLI219 $SCR/$JOB.F219
setenv  OLI220 $SCR/$JOB.F220
setenv  OLI221 $SCR/$JOB.F221
setenv  OLI222 $SCR/$JOB.F222
setenv  OLI223 $SCR/$JOB.F223
setenv  OLI224 $SCR/$JOB.F224
setenv  OLI225 $SCR/$JOB.F225
setenv  OLI226 $SCR/$JOB.F226
setenv  OLI227 $SCR/$JOB.F227
setenv  OLI228 $SCR/$JOB.F228
setenv  OLI229 $SCR/$JOB.F229
setenv  OLI230 $SCR/$JOB.F230
setenv  OLI231 $SCR/$JOB.F231
setenv  OLI232 $SCR/$JOB.F232
setenv  OLI233 $SCR/$JOB.F233
setenv  OLI234 $SCR/$JOB.F234
setenv  OLI235 $SCR/$JOB.F235
setenv  OLI236 $SCR/$JOB.F236
setenv  OLI237 $SCR/$JOB.F237
setenv  OLI238 $SCR/$JOB.F238
setenv  OLI239 $SCR/$JOB.F239
unset echo

#    Next files are used only during divide-and-conquer runs
setenv   DCSUB $SCR/$JOB.F250
setenv   DCVEC $SCR/$JOB.F251
setenv   DCEIG $SCR/$JOB.F252
setenv   DCDM  $SCR/$JOB.F253
setenv   DCDMO $SCR/$JOB.F254
setenv   DCQ   $SCR/$JOB.F255
setenv   DCW   $SCR/$JOB.F256
setenv   DCEDM $SCR/$JOB.F257

#    Next files are used only during LMO hyperpolarizability analysis
setenv LHYPWRK $SCR/$JOB.F297
setenv LHYPWK2 $SCR/$JOB.F298
setenv BONDDPF $SCR/$JOB.F299

#    Next value is used only within the VB2000 add-on code
setenv GMSJOBNAME $JOB

#
#    "group DDI" should reset to true, below, if $GDDI input is found
set GDDIjob=false

#    data left over from a previous run might be precious, stop if found.
if ((-e $PUNCH) || (-e $MAKEFP) || (-e $TRAJECT) || (-e $RESTART) ) then
   echo "Please save, rename, or erase these files from a previous run:"
   echo "     $PUNCH,"
   echo "     $TRAJECT,"
   echo "     $RESTART, and/or"
   echo "     $MAKEFP,"
   echo "and then resubmit this computation."
   exit 4
endif

#  ---- the middle third of the script is to execute GAMESS ----
#
#   Most workstations run DDI over TCP/IP sockets, and therefore execute
#   according to the following clause.  The installer must
#      a) Set the path to point to the DDIKICK and GAMESS executables.
#      b) Build the HOSTLIST variable as a word separated string, i.e. ()'s.
#         There should be one host name for every compute process that is
#         to be run.  DDIKICK will automatically generate a set of data
#         server processes (if required) on the same hosts.
#   An extended explanation of the arguments to ddikick.x can be found
#   in the file gamess/ddi/readme.ddi, if you have any trouble executing.
#
if ($TARGET == sockets) then
#
#        set the path pointing to GAMESS and DDIKICK binaries
#     At Iowa State, we have many operating systems, and store files
#     in different partitions according to which system is being used.
#
#     One such partition is fed to all nodes of the same type and
#     operating system, e.g. the /cu directory below feeds all of
#     our 32 bit PCs running RedHat Fedora Core 1, by means of NFS.
#
#     Note that 'uname' is not a terribly specific way to choose
#     a list of all machines running a specific O/S release, so we
#     use the host names for finer control, for example, we have 
#     three different Solaris releases on two different chip types.
#
   set os=`uname`
#   if ($os == AIX)     set GMSPATH=/u1/mike/gamess
#   if ($os == Darwin)  set GMSPATH=/Users/mike/desktop/gamess
#   if ($os == HP-UX)   set GMSPATH=/zr/mike/gamess
#   if ($os == Linux)   set GMSPATH=/cu/mike/gamess
#   if ($os == OSF1)    set GMSPATH=/in/mike/gamess
#   if ($os == SunOS)   set GMSPATH=/hf/mike/gamess
#       special compilation for Sun E450 uSPARC       (uname also= SunOS)
#   if (`hostname` == sc.fi.ameslab.gov) set GMSPATH=/sc/mike/gamess
#       special compilation for Sun V40Z Opteron      (uname also= SunOS)
#   if (`hostname` == as.fi.ameslab.gov) set GMSPATH=/as/mike/gamess
#       special compilation for SGI Altix 450         (uname also= Linux)
#   if (`hostname` == br.fi.ameslab.gov) set GMSPATH=/br/mike/gamess
#       special compilation for SGI XE210             (uname also= Linux)
#   if (`hostname` == se.fi.ameslab.gov) set GMSPATH=/se/mike/gamess
#   if (`hostname` == sb.fi.ameslab.gov) set GMSPATH=/se/mike/gamess
#       special compilation for IBM OpenPower 720     (uname also= Linux)
#   if (`hostname` == ga.fi.ameslab.gov) set GMSPATH=/ge/mike/gamess
#   if (`hostname` == ge.fi.ameslab.gov) set GMSPATH=/ge/mike/gamess

#         it is unlikely that you would need to change DDI_VER from 'new'!
#         some antique system lacking pthreads, for example, might have
#         to use the old DDI code, so we keep an execution example below.
   set DDI_VER='new'
   if (`hostname` == antique.msg.chem.iastate.edu) set DDI_VER='old'

#      -- some special settings for certain operating systems --

#         IBM's AIX needs special setting if node is more than a 4-way SMP
   if ($os == AIX) setenv EXTSHM ON
#         Fedora Core 1 can't run DDI processes w/o placing a finite
#         but large limit on the stack size (2**27 bytes seems OK)
   if ($os == Linux) limit stacksize 131072
#         In case this Linux system is using Intel's Math Kernel Library
#         to obtain its BLAS, we insist each process runs single-threaded.
#         This variable worked up to MKL 9, MKL 10 on up should link serial.
   if ($os == Linux) setenv MKL_SERIAL YES

#
#       Five examples of how to build the HOSTLIST are shown....
#           terminology: CPU= processor, NODE= physical enclosure (box)
#
#       1. Sequential execution is sure to be on this very same host
#   if ($NCPUS == 1) then
#      set NNODES=1
#      set HOSTLIST=(`hostname`)
#   endif
#
#   The following will always run as is, giving you NCPUS number of 
#   processes on the launching computer. Activate the 4 lines above
#   and setup appropriate lines below this section with this section
#   commented out to run in paralle on multiple nodes.
# 
   set HOSTLIST=(localhost)
   @ n=2
   while ($n <= $NCPUS)
      set HOSTLIST=($HOSTLIST localhost)
      @ n++
   end
   @ n--
   set NNODES=$n
#
#       2. This is an example of how to run on a 4-way SMP enclosure,
#          where all CPUs are inside a single NODE.
   if ((`hostname` == ti.fi.ameslab.gov) && ($NCPUS > 1)) then
      set NNODES=1
      if ($NCPUS > 4) set NCPUS=4
      set HOSTLIST=(ti.fi.ameslab.gov:cpus=$NCPUS)
   endif
#          same example, just a different node, which is a 4-way SMP
   if ((`hostname` == sc.fi.ameslab.gov) && ($NCPUS > 1)) then
      set NNODES=1
      if ($NCPUS > 4) set NCPUS=4
      set HOSTLIST=(sc.fi.ameslab.gov:cpus=$NCPUS)
   endif
#          same example, my dual processor G5 Macintosh
   if ((`hostname` == he.fi.ameslab.gov) && ($NCPUS > 1)) then
      set NNODES=1
      if ($NCPUS > 2) set NCPUS=2
      set HOSTLIST=(he.fi.ameslab.gov:cpus=$NCPUS)
   endif
#          same example, a 8-core Altix we use to run four p=2 queues
   if ((`hostname` == br.fi.ameslab.gov) && ($NCPUS > 1)) then
      set NNODES=1
      if ($NCPUS > 2) set NCPUS=2
      set HOSTLIST=(br.fi.ameslab.gov:cpus=$NCPUS)
   endif
#          same example, for our pair of dual socket Woodcrest SGI blades,
#                        we default to running in parallel, so that each
#                        batch queue has double the (small) memory.
   if ((`hostname` == se.fi.ameslab.gov) && ($NCPUS > 1)) then
      set NNODES=1
      if ($NCPUS > 2) set NCPUS=2
      set HOSTLIST=(se.fi.ameslab.gov:cpus=$NCPUS)
   endif
   if ((`hostname` == sb.fi.ameslab.gov) && ($NCPUS > 1)) then
      set NNODES=1
      if ($NCPUS > 2) set NCPUS=2
      set HOSTLIST=(sb.fi.ameslab.gov:cpus=$NCPUS)
   endif
#
#       3. A phony example, of six uniprocessors (arbitrary names)
#          Because they are uniprocessors, we just set NNODES = NCPUS.
#          Since their names never change, we just can just specify them.
#          Note that we can use a short name like 'bb' if and only if 
#          system name resolution can map them onto the true host names.
   if (`hostname` == aa.msg.chem.iastate.edu) then
      set NNODES=$NCPUS
      set HOSTLIST=(aa bb cc dd ee ff)
   endif
#
#       4. An example of 16 uniprocessor boxes in a Beowulf-type cluster.
#          Because they are uniprocessors, we just set NNODES = NCPUS.
#          Their host names fall into the pattern fly1 to fly16,
#          which we can turn into a HOSTLIST with a small loop.
   if (`hostname` == fly1.fi.ameslab.gov) then
      set NNODES=$NCPUS
      set HOSTLIST=()
      set nmax=$NCPUS
      if ($nmax > 16) set nmax=16
      @ CPU=1
      while ($CPU <= $nmax)
         set HOSTLIST=($HOSTLIST fly$CPU)
         @ CPU++
      end
      unset $CPU
   endif
#
#       5. Scalable Computing Lab's clusters running PBS batch queues.
#          Here the task is to manipulate the dynamically assigned host
#          names into the HOSTLIST string for the kickoff program,
#          and to request the host name of the fast network adapters.
#
    if ($?PBS_JOBID) then
#
#         The IBM cluster has two Gigabit adapters in each 4-way SMP,
#         while the AXP cluster is based on a Myrinet network.
      if (`uname` == AIX)   set NETEXT=".gig,.gig2"
      if (`uname` == Linux) set NETEXT=".myri"
#
#         repeated host names in the PBS host file indicate being assigned
#         CPUs in the same SMP enclosure, which we must count up correctly.
#         Fortunately PBS gives duplicate host names in a row, not scrambled.
#         The number of hosts in the PBS node file (nmax) should equal the
#         requested processor count, NCPUS.  We need to count duplicates
#         in order to learn the number of SMP enclosures, NNODES, and how
#         many CPUs inside each SMP were assigned (NSMPCPU).  For example,
#         if we are assigned the host names "a a a b b c c c" we must build
#         the string "a:cpus=3 b:cpus=2 c:cpus=3" so that ddikick.x will
#         know the SMP structure of the assigned node names.  (C-shell handles
#         variable substitution followed by colon gracefully by ${HOST}:cpus.)
#
      set HOSTLIST=()
      set nmax=`wc -l $PBS_NODEFILE`
      set nmax=$nmax[1]
      if ($nmax != $NCPUS) then
         echo There is processor count confusion
         exit
      endif
#            1st host in the list is sure to be a new SMP enclosure
      set MYNODE=`sed -n -e "1 p" $PBS_NODEFILE`
      set MYNODE=`echo $MYNODE | awk '{split($0,a,"."); print a[1]}'`
#            IPROC counts assigned processors (up to NCPUS),
#            NNODES counts number of SMP enclosures.
#            NSMPCPU counts processors in the current SMP enclosure
      @ IPROC = 2
      @ NNODES = 1
      @ NSMPCPU = 1
      set spacer1=":cpus="
      set spacer2=":netext="
      while($IPROC <= $nmax)
         set MYPROC=`sed -n -e "$IPROC p" $PBS_NODEFILE`
         set MYPROC=`echo $MYPROC | awk '{split($0,a,"."); print a[1]}'`
         if($MYPROC != $MYNODE) then
            set HOSTLIST = ($HOSTLIST $MYNODE$spacer1$NSMPCPU$spacer2$NETEXT)
            set MYNODE=$MYPROC
            @ NSMPCPU = 0
            @ NNODES++
         endif
         @ IPROC++
         @ NSMPCPU++
      end
      set HOSTLIST = ($HOSTLIST $MYNODE$spacer1$NSMPCPU$spacer2$NETEXT)
   endif
#
#        we have now finished setting up a correct HOSTLIST.
#        uncomment the next two if you are doing script debugging.
#--echo "The generated host list is"
#--echo $HOSTLIST
#

#
#    If a $GDDI input group is present the calculation will be using
#    subgroups within DDI.  The master within each group must have a
#    copy of INPUT, and will create separate PUNCH and OUTPUT files.
#    Note that this separates the OUTPUT from the normal log file, and
#    at the end we must take steps to save the files from at least the
#    master of the first group.  Skip if no $GDDI is found in the input.
#          Note that some sites might prefer rcp over scp below.
#
   set ngddi=`grep -i '^ \$GDDI' $SCR/$JOB.F05 | grep -iv 'NGROUP=1 ' | wc -l`
   if ($ngddi > 0) then
      set GDDIjob=true
      echo "This is a GDDI run, keeping output files on local disks"
      echo "until the very end of the run, when they'll be saved from"
      echo "the master process in the first group, only."
      set echo
      setenv  MAKEFP $SCR/$JOB.F01
      setenv TRAJECT $SCR/$JOB.F04
      setenv  OUTPUT $SCR/$JOB.F06
      setenv   PUNCH $SCR/$JOB.F07
      setenv RESTART $SCR/$JOB.F35
      unset echo
      @ n=2   # master in master group already did 'cp' above
      while ($n <= $NNODES)
         set host=$HOSTLIST[$n]
         set host=`echo $host | cut -f 1 -d :` # drop anything behind a colon
         echo scp $SCR/$JOB.F05 ${host}:$SCR/$JOB.F05
         scp $SCR/$JOB.F05 ${host}:$SCR/$JOB.F05
         @ n++
      end
   else
      set GDDIjob=false
   endif
#
#        Just make sure we have the binaries, before we try to run
#
   if ((-x $GMSPATH/gamess.$VERNO.x) && (-x $GMSPATH/ddikick.x)) then
   else
      echo The GAMESS executable gamess.$VERNO.x or else
      echo the DDIKICK executable ddikick.x
      echo could not be found in directory $GMSPATH,
      echo or else they did not properly link to executable permission.
      exit 8
   endif
#
#        choose remote shell execution program.
#    Parallel run do initial launch of GAMESS on remote nodes by the
#    following program.  Note that the authentication keys for ssh
#    must have been set up correctly.
#    It is possible to select 'rsh' using .rhosts authentication.
   setenv DDI_RSH ssh
#
#        OK, now we are ready to execute!
#    The kickoff program initiates GAMESS process(es) on all CPUs/nodes.
#
   if ($DDI_VER == new) then
      set echo
      $GMSPATH/ddikick.x $GMSPATH/gamess.$VERNO.x $JOB \
          -ddi $NNODES $NCPUS $HOSTLIST \
          -scr $SCR < /dev/null
      unset echo
   else
      set path=($GMSPATH $path)
      set echo
      ddikick.x $JOB $GMSPATH gamess.$VERNO.x $SCR $NCPUS $HOSTLIST < /dev/null
      unset echo
   endif
endif

#      ------ end of the TCP/IP socket execution sections -------

#   Compaq Supercluster running SHMEM wants you to
#      a) set the path to point to the GAMESS executable
if ($TARGET == compaq-sc) then
   set GMSPATH=/u1/mike/gamess
   chdir $SCR
   set echo
   prun -n $NNODES $GMSPATH/gamess.$VERNO.x $JOB
   unset echo
endif
 
#   Cray T3E running SHMEM wants you to
#      a) set the path to point to the GAMESS executable
if ($TARGET == cray-t3e) then
   set GMSPATH=/u1/mike/gamess
   chdir $SCR
   set echo
   mpprun -n $NCPUS $GMSPATH/gamess.$VERNO.x $JOB
   unset echo
endif

#   Cray X1 running SHMEM wants you to
#      a) set the path to point to the GAMESS executable
if ($TARGET == cray-x1) then
   set GMSPATH=/u1/mike/gamess
   set OPTS="-m exclusive"
   if ($NCPUS > 16) then
      set PERNODE=16
   else
      set PERNODE=$NCPUS
   endif
   chdir $SCR
   set echo
   aprun -c core=0 -n $NCPUS -N $PERNODE $OPTS $GMSPATH/gamess.$VERNO.x $JOB
   unset echo
endif

#   SGI Origin running SHMEM wants you to
#      a) set the path to point to the GAMESS executable
#   We've heard that setting the environment variable
#       SMA_SYMMETRIC_SIZE to 2147483648 (2 GB)
#   may be helpful if you see DDI_SHPALLOC error messages.
#
if ($TARGET == sgi64) then
   set GMSPATH=/home/hbar4/people/schmidt/gamess
   chdir $SCR
   set echo
   mpirun -np $NCPUS $GMSPATH/gamess.$VERNO.x $JOB < /dev/null
   unset echo
endif

#   CRAY-XT3 running SHMEM/MPI wants you to
#      a) set the path to point to the GAMESS executable
#      b) review the memory settings below.
#   This implementation uses one-sided messaging, i.e. yod does not
#   start up data server processes, just compute processes.
#
#   Set the -shmem parameter to about 400MB smaller than the memory
#   available to each core to allow for MPI buffer space and other OS
#   buffers.  For example, if the XT3 node has 2GB,
#     single-core = 2000MB - 400MB = 1600MB
#     dual-core   = 2GB / 2 cores = 1000MB - 400MB = 600MB
#   You can sometimes drop to subtracting only 300MB per core,
#   depending on the job type.
#
#   The TPN variable below lets you use more memory, by wasting one
#   half of the processors, if that is needed to do your run.
#
#   If you get an error of the type 
#      Fatal error in MPI_Ssend: Invalid rank, error stack:
#   then you have run out of stack space.  Increase the -stack argument
#   from "-stack 24m" to "-stack 32m" or larger.  This usually only occurs
#   for very large jobs.
#
if ($TARGET == cray-xt3) then
   set GMSPATH=/u/boatzj/gamess
   set SMP_SIZE=2  # number of cpus/cores per node (XT3 = dual CPU nodes)
   set TPN=$4      # number of compute processes per node (TPN=tasks/node)
   if (null$TPN == null) set TPN=$SMP_SIZE
#
   if ($TPN == 1) then
      set CORTYP=-SN
      set SHMEM='-shmem 3600M'
   else
      set CORTYP=-VN
      set SHMEM='-shmem 1700M'
   endif
#
   setenv IOBUF_PARAMS '%stdout:ignoreflush'
   env | grep IOBUF
#
   chdir $SCR
   set echo
   yod -stack 24m $CORTYP $SHMEM -size $NCPUS $GMSPATH/gamess.$VERNO.x $JOB
   unset echo
endif


#   The IBM SP running DDI using mixed LAPI/MPI messaging wants you to
#      a) set the path to point to the GAMESS executable
#      b) define hosts in a host file, which are probably defined by
#         a batch queue system.  An example for LoadLeveler is given.
#   Please note that most IBM systems schedule their batch jobs with
#   the LoadLeveler software product.  Please see gamess/misc/llgms for
#   a "front-end" script that submits this script as a "back-end" job,
#   with all necessary LL accouterments inserted at the top of the job.
#
if ($TARGET == ibm64-sp) then
#
#     point this to where your GAMESS executable is located
   set path=($path /u1/mike/gamess)
#
#     error messages defaulted to American English, try C if lacking en_US
   setenv LOCPATH /usr/lib/nls/loc:/usr/vacpp/bin
   setenv LANG en_US
#
#     this value is picked up inside DDI, then used in a "chdir $SCR"
   setenv DDI_SCRATCH $SCR
#
#     define the name of a host name file.
#
   setenv HOSTFILE $SCR/$JOB.poehosts
   if (-e $HOSTFILE) rm -f $HOSTFILE
#
#        If the job was scheduled by LoadLeveler, let LL control everything.
#
   if ($?LOADLBATCH) then
#        just get POE to tell us what nodes we were dynamically assigned to.
      /usr/bin/poe hostname -stdoutmode ordered > $HOSTFILE
      set SMP_SIZE = $TPN
#
#        Otherwise, if this was not an LoadLeveler job, here's a hack!
#        It is unlikely this will match your SP's characteristics, as
#        we just guess its a 4-way node, 4 processors, run interactively.
#        It is here mainly to illustrate the sort of MP_XXX's you need.
   else
      set SMP_SIZE=4
      set NCPUS=4
      set NNODES=1
      echo `hostname`  > $HOSTFILE
      echo `hostname` >> $HOSTFILE
      echo `hostname` >> $HOSTFILE
      echo `hostname` >> $HOSTFILE

      echo "Variables controlling Parallel Environment process kickoff are"
      set echo
      setenv MP_NODES          $NNODES
      setenv MP_PROCS          $NCPUS
      setenv MP_HOSTFILE       $HOSTFILE
      setenv MP_CPU_USE        unique
      setenv MP_ADAPTER_USE    dedicated
#       GAMESS is implemented using both MPI and LAPI active messages.
      setenv MP_MSG_API        MPI,LAPI
      setenv MP_EUILIB         us
#       SP systems with one switch adapter might use css0, not striping csss
      setenv MP_EUIDEVICE      csss
      setenv MP_RESD           no
      unset echo
   endif

#       and now we are ready to execute, using poe to kick off the tasks.
   @ NNODES = ($NCPUS - 1) / $SMP_SIZE + 1
   echo "Running $NCPUS processes on $NNODES nodes ($SMP_SIZE-way SMP)."
   set echo
   /usr/bin/poe gamess.$VERNO.x $JOB -stdinmode none
   unset echo
endif

#                   - a very specific MPI example -
#
#         This section is customized to the Intel MPI library,
#         and also to the batch scheduler called Sun Grid Engine (SGE).
#         See ~/gamess/misc/sge-gms for a front-end script to submit
#         this back-end script as a batch job.
#
#                   if you are using some other MPI:
#         See ~/gamess/ddi/readme.ddi for information about launching
#         processes using other MPI libraries (every one is different).
#
#                   if you are using some other batch scheduler:
#         Illustrating other batch scheduler's way's of providing the
#         hostname list is considered beyond the scope of this script.
#         Suffice it to say that
#             a) you will be given hostnames at run time
#             b) a typical way is a disk file, named by an environment
#                variable, containing the names in some format.
#             c) another typical way is an blank separated list in some
#                environment variable.
#         Either way, whatever the batch schedular gives you must be
#         sliced-and-diced into the format required by your MPI kickoff.
#
if ($TARGET == mpi) then
   #
   #  Each node should have a local disk directory for work files.
   #  At the top of this script, set SCR equal to SGE's $TMPDIR!
   #                      set TARGET=mpi
   #                      set SCR=$TMPDIR
   #
   #      besides the usual three arguments to 'rungms' (see top),
   #      we'll pass in a "processers per node" value.  This will
   #      be a value from 1 to 8 on our 8-way nodes.
   set PPN=$4
   #
   #  Allow for compute process and data servers (one pair per core)
   #
   @ NPROCS = $NCPUS + $NCPUS
   #
   #  Intel's MPI is derived from Argonne's MPICH, so it shares the
   #  same kick-off procedures, guided by two disk files (A and B).
   #
   #  A. build HOSTFILE, saying which nodes will be in our MPI ring
   #
   setenv HOSTFILE $SCR/$JOB.nodes.mpd
   if (-e $HOSTFILE) rm $HOSTFILE
   touch $HOSTFILE
   #
   if ($NCPUS == 1) then
             # Serial run must be on this node itself!
      echo `hostname` >> $HOSTFILE
      set NNODES=1
   else
             # Parallel run gets node names from SGE's assigned list,
             # which is given to us in a disk file $TMPDIR/machines.
      uniq $TMPDIR/machines $HOSTFILE
      set NNODES=`wc -l $HOSTFILE`
      set NNODES=$NNODES[1]
   endif
   #           uncomment these if you are still setting up...
   #--echo '------------'
   #--echo HOSTFILE $HOSTFILE contains
   #--cat $HOSTFILE
   #--echo '------------'
   #
   #  B. the next file forces explicit "which process on what node" rules.
   #
   setenv PROCFILE $SCR/$JOB.processes.mpd
   if (-e $PROCFILE) rm $PROCFILE
   touch $PROCFILE
   #
   if ($NCPUS == 1) then
      @ NPROCS = 2
      echo "-n $NPROCS -host `hostname` /home/mike/gamess/gamess.$VERNO.x" >> $PROCFILE
   else
      @ NPROCS = $NCPUS + $NCPUS
      if ($PPN == 0) then
             # when our SGE is just asked to assign so many cores from one
             # node, PPN=0, we are launching compute processes and data
             # servers within our own node...simple.
         echo "-n $NPROCS -host `hostname` /home/mike/gamess/gamess.$VERNO.x" >> $PROCFILE
      else
             # when our SGE is asked to reserve entire nodes, 1<=PPN<=8,
             # the $TMPDIR/machines contains the assigned node names
             # once and only once.  We want PPN compute processes on
             # each node, and of course, PPN data servers on each.
             # Although DDI itself can assign c.p. and d.s. to the
             # hosts in any order, the GDDI logic below wants to have
             # all c.p. names before any d.s. names in the $HOSTFILE.
             #
             # thus, lay down a list of c.p.
         @ PPN2 = $PPN + $PPN
         @ n=1
         while ($n <= $NNODES)
            set host=`sed -n -e "$n p" $HOSTFILE`
            set host=$host[1]
            echo "-n $PPN2 -host $host /home/mike/gamess/gamess.$VERNO.x" >> $PROCFILE
            @ n++
         end
      endif
   endif
   #           uncomment these if you are still setting up...
   #--echo PROCFILE $PROCFILE contains
   #--cat $PROCFILE
   #--echo '------------'
   #
   echo "Intel MPI (iMPI) will be running GAMESS on $NNODES nodes."
   echo "The binary to be kicked off by 'mpiexec' is gamess.$VERNO.x"
   echo "iMPI will run $NCPUS compute processes and $NCPUS data servers."
   if ($PPN > 0) echo "iMPI will be running $PPN of each process per node."
   # 
   #  Next sets up MKL usage
   setenv LD_LIBRARY_PATH /opt/intel/mkl/10.0.3.020/lib/em64t
   #  force old MKL versions (version 9 and older) to run single threaded
   setenv MKL_SERIAL YES 
   #
   #  add Intel MPI to the library path and execution path
   setenv LD_LIBRARY_PATH /opt/intel/impi/3.2.1/lib64:$LD_LIBRARY_PATH
   set path=(/opt/intel/impi/3.2.1/bin64 $path)
   #
   #  Next sets up GDDI usage.
   #  If a $GDDI input group is present the calculation will be using
   #  subgroups within DDI.  The master within each group must have a
   #  copy of INPUT, and will create separate PUNCH and OUTPUT files.
   #  Note that this separates the OUTPUT from the normal log file, and
   #  at the end we must take steps to save the files from at least the
   #  master of the first group.  Skip if no $GDDI is found in the input.
   #
   set ngddi=`grep -i '^ \$GDDI' $SCR/$JOB.F05 | grep -iv 'NGROUP=0 ' | wc -l`
   if ($ngddi > 0) then
      set GDDIjob=true
      echo "This is a GDDI run, keeping output files on local disks"
      echo "until the very end of the run, when they'll be saved from"
      echo "the master process in the first group, only."
      set echo
      setenv  MAKEFP $SCR/$JOB.F01
      setenv TRAJECT $SCR/$JOB.F04
      setenv  OUTPUT $SCR/$JOB.F06
      setenv   PUNCH $SCR/$JOB.F07
      setenv RESTART $SCR/$JOB.F35 
      unset echo
#        copy input file to every node, a simple way to be sure of
#        including just the master nodes requested by $GDDI.
      set nmax=`wc -l $HOSTFILE`
      set nmax=$nmax[1]
      set lasthost=$master
      @ n=2   # input is already copied on the master node.
      while ($n <= $nmax)
         set host=`sed -n -e "$n p" $HOSTFILE`
         set host=$host[1]
         if ($host != $lasthost) then
            echo scp $SCR/$JOB.F05 ${host}:$SCR/$JOB.F05
            scp $SCR/$JOB.F05 ${host}:$SCR/$JOB.F05
            set lasthost=$host
         endif
         @ n++
      end
   else      
      set GDDIjob=false
   endif
   #     
   echo The scratch disk space on each node is $SCR
   chdir $SCR
   #
   #  Now, at last, we can actually launch the processes, in 3 steps.
   #  a) bring up a 'ring' of MPI demons
   #
   set echo
   mpdboot --rsh=ssh -n $NNODES -f $HOSTFILE
   #
   #  b) kick off the compute processes and the data servers
   #     trial and error shows process pinning slows down GAMESS runs,
   #     turn on the option to avoid polling for incoming messages 
   #        which allows us to compile DDI in pure "mpi" mode,
   #     set debug level to 2 or to 5 to see messages from kickoff stage,
   #     we need at least version 3.2 of iMPI to specify a DAPL 2.0 library.
   #
   setenv I_MPI_PIN disable
   setenv I_MPI_WAIT_MODE enable
   setenv I_MPI_DEBUG 0
   setenv I_MPI_DAT_LIBRARY libdat2.so
   mpiexec -configfile $PROCFILE < /dev/null
   #
   #  c) shut down the 'ring' of MPI demons
   #
   mpdallexit
   unset echo
   #
   #    HOSTFILE is passed to the file erasing step below
   rm -f $PROCFILE
   #           
endif

#   NEC SX Series wants you to
#      a) set the path variable to point to the GAMESS executable
#   this isn't correct, it uses MPI, but need to save the F_ stuff???
#
if ($TARGET == necsx) then
   set GMSPATH=/u1/mike/gamess

   chdir $SCR
   setenv F_RECLUNIT BYTE
   setenv F_ABORT YES
   setenv F_ERROPT1 252,252,2,2,1,1,2,1
   setenv F_PROGINF detail
   setenv F_SETBUF 4096
   echo Running $NCPUS compute processes and $NCPUS data server processes...
   @ NPROCS = $NCPUS + $NCPUS
   set echo
   #--mpirun -np $NPROCS $GMSPATH/gamess.$VERNO.x $JOB < /dev/null
   unset echo
endif

#   in the case of GDDI runs, we save the first group master's files only.
if ($GDDIjob == true) then
   cat $OUTPUT
   cp $SCR/$JOB.F06.* ~/scr
                        cp $SCR/$JOB.F07 ~/scr/$JOB.dat
   if(-e $SCR/$JOB.F04) cp $SCR/$JOB.F04 ~/scr/$JOB.trj
   if(-e $SCR/$JOB.F35) cp $SCR/$JOB.F35 ~/scr/$JOB.rst
endif

#
#  ---- the bottom third of the script is to clean up all disk files ----
#
echo ----- accounting info -----
#
#   Clean up the master's scratch directory.
#
echo Files used on the master node $master were:
ls -lF $SCR/$JOB.*
rm -f  $SCR/$JOB.F*
#
#   Clean up scratch directory of remote nodes.
#
#   This may not be necessary, e.g. on a T3E where all files are in the
#   same directory, and just got cleaned out by the previous 'rm'.  Many
#   batch queue managers provide cleaning out of scratch directories.
#   It still may be interesting to the user to see the sizes of files.
#
#   The 'lasthost' business prevents multiple cleanup tries on SMP nodes.
#
if ($TARGET == sockets) then
   set nmax=${#HOSTLIST}
   set lasthost=$master
   @ n=2   # master already cleaned above
   while ($n <= $nmax)
      set host=$HOSTLIST[$n]
      set host=`echo $host | cut -f 1 -d :`   # drop anything behind a colon
      if ($host != $lasthost) then
         echo Files from $host are:
         $DDI_RSH $host -l $USER -n "ls -l $SCR/$JOB.*"
         $DDI_RSH $host -l $USER -n "rm -f $SCR/$JOB.F*"
         set lasthost=$host
      endif
      @ n++
   end
endif
#
#    this particular example is for the combination iMPI/SGE 
#    we have inherited a file of unique node names from above.
if ($TARGET == mpi) then
   set nnodes=`wc -l $HOSTFILE`
   set nnodes=$nnodes[1]
   @ n=1
   set master=`hostname`
   set master=$master:r    # burn off the .local suffix in our cluster
   while ($n <= $nnodes)
      set host=`sed -n -e "$n p" $HOSTFILE`
      if ($host != $master) then
         echo Files used on node $host were:
         ssh $host -l $USER "ls -l $SCR/*"
         ssh $host -l $USER "rm -f $SCR/*"
      endif
      @ n++
   end
#          clean off the last file on the master's scratch disk.
   rm -f $HOSTFILE
endif
#
#   IBM SP cleanup code...might need to be something other than 'rsh'.
#
if ($TARGET == ibm64-sp) then
   set lasthost=$master
   @ n=2   # we already cleaned up the master node just above.
   @ nmax=$NCPUS
   while ($n <= $nmax)
      set host=`sed -n -e "$n p" $HOSTFILE`
      if ($host != $lasthost) then
         echo Files used on node $host were:
         rsh $host "ls -l $SCR/$JOB.*"
         rsh $host "rm -f $SCR/$JOB.F*"
         set lasthost=$host
      endif
      @ n++
   end
   rm -f $HOSTFILE
endif
#
#  and this is the end
#
date
time
exit