Hints and FAQ for Grendel

   #!/bin/sh
   #PBS -q qexp
   #PBS -l nodes=2:nehalem:ppn=8
   #PBS -N MPIjob
   ...

    qsub -q qfat -l nodes=5:ppn=8 jobscript

    qsub -q q4 -l nodes=5:ppn=4 jobscript

   #!/bin/bash
   echo "========= Job started  at `date` =========="
   echo "Host: `hostname -s` has `cpus` CPUs"
   cd some/where
   for i in $(seq `cpus`) ; do
     ./myprogram < input.$i > output.$i &
   done
   wait
   echo "========= Job finished at `date` =========="

   #!/bin/bash
   echo "========= Job started  at `date` =========="
   echo "Host: `hostname -s` has `mem` GB memory"
   echo "Host: `hostname -s` has `cpus` CPUs"
   Mreq=2.5  # 'myprogram' requires Mreq GB.
   instances=$(echo "scale=0; `mem` / $Mreq" | bc)
   [ $instances -gt `cpus` ] && instances=`cpus`
   if [ $instances -lt 1 ]; then
     echo "Insufficient memory to run program. Exiting"
     exit
   fi
   cd some/where
   for i in $(seq $instances) ; do
     ./myprogram < input.$i > output.$i &
   done
   wait
   echo "========= Job finished at `date` =========="

   #!/bin/bash
   echo "========= Job started  at `date` =========="
   ./myprogram > /scratch/$PBS_JOBID/out
   grep "Energy minimum" /scratch/$PBS_JOBID/out > results
   cp /scratch/$PBS_JOBID/out out.$PBS_JOBID
   echo "========= Job finished at `date` =========="

Examples:
   subg03 -h                     # List all options to subg03.
   subg03 gaussjob.com           # Submit a Gaussian job.
   subg03 -q q8 gaussjob.com     # Submit the job to queue q8

First of all, include the path to MPICH2 to your PATH environment variable. Choose one of these:

   # To use MPICH2 built w. Portland compilers
   export PATH=/com/mpich2-1.0.4p1-pgi/bin:$PATH   (Bourne-shell syntax)
   set path = (/com/mpich2-1.0.4p1-pgi/bin $path)  (C-shell syntax)

   # To use MPICH2 built w. Intel compilers
   export PATH=/com/mpich2-1.0.4p1/bin:$PATH       (Bourne-shell syntax)
   set path = (/com/mpich2-1.0.4p1/bin $path)      (C-shell syntax)

   mpif90 -c progfrag1.f -O2
   mpif90 -c progfrag2.f -O2
   mpif90 -c progfrag3.f -O2

   mpif90 -o program.x progfrag1.o progfrag2.o progfrag3.o

If running MPICH2 for the first time, create a file ~/.mpd.conf in the login-directory, f.ex.:

   echo "MPD_SECRETWORD=SW92seSam" > ~/.mpd.conf
   chmod 600 ~/.mpd.conf

   #!/bin/sh
   #PBS -l nodes=16:ppn=2
   #PBS -N MPIjob
   export PATH=/com/mpich2-1.0.4p1-pgi/bin:$PATH
   cd $PBS_O_WORKDIR
   # Generate the hostfile, with full-qualified-domainnames:
   mpdfile=mpd.$$
   awk '{printf("%s.grendel.cscaa.dk\n", $1)}' $PBS_NODEFILE |\
     sort -u > $mpdfile
   # Start the virtual machine on the nodes:
   mpdboot --totalnum=16 --mpd=/com/mpich2-1.0.4p1-pgi/bin/mpd \
           --file=$mpdfile --rsh=rsh
   # Optional, show the participating nodes:
   echo '=====   ============================='
   mpdtrace -l
   echo '=====  ============================='
   # NB: If we used Intel compilers we must define LD_LIBRARY_PATH first:
   # export LD_LIBRARY_PATH=/com/intel/fce/9.0/lib:$LD_LIBRARY_PATH
   # Now, run the MPI-program:
   mpiexec -n 32 ./mpiprogram arg1 arg2
   # Shutdown the virtual machine, and clean up:
   mpdallexit
   rm -f $mpdfile
   #

First of all, setup the OpenMPI environment:

  source /com/OpenMPI/1.4.1/intel/bin/openmpi.sh    (Bourne-shell)
  source /com/OpenMPI/1.4.1/intel/bin/openmpi.csh   (C-shell)

   mpif90 -c progfrag1.f -O2
   mpif90 -c progfrag2.f -O2
   mpif90 -c progfrag3.f -O2
   mpif90 -o OpenMPI-program progfrag1.o progfrag2.o progfrag3.o

   mpif90 -o OpenMPI-program progfrag1.o progfrag2.o progfrag3.o -limf -lm

   #!/bin/sh
   #PBS -l nodes=16:ppn=2
   #PBS -N OpenMPIjob
   echo "========= Job started  at `date` =========="
   # To enable OpenMPI:
   source /com/OpenMPI/1.4.1/intel/bin/openmpi.sh
   # To enable MKL:
   source /com/intel/Compiler/11.1/064/mkl/tools/environment/mklvarsem64t.sh
   cd $PBS_O_WORKDIR
   mpiexec ./OpenMPI-program arg1 arg2
   echo "========= Job finished at `date` =========="
   #

If the program requires more memory, it may be necessary to reserve a whole node for each instance of the program. I this case change the mpiexec command to:

   mpiexec -bynode -n 16 ./OpenMPI-program arg1 arg2

If possible, OpenMPI will first try to make any inter-node communication via Infiniband. If that isn't possible (the nodes may not posses any Infiniband hardware) it will communicate via gigabit Ethernet. If two processes are located within the same node, OpenMPI will communicate via Shared Memory regions.
The latency is ca. 1000 times smaller going from gigabit Ethernet to Infiniband.
The effective bandwidth is ca. 20-25 times bigger going from gigabit Ethernet to Infiniband.

   ifort -o prog.exe prog.f \
     -L/com/intel/mkl/10.1.1.019/lib/em64t \
     -lmkl_lapack -lmkl -lguide   -lpthread

   export LD_LIBRARY_PATH=/com/intel/mkl/10.1.1.019/lib/em64t:$LD_LIBRARY_PATH
   ./prog.exe

   #!/bin/sh
   #PBS -l nodes=1:ppn=2
   export LD_LIBRARY_PATH=/com/intel/mkl/10.1.1.019/lib/em64t:$LD_LIBRARY_PATH
   export OMP_NUM_THREADS=2
   ./prog.exe

   % ifort -p -g -c x1.f
   % ifort -p -g -c x2.f
   % ifort -o x.out -p x1.o x2.o

   % gprof x.out gmon.out

   # Makefile for making a Scalapack -program
   OBJS = pdscaex.o pdscaexinfo.o pdlaread.o pdlawrite.o
   EXE  = xdscaex
   #
   PATH := /com/OpenMPI/1.4.1/intel/bin:${PATH}
   include /com/lib/scalapack_openmpi/SLmake.inc
   MYFFLAGS = -i_dynamic
   all: $(EXE)
   .f.o : ; $(F77) -c $(F77FLAGS) $*.f
   .c.o : ; $(CC) -c $(CCFLAGS) $(CDEFS) $*.c
   $(EXE): $(OBJS)
            $(FCLOADER) $(FCLOADFLAGS) $(MYFFLAGS) -o $@ $(OBJS) $(STLIBS)

To run the program, use a jobscript like this:

   #!/bin/sh
   #PBS -l nodes=4:ppn=1
   #PBS -N scalapack
   #
   export PATH=/com/OpenMPI/1.4.1/intel/bin:$PATH
   cd ${PBS_O_WORKDIR}
   #
   mkl=/com/intel/mkl/9.1.023/lib/em64t
   openmpi=/com/OpenMPI/1.4.1/intel/lib
   intel=/com/intel/Compiler/11.1/064/lib/intel64
   export LD_LIBRARY_PATH=${mkl}:${openmpi}:${intel}:$LD_LIBRARY_PATH
   #
   mpiexec ./xdscaex
   #

To link with Intel's math. library, MKL, (eg. if a program needs some Lapack-routines) do:

   ifort myprogram.f \
     -L/com/intel/Compiler/11.1/046/mkl/lib/em64t        \
     -Wl,--start-group                                   \
        -lmkl_intel_lp64 -lmkl_intel_thread -lmkl_core  \
     -Wl,--end-group                                     \
     -liomp5 -lpthread

Intel has a very convenient webpage which shows how to link with MKL.
In a browser, open how to link and make these selections:

• How to start EP-tasks transparrantly on the available hardware resources? There should be no difference in starting EP-tasks whether the tasks will be running on one or more cluster-nodes.
• How to "queue" tasks, if the number of tasks is larger than the number of available processors (ie. CPU-cores). When a task finishes, a waiting task is allowed to start, but never before! The only UNIX/Linux command having an analogous capabillity is probably 'make -jN' (N>1).
• If the EP-job spans more than one node, how to distribute the tasks in a balanced manner between the nodes.
• How to specify the EP-job topology wrt. nodes and CPUcores.

    #!/bin/bash
    cd `dirname $0`
    ./$1 > $1.log 2>&1

    #PBS -l nodes=N:ppn=P
    dispatch -s /absolute/path/to/master  script1 script2 ... scriptT

    #PBS -l nodes=N:ppn=P
    dispatch /absolute/path/to/mydir run  subdir1 subdir2 ... subdirT