HPC

Difference between revisions of "Applications/Ansys"

From HPC
Jump to: navigation , search
Line 9: Line 9:
 
#!/bin/bash
 
#!/bin/bash
  
−
#SBATCH -J ANSYS
+
#SBATCH -J ANSYS # Job Name
−
#SBATCH -N 1      # number of cores (nodes) to use
+
#SBATCH -N 1      # Number of Nodes to use
−
#SBATCH --ntasks-per-node 28
+
#SBATCH -n 28   # Number of CPUs
−
#SBATCH -o %N.%j.%a.out
+
#SBATCH -o %N.%j.%a.out # Output file name
−
#SBATCH -e %N.%j.%a.err
+
#SBATCH -e %N.%j.%a.err # Error file name
−
#SBATCH -p compute     # use "highmem" or "compute" node
+
#SBATCH -p compute       # Partition to run on
−
# maximum execution time
 
 
#SBATCH --exclusive      # Instructs SLURM to not run any other job in the node(s) selected
 
#SBATCH --exclusive      # Instructs SLURM to not run any other job in the node(s) selected
  
−
module load ansys/v172
+
# Load Ansys Version 17.2<br/>
 +
module add ansys/v172
 +
 
 +
# This is the run command the -b instructs Ansys to run in batch mode the -i specifies the input file
 
ansys172 -b -i /home/test/Input.lgw
 
ansys172 -b -i /home/test/Input.lgw
 
</pre>
 
</pre>
−
 
−
Details:<br/>
 
−
Line 1 – Just a standard line that needs to be at the top of the file<br/>
 
−
Line 2 – The -J sets the name of the job, in this case to ANSYS. This doesn’t impact on the job and doesn’t have to be unique. It helps distinguish tasks when looking in squeue (LINK) <br/>
 
−
Line 3 – This requests you are allocated 1 compute node. <br/>
 
−
Line 4 – This requests 28 slots on the node. (the full compute node)<br/>
 
−
Line 5 and 6 – These set the output and error files. The log file will contain Fluent console output, the error file will contain information that may be useful if things don’t work as expected (at a cluster level)<br/>
 
−
Line 7 – This requests the job runs on one of the compute nodes on the compute queue.<br/>
 
−
Line 8 – this requests the job runs exclusively on a node i.e (no other jobs)<br/>
 
−
Line 9- Load Ansys Version 17.2<br/>
 
−
Line 10– This is the run command the -b instructs Ansys to run in batch mode the -i specifies the input file<br/>
 
  
 
===Ansys Fluent===
 
===Ansys Fluent===
Line 64: Line 55:
 
<pre style="background-color: #E5E4E2; color: black; font-family: monospace, sans-serif;">
 
<pre style="background-color: #E5E4E2; color: black; font-family: monospace, sans-serif;">
 
#!/bin/bash
 
#!/bin/bash
−
#SBATCH -J ANSYS_FLUENT # sensible name for the job
+
#SBATCH -J ANSYS_FLUENT # Job Name
−
#SBATCH -N 1
+
#SBATCH -N 1 # Number of  Nodes to use
−
#SBATCH -n 28
+
#SBATCH -n 28 # Number of CPUs
−
#SBATCH -o %N.%j.%a.out
+
#SBATCH -o %N.%j.%a.out # Output file name
−
#SBATCH -e %N.%j.%a.err
+
#SBATCH -e %N.%j.%a.err # Error file name
−
#SBATCH -p compute
+
#SBATCH -p compute # Partition to run on
−
#SBATCH --exclusive
+
#SBATCH --exclusive # Instructs SLURM to not run any other job in the node(s) selected
 
    
 
    
−
# load the relevant module files
+
# Remove all currently running modules and load Intel MPI and Ansys V17.2
 
module purge
 
module purge
 
module load intel/mpi/64/5.1.3.181
 
module load intel/mpi/64/5.1.3.181
 
module load ansys/v172
 
module load ansys/v172
 
   
 
   
−
export FLUENT_GUI=off
+
export FLUENT_GUI=off  #Turns the Fluent GUI off
−
export I_MPI_ROOT=/trinity/clustervision/CentOS/7/apps/intel/impi/5.1.3.181
+
export I_MPI_ROOT=/trinity/clustervision/CentOS/7/apps/intel/impi/5.1.3.181 #Tells fluent where Intel MPI is located
−
export I_MPI_DEBUG=5
+
export I_MPI_DEBUG=5 # Intel MPI level of error messages
−
export I_MPI_FABRICS=shm:tmi
+
export I_MPI_FABRICS=shm:tmi # Sets the Omnipath interconnect message protocol
−
export I_MPI_FALLBACK=no
+
export I_MPI_FALLBACK=no # No fallback to ethernet
 
   
 
   
 +
#Checks number of tasks and sets number of processes
 
if [ -z "$SLURM_NPROCS" ]; then
 
if [ -z "$SLURM_NPROCS" ]; then
 
N=$(( $(echo $SLURM_TASKS_PER_NODE | sed -r 's/([0-9]+)\(x([0-9]+)\)/\1 * \2/') ))
 
N=$(( $(echo $SLURM_TASKS_PER_NODE | sed -r 's/([0-9]+)\(x([0-9]+)\)/\1 * \2/') ))
Line 88: Line 80:
 
  N=$SLURM_NPROCS
 
  N=$SLURM_NPROCS
 
fi
 
fi
−
echo $SLURM_JOB_NODELIST
+
 
 +
echo $SLURM_JOB_NODELIST # Prints Node range to output file
 +
# Prints number of processes to output file
 
echo $SLURM_NPROCS
 
echo $SLURM_NPROCS
 
echo -e "N: $N\n";
 
echo -e "N: $N\n";
 
   
 
   
−
  # run fluent in batch on the allocated node(s)
+
# run fluent in batch on the allocated node(s)
 
srun hostname -s > hostfile
 
srun hostname -s > hostfile
 +
# Set architecture of the CPU (in this case amd64)
 
FLUENT_ARCH=lnamd64 export FLUENT_ARCH
 
FLUENT_ARCH=lnamd64 export FLUENT_ARCH
 +
# Appends to the library path psm2 library file
 
export LD_LIBRARY_PATH=/usr/lib64/psm2-compat:$LD_LIBRARY_PATH
 
export LD_LIBRARY_PATH=/usr/lib64/psm2-compat:$LD_LIBRARY_PATH
−
 
+
# This is the run command. Note -i specifies the name of the input journal file
 
fluent -ssh  3ddp -g -t$N -mpi=intel -pib.infinipath  -cnf=hostfile -i my_fluent_file
 
fluent -ssh  3ddp -g -t$N -mpi=intel -pib.infinipath  -cnf=hostfile -i my_fluent_file
 
</pre>
 
</pre>
−
 
−
Details:<br/>
 
−
Line 1 – Just a standard line that needs to be at the top of the file<br/>
 
−
Line 2 – The -J sets the name of the job, in this case to ANSYS_FLUENT. This doesn’t impact on the job and doesn’t have to be unique. It helps distinguish tasks when looking in squeue (LINK) <br/>
 
−
Line 3 – This requests you are allocated 1 compute node. <br/>
 
−
Line 4 – This requests 28 slots on the node. (the full compute node)<br/>
 
−
Line 5 and 6 – These set the output and error files. The log file will contain Fluent console output, the error file will contain information that may be useful if things don’t work as expected (at a cluster level)<br/>
 
−
Line 7 – This requests the job runs on one of the compute nodes on the compute queue.<br/>
 
−
Line 8 – this requests the job runs exclusively on a node i.e (no other jobs)<br/>
 
−
Line 9 – Comment<br/>
 
−
Line 10 – Remove all currently running modules<br/>
 
−
Line 11 – Load Intel MPI module<br/>
 
−
Line 12- Load Ansys Version 17.2<br/>
 
−
Line 13 – Turns the Fluent GUI off<br/>
 
−
Line 14 – Tells fluent where Intel MPI is located<br/>
 
−
Line 15 – Intel MPI level of error messages<br/>
 
−
Line 16 – Sets the Omnipath interconnect message protocol<br/>
 
−
Line 17 – Tells Intel MPI not to fallback to Ethernet<br/>
 
−
Line 18 - 20 – Checks number of tasks and sets number of processes<br/>
 
−
Line 21 – Prints Node range to output file<br/>
 
−
Line 22 – 23 – Prints number of processes to output file<br/>
 
−
Line 24 – Comment<br/>
 
−
Line 25 – Produces a hostfile listing nodes to run on<br/>
 
−
Line 26 – Tells fluent the architecture of the CPU (in this case amd64)</br>
 
−
Line 27 – Appends to the library path psm2 library file<br/>
 
−
Line 28 – This is the run command. Note -i specifies the name of the input journal file<br/>
 
  
 
===Job Submission===
 
===Job Submission===

Revision as of 11:17, 3 February 2017

Application Details

  • Versions: V17.0, V17.2
  • Module names: ansys/v170, ansys/v172
  • License:


Job Submission Script

#!/bin/bash

#SBATCH -J ANSYS # Job Name
#SBATCH -N 1      # Number of  Nodes to use
#SBATCH  -n 28   # Number of CPUs
#SBATCH -o %N.%j.%a.out # Output file name
#SBATCH -e %N.%j.%a.err  # Error file name
#SBATCH -p compute       # Partition to run on
#SBATCH --exclusive       # Instructs SLURM to not run any other job in the node(s) selected

# Load Ansys Version 17.2<br/>
module add ansys/v172

# This is the run command the -b instructs Ansys to run in batch mode the -i specifies the input file
ansys172 -b -i /home/test/Input.lgw

Ansys Fluent

Usage Examples

Interactive
[username@login01 ~]$ interactive
salloc: Granted job allocation 296769
Job ID 296769 connecting to c170, please wait...
Last login: Wed Jan 25 09:10:51 2017 from 10.254.5.246
[username@c170 ~]$ module add ansys/v172

[username@c170 ~]$ fluent

Journal File

A journal file needs to be created to control the job when running 

rcd "hw_RANS"
/solve/dual-time-iterate 2000 10
wcd "hw_LESHot"

Line 1 - Read case and data from hw_RANS Line 2 - Number of iterations Line 3 - Write case and date to hw_LESHOT

Job Submission Script

#!/bin/bash
#SBATCH -J ANSYS_FLUENT # Job Name
#SBATCH -N 1 # Number of  Nodes to use
#SBATCH -n 28 # Number of CPUs
#SBATCH -o %N.%j.%a.out # Output file name
#SBATCH -e %N.%j.%a.err # Error file name
#SBATCH -p compute # Partition to run on
#SBATCH --exclusive # Instructs SLURM to not run any other job in the node(s) selected
  
# Remove all currently running modules and load Intel MPI and Ansys V17.2
module purge
module load intel/mpi/64/5.1.3.181
module load ansys/v172
 
export FLUENT_GUI=off  #Turns the Fluent GUI off
export I_MPI_ROOT=/trinity/clustervision/CentOS/7/apps/intel/impi/5.1.3.181 #Tells fluent where Intel MPI is located
export I_MPI_DEBUG=5 # Intel MPI level of error messages
export I_MPI_FABRICS=shm:tmi # Sets the Omnipath interconnect message protocol
export I_MPI_FALLBACK=no # No fallback to ethernet
 
#Checks number of tasks and sets number of processes
if [ -z "$SLURM_NPROCS" ]; then
N=$(( $(echo $SLURM_TASKS_PER_NODE | sed -r 's/([0-9]+)\(x([0-9]+)\)/\1 * \2/') ))
else
 		N=$SLURM_NPROCS
fi

echo $SLURM_JOB_NODELIST # Prints Node range to output file
# Prints number of processes to output file
echo $SLURM_NPROCS
echo -e "N: $N\n";
 
# run fluent in batch on the allocated node(s)
srun hostname -s > hostfile
# Set architecture of the CPU (in this case amd64)
FLUENT_ARCH=lnamd64 export FLUENT_ARCH
# Appends to the library path psm2 library file
export LD_LIBRARY_PATH=/usr/lib64/psm2-compat:$LD_LIBRARY_PATH
# This is the run command. Note -i specifies the name of the input journal file
fluent -ssh  3ddp -g -t$N -mpi=intel -pib.infinipath  -cnf=hostfile -i my_fluent_file

Job Submission

[username@login01 ~]$ sbatch fluent.job
Submitted batch job 289535

Tuning Fluent tasks

Number of Nodes and Cores

The number of nodes and CPU cores Fluent can be run across can be increased, which may result in performance improvements. This is performed by altering the number of nodes (#SBATCH -N XX) in the submission script. In order to make use of the additional CPU cores allocated alter #SBATCH -n xx, the number of CPUS should be Number of nodes * 28.

MPI and Interconnect

Please use Fluent with Intel MPI and the Omnipath interconnect for best performance.

Further Information

Forum Support: Viper Ansys Forum
Ansys Website: Ansys

Retrieved from ‘http://hpc.mediawiki.hull.ac.uk/index.php?title=Applications/Ansys&oldid=424’