Example Program and Makefile for Theta

Here is a simple example program and makefile for building on the ALCF's Theta system. When you compile your code for Theta, you are normally cross-compiling for the compute nodes (Intel Xeon Phi 2nd Generation [KNL]), rather than directly compiling for the login nodes (non-KNL). The default, you are using the Intel compiler suite. We also provide the Cray, GNU, and Clang compiler suites via modules.

C Program

/* File pi.c */
#include <mpi.h>
#include <math.h>
#include <stdio.h>

int main(int argc, char** argv)
{
  int n, myid, numprocs, i;
  double PI25DT = 3.141592653589793238462643;
  double mypi, pi, h, sum, x;

  MPI_Init(&argc, &argv);
  MPI_Comm_size(MPI_COMM_WORLD, &numprocs);
  MPI_Comm_rank(MPI_COMM_WORLD, &myid);

  /* Number of intervals */
  n = 25;

  MPI_Bcast(&n, 1, MPI_INT, 0, MPI_COMM_WORLD);
  h = 1.0 / (double) n;
  sum = 0.0;
  for ( i = myid + 1; i <= n; i += numprocs )
  {
       x = h * ((double) i - 0.5);
       sum += ( 4.0 / (1.0 + x*x) );
  }
  mypi = h * sum;
  MPI_Reduce(&mypi, &pi, 1, MPI_DOUBLE, MPI_SUM, 0, MPI_COMM_WORLD);

  if (myid == 0)
  {
      printf("pi is approximately %.16f, Error is %.16f\n",
               pi, fabs(pi - PI25DT));
  }

  MPI_Finalize();
  return 0;
}

Makefile

# File makefile
CC = cc

OPTFLAGS = -O3
CFLAGS = $(OPTFLAGS) -g

all: pi

pi: pi.o
  $(CC) $(CFLAGS) -o pi pi.o

pi.o: pi.o
    $(CC) $(CFLAGS) -c pi.c

clean:
    rm pi pi.o

Usage

Session screenshot shows building and running, using an example job script:

thetalogin6(pi)> ls
makefile pi.c runpi.bash* 

thetalogin6(pi)> make 
cc -O3 -g -c pi.c 
cc -O3 -g -o pi pi.o 

thetalogin6(pi)> ls 
makefile pi* pi.c pi.o runpi.bash* 
thetalogin6(pi)> cat runpi.bash

 #!/bin/bash 
echo "Starting Cobalt job script" 
export n_nodes=$COBALT_JOBSIZE 
export n_mpi_ranks_per_node=64 
export n_mpi_ranks=$(($n_nodes * $n_mpi_ranks_per_node)) 
aprun -n $n_mpi_ranks -N $n_mpi_ranks_per_node -cc depth -d 4 -j 4 ./pi 

thetalogin6(pi)> qsub -A AProject -n 1 -t 20 runpi.bash 

106040 thetalogin6(pi)> qstat -u user1 

JobID JobName User WallTime QueuedTime RunTime Nodes State attrs Queue Score Project 
====================================================================================
106040 N/A user1 00:20:00 00:00:05 N/A 1 queued {'numa': 'quad', 'mcdram': 'cache'} default 51.0 AProject 

thetalogin6(pi)> qstat -u user1 

JobID JobName User WallTime QueuedTime RunTime Nodes State attrs Queue Score Project 
====================================================================================
106040 N/A user1 00:20:00 00:00:25 00:00:19 1 starting {'numa': 'quad', 'mcdram': 'cache'} default 51.1 AProject 

thetalogin6(pi)> qstat -u user1 

JobID JobName User WallTime QueuedTime RunTime Nodes State attrs Queue Score Project 
====================================================================================
106040 N/A user1 00:20:00 00:00:32 00:00:59 1 running {'numa': 'quad', 'mcdram': 'cache'} default 51.2 AProject 

thetalogin6(pi)> ls -lt 
total 18240 
-rw-r--r-- 1 user1 cobalt 1671 Jun 27 23:08 106040.cobaltlog 
-rw-r--r-- 1 user1 users 185 Jun 27 23:07 106040.output 
-rw-r--r-- 1 user1 users 0 Jun 27 23:07 106040.error 
-rwxr-xr-x 1 user1 users 9300576 Jun 27 23:06 pi* 
-rw-r--r-- 1 user1 users 10256 Jun 27 23:06 pi.o 
-rw-r--r-- 1 user1 users 152 Jun 27 23:05 makefile 
-rwxr-xr-x 1 user1 users 237 Jun 27 23:05 runpi.bash* 
-rw-r--r-- 1 user1 users 816 Jun 27 22:20 pi.c

 thetalogin6(pi)> cat 106040.output 

Starting Cobalt job script pi is approximately 3.1417259869152536, Error is 0.0001333333254605 
Application 3373484 resources: utime ~6s, stime ~6s, Rss ~5036, inblocks ~0, outblocks ~8 
thetalogin6(pi)>