Initial commit

Hands-on-1/.example.yml

+stages:
+   - build
+   - test
+
+default:
+  tags: ["hpc"]
+
+
+.template:
+   before_script:
+     - module use /pc2/groups/hpc-prf-nhrgs/modules
+     - module load MUST
+      
+build:
+   stage: build
+   extends: .template
+   script:
+     - cd Hands-on-1
+     - module li
+     - mpicc -g -O3 -I. -Wall -o example.exe example.c
+     
+   artifacts:
+     paths:
+      - Hands-on-1/example.exe
+      
+test:
+   stage: test
+   
+   extends: .template
+   dependencies: 
+    - build
+
+   variables:
+     SLURM_PARAM_TASKS: "-n 9"
+     SLURM_PARAM_ACCOUNT: "-A hpc-prf-nhrgs"
+
+   script:
+     - cd Hands-on-1
+     - EXIT_CODE=0
+     - mustrun --must:timeout 90 -np 8 ./example.exe || EXIT_CODE=$?
+     - mv MUST_Output.html MUST_Output.htm
+     - exit $EXIT_CODE
+     
+   artifacts:
+     when: always
+     paths:
+      - Hands-on-1/MUST_Output.htm

Hands-on-1/example.c

+#include <mpi.h>
+#include <stdio.h>
+
+int main (int argc, char** argv)
+{
+    int rank, size, buf[8];
+
+    MPI_Init(&argc,&argv);
+    MPI_Comm_rank (MPI_COMM_WORLD, &rank);
+    MPI_Comm_size (MPI_COMM_WORLD, &size);
+    MPI_Datatype type;
+
+    MPI_Type_contiguous(2, MPI_INTEGER, &type);
+
+    MPI_Recv(buf, 2, MPI_INT, size-rank-1, 123, MPI_COMM_WORLD, MPI_STATUS_IGNORE);
+
+    MPI_Send(buf, 2, type, size-rank-1, 123, MPI_COMM_WORLD);
+
+    printf ("Hello, I am rank %d of %d.\n", rank, size);
+
+    MPI_Finalize ();
+
+    return 0;
+}

Hands-on-2/.lulesh.yml

+stages:
+   - build
+   - test
+   - performance
+
+default:
+  tags: ["hpc"]
+
+
+.template:
+   before_script:
+     - module use /pc2/groups/hpc-prf-nhrgs/modules
+     - module load MUST
+      
+build-lulesh:
+   stage: build
+   extends: .template
+   script:
+     - cd Hands-on-2
+     - mkdir build-openmp
+     - cd build-openmp
+     - cmake -DCMAKE_BUILD_TYPE=Release -DWITH_OPENMP=On ..
+     - make 
+     
+   artifacts:
+     paths:
+      - Hands-on-2/build-openmp/lulesh2.0-mpi
+
+build-tsan-lulesh:
+   stage: build
+   extends: .template
+   script:
+     - cd Hands-on-2
+     - export MUST_ROOT=/pc2/groups/hpc-prf-nhrgs/MUST/install-new
+     - make lulesh2.0-mpi-tsan2
+     
+   artifacts:
+     paths:
+      - Hands-on-2/lulesh2.0-mpi-tsan2
+
+test-must-tsan-lulesh:
+   stage: test
+   
+   extends: .template
+   dependencies: 
+    - build-tsan-lulesh
+
+   variables:
+     OMP_NUM_THREADS: 4
+     SLURM_PARAM_THREADS: "-c 4"
+     SLURM_PARAM_TASKS: "-n 9"
+     SLURM_PARAM_ACCOUNT: "-A hpc-prf-nhrgs"
+
+   script:
+     - cd Hands-on-2
+     - EXIT_CODE=0
+     - export OMP_NUM_THREADS=$OMP_NUM_THREADS
+     - mustrun --must:tsan -np 8 ./lulesh2.0-mpi-tsan2 -i 10 || EXIT_CODE=$?
+     - mv MUST_Output.html MUST_Output.htm
+     - exit $EXIT_CODE
+     
+   artifacts:
+     when: always
+     paths:
+      - Hands-on-2/MUST_Output.htm
+
+test-must-lulesh:
+   stage: test
+   
+   extends: .template
+   dependencies: 
+    - build-lulesh
+
+   variables:
+     OMP_NUM_THREADS: 4
+     SLURM_PARAM_THREADS: "-c 4"
+     SLURM_PARAM_TASKS: "-n 9"
+     SLURM_PARAM_ACCOUNT: "-A hpc-prf-nhrgs"
+
+   script:
+     - cd Hands-on-2
+     - EXIT_CODE=0
+     - export OMP_NUM_THREADS=$OMP_NUM_THREADS
+     - mustrun -np 8 ./build-openmp/lulesh2.0-mpi -i 10 || EXIT_CODE=$?
+     - mv MUST_Output.html MUST_Output.htm
+     - exit $EXIT_CODE
+     
+   artifacts:
+     when: always
+     paths:
+      - Hands-on-2/MUST_Output.htm
+
+run-lulesh:
+   stage: performance
+   
+   extends: .template
+   dependencies: 
+    - build-lulesh
+
+   variables:
+     OMP_NUM_THREADS: 4
+     SLURM_PARAM_THREADS: "-c 4"
+     SLURM_PARAM_TASKS: "-n 8"
+     SLURM_PARAM_ACCOUNT: "-A hpc-prf-nhrgs"
+
+   script:
+     - cd Hands-on-2
+     - export OMP_NUM_THREADS=$OMP_NUM_THREADS
+     - mpirun -np 8 ./build-openmp/lulesh2.0-mpi -i 100
+
+build-lulesh-typeart:
+   stage: build
+   extends: .template
+   script:
+     - cd Hands-on-2
+     - WITH_FLOAT=1 MPICXX=typeart-mpic++ make 
+     
+   artifacts:
+     paths:
+      - Hands-on-2/lulesh2.0-mpi
+      
+test-typeart-lulesh:
+   stage: test
+   
+   extends: .template
+   dependencies: 
+    - build-lulesh-typeart
+
+   variables:
+     SLURM_PARAM_TASKS: "-n 9"
+     SLURM_PARAM_ACCOUNT: "-A hpc-prf-nhrgs"
+
+   script:
+     - cd Hands-on-2
+     - EXIT_CODE=0
+     - export OMP_NUM_THREADS=1
+     - mustrun -np 8 --must:typeart ./lulesh2.0-mpi -i 10 || EXIT_CODE=$?
+     - mv MUST_Output.html MUST_Output.htm
+     - exit $EXIT_CODE
+     
+   artifacts:
+     when: always
+     paths:
+      - Hands-on-2/MUST_Output.htm

Hands-on-2/.sol.yml

+stages:
+   - build
+   - test
+   - performance
+
+default:
+  tags: ["hpc"]
+
+
+.template:
+   before_script:
+     - module use /pc2/groups/hpc-prf-nhrgs/modules
+     - module load MUST
+      
+build-lulesh:
+   stage: build
+   extends: .template
+   script:
+     - cd Hands-on-2
+     - mkdir build-openmp
+     - cd build-openmp
+     - cmake -DCMAKE_BUILD_TYPE=Release -DWITH_OPENMP=On ..
+     - make 
+     
+   artifacts:
+     paths:
+      - Hands-on-2/build-openmp/lulesh2.0-mpi
+      
+build-tsan-lulesh:
+   stage: build
+   extends: .template
+   script:
+     - cd Hands-on-2
+     - export MUST_ROOT=/pc2/groups/hpc-prf-nhrgs/MUST/install-new
+     - make lulesh2.0-mpi-tsan2
+     
+   artifacts:
+     paths:
+      - Hands-on-2/lulesh2.0-mpi-tsan2
+
+test-must-tsan-lulesh:
+   stage: test
+   
+   extends: .template
+   dependencies: 
+    - build-tsan-lulesh
+
+   variables:
+     OMP_NUM_THREADS: 4
+     SLURM_PARAM_THREADS: "-c 4"
+     SLURM_PARAM_TASKS: "-n 9"
+     SLURM_PARAM_ACCOUNT: "-A hpc-prf-nhrgs"
+
+   script:
+     - cd Hands-on-2
+     - EXIT_CODE=0
+     - export OMP_NUM_THREADS=$OMP_NUM_THREADS
+     - env MUST_FILTER_FILE=./Hands-on-2/filterfile.txt mustrun --must:tsan -np 8 ./lulesh2.0-mpi-tsan2 -i 10 || EXIT_CODE=$?
+     - mv MUST_Output.html MUST_Output.htm
+     - exit $EXIT_CODE
+     
+   artifacts:
+     when: always
+     paths:
+      - Hands-on-2/MUST_Output.htm
+
+
+test-must-lulesh:
+   stage: test
+   
+   extends: .template
+   dependencies: 
+    - build-lulesh
+
+   variables:
+     OMP_NUM_THREADS: 4
+     SLURM_PARAM_THREADS: "-c 4"
+     SLURM_PARAM_TASKS: "-n 9"
+     SLURM_PARAM_ACCOUNT: "-A hpc-prf-nhrgs"
+
+   script:
+     - cd Hands-on-2
+     - EXIT_CODE=0
+     - export OMP_NUM_THREADS=$OMP_NUM_THREADS
+     - env MUST_FILTER_FILE=./Hands-on-2/filterfile.txt mustrun -np 8 ./build-openmp/lulesh2.0-mpi -i 10 || EXIT_CODE=$?
+     - mv MUST_Output.html MUST_Output.htm
+     - exit $EXIT_CODE
+     
+   artifacts:
+     when: always
+     paths:
+      - Hands-on-2/MUST_Output.htm
+
+run-lulesh:
+   stage: performance
+   
+   extends: .template
+   dependencies: 
+    - build-lulesh
+
+   variables:
+     OMP_NUM_THREADS: 4
+     SLURM_PARAM_THREADS: "-c 4"
+     SLURM_PARAM_TASKS: "-n 8"
+     SLURM_PARAM_ACCOUNT: "-A hpc-prf-nhrgs"
+
+   script:
+     - cd Hands-on-2
+     - export OMP_NUM_THREADS=$OMP_NUM_THREADS
+     - mpirun -np 8 ./build-openmp/lulesh2.0-mpi -i 100
+
+build-lulesh-typeart:
+   stage: build
+   extends: .template
+   script:
+     - cd Hands-on-2
+     - WITH_FLOAT=1 MPICXX=typeart-mpic++ make 
+     
+   artifacts:
+     paths:
+      - Hands-on-2/lulesh2.0-mpi
+      
+test-typeart-lulesh:
+   stage: test
+   
+   extends: .template
+   dependencies: 
+    - build-lulesh-typeart
+
+   variables:
+     SLURM_PARAM_TASKS: "-n 9"
+     SLURM_PARAM_ACCOUNT: "-A hpc-prf-nhrgs"
+
+   script:
+     - cd Hands-on-2
+     - EXIT_CODE=0
+     - export OMP_NUM_THREADS=1
+     - env MUST_FILTER_FILE=./Hands-on-2/filterfile.txt mustrun -np 8 --must:typeart ./lulesh2.0-mpi -i 10 || EXIT_CODE=$?
+     - mv MUST_Output.html MUST_Output.htm
+     - exit $EXIT_CODE
+     
+   artifacts:
+     when: always
+     paths:
+      - Hands-on-2/MUST_Output.htm

Hands-on-2/CMakeLists.txt

+cmake_minimum_required(VERSION 3.0)
+
+project(LULESH CXX)
+
+#option(WITH_MPI    "Build LULESH with MPI"          TRUE)
+option(WITH_OPENMP "Build LULESH with OpenMP"       TRUE)
+option(WITH_SILO   "Build LULESH with silo support" FALSE)
+option(WITH_DOUBLE "Build LULESH with double float" TRUE)
+
+#if (WITH_MPI)
+  find_package(MPI REQUIRED)
+  include_directories(${MPI_C_INCLUDE_PATH} ${MPI_CXX_INCLUDE_PATH})
+#  add_definitions("-DUSE_MPI=1")
+  list(APPEND LULESH_MPI_LIBS ${MPI_C_LIBRARIES} ${MPI_CXX_LIBRARIES})
+#else()
+#  add_definitions("-DUSE_MPI=0")
+#endif()
+
+if (WITH_DOUBLE)
+  add_definitions("-DUSE_DOUBLE=1")
+endif()
+
+
+if (WITH_OPENMP)
+  find_package(OpenMP REQUIRED)
+  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${OpenMP_CXX_FLAGS}")
+endif()
+
+if (WITH_SILO)
+  find_path(SILO_INCLUDE_DIR silo.h
+    HINTS ${SILO_DIR}/include)
+  find_library(SILO_LIBRARY
+    NAMES siloh5
+    HINTS ${SILO_DIR}/lib)
+
+  include(FindPackageHandleStandardArgs)
+  find_package_handle_standard_args(SILO DEFAULT_MSG
+    SILO_LIBRARY
+    SILO_INCLUDE_DIR)
+
+  if (SILO_FOUND)
+    add_definitions("-DVIZ_MESH")
+    include_directories(${SILO_INCLUDE_DIR})
+    #   Note: silo needs to be built as a dynamic lib, otherwise 
+    # there are additional dependencies (hdf5) which we don't know.
+    # This would be fixed by silo providing a CMake package.
+    list(APPEND LULESH_EXTERNAL_LIBS ${SILO_LIBRARY})
+  endif()
+endif()
+
+set(LULESH_SOURCES
+  lulesh-comm.cc
+  lulesh-init.cc
+  lulesh-util.cc
+  lulesh-viz.cc
+  lulesh.cc)
+
+set(LULESH_EXEC lulesh2.0)
+
+add_executable(${LULESH_EXEC} ${LULESH_SOURCES})
+add_executable(${LULESH_EXEC}-tsan ${LULESH_SOURCES})
+add_executable(${LULESH_EXEC}-mpi ${LULESH_SOURCES})
+target_link_libraries(${LULESH_EXEC} ${LULESH_EXTERNAL_LIBS})
+target_link_libraries(${LULESH_EXEC}-tsan ${LULESH_EXTERNAL_LIBS})
+target_link_libraries(${LULESH_EXEC}-mpi ${LULESH_EXTERNAL_LIBS} ${LULESH_MPI_LIBS})
+
+target_compile_options(${LULESH_EXEC}-tsan PRIVATE -fsanitize=thread)
+target_link_options(${LULESH_EXEC}-tsan PRIVATE -fsanitize=thread)
+target_compile_definitions(${LULESH_EXEC} PRIVATE -DUSE_MPI=0)
+target_compile_definitions(${LULESH_EXEC}-tsan PRIVATE -DUSE_MPI=0)
+target_compile_definitions(${LULESH_EXEC}-mpi PRIVATE -DUSE_MPI=1)

Hands-on-2/Makefile

+#default build suggestion of MPI + OPENMP with gcc on Livermore machines you might have to change the compiler name
+
+SHELL = /bin/sh
+.SUFFIXES: .cc .o
+
+LULESH_EXEC = lulesh2.0
+
+MPI_INC = /opt/local/include/openmpi
+MPI_LIB = /opt/local/lib
+
+SERCXX = clang++
+MPICXX ?= mpicxx
+CXX ?= $(SERCXX)
+
+SOURCES2.0 = \
+	lulesh.cc \
+	lulesh-comm.cc \
+	lulesh-viz.cc \
+	lulesh-util.cc \
+	lulesh-init.cc
+OBJECTS2.0 = $(SOURCES2.0:.cc=.o)
+OBJECTS2.0-tsan = $(SOURCES2.0:.cc=.tsan.o)
+OBJECTS2.0-mpi-tsan = $(SOURCES2.0:.cc=.mpi-tsan.o)
+OBJECTS2.0-mpi = $(SOURCES2.0:.cc=.mpi.o)
+
+#Default build suggestions with OpenMP for g++
+CXXFLAGS = -g -O3 -fopenmp -I. -Wall
+LDFLAGS = -g -O3 -fopenmp
+
+ifndef WITH_FLOAT
+CXXFLAGS += -DUSE_DOUBLE=1
+endif
+
+#Below are reasonable default flags for a serial build
+#CXXFLAGS = -g -O3 -I. -Wall
+#LDFLAGS = -g -O3 
+
+#common places you might find silo on the Livermore machines.
+#SILO_INCDIR = /opt/local/include
+#SILO_LIBDIR = /opt/local/lib
+#SILO_INCDIR = ./silo/4.9/1.8.10.1/include
+#SILO_LIBDIR = ./silo/4.9/1.8.10.1/lib
+
+#If you do not have silo and visit you can get them at:
+#silo:  https://wci.llnl.gov/codes/silo/downloads.html
+#visit: https://wci.llnl.gov/codes/visit/download.html
+
+#below is and example of how to make with silo, hdf5 to get vizulization by default all this is turned off.  All paths are Livermore specific.
+#CXXFLAGS = -g -DVIZ_MESH -I${SILO_INCDIR} -Wall -Wno-pragmas
+#LDFLAGS = -g -L${SILO_LIBDIR} -Wl,-rpath -Wl,${SILO_LIBDIR} -lsiloh5 -lhdf5
+
+%.o: %.cc lulesh.h
+	@echo "Building $<"
+	$(CXX) -c $(CXXFLAGS) -DUSE_MPI=0 -o $@  $<
+
+%.tsan.o: %.cc lulesh.h
+	@echo "Building $<"
+	$(CXX) -c $(CXXFLAGS) -fsanitize=thread -DUSE_MPI=0 -o $@  $<
+
+%.mpi-tsan.o: %.cc lulesh.h
+	@echo "Building $<"
+	MPICH_CXX=$(CXX) $(MPICXX) -c $(CXXFLAGS) -fsanitize=thread -DUSE_MPI=1 -o $@  $<
+
+%.mpi.o: %.cc lulesh.h
+	@echo "Building $<"
+	$(MPICXX) -c $(CXXFLAGS) -DUSE_MPI=1 -o $@  $<
+
+all: $(LULESH_EXEC) $(LULESH_EXEC)-mpi $(LULESH_EXEC)-tsan
+
+$(LULESH_EXEC): $(OBJECTS2.0)
+	@echo "Linking"
+	$(CXX) $(OBJECTS2.0) $(LDFLAGS) -lm -o $@
+
+$(LULESH_EXEC)-mpi: $(OBJECTS2.0-mpi)
+	@echo "Linking"
+	$(MPICXX) $(OBJECTS2.0-mpi) $(LDFLAGS) -lm -o $@
+
+$(LULESH_EXEC)-tsan: $(OBJECTS2.0-tsan)
+	@echo "Linking"
+	$(CXX) $(OBJECTS2.0-tsan) $(LDFLAGS) -fsanitize=thread -lm -o $@
+
+$(LULESH_EXEC)-mpi-tsan: $(OBJECTS2.0-mpi-tsan)
+	@echo "Linking"
+	$(MPICXX) $(OBJECTS2.0-mpi-tsan) $(LDFLAGS) -fsanitize=thread -lm -o $@
+
+$(LULESH_EXEC)-mpi-tsan2: $(OBJECTS2.0-mpi-tsan)
+	@echo "Linking"
+	$(MPICXX) -Wl,--whole-archive ${MUST_ROOT}/lib/libonReportLoader.a -Wl,--no-whole-archive $(OBJECTS2.0-mpi-tsan) $(LDFLAGS) -fsanitize=thread -lm -o $@
+
+clean:
+	/bin/rm -f *.o *~ $(OBJECTS) $(LULESH_EXEC)*
+	/bin/rm -rf *.dSYM
+
+tar: clean
+	cd .. ; tar cvf lulesh-2.0.tar LULESH-2.0 ; mv lulesh-2.0.tar LULESH-2.0
+

Hands-on-2/README

+This is the README for LULESH 2.0
+
+More information including LULESH 1.0 can be found at https://codesign.llnl.gov/lulesh.php
+
+If you have any questions or problems please contact:
+
+Ian Karlin <karlin1@llnl.gov> or
+Rob Neely <neely4@llnl.gov>
+
+Also please send any notable results to Ian Karlin <karlin1@llnl.gov> as we are still evaluating the performance of this code.
+
+A Makefile and a CMake build system are provided.
+
+*** Building with CMake ***
+
+Create a build directory and run cmake. Example:
+
+  $ mkdir build; cd build; cmake -DCMAKE_BUILD_TYPE=Release -DMPI_CXX_COMPILER=`which mpicxx` ..
+
+CMake variables:
+
+  CMAKE_BUILD_TYPE      "Debug", "Release", or "RelWithDebInfo"
+
+  CMAKE_CXX_COMPILER    Path to the C++ compiler
+  MPI_CXX_COMPILER      Path to the MPI C++ compiler
+
+  WITH_MPI=On|Off       Build with MPI (Default: On)
+  WITH_OPENMP=On|Off    Build with OpenMP support (Default: On)
+  WITH_SILO=On|Off      Build with support for SILO. (Default: Off).
+  
+  SILO_DIR              Path to SILO library (only needed when WITH_SILO is "On")
+
+*** Notable changes in LULESH 2.0 ***
+
+Split functionality into different files
+lulesh.cc - where most (all?) of the timed functionality lies
+lulesh-comm.cc - MPI functionality
+lulesh-init.cc - Setup code
+lulesh-viz.cc  - Support for visualization option
+lulesh-util.cc - Non-timed functions
+
+The concept of "regions" was added, although every region is the same ideal gas material, and the same sedov blast wave problem is still the only problem its hardcoded to solve. Regions allow two things important to making this proxy app more representative:
+
+Four of the LULESH routines are now performed on a region-by-region basis, making the memory access patterns non-unit stride
+
+Artificial load imbalances can be easily introduced that could impact parallelization strategies.  
+   * The load balance flag changes region assignment.  Region number is raised to the power entered for assignment probability.  Most likely regions changes with MPI process id.
+   * The cost flag raises the cost of ~45% of the regions to evaluate EOS by the entered multiple.  The cost of 5% is 10x the entered
+ multiple.
+
+MPI and OpenMP were added, and coalesced into a single version of the source that can support serial builds, MPI-only, OpenMP-only, and MPI+OpenMP
+
+Added support to write plot files using "poor mans parallel I/O" when linked with the silo library, which in turn can be read by VisIt.
+
+Enabled variable timestep calculation by default (courant condition), which results in an additional reduction.  Also, seeded the initial timestep based on analytical equation to allow scaling to arbitrary size.  Therefore steps to solution will differ from LULESH 1.0.
+
+Default domain (mesh) size reduced from 45^3 to 30^3
+
+Command line options to allow for numerous test cases without needing to recompile
+
+Performance optimizations and code cleanup uncovered during study of LULESH 1.0
+
+Added a "Figure of Merit" calculation (elements solved per microsecond) and output in support of using LULESH 2.0 for the 2017 CORAL procurement
+
+*** Notable changes in LULESH 2.1 ***
+
+Minor bug fixes.
+Code cleanup to add consitancy to variable names, loop indexing, memory allocation/deallocation, etc.
+Destructor added to main class to clean up when code exits.
+
+
+Possible Future 2.0 minor updates (other changes possible as discovered)
+
+* Different default parameters
+* Minor code performance changes and cleanupS
+
+TODO in future versions
+* Add reader for (truly) unstructured meshes, probably serial only
+
+

Hands-on-2/README.md

+# Hands-On 2
+## Prepare environment
+* `module use /pc2/groups/hpc-prf-nhrgs/modules`
+* `module load MUST`
+* `export MPICH_CXX=clang++`
+* `export TSAN_OPTIONS='ignore_noninstrumented_modules=1'`
+
+## Build Application
+* `MPICXX=mpigxx make -j4`
+
+## Test for DataRace with Archer
+* `OMP_NUM_THREADS=2 ./lulesh2.0-tsan -s 4 -i 2`
+
+Look into the code and try to understand the data race (do not fix them yet)
+
+### Test for DataRace in MUST
+* `MPICXX=mpigxx make -j4 lulesh2.0-mpi-tsan`
+* `OMP_NUM_THREADS=2 mustrun -n 8 --must:hybrid ./lulesh2.0-mpi-tsan -s 4 -i 2`
+
+All TSan output goes to stderr
+
+### Test for DataRace in MUST report
+* `MPICXX=mpigxx make -j4 lulesh2.0-mpi-tsan2`
+* `OMP_NUM_THREADS=2 mustrun -n 8 --must:hybrid --must:tsan ./lulesh2.0-mpi-tsan2 -s 4 -i 2`
+
+TSan output in MUST_Output.html
+
+## Type checking in MUST
+Recompile Lulesh with TypeART:
+* `MPICXX=typeart-mpic++ make -j4 -B lulesh2.0-mpi`
+Execute with TypeART analysis:
+* `OMP_NUM_THREADS=1 mustrun -n 8 --must:typeart --must:output stdout ./lulesh2.0-mpi -i 2`
+
+
+
+## References
+* MUST: https://itc.rwth-aachen.de/must
+* Archer: https://github.com/llvm/llvm-project/tree/main/openmp/tools/archer
+* TypeART: https://github.com/tudasc/TypeART
\ No newline at end of file

Hands-on-2/filterfile.txt

+messageType: MUST_WARNING_REQUEST_NULL_OR_INACTIVE_ARRAY:*
\ No newline at end of file

Hands-on-2/lulesh-util.cc

+#include <string.h>
+#include <stdlib.h>
+#include <ctype.h>
+#include <stdio.h>
+#include <iostream>
+#include <iomanip>
+#if USE_MPI
+#include <mpi.h>
+#endif
+#include "lulesh.h"
+
+/* Helper function for converting strings to ints, with error checking */
+template<typename IntT>
+int StrToInt(const char *token, IntT *retVal)
+{
+   const char *c ;
+   char *endptr ;
+   const int decimal_base = 10 ;
+
+   if (token == NULL)
+      return 0 ;
+   
+   c = token ;
+   *retVal = strtol(c, &endptr, decimal_base) ;
+   if((endptr != c) && ((*endptr == ' ') || (*endptr == '\0')))
+      return 1 ;
+   else
+      return 0 ;
+}
+
+static void PrintCommandLineOptions(char *execname, int myRank)
+{
+   if (myRank == 0) {
+
+      printf("Usage: %s [opts]\n", execname);
+      printf(" where [opts] is one or more of:\n");
+      printf(" -q              : quiet mode - suppress all stdout\n");
+      printf(" -i <iterations> : number of cycles to run\n");
+      printf(" -s <size>       : length of cube mesh along side\n");
+      printf(" -r <numregions> : Number of distinct regions (def: 11)\n");
+      printf(" -b <balance>    : Load balance between regions of a domain (def: 1)\n");
+      printf(" -c <cost>       : Extra cost of more expensive regions (def: 1)\n");
+      printf(" -f <numfiles>   : Number of files to split viz dump into (def: (np+10)/9)\n");
+      printf(" -p              : Print out progress\n");
+      printf(" -v              : Output viz file (requires compiling with -DVIZ_MESH\n");
+      printf(" -h              : This message\n");
+      printf("\n\n");
+   }
+}
+
+static void ParseError(const char *message, int myRank)
+{
+   if (myRank == 0) {
+      printf("%s\n", message);
+#if USE_MPI      
+      MPI_Abort(MPI_COMM_WORLD, -1);
+#else
+      exit(-1);
+#endif
+   }
+}
+
+void ParseCommandLineOptions(int argc, char *argv[],
+                             Int_t myRank, struct cmdLineOpts *opts)
+{
+   if(argc > 1) {
+      int i = 1;
+
+      while(i < argc) {
+         int ok;
+         /* -i <iterations> */
+         if(strcmp(argv[i], "-i") == 0) {
+            if (i+1 >= argc) {
+               ParseError("Missing integer argument to -i", myRank);
+            }
+            ok = StrToInt(argv[i+1], &(opts->its));
+            if(!ok) {
+               ParseError("Parse Error on option -i integer value required after argument\n", myRank);
+            }
+            i+=2;
+         }
+         /* -s <size, sidelength> */
+         else if(strcmp(argv[i], "-s") == 0) {
+            if (i+1 >= argc) {
+               ParseError("Missing integer argument to -s\n", myRank);
+            }
+            ok = StrToInt(argv[i+1], &(opts->nx));
+            if(!ok) {
+               ParseError("Parse Error on option -s integer value required after argument\n", myRank);
+            }
+            i+=2;
+         }
+	 /* -r <numregions> */
+         else if (strcmp(argv[i], "-r") == 0) {
+            if (i+1 >= argc) {
+               ParseError("Missing integer argument to -r\n", myRank);
+            }
+            ok = StrToInt(argv[i+1], &(opts->numReg));
+            if (!ok) {
+               ParseError("Parse Error on option -r integer value required after argument\n", myRank);
+            }
+            i+=2;
+         }
+	 /* -f <numfilepieces> */
+         else if (strcmp(argv[i], "-f") == 0) {
+            if (i+1 >= argc) {
+               ParseError("Missing integer argument to -f\n", myRank);
+            }
+            ok = StrToInt(argv[i+1], &(opts->numFiles));
+            if (!ok) {
+               ParseError("Parse Error on option -f integer value required after argument\n", myRank);
+            }
+            i+=2;
+         }
+         /* -p */
+         else if (strcmp(argv[i], "-p") == 0) {
+            opts->showProg = 1;
+            i++;
+         }
+         /* -q */
+         else if (strcmp(argv[i], "-q") == 0) {
+            opts->quiet = 1;
+            i++;
+         }
+         else if (strcmp(argv[i], "-b") == 0) {
+            if (i+1 >= argc) {
+               ParseError("Missing integer argument to -b\n", myRank);
+            }
+            ok = StrToInt(argv[i+1], &(opts->balance));
+            if (!ok) {
+               ParseError("Parse Error on option -b integer value required after argument\n", myRank);
+            }
+            i+=2;
+         }
+         else if (strcmp(argv[i], "-c") == 0) {
+            if (i+1 >= argc) {
+               ParseError("Missing integer argument to -c\n", myRank);
+            }
+            ok = StrToInt(argv[i+1], &(opts->cost));
+            if (!ok) {
+               ParseError("Parse Error on option -c integer value required after argument\n", myRank);
+            }
+            i+=2;
+         }
+         /* -v */
+         else if (strcmp(argv[i], "-v") == 0) {
+#if VIZ_MESH            
+            opts->viz = 1;
+#else
+            ParseError("Use of -v requires compiling with -DVIZ_MESH\n", myRank);
+#endif
+            i++;
+         }
+         /* -h */
+         else if (strcmp(argv[i], "-h") == 0) {
+            PrintCommandLineOptions(argv[0], myRank);
+#if USE_MPI            
+            MPI_Abort(MPI_COMM_WORLD, 0);
+#else
+            exit(0);
+#endif
+         }
+         else {
+            char msg[80];
+            PrintCommandLineOptions(argv[0], myRank);
+            sprintf(msg, "ERROR: Unknown command line argument: %s\n", argv[i]);
+            ParseError(msg, myRank);
+         }
+      }
+   }
+}
+
+/////////////////////////////////////////////////////////////////////
+
+void VerifyAndWriteFinalOutput(Real_t elapsed_time,
+                               Domain& locDom,
+                               Int_t nx,
+                               Int_t numRanks)
+{
+   // GrindTime1 only takes a single domain into account, and is thus a good way to measure
+   // processor speed indepdendent of MPI parallelism.
+   // GrindTime2 takes into account speedups from MPI parallelism.
+   // Cast to 64-bit integer to avoid overflows.
+   Int8_t nx8 = nx;
+   Real_t grindTime1 = ((elapsed_time*1e6)/locDom.cycle())/(nx8*nx8*nx8);
+   Real_t grindTime2 = ((elapsed_time*1e6)/locDom.cycle())/(nx8*nx8*nx8*numRanks);
+
+   Index_t ElemId = 0;
+   std::cout << "Run completed:\n";
+   std::cout << "   Problem size        =  " << nx       << "\n";
+   std::cout << "   MPI tasks           =  " << numRanks << "\n";
+   std::cout << "   Iteration count     =  " << locDom.cycle() << "\n";
+   std::cout << "   Final Origin Energy =  ";
+   std::cout << std::scientific << std::setprecision(6);
+   std::cout << std::setw(12) << locDom.e(ElemId) << "\n";
+
+   Real_t   MaxAbsDiff = Real_t(0.0);
+   Real_t TotalAbsDiff = Real_t(0.0);
+   Real_t   MaxRelDiff = Real_t(0.0);
+
+   for (Index_t j=0; j<nx; ++j) {
+      for (Index_t k=j+1; k<nx; ++k) {
+         Real_t AbsDiff = FABS(locDom.e(j*nx+k)-locDom.e(k*nx+j));
+         TotalAbsDiff  += AbsDiff;
+
+         if (MaxAbsDiff <AbsDiff) MaxAbsDiff = AbsDiff;
+
+         Real_t RelDiff = AbsDiff / locDom.e(k*nx+j);
+
+         if (MaxRelDiff <RelDiff)  MaxRelDiff = RelDiff;
+      }
+   }
+
+   // Quick symmetry check
+   std::cout << "   Testing Plane 0 of Energy Array on rank 0:\n";
+   std::cout << "        MaxAbsDiff   = " << std::setw(12) << MaxAbsDiff   << "\n";
+   std::cout << "        TotalAbsDiff = " << std::setw(12) << TotalAbsDiff << "\n";
+   std::cout << "        MaxRelDiff   = " << std::setw(12) << MaxRelDiff   << "\n";
+
+   // Timing information
+   std::cout.unsetf(std::ios_base::floatfield);
+   std::cout << std::setprecision(2);
+   std::cout << "\nElapsed time         = " << std::setw(10) << elapsed_time << " (s)\n";
+   std::cout << std::setprecision(8);
+   std::cout << "Grind time (us/z/c)  = "  << std::setw(10) << grindTime1 << " (per dom)  ("
+             << std::setw(10) << elapsed_time << " overall)\n";
+   std::cout << "FOM                  = " << std::setw(10) << 1000.0/grindTime2 << " (z/s)\n\n";
+
+   return ;
+}

Hands-on-2/lulesh-viz.cc

+#include <stdio.h>
+#include <string.h>
+#include <stdlib.h>
+#include <math.h>
+#include "lulesh.h"
+
+#ifdef VIZ_MESH
+
+#ifdef __cplusplus
+  extern "C" {
+#endif
+#include "silo.h"
+#if USE_MPI
+# include "pmpio.h"
+#endif
+#ifdef __cplusplus
+  }
+#endif
+
+// Function prototypes
+static void 
+DumpDomainToVisit(DBfile *db, Domain& domain, int myRank);
+static
+
+
+#if USE_MPI
+// For some reason, earlier versions of g++ (e.g. 4.2) won't let me
+// put the 'static' qualifier on this prototype, even if it's done
+// consistently in the prototype and definition
+void
+DumpMultiblockObjects(DBfile *db, PMPIO_baton_t *bat, 
+                      char basename[], int numRanks);
+
+// Callback prototypes for PMPIO interface (only useful if we're
+// running parallel)
+static void *
+LULESH_PMPIO_Create(const char *fname,
+		     const char *dname,
+		     void *udata);
+static void *
+LULESH_PMPIO_Open(const char *fname,
+		   const char *dname,
+		   PMPIO_iomode_t ioMode,
+		   void *udata);
+static void
+LULESH_PMPIO_Close(void *file, void *udata);
+
+#else
+void
+DumpMultiblockObjects(DBfile *db, char basename[], int numRanks);
+#endif
+
+
+/**********************************************************************/
+void DumpToVisit(Domain& domain, int numFiles, int myRank, int numRanks) 
+{
+  char subdirName[32];
+  char basename[32];
+  DBfile *db;
+
+
+  sprintf(basename, "lulesh_plot_c%d", domain.cycle());
+  sprintf(subdirName, "data_%d", myRank);
+
+#if USE_MPI
+
+  PMPIO_baton_t *bat = PMPIO_Init(numFiles,
+				  PMPIO_WRITE,
+				  MPI_COMM_WORLD,
+				  10101,
+				  LULESH_PMPIO_Create,
+				  LULESH_PMPIO_Open,
+				  LULESH_PMPIO_Close,
+				  NULL);
+
+  int myiorank = PMPIO_GroupRank(bat, myRank);
+
+  char fileName[64];
+  
+  if (myiorank == 0) 
+    strcpy(fileName, basename);
+  else
+    sprintf(fileName, "%s.%03d", basename, myiorank);
+
+  db = (DBfile*)PMPIO_WaitForBaton(bat, fileName, subdirName);
+
+  DumpDomainToVisit(db, domain, myRank);
+
+  // Processor 0 writes out bit of extra data to its file that
+  // describes how to stitch all the pieces together
+  if (myRank == 0) {
+    DumpMultiblockObjects(db, bat, basename, numRanks);
+  }
+
+  PMPIO_HandOffBaton(bat, db);
+
+  PMPIO_Finish(bat);
+#else
+
+  db = (DBfile*)DBCreate(basename, DB_CLOBBER, DB_LOCAL, NULL, DB_HDF5X);
+
+  if (db) {
+     DBMkDir(db, subdirName);
+     DBSetDir(db, subdirName);
+     DumpDomainToVisit(db, domain, myRank);
+     DumpMultiblockObjects(db, basename, numRanks);
+     DBClose(db);
+  }
+  else {
+     printf("Error writing out viz file - rank %d\n", myRank);
+  }
+
+#endif
+}
+
+
+
+/**********************************************************************/
+
+static void 
+DumpDomainToVisit(DBfile *db, Domain& domain, int myRank)
+{
+   int ok = 0;
+   
+   /* Create an option list that will give some hints to VisIt for
+    * printing out the cycle and time in the annotations */
+   DBoptlist *optlist;
+
+
+   /* Write out the mesh connectivity in fully unstructured format */
+   int shapetype[1] = {DB_ZONETYPE_HEX};
+   int shapesize[1] = {8};
+   int shapecnt[1] = {domain.numElem()};
+   int *conn = new int[domain.numElem()*8] ;
+   int ci = 0 ;
+   for (int ei=0; ei < domain.numElem(); ++ei) {
+      Index_t *elemToNode = domain.nodelist(ei) ;
+      for (int ni=0; ni < 8; ++ni) {
+         conn[ci++] = elemToNode[ni] ;
+      }
+   }
+   ok += DBPutZonelist2(db, "connectivity", domain.numElem(), 3,
+                        conn, domain.numElem()*8,
+                        0,0,0, /* Not carrying ghost zones */
+                        shapetype, shapesize, shapecnt,
+                        1, NULL);
+   delete [] conn ;
+
+   /* Write out the mesh coordinates associated with the mesh */
+   const char* coordnames[3] = {"X", "Y", "Z"};
+   float *coords[3] ;
+   coords[0] = new float[domain.numNode()] ;
+   coords[1] = new float[domain.numNode()] ;
+   coords[2] = new float[domain.numNode()] ;
+   for (int ni=0; ni < domain.numNode() ; ++ni) {
+      coords[0][ni] = float(domain.x(ni)) ;
+      coords[1][ni] = float(domain.y(ni)) ;
+      coords[2][ni] = float(domain.z(ni)) ;
+   }
+   optlist = DBMakeOptlist(2);
+   ok += DBAddOption(optlist, DBOPT_DTIME, &domain.time());
+   ok += DBAddOption(optlist, DBOPT_CYCLE, &domain.cycle());
+   ok += DBPutUcdmesh(db, "mesh", 3, (char**)&coordnames[0], (float**)coords,
+                      domain.numNode(), domain.numElem(), "connectivity",
+                      0, DB_FLOAT, optlist);
+   ok += DBFreeOptlist(optlist);
+   delete [] coords[2] ;
+   delete [] coords[1] ;
+   delete [] coords[0] ;
+
+   /* Write out the materials */
+   int *matnums = new int[domain.numReg()];
+   int dims[1] = {domain.numElem()}; // No mixed elements
+   for(int i=0 ; i<domain.numReg() ; ++i)
+      matnums[i] = i+1;
+   
+   ok += DBPutMaterial(db, "regions", "mesh", domain.numReg(),
+                       matnums, domain.regNumList(), dims, 1,
+                       NULL, NULL, NULL, NULL, 0, DB_FLOAT, NULL);
+   delete [] matnums;
+
+   /* Write out pressure, energy, relvol, q */
+
+   float *e = new float[domain.numElem()] ; 
+   for (int ei=0; ei < domain.numElem(); ++ei) {
+      e[ei] = float(domain.e(ei)) ;
+   }
+   ok += DBPutUcdvar1(db, "e", "mesh", e,
+                      domain.numElem(), NULL, 0, DB_FLOAT, DB_ZONECENT,
+                      NULL);
+   delete [] e ;
+
+
+   float *p = new float[domain.numElem()] ; 
+   for (int ei=0; ei < domain.numElem(); ++ei) {
+      p[ei] = float(domain.p(ei)) ;
+   }
+   ok += DBPutUcdvar1(db, "p", "mesh", p,
+                      domain.numElem(), NULL, 0, DB_FLOAT, DB_ZONECENT,
+                      NULL);
+   delete [] p ;
+
+   float *v = new float[domain.numElem()] ; 
+   for (int ei=0; ei < domain.numElem(); ++ei) {
+      v[ei] = float(domain.v(ei)) ;
+   }
+   ok += DBPutUcdvar1(db, "v", "mesh", v,
+                      domain.numElem(), NULL, 0, DB_FLOAT, DB_ZONECENT,
+                      NULL);
+   delete [] v ;
+
+   float *q = new float[domain.numElem()] ; 
+   for (int ei=0; ei < domain.numElem(); ++ei) {
+      q[ei] = float(domain.q(ei)) ;
+   }
+   ok += DBPutUcdvar1(db, "q", "mesh", q,
+                      domain.numElem(), NULL, 0, DB_FLOAT, DB_ZONECENT,
+                      NULL);
+   delete [] q ;
+
+   /* Write out nodal speed, velocities */
+   float *zd    = new float[domain.numNode()];
+   float *yd    = new float[domain.numNode()];
+   float *xd    = new float[domain.numNode()];
+   float *speed = new float[domain.numNode()];
+   for(int ni=0 ; ni < domain.numNode() ; ++ni) {
+      xd[ni]    = float(domain.xd(ni));
+      yd[ni]    = float(domain.yd(ni));
+      zd[ni]    = float(domain.zd(ni));
+      speed[ni] = float(sqrt((xd[ni]*xd[ni])+(yd[ni]*yd[ni])+(zd[ni]*zd[ni])));
+   }
+
+   ok += DBPutUcdvar1(db, "speed", "mesh", speed,
+                      domain.numNode(), NULL, 0, DB_FLOAT, DB_NODECENT,
+                      NULL);
+   delete [] speed;
+
+
+   ok += DBPutUcdvar1(db, "xd", "mesh", xd,
+                      domain.numNode(), NULL, 0, DB_FLOAT, DB_NODECENT,
+                      NULL);
+   delete [] xd ;
+
+   ok += DBPutUcdvar1(db, "yd", "mesh", yd,
+                      domain.numNode(), NULL, 0, DB_FLOAT, DB_NODECENT,
+                      NULL);
+   delete [] yd ;
+
+   ok += DBPutUcdvar1(db, "zd", "mesh", zd,
+                      domain.numNode(), NULL, 0, DB_FLOAT, DB_NODECENT,
+                      NULL);
+   delete [] zd ;
+
+
+   if (ok != 0) {
+      printf("Error writing out viz file - rank %d\n", myRank);
+   }
+}
+
+/**********************************************************************/
+
+#if USE_MPI     
+void
+   DumpMultiblockObjects(DBfile *db, PMPIO_baton_t *bat, 
+                         char basename[], int numRanks)
+#else
+void
+  DumpMultiblockObjects(DBfile *db, char basename[], int numRanks)
+#endif
+{
+   /* MULTIBLOCK objects to tie together multiple files */
+  char **multimeshObjs;
+  char **multimatObjs;
+  char ***multivarObjs;
+  int *blockTypes;
+  int *varTypes;
+  int ok = 0;
+  // Make sure this list matches what's written out above
+  char vars[][10] = {"p","e","v","q", "speed", "xd", "yd", "zd"};
+  int numvars = sizeof(vars)/sizeof(vars[0]);
+
+  // Reset to the root directory of the silo file
+  DBSetDir(db, "/");
+
+  // Allocate a bunch of space for building up the string names
+  multimeshObjs = new char*[numRanks];
+  multimatObjs = new char*[numRanks];
+  multivarObjs = new char**[numvars];
+  blockTypes = new int[numRanks];
+  varTypes = new int[numRanks];
+
+  for(int v=0 ; v<numvars ; ++v) {
+     multivarObjs[v] = new char*[numRanks];
+  }
+  
+  for(int i=0 ; i<numRanks ; ++i) {
+     multimeshObjs[i] = new char[64];
+     multimatObjs[i] = new char[64];
+     for(int v=0 ; v<numvars ; ++v) {
+        multivarObjs[v][i] = new char[64];
+     }
+     blockTypes[i] = DB_UCDMESH;
+     varTypes[i] = DB_UCDVAR;
+  }
+      
+  // Build up the multiobject names
+  for(int i=0 ; i<numRanks ; ++i) {
+#if USE_MPI     
+    int iorank = PMPIO_GroupRank(bat, i);
+#else
+    int iorank = 0;
+#endif
+
+    //delete multivarObjs[i];
+    if (iorank == 0) {
+      snprintf(multimeshObjs[i], 64, "/data_%d/mesh", i);
+      snprintf(multimatObjs[i], 64, "/data_%d/regions",i);
+      for(int v=0 ; v<numvars ; ++v) {
+	snprintf(multivarObjs[v][i], 64, "/data_%d/%s", i, vars[v]);
+      }
+     
+    }
+    else {
+      snprintf(multimeshObjs[i], 64, "%s.%03d:/data_%d/mesh",
+               basename, iorank, i);
+      snprintf(multimatObjs[i], 64, "%s.%03d:/data_%d/regions", 
+	       basename, iorank, i);
+      for(int v=0 ; v<numvars ; ++v) {
+         snprintf(multivarObjs[v][i], 64, "%s.%03d:/data_%d/%s", 
+                  basename, iorank, i, vars[v]);
+      }
+    }
+  }
+
+  // Now write out the objects
+  ok += DBPutMultimesh(db, "mesh", numRanks,
+		       (char**)multimeshObjs, blockTypes, NULL);
+  ok += DBPutMultimat(db, "regions", numRanks,
+		      (char**)multimatObjs, NULL);
+  for(int v=0 ; v<numvars ; ++v) {
+     ok += DBPutMultivar(db, vars[v], numRanks,
+                         (char**)multivarObjs[v], varTypes, NULL);
+  }
+
+  for(int v=0; v < numvars; ++v) {
+    for(int i = 0; i < numRanks; i++) {
+      delete multivarObjs[v][i];
+    }
+    delete multivarObjs[v];
+  }
+
+  // Clean up
+  for(int i=0 ; i<numRanks ; i++) {
+    delete multimeshObjs[i];
+    delete multimatObjs[i];
+  }
+  delete [] multimeshObjs;
+  delete [] multimatObjs;
+  delete [] multivarObjs;
+  delete [] blockTypes;
+  delete [] varTypes;
+
+  if (ok != 0) {
+    printf("Error writing out multiXXX objs to viz file - rank 0\n");
+  }
+}
+
+# if USE_MPI
+
+/**********************************************************************/
+
+static void *
+LULESH_PMPIO_Create(const char *fname,
+		     const char *dname,
+		     void *udata)
+{
+   /* Create the file */
+   DBfile* db = DBCreate(fname, DB_CLOBBER, DB_LOCAL, NULL, DB_HDF5X);
+
+   /* Put the data in a subdirectory, so VisIt only sees the multimesh
+    * objects we write out in the base file */
+   if (db) {
+     DBMkDir(db, dname);
+     DBSetDir(db, dname);
+   }
+   return (void*)db;
+}
+
+   
+/**********************************************************************/
+
+static void *
+LULESH_PMPIO_Open(const char *fname,
+		   const char *dname,
+		   PMPIO_iomode_t ioMode,
+		   void *udata)
+{
+   /* Open the file */
+  DBfile* db = DBOpen(fname, DB_UNKNOWN, DB_APPEND);
+
+   /* Put the data in a subdirectory, so VisIt only sees the multimesh
+    * objects we write out in the base file */
+   if (db) {
+     DBMkDir(db, dname);
+     DBSetDir(db, dname);
+   }
+   return (void*)db;
+}
+
+   
+/**********************************************************************/
+
+static void
+LULESH_PMPIO_Close(void *file, void *udata)
+{
+  DBfile *db = (DBfile*)file;
+  if (db)
+    DBClose(db);
+}
+# endif
+
+   
+#else
+
+void DumpToVisit(Domain& domain, int numFiles, int myRank, int numRanks)
+{
+   if (myRank == 0) {
+      printf("Must enable -DVIZ_MESH at compile time to call DumpDomain\n");
+   }
+}
+
+#endif
+