Elmer compilation problem on Cray cluster

Discussion about building and installing Elmer
Post Reply
han2014
Posts: 11
Joined: 20 Jun 2014, 13:51
Antispam: Yes

Elmer compilation problem on Cray cluster

Post by han2014 »

Hello,

I am having problems with compilation of Elmer on a Cray XC30 cluster.

I've downloaded the source code from the repository git://www.github.com/ElmerCSC/elmerfem

My installation script is:

Code: Select all

#!/bin/bash

module swap PrgEnv-cray PrgEnv-gnu
module load cray-libsci
module load cray-tpsl

rm -r CMakeFiles CMakeCache.txt

CMAKE=cmake

# Installation directory (set these!)
TIMESTAMP=$(date +"%m-%d-%y")
ELMER_REV="Elmer_devel_${TIMESTAMP}"
ELMERSRC="/work/e305/e305/zss/elmerfem"
BUILDDIR="$ELMERSRC/builddir"
IDIR="/work/e305/e305/zss/$ELMER_REV"
TOOLCHAIN="$ELMERSRC/toolchain.cmake"

echo "Building Elmer from within " ${BUILDDIR}
echo "using following toolchain file " ${TOOLCHAIN}
echo "installation into " ${IDIR}
cd ${BUILDDIR}
pwd
ls -ltr

echo $CMAKE $ELMERSRC -DCMAKE_TOOLCHAIN_FILE=$TOOLCHAIN

$CMAKE $ELMERSRC -DCMAKE_TOOLCHAIN_FILE=$TOOLCHAIN \
    -DCMAKE_INSTALL_PREFIX=$IDIR \
    -DCMAKE_C_COMPILER=cc \
    -DCMAKE_Fortran_COMPILER=ftn \
    -DWITH_MPI:BOOL=TRUE \
    -DWITH_Mumps:BOOL=TRUE \
    -DWITH_Hypre:BOOL=TRUE \
    -DWITH_TRILINOS:BOOL=FALSE \
    -DWITH_ELMERGUI:BOOL=FALSE \
    -DWITH_OCC:BOOL=TRUE \
    -DWITH_PARAVIEW:BOOL=FALSE \
    -DWITH_QWT:BOOL=TRUE \
    -DWITH_VTK:BOOL=TRUE \
    -DWITH_PYTHONQT:BOOL=FALSE \
    -DWITH_MATC:BOOL=TRUE

make -j4 && make install

rm /work/e305/e305/zss/Elmer-devel
ln -s $IDIR /work/e305/e305/zss/Elmer-devel
My toolchain file (toolchain.cmake) is:

Code: Select all


# CMake toolchain file for building with GNU compilers
#

SET(CMAKE_SYSTEM_NAME Linux)
SET(CMAKE_SYSTEM_PROCESSOR x86_64)
SET(CMAKE_SYSTEM_VERSION 1)

# Specify the cross compilers (serial)
SET(CMAKE_C_COMPILER cc)
SET(CMAKE_Fortran_COMPILER ftn)
SET(CMAKE_CXX_COMPILER CC)

# Specify the cross compilers (parallel)
SET(MPI_C_COMPILER cc)
SET(MPI_CXX_COMPILER CC)
SET(MPI_Fortran_COMPILER ftn)

# Compilation flags (i.e. with optimization)
SET(CMAKE_C_FLAGS "-g -fPIC" CACHE STRING "")
SET(CMAKE_CXX_FLAGS "-g -fPIC" CACHE STRING "")
SET(CMAKE_Fortran_FLAGS "-g -fPIC" CACHE STRING "")

# Reset the default build type flags, all flags should be set above
SET(CMAKE_C_FLAGS_RELWITHDEBINFO "" CACHE STRING "")
SET(CMAKE_CXX_FLAGS_RELWITHDEBINFO "" CACHE STRING "")
SET(CMAKE_Fortran_FLAGS_RELWITHDEBINFO "" CACHE STRING "")

# Compilation flags for native build using C compiler, used for ElmerGrid
SET(NATIVE_BUILD_FLAGS "-g -fPIC -target-cpu=sandybridge" CACHE STRING "")

# Path of Blas and Lapack
SET(BLAS_LIBRARIES "m" CACHE STRING "")
SET(LAPACK_LIBRARIES "m" CACHE STRING "")

# Path of MUMPS
SET(Mumps_INCLUDE_DIR /opt/cray/tpsl/1.4.3/GNU/49/sandybridge/include)
SET(Mumps_LIBRARIES /opt/cray/tpsl/1.4.3/GNU/49/sandybridge/lib/libdmumps.a)
LIST(APPEND Mumps_LIBRARIES /opt/cray/tpsl/1.4.3/GNU/49/sandybridge/lib/libmumps_common.a)
LIST(APPEND Mumps_LIBRARIES /opt/cray/tpsl/1.4.3/GNU/49/sandybridge/lib/libpord.a)
LIST(APPEND Mumps_LIBRARIES /opt/cray/tpsl/1.4.3/GNU/49/sandybridge/lib/libmetis.a)

# Path of Hypre
SET(Hypre_INCLUDE_DIR /opt/cray/tpsl/1.4.3/GNU/49/sandybridge/include)
SET(Hypre_LIBRARIES /opt/cray/tpsl/1.4.3/GNU/49/sandybridge/lib/libHYPRE.a)

# Set the result of GFortran version test for cross compiling
SET( GFORTRAN_VERSIONTEST_RUN_RESULT 
     "0"
     CACHE STRING "Result from TRY_RUN" FORCE)

SET( GFORTRAN_VERSIONTEST_RUN_RESULT__TRYRUN_OUTPUT 
     "4.9"
     CACHE STRING "Output from TRY_RUN" FORCE)
SET ( MPIEXEC "aprun" CACHE STRING "")
SET ( MPIEXEC_NUMPROC_FLAG  "-n" CACHE STRING "")

and this is the log file with compilation error that I got:
cmake /work/e305/e305/zss/elmerfem -DCMAKE_TOOLCHAIN_FILE=/work/e305/e305/zss/elmerfem/toolchain.cmake
-- The Fortran compiler identification is GNU
-- The C compiler identification is GNU 4.9.2
-- The CXX compiler identification is GNU 4.9.2
-- Check for working Fortran compiler: /opt/cray/craype/2.2.1/bin/ftn
-- Check for working Fortran compiler: /opt/cray/craype/2.2.1/bin/ftn -- works
-- Detecting Fortran compiler ABI info
-- Detecting Fortran compiler ABI info - done
-- Checking whether /opt/cray/craype/2.2.1/bin/ftn supports Fortran 90
-- Checking whether /opt/cray/craype/2.2.1/bin/ftn supports Fortran 90 -- yes
-- Check for working C compiler: /opt/cray/craype/2.2.1/bin/cc
-- Check for working C compiler: /opt/cray/craype/2.2.1/bin/cc -- works
-- Detecting C compiler ABI info
-- Detecting C compiler ABI info - done
-- Check for working CXX compiler: /opt/cray/craype/2.2.1/bin/CC
-- Check for working CXX compiler: /opt/cray/craype/2.2.1/bin/CC -- works
-- Detecting CXX compiler ABI info
-- Detecting CXX compiler ABI info - done
-- Checking whether GFortran version >= 4.8 -- yes
-- Unable to determine MPI from MPI driver /opt/cray/craype/2.2.1/bin/cc
-- Found MPI_C: /opt/cray/craype/2.2.1/bin/cc
-- Unable to determine MPI from MPI driver /opt/cray/craype/2.2.1/bin/CC
-- Found MPI_CXX: /opt/cray/craype/2.2.1/bin/CC
-- Unable to determine MPI from MPI driver /opt/cray/craype/2.2.1/bin/ftn
-- Found MPI_Fortran: /opt/cray/craype/2.2.1/bin/ftn
-- Looking for include file pthread.h
-- Looking for include file pthread.h - found
-- Looking for pthread_create
-- Looking for pthread_create - found
-- Found Threads: TRUE
-- A library with BLAS API found.
-- A library with BLAS API found.
-- A library with LAPACK API found.
-- Checking whether /opt/cray/craype/2.2.1/bin/ftn supports PROCEDURE POINTER
-- Checking whether /opt/cray/craype/2.2.1/bin/ftn supports PROCEDURE POINTER -- yes
-- Checking whether /opt/cray/craype/2.2.1/bin/ftn supports CONTIGUOUS
-- Checking whether /opt/cray/craype/2.2.1/bin/ftn supports CONTIGUOUS -- yes
-- Checking whether /opt/cray/craype/2.2.1/bin/ftn supports EXECUTE_COMMAND_LINE
-- Checking whether /opt/cray/craype/2.2.1/bin/ftn supports EXECUTE_COMMAND_LINE -- yes
-- Looking for include file inttypes.h
-- Looking for include file inttypes.h - found
-- Looking for sys/types.h
-- Looking for sys/types.h - found
-- Looking for stdint.h
-- Looking for stdint.h - found
-- Looking for stddef.h
-- Looking for stddef.h - found
-- Check size of long
-- Check size of long - done
-- Skipping test WinkelPoissonMetisKwayDual with 16 procs
-- Skipping test WinkelPoissonMetisKwayNodal with 16 procs
-- Resetting build flags for Elmergrid: -g -fPIC -target-cpu=sandybridge
-- ------------------------------------------------
-- BLAS library: m
-- LAPACK library: m
-- ------------------------------------------------
-- Fortran compiler: /opt/cray/craype/2.2.1/bin/ftn
-- Fortran flags: -g -fPIC
-- ------------------------------------------------
-- C compiler: /opt/cray/craype/2.2.1/bin/cc
-- C flags: -g -fPIC
-- ------------------------------------------------
-- CXX compiler: /opt/cray/craype/2.2.1/bin/CC
-- CXX flags: -g -fPIC
-- ------------------------------------------------
-- MPI Fortran: FALSE
-- MPI Fortran compiler: /opt/cray/craype/2.2.1/bin/ftn
-- MPI Fortran flags:
-- MPI Fortran include dir:
-- MPI Fortran libraries:
-- MPI Fortran link flags:
-- ------------------------------------------------
-- MPI C: TRUE
-- MPI C compiler: /opt/cray/craype/2.2.1/bin/cc
-- MPI C flags:
-- MPI C include dir:
-- MPI C libraries:
-- MPI C flags:
-- ------------------------------------------------
-- Mumps: TRUE
-- Mumps include: /opt/cray/tpsl/1.4.3/GNU/49/sandybridge/include
-- Mumps libraries: /opt/cray/tpsl/1.4.3/GNU/49/sandybridge/lib/libdmumps.a;/opt/cray/tpsl/1.4.3/GNU/49/sandybridge/lib/libmumps_common.a;/opt/cray/tpsl/1.4.3/GNU/49/sandybridge/lib/libpord.a;/opt/cray/tpsl/1.4.3/GNU/49/sandybridge/lib/libmetis.a
-- ------------------------------------------------
-- Hypre: TRUE
-- Hypre include: /opt/cray/tpsl/1.4.3/GNU/49/sandybridge/include
-- Hypre libraries: /opt/cray/tpsl/1.4.3/GNU/49/sandybridge/lib/libHYPRE.a
-- ------------------------------------------------
-- ------------------------------------------------
-- Package filename: elmerfem-8.1-a450ffc-20160303_Linux-x86_64
-- Patch version: 8.1-a450ffc
-- Configuring done
-- Generating done
CMake Warning:
Manually-specified variables were not used by the project:

WITH_MATC
WITH_PYTHONQT
WITH_TRILINOS


-- Build files have been written to: /work/e305/e305/zss/elmerfem
[ 0%] [ 0%] Built target amdf77
Built target umfpack_srcs
[ 3%] Built target matc
[ 5%] Built target Mesh2D
[ 12%] Built target arpack
Scanning dependencies of target binio
[ 13%] Built target fhuti
[ 13%] Built target mpi_stubs
[ 13%] Scanning dependencies of target view3d
Building C object fem/src/binio/CMakeFiles/binio.dir/binio.c.o
Scanning dependencies of target viewaxis
[ 14%] [ 14%] Building CXX object fem/src/viewaxis/CMakeFiles/viewaxis.dir/viewfact.cpp.o
Building C object fem/src/view3d/CMakeFiles/view3d.dir/ViewFactors.c.o
[ 17%] Built target amd
[ 18%] Building C object fem/src/view3d/CMakeFiles/view3d.dir/TestModel.c.o
Linking Fortran static library libbinio.a
[ 18%] Built target binio
[ 18%] Linking CXX static library libviewaxis.a
Building C object fem/src/view3d/CMakeFiles/view3d.dir/LineUtil.c.o
[ 18%] Building C object elmergrid/src/metis/CMakeFiles/metis.dir/balance.c.o
[ 18%] Building C object fem/src/view3d/CMakeFiles/view3d.dir/TriangleUtil.c.o
[ 18%] Built target viewaxis
[ 18%] /work/e305/e305/zss/elmerfem/elmergrid/src/metis/balance.c:21:18: error: unknown type name 'CtrlType'
void Balance2Way(CtrlType *ctrl, GraphType *graph, int *tpwgts, float ubfactor)
^
/work/e305/e305/zss/elmerfem/elmergrid/src/metis/balance.c:21:34: error: unknown type name 'GraphType'
void Balance2Way(CtrlType *ctrl, GraphType *graph, int *tpwgts, float ubfactor)
^
/work/e305/e305/zss/elmerfem/elmergrid/src/metis/balance.c:48:21: error: unknown type name 'CtrlType'
void Bnd2WayBalance(CtrlType *ctrl, GraphType *graph, int *tpwgts)
^
/work/e305/e305/zss/elmerfem/elmergrid/src/metis/balance.c:48:37: error: unknown type name 'GraphType'
void Bnd2WayBalance(CtrlType *ctrl, GraphType *graph, int *tpwgts)
^
/work/e305/e305/zss/elmerfem/elmergrid/src/metis/balance.c:174:25: error: unknown type name 'CtrlType'
void General2WayBalance(CtrlType *ctrl, GraphType *graph, int *tpwgts)
^
/work/e305/e305/zss/elmerfem/elmergrid/src/metis/balance.c:174:41: error: unknown type name 'GraphType'
void General2WayBalance(CtrlType *ctrl, GraphType *graph, int *tpwgts)
^
make[2]: *** [elmergrid/src/metis/CMakeFiles/metis.dir/balance.c.o] Error 1
make[1]: *** [elmergrid/src/metis/CMakeFiles/metis.dir/all] Error 2
make[1]: *** Waiting for unfinished jobs....
Building C object fem/src/view3d/CMakeFiles/view3d.dir/BiLinearUtil.c.o
[ 18%] Building C object fem/src/view3d/CMakeFiles/view3d.dir/BiQuadraticUtil.c.o
[ 18%] Building C object fem/src/view3d/CMakeFiles/view3d.dir/BiCubicUtil.c.o
[ 18%] Building C object fem/src/view3d/CMakeFiles/view3d.dir/RayTrace.c.o
[ 18%] Building C object fem/src/view3d/CMakeFiles/view3d.dir/VectorUtil.c.o
[ 18%] Building C object fem/src/view3d/CMakeFiles/view3d.dir/second.c.o
Linking C static library libview3d.a
[ 18%] Built target view3d
make: *** [all] Error 2
The error message suggests that certain data types such as CtrlType and GraphType are not recognised when cmake attempts to compile source files within elmerfem/elmergrid/src/metis. I've also tried to compile Elmer release 8.0 from this source: https://github.com/ElmerCSC/elmerfem/releases, but I encountered the same error.

Can anyone please suggest what can be done to fix this error?

Many thanks.

Kind regards,

-Han-
KaiMartin
Posts: 42
Joined: 02 Jun 2013, 00:07
Antispam: Yes
Location: Hannover, Germany
Contact:

Re: Elmer compilation problem on Cray cluster

Post by KaiMartin »

I probably don't see this because my elmer build takes mepis precompiled from the Debian distribution. Maybe, this provides a way to work around the bug: Compile mepis separately from source directly pulled http://glaros.dtc.umn.edu/gkhome/metis/metis/download .

---<)kaimartin(>---
---<)kaimartin(>---
tzwinger
Site Admin
Posts: 99
Joined: 24 Aug 2009, 12:20
Antispam: Yes

Re: Elmer compilation problem on Cray cluster

Post by tzwinger »

Hello,
I do not know if this is of particular help in your case, but here is the compilation script for GNU environment on our Cray XC40 attached at the end of this message

I think the essential two lines are

Code: Select all

CRAY_ADD_RPATH=yes
export CRAYPE_LINK_TYPE=dynamic
Naturally, you need to change a few paths.

Best Regards,

Thomas


Code: Select all

#!/bin/bash -l
# ################################
#                                #
# Setup and compilation          #
# of Elmer on Cray XC40          #
#                                #
# T. Zwinger & S. Ilvonen        #
# CSC-IT Center for Science Ltd. #
#                                #
# ################################
set -x
# RUN THESE MODULE COMMAND BEFORE YOU START!
module swap PrgEnv-cray PrgEnv-gnu/5.2.56
module load cray-tpsl/1.5.2
module load cray-libsci/13.3.0  #earlier 13.2.0
module load cray-netcdf/4.3.3.1         #(only GriddataReader)
module load cmake/3.2.3
module load git/2.1.0
echo "Setup and compilation"
echo "of Elmer on Cray XC40"
echo "on" date
echo "-------------------------"
echo "Following modules loaded:"
module list
echo "-------------------------"
# necessary declarations
export Hypre_LIBRARIES="-lm"
export Hypre_INCLUDE_DIR=" "
CRAY_ADD_RPATH=yes
export CRAYPE_LINK_TYPE=dynamic
CMAKE=cmake


# Directories (set these!)
ELMERSRC="/wrk/zwinger/elmerfem"
BUILDDIR="/wrk/zwinger/builddir_sisu_gnu"

# use toolchain file for x-compilation to CNL-target
TOOLCHAIN="$ELMERSRC/cmake/Toolchains/Elmer-CNL-sisu.cmake"

# get GIT-tag and construct installation name from it
cd ${ELMERSRC}
echo "git branch -a:"
git branch -a
TIMESTAMP=$(date +"%m-%d-%y")
VERSION=$(git log -1 --pretty=format:%h)
ELMER_REV="Elmer_${VERSION}_${TIMESTAMP}"
IDIR="/appl/math/elmer/gnu_5.2.56/$ELMER_REV"

echo "Building Elmer from within " ${BUILDDIR}
echo "using following toolchain file " ${TOOLCHAIN}
echo "installation into: " ${IDIR}
echo "-------------------------"
cd ${BUILDDIR}
pwd
ls -ltr

echo $CMAKE $ELMERSRC 

# configure it
$CMAKE -Wno-dev $ELMERSRC \
    -DCMAKE_INSTALL_PREFIX=$IDIR \
    -DCMAKE_TOOLCHAIN_FILE=$TOOLCHAIN \
    -DWITH_MPI:BOOL=TRUE \
    -DWITH_Trilinos:BOOL=FALSE \
    -DWITH_Mumps:BOOL=TRUE \
    -DWITH_Hypre:BOOL=TRUE \
    -DWITH_ELMERGUI:BOOL=FALSE \
    -DWITH_ElmerIce:BOOL=TRUE \
    -DHypre_LIBRARIES="-lm" \
    -DHypre_INCLUDE_DIR=" " \
    -DMumps_LIBRARIES="-lm" \
    -DMumps_INCLUDE_DIR=" "\
    -DCMAKE_CXX_FLAGS_RELWITHDEBINFO="-g -O3"\
    -DCMAKE_C_FLAGS_RELWITHDEBINFO="-g -O3"\
    -DCMAKE_Fortran_FLAGS_RELWITHDEBINFO="-g -O3"
echo "Done configuring"
echo "-------------------------"
echo "Starting to build:"

# compile and install it (last step might be in need of a sudo, perhaps)
make -j 4 && make install && cp ~/Source/Elmer/Sisu/compilation_sisu_gnuCmake.sh $IDIR && /appl/opt/fix-perms.sh $IDIR
han2014
Posts: 11
Joined: 20 Jun 2014, 13:51
Antispam: Yes

Re: Elmer compilation problem on Cray cluster

Post by han2014 »

Hi KaiMartin & Thomas,

Thanks a lot for the hints. I will give it a try again.

Regards,

Han
Post Reply