CMake cannot find MPI in standard installation path

The following is my CMakeLists.txt file and actually it works well on my server4. However, I tried to configure it on my server3 and then it could not find the MPI package.


project(CUDA_MPI LANGUAGES CXX CUDA)
set(CMAKE_CXX_FLAGS "-lrt")
find_package(CUDA REQUIRED)
find_package(MPI REQUIRED)

add_library(Timer STATIC timer.h 
						 timer.cpp)
add_library(Coll STATIC  collectives.h 
						collectives.cu)
add_executable(CUDA_MPI ${CMAKE_CURRENT_SOURCE_DIR}/test/test.cpp)

target_link_libraries(Coll PUBLIC Timer)
target_link_libraries(CUDA_MPI PUBLIC Coll MPI_LIB ${CUDA_LIBRARIES})

I have checked the installation path of OpenMPI and it seems that both server are exact same. The path is

openmpi: /usr/lib/x86_64-linux-gnu/openmpi

I want to provide any hints for my CMakeLists file and it also failed. I am confused that why I cannot find the MPI on the server3 but I can do it on another machine. Is it because my different installation way?

The error log is as the following.

Performing C++ SOURCE FILE Test CMAKE_HAVE_LIBC_PTHREAD failed with the following output:
Change Dir: /home/szhangcj/HPC/MPI/baidu-allreduce/build/CMakeFiles/CMakeTmp

Run Build Command(s):/usr/bin/make cmTC_c1a8d/fast && /usr/bin/make -f CMakeFiles/cmTC_c1a8d.dir/build.make CMakeFiles/cmTC_c1a8d.dir/build
make[1]: Entering directory '/home/szhangcj/HPC/MPI/baidu-allreduce/build/CMakeFiles/CMakeTmp'
Building CXX object CMakeFiles/cmTC_c1a8d.dir/src.cxx.o
/home/szhangcj/.conda/envs/distributed/bin/x86_64-conda_cos6-linux-gnu-c++    -lrt -L/home/szhangcj/.conda/envs/distributed/lib -DCMAKE_HAVE_LIBC_PTHREAD   -o CMakeFiles/cmTC_c1a8d.dir/src.cxx.o -c /home/szhangcj/HPC/MPI/baidu-allreduce/build/CMakeFiles/CMakeTmp/src.cxx
Linking CXX executable cmTC_c1a8d
/usr/local/bin/cmake -E cmake_link_script CMakeFiles/cmTC_c1a8d.dir/link.txt --verbose=1
/home/szhangcj/.conda/envs/distributed/bin/x86_64-conda_cos6-linux-gnu-c++  -lrt -L/home/szhangcj/.conda/envs/distributed/lib -DCMAKE_HAVE_LIBC_PTHREAD  -Wl,-O2 -Wl,--sort-common -Wl,--as-needed -Wl,-z,relro -Wl,-z,now -Wl,--disable-new-dtags -Wl,--gc-sections -Wl,-rpath,/home/szhangcj/.conda/envs/distributed/lib -Wl,-rpath-link,/home/szhangcj/.conda/envs/distributed/lib -L/home/szhangcj/.conda/envs/distributed/lib  CMakeFiles/cmTC_c1a8d.dir/src.cxx.o  -o cmTC_c1a8d 
/home/szhangcj/.conda/envs/distributed/bin/../lib/gcc/x86_64-conda_cos6-linux-gnu/7.3.0/../../../../x86_64-conda_cos6-linux-gnu/bin/ld: CMakeFiles/cmTC_c1a8d.dir/src.cxx.o: undefined reference to symbol 'pthread_create@@GLIBC_2.2.5'
/home/szhangcj/.conda/envs/distributed/bin/../lib/gcc/x86_64-conda_cos6-linux-gnu/7.3.0/../../../../x86_64-conda_cos6-linux-gnu/bin/ld: /home/szhangcj/.conda/envs/distributed/bin/../x86_64-conda_cos6-linux-gnu/sysroot/lib/libpthread.so.0: error adding symbols: DSO missing from command line
collect2: error: ld returned 1 exit status
CMakeFiles/cmTC_c1a8d.dir/build.make:86: recipe for target 'cmTC_c1a8d' failed
make[1]: *** [cmTC_c1a8d] Error 1
make[1]: Leaving directory '/home/szhangcj/HPC/MPI/baidu-allreduce/build/CMakeFiles/CMakeTmp'
Makefile:121: recipe for target 'cmTC_c1a8d/fast' failed
make: *** [cmTC_c1a8d/fast] Error 2


Source file was:
#include <pthread.h>

void* test_func(void* data)
{
  return data;
}

int main(void)
{
  pthread_t thread;
  pthread_create(&thread, NULL, test_func, NULL);
  pthread_detach(thread);
  pthread_join(thread, NULL);
  pthread_atfork(NULL, NULL, NULL);
  pthread_exit(NULL);

  return 0;
}

Determining if the function pthread_create exists in the pthreads failed with the following output:
Change Dir: /home/szhangcj/HPC/MPI/baidu-allreduce/build/CMakeFiles/CMakeTmp

Run Build Command(s):/usr/bin/make cmTC_5b4c9/fast && /usr/bin/make -f CMakeFiles/cmTC_5b4c9.dir/build.make CMakeFiles/cmTC_5b4c9.dir/build
make[1]: Entering directory '/home/szhangcj/HPC/MPI/baidu-allreduce/build/CMakeFiles/CMakeTmp'
Building CXX object CMakeFiles/cmTC_5b4c9.dir/CheckFunctionExists.cxx.o
/home/szhangcj/.conda/envs/distributed/bin/x86_64-conda_cos6-linux-gnu-c++    -lrt -L/home/szhangcj/.conda/envs/distributed/lib -DCHECK_FUNCTION_EXISTS=pthread_create   -o CMakeFiles/cmTC_5b4c9.dir/CheckFunctionExists.cxx.o -c /home/szhangcj/HPC/MPI/baidu-allreduce/build/CMakeFiles/CheckLibraryExists/CheckFunctionExists.cxx
Linking CXX executable cmTC_5b4c9
/usr/local/bin/cmake -E cmake_link_script CMakeFiles/cmTC_5b4c9.dir/link.txt --verbose=1
/home/szhangcj/.conda/envs/distributed/bin/x86_64-conda_cos6-linux-gnu-c++  -lrt -L/home/szhangcj/.conda/envs/distributed/lib -DCHECK_FUNCTION_EXISTS=pthread_create  -Wl,-O2 -Wl,--sort-common -Wl,--as-needed -Wl,-z,relro -Wl,-z,now -Wl,--disable-new-dtags -Wl,--gc-sections -Wl,-rpath,/home/szhangcj/.conda/envs/distributed/lib -Wl,-rpath-link,/home/szhangcj/.conda/envs/distributed/lib -L/home/szhangcj/.conda/envs/distributed/lib  CMakeFiles/cmTC_5b4c9.dir/CheckFunctionExists.cxx.o  -o cmTC_5b4c9 -lpthreads 
/home/szhangcj/.conda/envs/distributed/bin/../lib/gcc/x86_64-conda_cos6-linux-gnu/7.3.0/../../../../x86_64-conda_cos6-linux-gnu/bin/ld: cannot find -lpthreads
collect2: error: ld returned 1 exit status
CMakeFiles/cmTC_5b4c9.dir/build.make:86: recipe for target 'cmTC_5b4c9' failed
make[1]: *** [cmTC_5b4c9] Error 1
make[1]: Leaving directory '/home/szhangcj/HPC/MPI/baidu-allreduce/build/CMakeFiles/CMakeTmp'
Makefile:121: recipe for target 'cmTC_5b4c9/fast' failed
make: *** [cmTC_5b4c9/fast] Error 2



The MPI test test_mpi for CXX in mode normal failed to compile with the following output:
Change Dir: /home/szhangcj/HPC/MPI/baidu-allreduce/build/CMakeFiles/CMakeTmp

Run Build Command(s):/usr/bin/make cmTC_01687/fast && /usr/bin/make -f CMakeFiles/cmTC_01687.dir/build.make CMakeFiles/cmTC_01687.dir/build
make[1]: Entering directory '/home/szhangcj/HPC/MPI/baidu-allreduce/build/CMakeFiles/CMakeTmp'
Building CXX object CMakeFiles/cmTC_01687.dir/test_mpi.cpp.o
/home/szhangcj/.conda/envs/distributed/bin/x86_64-conda_cos6-linux-gnu-c++    -lrt -L/home/szhangcj/.conda/envs/distributed/lib    -pthread -o CMakeFiles/cmTC_01687.dir/test_mpi.cpp.o -c /home/szhangcj/HPC/MPI/baidu-allreduce/build/CMakeFiles/FindMPI/test_mpi.cpp
/home/szhangcj/HPC/MPI/baidu-allreduce/build/CMakeFiles/FindMPI/test_mpi.cpp:1:10: fatal error: mpi.h: No such file or directory
 #include <mpi.h>
          ^~~~~~~
compilation terminated.
CMakeFiles/cmTC_01687.dir/build.make:65: recipe for target 'CMakeFiles/cmTC_01687.dir/test_mpi.cpp.o' failed
make[1]: *** [CMakeFiles/cmTC_01687.dir/test_mpi.cpp.o] Error 1
make[1]: Leaving directory '/home/szhangcj/HPC/MPI/baidu-allreduce/build/CMakeFiles/CMakeTmp'
Makefile:121: recipe for target 'cmTC_01687/fast' failed
make: *** [cmTC_01687/fast] Error 2

Do you have the actual error output available? The literal MPI_LIB looks…suspicious to me.

I have edited my question. Please check it.

It seems like something went wrong with the MPI compiler wrapper detection? @chuckatkins Thoughts?

But I can check the configure of my mpicc as the following.

Using built-in specs.
COLLECT_GCC=/usr/bin/gcc
COLLECT_LTO_WRAPPER=/usr/lib/gcc/x86_64-linux-gnu/7/lto-wrapper
OFFLOAD_TARGET_NAMES=nvptx-none
OFFLOAD_TARGET_DEFAULT=1
Target: x86_64-linux-gnu
Configured with: ../src/configure -v --with-pkgversion='Ubuntu 7.4.0-1ubuntu1~18.04.1' --with-bugurl=file:///usr/share/doc/gcc-7/README.Bugs --enable-languages=c,ada,c++,go,brig,d,fortran,objc,obj-c++ --prefix=/usr --with-gcc-major-version-only --program-suffix=-7 --program-prefix=x86_64-linux-gnu- --enable-shared --enable-linker-build-id --libexecdir=/usr/lib --without-included-gettext --enable-threads=posix --libdir=/usr/lib --enable-nls --with-sysroot=/ --enable-clocale=gnu --enable-libstdcxx-debug --enable-libstdcxx-time=yes --with-default-libstdcxx-abi=new --enable-gnu-unique-object --disable-vtable-verify --enable-libmpx --enable-plugin --enable-default-pie --with-system-zlib --with-target-system-zlib --enable-objc-gc=auto --enable-multiarch --disable-werror --with-arch-32=i686 --with-abi=m64 --with-multilib-list=m32,m64,mx32 --enable-multilib --with-tune=generic --enable-offload-targets=nvptx-none --without-cuda-driver --enable-checking=release --build=x86_64-linux-gnu --host=x86_64-linux-gnu --target=x86_64-linux-gnu
Thread model: posix
gcc version 7.4.0 (Ubuntu 7.4.0-1ubuntu1~18.04.1)

By the way, I still cannot find the MPI package even I tried to install the MPI by using conda.

The problem seems to be that FindMPI is not extracting the information directly from it properly or something. It is good that the wrappers do work though :slight_smile: .

Yes. I do not know why it is.