Hi,
We are trying to use the PkgConfig module with our GridPACK project to link with builds of the PETSc library. This works fine for builds using shared libraries, but fails completely for static builds. We need to get static builds working before we can release this functionality. The PETSc libraries appear to produce viable petsc.pc libraries that correctly define a number of variables that can be used in our application builds. After running pkg_check_modules, the following variables are defined
– Checking PETSc …
– Found PkgConfig: /usr/bin/pkg-config (found version “0.23”)
– Checking for module ‘PETSc’
– Found PETSc, version 3.16.3
– Using PETSc version 3.16.3
– PETSC_FOUND: 1
– PETSC_LIBRARIES: petsc
– PETSC_LINK_LIBRARIES: /pic/projects/gridpack/software/petsc-3.16.3/linux-openmpi-gnu-cxx-complex-opt/lib/libpetsc.a
– PETSC_LIBRARY_DIRS: /pic/projects/gridpack/software/petsc-3.16.3/linux-openmpi-gnu-cxx-complex-opt/lib
– PETSC_LDFLAGS: -L/pic/projects/gridpack/software/petsc-3.16.3/linux-openmpi-gnu-cxx-complex-opt/lib;-lpetsc
– PETSC_LDFLAGS_OTHER:
– PETSC_INCLUDE_DIRS: /pic/projects/gridpack/software/petsc-3.16.3/linux-openmpi-gnu-cxx-complex-opt/include;/pic/projects/gridpack/software/petsc-3.16.3/include
– PETSC_CFLAGS: -I/pic/projects/gridpack/software/petsc-3.16.3/linux-openmpi-gnu-cxx-complex-opt/include;-I/pic/projects/gridpack/software/petsc-3.16.3/include
– PETSC_CFLAGS_OTHER:
– PETSC_STATIC_LIBRARIES: petsc;spqr;umfpack;klu;cholmod;btf;ccolamd;colamd;camd;amd;suitesparseconfig;rt;superlu;superlu_dist;f2clapack;f2cblas;parmetis;metis;mpi_usempif08;mpi_usempi_ignore_tkr;mpi_mpifh;mpi;gfortran;m;gcc_s;pthread;quadmath;stdc++;dl
– PETSC_STATIC_LINK_LIBRARIES:
– PETSC_STATIC_LDFLAGS: -L/pic/projects/gridpack/software/petsc-3.16.3/linux-openmpi-gnu-cxx-complex-opt/lib;-L/share/apps/openmpi/3.0.1/gcc/6.1.0/lib;-L/qfs/projects/ops/rh6/gcc/6.1.0/lib/gcc/x86_64-pc-linux-gnu/6.1.0;-L/qfs/projects/ops/rh6/gcc/6.1.0/lib/gcc;-L/qfs/projects/ops/rh6/gcc/6.1.0/lib64;-L/qfs/projects/ops/rh6/gcc/6.1.0/lib;-lpetsc;-lspqr;-lumfpack;-lklu;-lcholmod;-lbtf;-lccolamd;-lcolamd;-lcamd;-lamd;-lsuitesparseconfig;-lrt;-lsuperlu;-lsuperlu_dist;-lf2clapack;-lf2cblas;-lparmetis;-lmetis;-lmpi_usempif08;-lmpi_usempi_ignore_tkr;-lmpi_mpifh;-lmpi;-lgfortran;-lm;-lgcc_s;-lpthread;-lquadmath;-lstdc++;-ldl
Apart from adding some include directories, the only thing needed to get the builds to work with shared libraries is to add a line
set(target_libraries PkgConfig::PETSC ${target_libraries})
and
target_link_libraries(gridpack_math
PUBLIC
gridpack_configuration
PUBLIC
gridpack_timer
PUBLIC
gridpack_parallel
PUBLIC
PkgConfig::PETSC
)
to one of the GridPACK module CMakeLists.txt file. However, this does not work for static builds. In particular, a lot of libraries needed by PETSc do not seem to be included in the line line. I’ve tried fixing things by hand by trying to add the PETSC_STATIC_LIBRARIES and PETSC_STATIC_LDFLAGS variables to the build. If I add the lines
target_link_libraries(gridpack_math
PUBLIC
${PETSC_STATIC_LIBRARIES}$
)
target_link_directories(gridpack_math PUBLIC ${PETSC_LIBRARY_DIRS})
to the build, the libraries are included in the link line but the build cannot find the MPI libraries.
/share/apps/gcc/6.1.0/bin/g++ -pthread -g -rdynamic CMakeFiles/greetings.dir/test/greetings.cpp.o -o greetings -L/pic/projects/gridpack/software/petsc-3.16.3/linux-openmpi-gnu-cxx-complex-opt/lib -Wl,-rpath,/pic/projects/gridpack/software/petsc-3.16.3/linux-openmpi-gnu-cxx-complex-opt/lib:/qfs/projects/ops/rh6/openmpi/3.0.1/gcc/6.1.0/lib …/math/libgridpack_math.a libgridpack_parallel.a …/timer/libgridpack_timer.a …/environment/libgridpack_environment.a …/math/libgridpack_math.a /pic/projects/gridpack/software/boost_1_65_0/lib/libboost_mpi.a /pic/projects/gridpack/software/boost_1_65_0/lib/libboost_serialization.a /pic/projects/gridpack/software/boost_1_65_0/lib/libboost_random.a /pic/projects/gridpack/software/boost_1_65_0/lib/libboost_filesystem.a /pic/projects/gridpack/software/boost_1_65_0/lib/libboost_system.a /pic/projects/gridpack/software/ga-5.7/build_pr/lib/libga++.a /pic/projects/gridpack/software/ga-5.7/build_pr/lib/libga.a /pic/projects/gridpack/software/ga-5.7/build_pr/lib/libarmci.a -lrt /usr/lib64/librt.so /usr/lib64/libdl.so /qfs/projects/ops/rh6/openmpi/3.0.1/gcc/6.1.0/lib/libmpi.so …/timer/libgridpack_timer.a libgridpack_parallel.a …/configuration/libgridpack_configuration.a /pic/projects/gridpack/software/petsc-3.16.3/linux-openmpi-gnu-cxx-complex-opt/lib/libpetsc.a -lpetsc -lspqr -lumfpack -lklu -lcholmod -lbtf -lccolamd -lcolamd -lcamd -lamd -lsuitesparseconfig -lrt -lsuperlu -lsuperlu_dist -lf2clapack -lf2cblas -lparmetis -lmetis -lmpi_usempif08 -lmpi_usempi_ignore_tkr -lmpi_mpifh -lmpi -lgfortran -lm -lgcc_s -lpthread -lquadmath -lstdc++ -ldl$
/share/apps/binutils/2.24/bin/ld: cannot find -lmpi_usempif08
/share/apps/binutils/2.24/bin/ld: cannot find -lmpi_usempi_ignore_tkr
/share/apps/binutils/2.24/bin/ld: cannot find -lmpi_mpifh
/share/apps/binutils/2.24/bin/ld: cannot find -lmpi
collect2: error: ld returned 1 exit status
If I add the LDFLAGS to the build using
target_link_options(gridpack_math PUBLIC ${PETSC_STATIC_LDFLAGS})
the build fails since the link options show up in the wrong place in the compilation line and the build complains about missing symbols that are used in the libraries that follow
/share/apps/gcc/6.1.0/bin/g++ -pthread -g -rdynamic -L/pic/projects/gridpack/software/petsc-3.16.3/linux-openmpi-gnu-cxx-complex-opt/lib -L/share/apps/openmpi/3.0.1/gcc/6.1.0/lib -L/qfs/projects/ops/rh6/gcc/6.1.0/lib/gcc/x86_64-pc-linux-gnu/6.1.0 -L/qfs/projects/ops/rh6/gcc/6.1.0/lib/gcc -L/qfs/projects/ops/rh6/gcc/6.1.0/lib64 -L/qfs/projects/ops/rh6/gcc/6.1.0/lib -lpetsc -lspqr -lumfpack -lklu -lcholmod -lbtf -lccolamd -lcolamd -lcamd -lamd -lsuitesparseconfig -lrt -lsuperlu -lsuperlu_dist -lf2clapack -lf2cblas -lparmetis -lmetis -lmpi_usempif08 -lmpi_usempi_ignore_tkr -lmpi_mpifh -lmpi -lgfortran -lm -lgcc_s -lpthread -lquadmath -lstdc++ -ldl CMakeFiles/greetings.dir/test/greetings.cpp.o -o greetings -L/pic/projects/gridpack/software/petsc-3.16.3/linux-openmpi-gnu-cxx-complex-opt/lib -Wl,-rpath,/pic/projects/gridpack/software/petsc-3.16.3/linux-openmpi-gnu-cxx-complex-opt/lib:/qfs/projects/ops/rh6/openmpi/3.0.1/gcc/6.1.0/lib …/math/libgridpack_math.a libgridpack_parallel.a …/timer/libgridpack_timer.a …/environment/libgridpack_environment.a …/math/libgridpack_math.a /pic/projects/gridpack/software/boost_1_65_0/lib/libboost_mpi.a /pic/projects/gridpack/software/boost_1_65_0/lib/libboost_serialization.a /pic/projects/gridpack/software/boost_1_65_0/lib/libboost_random.a /pic/projects/gridpack/software/boost_1_65_0/lib/libboost_filesystem.a /pic/projects/gridpack/software/boost_1_65_0/lib/libboost_system.a /pic/projects/gridpack/software/ga-5.7/build_pr/lib/libga++.a /pic/projects/gridpack/software/ga-5.7/build_pr/lib/libga.a /pic/projects/gridpack/software/ga-5.7/build_pr/lib/libarmci.a -lrt /usr/lib64/librt.so /usr/lib64/libdl.so /qfs/projects/ops/rh6/openmpi/3.0.1/gcc/6.1.0/lib/libmpi.so …/timer/libgridpack_timer.a libgridpack_parallel.a …/configuration/libgridpack_configuration.a /pic/projects/gridpack/software/petsc-3.16.3/linux-openmpi-gnu-cxx-complex-opt/lib/libpetsc.a
/pic/projects/gridpack/software/petsc-3.16.3/linux-openmpi-gnu-cxx-complex-opt/lib/libpetsc.a(svd.o): In function `PCSetUp_SVD’:
/pic/projects/gridpack/software/petsc-3.16.3/src/ksp/pc/impls/svd/svd.c:90: undefined reference to `zgesvd_’
/pic/projects/gridpack/software/petsc-3.16.3/linux-openmpi-gnu-cxx-complex-opt/lib/libpetsc.a(bddcprivate.o): In function `MatDenseOrthogonalRangeOrComplement(_p_Mat*, PetscBool, int, std::complex, double, _p_Mat**)’:
/pic/projects/gridpack/software/petsc-3.16.3/src/ksp/pc/impls/bddc/bddcprivate.c:56: undefined reference to `zgesvd_’
/pic/projects/gridpack/software/petsc-3.16.3/linux-openmpi-gnu-cxx-complex-opt/lib/libpetsc.a(bddcprivate.o): In function `PCBDDCAdaptiveSelection(_p_PC*)’:
/pic/projects/gridpack/software/petsc-3.16.3/src/ksp/pc/impls/bddc/bddcprivate.c:3215: undefined reference to `zhegvx_’
I have not been able to get the LDFLAGS to show up at the end of the link line.
The PETSc developers seem to feel that this is an issue with CMake. Is there some way to get this working with static builds? I can’t seem to find a solution with the existing CMake framework.
Bruce Palmer