Linking multiarch libraries in cross-compilation toolchain

tambre · December 19, 2019, 6:32am

I’m creating a toolchain for cross-compiling on x64 for an ARM64 device.
I find some libraries using PkgConfig in the actual application and in the final linking command they’re linked like -lglib-2.0. Unfortunately this results in cannot find -lglib-2.0 despite the ARM64 variants existing under /usr/lib/aarch64-linux-gnu.
What would be the correct way to get CMake to add /usr/lib/aarch64-linux-gnu as a link directory on the command line?

Toolchain file

set(CMAKE_SYSTEM_NAME "Linux")
set(CMAKE_SYSTEM_PROCESSOR "aarch64")
set(CMAKE_LIBRARY_ARCHITECTURE "aarch64-linux-gnu")

set(SDK "<some path>" CACHE STRING "Path to the SDK")    

set(TOOLCHAIN "${SDK}/../toolchains/gcc-linaro-7.3.1-2018.05-x86_64_aarch64-linux-gnu")
set(CMAKE_CXX_COMPILER "${TOOLCHAIN}/bin/aarch64-linux-gnu-g++")
set(CMAKE_POSITION_INDEPENDENT_CODE TRUE)

include_directories(BEFORE SYSTEM "${SDK}/include")

list(APPEND CMAKE_SYSTEM_PREFIX_PATH "/usr")
list(APPEND CMAKE_SYSTEM_LIBRARY_PATH "${SDK}/lib-target" "/usr/lib/${CMAKE_LIBRARY_ARCHITECTURE}")

set(CMAKE_FIND_ROOT_PATH "${SDK}" "/usr/local")
set(CMAKE_FIND_ROOT_PATH_MODE_PROGRAM NEVER)
set(CMAKE_FIND_ROOT_PATH_MODE_LIBRARY ONLY)
set(CMAKE_FIND_ROOT_PATH_MODE_INCLUDE ONLY)
set(CMAKE_FIND_ROOT_PATH_MODE_PACKAGE ONLY)

EDIT:

list(APPEND CMAKE_SYSTEM_PREFIX_PATH "/usr")
list(APPEND CMAKE_SYSTEM_LIBRARY_PATH "${SDK}/lib-target" "/usr/lib/${CMAKE_LIBRARY_ARCHITECTURE}" "${SDK}/targetfs_a/lib/${CMAKE_LIBRARY_ARCHITECTURE}" "${SDK}/targetfs_a/usr/lib/${CMAKE_LIBRARY_ARCHITECTURE}")
list(APPEND CMAKE_FIND_ROOT_PATH "${SDK}" "/usr/local")

foreach(DIRECTORY IN LISTS CMAKE_SYSTEM_LIBRARY_PATH)
	list(APPEND LINK_DIRS "-L${DIRECTORY}")
	list(APPEND RPATHS "-rpath,${DIRECTORY}")
endforeach()

string(JOIN " " LINK_DIRS ${LINK_DIRS})
string(JOIN "," RPATHS ${RPATHS})
set(LINKER_FLAGS "${LINK_DIRS} -Wl,${RPATHS}")

set(CMAKE_SHARED_LINKER_FLAGS ${LINKER_FLAGS})
set(CMAKE_EXE_LINKER_FLAGS    ${LINKER_FLAGS})

Makes it work like I need it to, but it feels there ought to be a better solution.

erk · December 19, 2019, 8:18am

You can use: link_directories, it is not recommended as explained in the doc
https://cmake.org/cmake/help/latest/command/link_directories.html
but it should be safe in your case.

Concerning the use of pkg-config it may be that the .pc file used is either the one from the host and/or it does not give enough information about link directory.

You could try cross-configure the following dummy project and observe outputed values:

cmake_minimum_required(VERSION 3.13)

project(WithPkg CXX)

include(CMakePrintHelpers)

cmake_print_properties(DIRECTORIES .
  PROPERTIES
  LINK_DIRECTORIES)

find_package(PkgConfig REQUIRED)
pkg_search_module(GLIB2 REQUIRED  glib-2.0 IMPORTED_TARGET)
cmake_print_variables(GLIB2_LIBRARY_DIRS GLIB2_LIBDIR)
cmake_print_properties(TARGETS PkgConfig::GLIB2
  PROPERTIES
  INTERFACE_COMPILE_OPTIONS
  INTERFACE_LINK_DEPENDS
  INTERFACE_LINK_DIRECTORIES
  INTERFACE_LINK_LIBRARIES
  INTERFACE_LINK_OPTIONS)

note that you may need to help pkg-config to tell apart host vs target https://autotools.io/pkgconfig/cross-compiling.html in order to make pkg-config use proper system root.
You can specify CMAKE_SYSROOT in your toolchain too but I don’t know if it is automatically used by the FindPkgConfig module.

tambre · December 19, 2019, 9:07am

Toolchain

set(CMAKE_SYSTEM_NAME "Linux")
set(CMAKE_SYSTEM_PROCESSOR "aarch64")
set(CMAKE_LIBRARY_ARCHITECTURE "aarch64-linux-gnu")

set(SDK "<REDACTED>" CACHE STRING "Path to the SDK")    

set(TOOLCHAIN "${SDK}/../toolchains/gcc-linaro-7.3.1-2018.05-x86_64_aarch64-linux-gnu")
set(CMAKE_CXX_COMPILER "${TOOLCHAIN}/bin/aarch64-linux-gnu-g++")

include_directories(SYSTEM "${SDK}/include")
link_directories("/usr/lib/${CMAKE_LIBRARY_ARCHITECTURE}")

list(APPEND CMAKE_SYSTEM_PREFIX_PATH "/usr")
list(APPEND CMAKE_SYSTEM_LIBRARY_PATH "${SDK}/lib-target") # SDK libraries
list(APPEND CMAKE_SYSTEM_LIBRARY_PATH "${SDK}/targetfs/lib/${CMAKE_LIBRARY_ARCHITECTURE}" "${SDK}/targetfs/usr/lib/${CMAKE_LIBRARY_ARCHITECTURE}") # Misc target libraries
list(APPEND CMAKE_FIND_ROOT_PATH "${SDK}" "/usr/local")

foreach(DIRECTORY IN LISTS CMAKE_SYSTEM_LIBRARY_PATH)
	list(APPEND LINK_DIRS "-L${DIRECTORY}")
	list(APPEND RPATHS "-rpath,${DIRECTORY}")
endforeach()

string(JOIN " " LINK_DIRS ${LINK_DIRS})
string(JOIN "," RPATHS ${RPATHS})
set(LINKER_FLAGS "${LINK_DIRS} -Wl,${RPATHS}")

set(CMAKE_SHARED_LINKER_FLAGS ${LINKER_FLAGS})
set(CMAKE_EXE_LINKER_FLAGS    ${LINKER_FLAGS})
set(CMAKE_POSITION_INDEPENDENT_CODE TRUE)

set(CMAKE_FIND_ROOT_PATH_MODE_PROGRAM NEVER)
set(CMAKE_FIND_ROOT_PATH_MODE_LIBRARY ONLY)
set(CMAKE_FIND_ROOT_PATH_MODE_INCLUDE ONLY)
set(CMAKE_FIND_ROOT_PATH_MODE_PACKAGE ONLY)

Output

# cmake ../var_test/ -GNinja -DCMAKE_TOOLCHAIN_FILE=<REDACTED>
-- The CXX compiler identification is GNU 7.3.1
-- Check for working CXX compiler: <REDACTED>../toolchains/gcc-linaro-7.3.1-2018.05-x86_64_aarch64-linux-gnu/bin/aarch64-linux-gnu-g++
-- Check for working CXX compiler: <REDACTED>../toolchains/gcc-linaro-7.3.1-2018.05-x86_64_aarch64-linux-gnu/bin/aarch64-linux-gnu-g++ -- works
-- Detecting CXX compiler ABI info
-- Detecting CXX compiler ABI info - done
-- Detecting CXX compile features
-- Detecting CXX compile features - done
--
 Properties for DIRECTORY .:
   ..LINK_DIRECTORIES = "/usr/lib/aarch64-linux-gnu;/usr/lib/aarch64-linux-gnu"

-- Found PkgConfig: /usr/bin/pkg-config (found version "0.29.1")
-- Checking for one of the modules 'glib-2.0'
-- GLIB2_LIBRARY_DIRS="" ; GLIB2_LIBDIR="/usr/lib/x86_64-linux-gnu"
--
 Properties for TARGET PkgConfig::GLIB2:
   PkgConfig::GLIB2.INTERFACE_COMPILE_OPTIONS = <NOTFOUND>
   PkgConfig::GLIB2.INTERFACE_LINK_DEPENDS = <NOTFOUND>
   PkgConfig::GLIB2.INTERFACE_LINK_DIRECTORIES = <NOTFOUND>
   PkgConfig::GLIB2.INTERFACE_LINK_LIBRARIES = <NOTFOUND>
   PkgConfig::GLIB2.INTERFACE_LINK_OPTIONS = <NOTFOUND>

-- Configuring done
-- Generating done
-- Build files have been written to: /opt/build

Eric, thank you very much for the help!
The reason I don’t set CMAKE_SYSROOT is that the targetfs available on the host system is provided by the hardware vendor and doesn’t have some of the libraries I need to link to (but which are available on the actual system). As far as I understand, setting the sysroot would hinder me from linking to multiarch libraries in /usr/lib/aarch64-linux-gnu.

The ~~include_directories()~~ link_directories() unfortunately doesn’t seem to add the directories to the final link command.
FindPkgConfig uses the architecture-appropriate pkgconfig file, but only when not cross-compiling, which I am.

tambre · December 19, 2019, 9:23am

It seems like it should be fairly easy for me to create a custom sysroot that also includes the ARM64 libraries I need on top of the vendor’s. I’ll give that go.

McMartin · December 19, 2019, 9:39am

Be careful not to confuse include_directories() (for headers) and link_directories() (for libraries).

erk · December 19, 2019, 9:56am

– Found PkgConfig: /usr/bin/pkg-config (found version “0.29.1”)
– Checking for one of the modules ‘glib-2.0’
– GLIB2_LIBRARY_DIRS=“” ; GLIB2_LIBDIR=“/usr/lib/x86_64-linux-gnu”

somehow pkg-config got it wrong.

Properties for TARGET PkgConfig::GLIB2:
PkgConfig::GLIB2.INTERFACE_COMPILE_OPTIONS =
PkgConfig::GLIB2.INTERFACE_LINK_DEPENDS =
PkgConfig::GLIB2.INTERFACE_LINK_DIRECTORIES =
PkgConfig::GLIB2.INTERFACE_LINK_LIBRARIES =
PkgConfig::GLIB2.INTERFACE_LINK_OPTIONS =

and CMake module even worse as you noted this module does not seem cross-compiling ready or I missed something.

tambre · December 19, 2019, 12:52pm

Toolchain

set(CMAKE_SYSTEM_NAME "Linux")
set(CMAKE_SYSTEM_PROCESSOR "aarch64")
set(CMAKE_LIBRARY_ARCHITECTURE "aarch64-linux-gnu")

set(SDK "<REDACATED>" CACHE STRING "Path to the SDK")    

set(TOOLCHAIN "${SDK}/../toolchains/gcc-linaro-7.3.1-2018.05-x86_64_aarch64-linux-gnu")
set(CMAKE_CXX_COMPILER "${TOOLCHAIN}/bin/aarch64-linux-gnu-g++")
set(CMAKE_SYSROOT "/opt/<REDACTED>")

include_directories(SYSTEM "${SDK}/include")

set(CMAKE_SYSTEM_PREFIX_PATH "${CMAKE_SYSROOT}/usr")

list(APPEND CMAKE_SYSTEM_LIBRARY_PATH "${CMAKE_SYSROOT}/lib/${CMAKE_LIBRARY_ARCHITECTURE}")
list(APPEND CMAKE_SYSTEM_LIBRARY_PATH "${CMAKE_SYSROOT}/usr/lib/${CMAKE_LIBRARY_ARCHITECTURE}")

foreach(DIRECTORY IN LISTS CMAKE_SYSTEM_LIBRARY_PATH)
	list(APPEND LINK_DIRS "-L${DIRECTORY}")
	list(APPEND RPATHS "-rpath,${DIRECTORY}")
endforeach()

string(JOIN " " LINK_DIRS ${LINK_DIRS})
string(JOIN "," RPATHS ${RPATHS})
set(LINKER_FLAGS "${LINK_DIRS} -Wl,${RPATHS}")

set(CMAKE_SHARED_LINKER_FLAGS ${LINKER_FLAGS})
set(CMAKE_EXE_LINKER_FLAGS    ${LINKER_FLAGS})
set(CMAKE_POSITION_INDEPENDENT_CODE TRUE)

set(CMAKE_FIND_ROOT_PATH "/usr/local")
set(CMAKE_FIND_ROOT_PATH_MODE_PROGRAM NEVER)
set(CMAKE_FIND_ROOT_PATH_MODE_LIBRARY ONLY)
set(CMAKE_FIND_ROOT_PATH_MODE_INCLUDE ONLY)
set(CMAKE_FIND_ROOT_PATH_MODE_PACKAGE ONLY)

That’s the toolchain I’ve now ended up with and that works.
Creating my own sysroot was much easier than I thought and simplified the toolchain.
I appreciate everyone’s responses and help!

However,

link_directories("${CMAKE_SYSROOT}/lib/${CMAKE_LIBRARY_ARCHITECTURE}")
link_directories("${CMAKE_SYSROOT}/usr/lib/${CMAKE_LIBRARY_ARCHITECTURE}")

instead of

list(APPEND CMAKE_SYSTEM_LIBRARY_PATH "${CMAKE_SYSROOT}/lib/${CMAKE_LIBRARY_ARCHITECTURE}")
list(APPEND CMAKE_SYSTEM_LIBRARY_PATH "${CMAKE_SYSROOT}/usr/lib/${CMAKE_LIBRARY_ARCHITECTURE}")

doesn’t work. The directories simply aren’t added to the final linking command. Why?
Being able to do the former would simplify the toolchain (as long as it also adds them to the rpath).

Keen eyes! Fixed the typo.