I explained in my previous question that I am trying to build a dynamic library ‘libdpdk.so’ of my own functions linked to the 3rd party DPDK libraries. I would then link ‘libdpdk.so’ to my own main.cpp test program. The problem I have is that ‘libdpdk.so’ links without error but, when I link it to main.cpp I see undefined symbols. In fact, if I run ‘nm -u’ on ‘libdpdk.so’ I see quite a few undefined symbols.
The DPDK examples use gnu make, not CMake, so it is not easy to see what is wrong with my CMake files.
I am going to paste my CMakeLists.txt files in their entirety here - sorry about the length - but I think that is safer than showing snippets.
Top level CMakeLists.txt
cmake_minimum_required(VERSION 3.1...3.16)
project(main VERSION 1.0 LANGUAGES CXX C)
# Specify the executable name
set(_target_name dpdk_test)
set(CMAKE_CXX_COMPILER /opt/rh/devtoolset-7/root/usr/bin/g++)
set(CMAKE_C_COMPILER /opt/rh/devtoolset-7/root/usr/bin/gcc)
# Require C++14 tooling (minimum) for all targets in this project
set(CMAKE_CXX_STANDARD 14)
set(CMAKE_CXX_STANDARD_REQUIRED ON)
set(CMAKE_CXX_EXTENSIONS OFF)
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -march=native -g")
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -march=native -O0 -ggdb")
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wno-deprecated-declarations")
# DPDK version
set(_dpdk_version dpdk-stable-18.11.8)
add_subdirectory(dpdk dpdk)
add_executable(${_target_name}
main.cpp
Threads.cpp
ThreadMon.cpp
ThreadMSG.cpp
_kbhit.cpp
)
target_include_directories(${_target_name} PRIVATE dpdk/include)
target_include_directories(${_target_name} PRIVATE include)
target_include_directories(${_target_name} PRIVATE /opt/dpdk/${_dpdk_version}/x86_64-native-linuxapp-gcc/include)
#target_link_libraries(${_target_name} PRIVATE dpdk)
# Enable the -pthread compiler and linker flag
set(THREADS_PREFER_PTHREAD_FLAG ON)
find_package(Threads REQUIRED)
target_link_libraries(${_target_name}
dpdk
Threads::Threads)
’libdpdk.so’ CMakeLists.txt
cmake_minimum_required(VERSION 3.1...3.16)
# The library's name
set (_lib_name "dpdk")
# DPDK version
set(_dpdk_version dpdk-stable-18.11.8)
add_definitions(
-DRTE_MACHINE_CPUFLAG_SSE
-DRTE_MACHINE_CPUFLAG_SSE2
-DRTE_MACHINE_CPUFLAG_SSE3
-DRTE_MACHINE_CPUFLAG_SSSE3
-DRTE_MACHINE_CPUFLAG_SSE4_1
-DRTE_MACHINE_CPUFLAG_SSE4_2
-DRTE_MACHINE_CPUFLAG_AES
-DRTE_MACHINE_CPUFLAG_PCLMULQDQ
-DRTE_MACHINE_CPUFLAG_AVX
-DRTE_MACHINE_CPUFLAG_RDRAND
-DRTE_MACHINE_CPUFLAG_FSGSBASE
-DRTE_MACHINE_CPUFLAG_F16C
-DRTE_MACHINE_CPUFLAG_AVX2
)
# The library to be built
add_library(${_lib_name} SHARED)
# The sources to use when building the library
target_sources(${_lib_name}
PRIVATE
ThreadDpdk.cpp
DpdkSock.cpp
Log.c
cfg.cpp
wrap_eth.cpp
wrap_ip.cpp
SocketClass.cpp
RingClass.cpp
)
target_include_directories(${_lib_name} PRIVATE ../include)
target_include_directories(${_lib_name} PRIVATE include)
target_include_directories(${_lib_name} PRIVATE /opt/dpdk/${_dpdk_version}/x86_64-native-linuxapp-gcc/include)
# Enable the -pthread compiler and linker flag
set(THREADS_PREFER_PTHREAD_FLAG ON)
find_package(Threads REQUIRED)
#target_link_libraries(${_lib_name} PRIVATE
# Threads::Threads)
# Library paths
set(_lib64_path /usr/lib64/)
set(_dpdk_path /opt/dpdk/${_dpdk_version}/x86_64-native-linuxapp-gcc/lib/)
target_link_libraries(${_lib_name}
PRIVATE ${_lib64_path}libdl.so
PRIVATE ${_lib64_path}libutil.so
PRIVATE ${_lib64_path}libm.so
-Xlinker -export-dynamic
# dpdk libraries
PRIVATE ${_dpdk_path}librte_flow_classify.so
-Wl,--whole-archive
PRIVATE ${_dpdk_path}librte_pipeline.so
PRIVATE ${_dpdk_path}librte_table.so
PRIVATE ${_dpdk_path}librte_port.so
PRIVATE ${_dpdk_path}librte_pdump.so
PRIVATE ${_dpdk_path}librte_distributor.so
PRIVATE ${_dpdk_path}librte_ip_frag.so
PRIVATE ${_dpdk_path}librte_meter.so
PRIVATE ${_dpdk_path}librte_lpm.so
PRIVATE ${_dpdk_path}librte_acl.so
PRIVATE ${_dpdk_path}librte_jobstats.so
PRIVATE ${_dpdk_path}librte_metrics.so
PRIVATE ${_dpdk_path}librte_bitratestats.so
PRIVATE ${_dpdk_path}librte_latencystats.so
PRIVATE ${_dpdk_path}librte_power.so
PRIVATE ${_dpdk_path}librte_efd.so
PRIVATE ${_dpdk_path}librte_bpf.so
PRIVATE ${_dpdk_path}librte_cfgfile.so
PRIVATE ${_dpdk_path}librte_gro.so
PRIVATE ${_dpdk_path}librte_gso.so
PRIVATE ${_dpdk_path}librte_hash.so
PRIVATE ${_dpdk_path}librte_member.so
PRIVATE ${_dpdk_path}librte_vhost.so
PRIVATE ${_dpdk_path}librte_kvargs.so
PRIVATE ${_dpdk_path}librte_mbuf.so
PRIVATE ${_dpdk_path}librte_net.so
PRIVATE ${_dpdk_path}librte_ethdev.so
PRIVATE ${_dpdk_path}librte_bbdev.so
PRIVATE ${_dpdk_path}librte_cryptodev.so
PRIVATE ${_dpdk_path}librte_security.so
PRIVATE ${_dpdk_path}librte_compressdev.so
PRIVATE ${_dpdk_path}librte_eventdev.so
PRIVATE ${_dpdk_path}librte_rawdev.so
PRIVATE ${_dpdk_path}librte_timer.so
PRIVATE ${_dpdk_path}librte_mempool.so
PRIVATE ${_dpdk_path}librte_mempool_ring.so
PRIVATE ${_dpdk_path}librte_ring.so
PRIVATE ${_dpdk_path}librte_pci.so
PRIVATE ${_dpdk_path}librte_eal.so
PRIVATE ${_dpdk_path}librte_cmdline.so
PRIVATE ${_dpdk_path}librte_reorder.so
PRIVATE ${_dpdk_path}librte_sched.so
PRIVATE ${_dpdk_path}librte_kni.so
PRIVATE ${_dpdk_path}librte_common_cpt.so
PRIVATE ${_dpdk_path}librte_common_octeontx.so
PRIVATE ${_dpdk_path}librte_common_dpaax.so
PRIVATE ${_dpdk_path}librte_bus_pci.so
PRIVATE ${_dpdk_path}librte_bus_vdev.so
PRIVATE ${_dpdk_path}librte_bus_dpaa.so
PRIVATE ${_dpdk_path}librte_bus_fslmc.so
PRIVATE ${_dpdk_path}librte_mempool_bucket.so
PRIVATE ${_dpdk_path}librte_mempool_stack.so
PRIVATE ${_dpdk_path}librte_mempool_dpaa.so
PRIVATE ${_dpdk_path}librte_mempool_dpaa2.so
PRIVATE ${_dpdk_path}librte_pmd_af_packet.so
PRIVATE ${_dpdk_path}librte_pmd_ark.so
# PRIVATE ${_dpdk_path}librte_pmd_atlantic.so
PRIVATE ${_dpdk_path}librte_pmd_avf.so
PRIVATE ${_dpdk_path}librte_pmd_avp.so
PRIVATE ${_dpdk_path}librte_pmd_axgbe.so
PRIVATE ${_dpdk_path}librte_pmd_bnxt.so
PRIVATE ${_dpdk_path}librte_pmd_bond.so
PRIVATE ${_dpdk_path}librte_pmd_cxgbe.so
PRIVATE ${_dpdk_path}librte_pmd_dpaa.so
PRIVATE ${_dpdk_path}librte_pmd_dpaa2.so
PRIVATE ${_dpdk_path}librte_pmd_e1000.so
PRIVATE ${_dpdk_path}librte_pmd_ena.so
#PRIVATE ${_dpdk_path}librte_pmd_enetc.so
PRIVATE ${_dpdk_path}librte_pmd_enic.so
PRIVATE ${_dpdk_path}librte_pmd_fm10k.so
PRIVATE ${_dpdk_path}librte_pmd_failsafe.so
PRIVATE ${_dpdk_path}librte_pmd_i40e.so
PRIVATE ${_dpdk_path}librte_pmd_ixgbe.so
PRIVATE ${_dpdk_path}librte_pmd_kni.so
PRIVATE ${_dpdk_path}librte_pmd_lio.so
PRIVATE ${_dpdk_path}librte_pmd_nfp.so
PRIVATE ${_dpdk_path}librte_pmd_null.so
PRIVATE ${_dpdk_path}librte_pmd_qede.so
PRIVATE ${_dpdk_path}librte_pmd_ring.so
PRIVATE ${_dpdk_path}librte_pmd_softnic.so
PRIVATE ${_dpdk_path}librte_pmd_sfc_efx.so
PRIVATE ${_dpdk_path}librte_pmd_tap.so
PRIVATE ${_dpdk_path}librte_pmd_thunderx_nicvf.so
PRIVATE ${_dpdk_path}librte_pmd_vdev_netvsc.so
PRIVATE ${_dpdk_path}librte_pmd_virtio.so
PRIVATE ${_dpdk_path}librte_pmd_vhost.so
PRIVATE ${_dpdk_path}librte_pmd_ifc.so
PRIVATE ${_dpdk_path}librte_pmd_vmxnet3_uio.so
PRIVATE ${_dpdk_path}librte_bus_vmbus.so
PRIVATE ${_dpdk_path}librte_pmd_netvsc.so
PRIVATE ${_dpdk_path}librte_pmd_bbdev_null.so
PRIVATE ${_dpdk_path}librte_pmd_null_crypto.so
#PRIVATE ${_dpdk_path}librte_pmd_octeontx_crypto.so
PRIVATE ${_dpdk_path}librte_pmd_crypto_scheduler.so
PRIVATE ${_dpdk_path}librte_pmd_dpaa2_sec.so
PRIVATE ${_dpdk_path}librte_pmd_dpaa_sec.so
#PRIVATE ${_dpdk_path}librte_pmd_caam_jr.so
PRIVATE ${_dpdk_path}librte_pmd_virtio_crypto.so
PRIVATE ${_dpdk_path}librte_pmd_octeontx_zip.so
PRIVATE ${_dpdk_path}librte_pmd_qat.so
PRIVATE ${_dpdk_path}librte_pmd_skeleton_event.so
PRIVATE ${_dpdk_path}librte_pmd_sw_event.so
#PRIVATE ${_dpdk_path}librte_pmd_dsw_event.so
#PRIVATE ${_dpdk_path}librte_pmd_octeontx_ssovf.so
PRIVATE ${_dpdk_path}librte_pmd_dpaa_event.so
PRIVATE ${_dpdk_path}librte_pmd_dpaa2_event.so
PRIVATE ${_dpdk_path}librte_mempool_octeontx.so
PRIVATE ${_dpdk_path}librte_pmd_octeontx.so
PRIVATE ${_dpdk_path}librte_pmd_opdl_event.so
PRIVATE ${_dpdk_path}librte_pmd_skeleton_rawdev.so
PRIVATE ${_dpdk_path}librte_pmd_dpaa2_cmdif.so
PRIVATE ${_dpdk_path}librte_pmd_dpaa2_qdma.so
PRIVATE ${_dpdk_path}librte_bus_ifpga.so
PRIVATE ${_dpdk_path}librte_pmd_ifpga_rawdev.so
-Wl,--no-whole-archive
-lrt -lm -lnuma -ldl
${_lib64_path}libconfig.so
-Wl,-export
-Dynamic
#-L/opt/dpdk/${_dpdk_version}/examples/l2fwd/build/lib
-L/opt/dpdk/${_dpdk_version}/x86_64-native-linuxapp-gcc/lib
-Wl,--as-needed
PRIVATE Threads::Threads
-Wl,-Map=${_target_name}.map
-Wl,--cref
)
Linker output
[1/1] : && /opt/rh/devtoolset-7/root/usr/bin/g++ -march=native -g -O3 -DNDEBUG CMakeFiles/dpdk_test.dir/main.cpp.o CMakeFiles/dpdk_test.dir/Threads.cpp.o CMakeFiles/dpdk_test.dir/ThreadMon.cpp.o CMakeFiles/dpdk_test.dir/ThreadMSG.cpp.o CMakeFiles/dpdk_test.dir/_kbhit.cpp.o -o dpdk_test -Wl,-rpath,/data/daldrich/src/dpdk_test_library/build/dpdk dpdk/libdpdk.so -pthread && :
FAILED: dpdk_test
: && /opt/rh/devtoolset-7/root/usr/bin/g++ -march=native -g -O3 -DNDEBUG CMakeFiles/dpdk_test.dir/main.cpp.o CMakeFiles/dpdk_test.dir/Threads.cpp.o CMakeFiles/dpdk_test.dir/ThreadMon.cpp.o CMakeFiles/dpdk_test.dir/ThreadMSG.cpp.o CMakeFiles/dpdk_test.dir/_kbhit.cpp.o -o dpdk_test -Wl,-rpath,/data/daldrich/src/dpdk_test_library/build/dpdk dpdk/libdpdk.so -pthread && :
/opt/rh/devtoolset-7/root/usr/libexec/gcc/x86_64-redhat-linux/7/ld: CMakeFiles/dpdk_test.dir/main.cpp.o: undefined reference to symbol 'rte_cpu_get_flag_enabled@@DPDK_2.0'
//opt/dpdk/dpdk-stable-18.11.8/x86_64-native-linuxapp-gcc/lib/librte_eal.so.9.1: error adding symbols: DSO missing from command line
collect2: error: ld returned 1 exit status
ninja: build stopped: subcommand failed.
nm -u dpdk/libdpdk.so
$ nm -u dpdk/libdpdk.so
U access@@GLIBC_2.2.5
U bzero@@GLIBC_2.2.5
U calloc@@GLIBC_2.2.5
U config_destroy
U config_init
U config_lookup
U config_read_file
U config_setting_get_elem
U config_setting_length
U config_setting_lookup_int
U config_setting_lookup_string
U __cxa_atexit@@GLIBC_2.2.5
U __cxa_begin_catch@@CXXABI_1.3
U __cxa_end_catch@@CXXABI_1.3
w __cxa_finalize@@GLIBC_2.2.5
U exit@@GLIBC_2.2.5
U fclose@@GLIBC_2.2.5
U fflush@@GLIBC_2.2.5
U fopen@@GLIBC_2.2.5
U fprintf@@GLIBC_2.2.5
U fputs@@GLIBC_2.2.5
U free@@GLIBC_2.2.5
U fwrite@@GLIBC_2.2.5
U gettimeofday@@GLIBC_2.2.5
w __gmon_start__
U gp_threadDpdk
U g_shutdownNow
U g_shutdownNowMutex
U g_threadsMayRunCondVar
U g_threadsMayRunFlag
U g_threadsMayRunMutex
U __gxx_personality_v0@@CXXABI_1.3
U inet_addr@@GLIBC_2.2.5
w _ITM_deregisterTMCloneTable
w _ITM_registerTMCloneTable
U lcore_config@@DPDK_2.0
U localtime@@GLIBC_2.2.5
U memcpy@@GLIBC_2.14
U memset@@GLIBC_2.2.5
U per_lcore__lcore_id@@DPDK_2.0
U printf@@GLIBC_2.2.5
w __pthread_key_create@@GLIBC_2.2.5
w pthread_mutex_lock@@GLIBC_2.2.5
w pthread_mutex_unlock@@GLIBC_2.2.5
U pthread_self@@GLIBC_2.2.5
U pthread_spin_init@@GLIBC_2.2.5
U pthread_spin_lock@@GLIBC_2.2.5
U pthread_spin_unlock@@GLIBC_2.2.5
U putchar@@GLIBC_2.2.5
U puts@@GLIBC_2.2.5
U rte_cpu_get_flag_enabled@@DPDK_2.0
U rte_cycles_vmware_tsc_map@@DPDK_2.0
U rte_delay_us@@DPDK_2.0
U rte_eal_get_configuration@@DPDK_2.0
U rte_eal_init@@DPDK_2.0
U rte_eth_dev_configure@@DPDK_2.2
U rte_eth_dev_count_avail@@DPDK_18.05
U rte_eth_devices@@DPDK_2.2
U rte_eth_dev_info_get@@DPDK_2.2
U rte_eth_dev_is_valid_port@@DPDK_2.2
U rte_eth_dev_start@@DPDK_2.2
U rte_eth_link_get_nowait@@DPDK_2.2
U rte_eth_macaddr_get@@DPDK_2.2
U rte_eth_promiscuous_enable@@DPDK_2.2
U rte_eth_rx_queue_setup@@DPDK_2.2
U rte_eth_tx_queue_setup@@DPDK_2.2
U rte_get_tsc_hz@@DPDK_2.0
U rte_hash_add_key@@DPDK_2.0
U rte_hash_create@@DPDK_2.0
U rte_hash_iterate@@DPDK_2.1
U rte_hash_lookup@@DPDK_2.0
U rte_ip_frag_free_death_row@@DPDK_2.0
U rte_ip_frag_table_create@@DPDK_2.0
U rte_ipv4_fragment_packet@@DPDK_2.0
U rte_ipv4_frag_reassemble_packet@@DPDK_2.0
U rte_ipv6_frag_reassemble_packet@@DPDK_2.0
U rte_lpm6_create@@DPDK_2.0
U rte_lpm_add@@DPDK_16.04
U rte_lpm_create@@DPDK_16.04
U rte_mempool_create@@DPDK_2.0
U rte_mempool_lookup@@DPDK_2.0
U rte_mempool_ops_table@@DPDK_16.07
U __rte_panic@@DPDK_2.0
U rte_pktmbuf_dump@@DPDK_2.0
U rte_pktmbuf_init@@DPDK_2.0
U rte_pktmbuf_pool_create@@DPDK_2.1
U rte_pktmbuf_pool_init@@DPDK_2.0
U rte_ring_create@@DPDK_2.0
U rte_socket_id@@DPDK_2.0
U rte_zmalloc_socket@@DPDK_2.0
U sched_getcpu@@GLIBC_2.6
U setbuf@@GLIBC_2.2.5
U snprintf@@GLIBC_2.2.5
U sprintf@@GLIBC_2.2.5
U sscanf@@GLIBC_2.2.5
U stderr@@GLIBC_2.2.5
U stdout@@GLIBC_2.2.5
U __strdup@@GLIBC_2.2.5
U strerror@@GLIBC_2.2.5
U strftime@@GLIBC_2.2.5
U __tls_get_addr@@GLIBC_2.3
U _Unwind_Resume@@GCC_3.0
U vsnprintf@@GLIBC_2.2.5
U __xpg_basename@@GLIBC_2.2.5
U _Z14get_sched_attrmRSs
U _Z5fatalRKSs
U _ZdlPv@@GLIBCXX_3.4
U _ZNKSt15basic_stringbufIcSt11char_traitsIcESaIcEE3strEv@@GLIBCXX_3.4
U _ZNSo9_M_insertImEERSoT_@@GLIBCXX_3.4.9
U _ZNSs4_Rep10_M_destroyERKSaIcE@@GLIBCXX_3.4
U _ZNSs4_Rep20_S_empty_rep_storageE@@GLIBCXX_3.4
U _ZNSs4_Rep9_S_createEmmRKSaIcE@@GLIBCXX_3.4
U _ZNSs4swapERSs@@GLIBCXX_3.4
U _ZNSs6assignERKSs@@GLIBCXX_3.4
U _ZNSsC1EPKcRKSaIcE@@GLIBCXX_3.4
U _ZNSsC1ERKSs@@GLIBCXX_3.4
U _ZNSt18basic_stringstreamIcSt11char_traitsIcESaIcEEC1ESt13_Ios_Openmode@@GLIBCXX_3.4
U _ZNSt18basic_stringstreamIcSt11char_traitsIcESaIcEED1Ev@@GLIBCXX_3.4
U _ZNSt18condition_variable4waitERSt11unique_lockISt5mutexE@@GLIBCXX_3.4.11
U _ZNSt6localeC1Ev@@GLIBCXX_3.4
U _ZNSt6localeD1Ev@@GLIBCXX_3.4
U _ZNSt8ios_base4InitC1Ev@@GLIBCXX_3.4
U _ZNSt8ios_base4InitD1Ev@@GLIBCXX_3.4
U _ZNSt8ios_baseC2Ev@@GLIBCXX_3.4
U _ZNSt8ios_baseD2Ev@@GLIBCXX_3.4
U _ZNSt9basic_iosIcSt11char_traitsIcEE4initEPSt15basic_streambufIcS1_E@@GLIBCXX_3.4
U _ZSt16__ostream_insertIcSt11char_traitsIcEERSt13basic_ostreamIT_T0_ES6_PKS3_l@@GLIBCXX_3.4.9
U _ZSt19__throw_logic_errorPKc@@GLIBCXX_3.4
U _ZSt20__throw_system_errori@@GLIBCXX_3.4.11
U _ZStlsISt11char_traitsIcEERSt13basic_ostreamIcT_ES5_PKc@@GLIBCXX_3.4
U _ZTISt9exception@@GLIBCXX_3.4
U _ZTTSt18basic_stringstreamIcSt11char_traitsIcESaIcEE@@GLIBCXX_3.4
U _ZTVN10__cxxabiv117__class_type_infoE@@CXXABI_1.3
U _ZTVSt15basic_streambufIcSt11char_traitsIcEE@@GLIBCXX_3.4
U _ZTVSt15basic_stringbufIcSt11char_traitsIcESaIcEE@@GLIBCXX_3.4
U _ZTVSt18basic_stringstreamIcSt11char_traitsIcESaIcEE@@GLIBCXX_3.4
U _ZTVSt9basic_iosIcSt11char_traitsIcEE@@GLIBCXX_3.4
Any advice would be much appreciated.