How to use ctest --build-and-test for single and multi config generators?

Code
1

I found a problem wenn generate a visual studio project instead a ninja project:

The CMAKE_BUILD_TYPE is not set and I must use the --build-config options!

diff --git a/src/beman/execution26/tests/CMakeLists.txt b/src/beman/execution26/tests/CMakeLists.txt
index a497211..0efcefb 100644
--- a/src/beman/execution26/tests/CMakeLists.txt
+++ b/src/beman/execution26/tests/CMakeLists.txt
@@ -107,18 +107,26 @@ endforeach()
 if(NOT PROJECT_IS_TOP_LEVEL)
     # test if the targets are findable from the build directory
     # cmake-format: off
+    if(NOT DEFINED CMAKE_BUILD_TYPE)
+        set(CMAKE_BUILD_TYPE Release)
+    endif()
     add_test(find-package-test
         ${CMAKE_CTEST_COMMAND}
+        --verbose
+        # TODO: --output-on-failure
         -C ${CMAKE_BUILD_TYPE}
         --build-and-test
             "${CMAKE_CURRENT_SOURCE_DIR}"
             "${CMAKE_CURRENT_BINARY_DIR}/find-package-test"
         --build-generator ${CMAKE_GENERATOR}
         --build-makeprogram ${CMAKE_MAKE_PROGRAM}
+        # XXX --build-noclean
         --build-options
+            # XXX --clean-first
             "-DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER}"
             "-DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}"
             "-DCMAKE_PREFIX_PATH=${CMAKE_PREFIX_PATH}"
+            # FIXME: "-C ${CMAKE_BUILD_TYPE}"
     )
     # cmake-format: on
 endif()

I noted too, that with visual studio generator, the project is not cleaned befor build?

2

Use the NAME and COMMAND keyword form of add_test(), which allows you to use generator expressions. Then use $<CONFIG> instead of ${CMAKE_BUILD_TYPE}. I think that should get you most of the way there. I don’t know if the --build-config-sample might also be useful to you, but maybe you don’t need that.

3

And what is about --clean-first, it seams to me, I should use it to get same behaviour with all generators?

But with this build-options, it always fails with Ninja generator?

Start testing: Oct 31 10:41 CET
----------------------------------------------------------
80/80 Testing: find-package-test
80/80 Test: find-package-test
Command: "/usr/local/bin/ctest" "--output-on-failure" "-C" "RelWithDebInfo" "--build-and-test" "/Users/clausklein/Workspace/cpp/beman-project/execution26/src/beman/execution26/tests" "/Users/clausklein/Workspace/cpp/beman-project/execution26/build/release/src/beman/execution26/tests/find-package-test" "--build-generator" "Ninja" "--build-makeprogram" "/usr/local/bin/ninja" "--build-options" "--clean-first" "-DCMAKE_CXX_COMPILER=/usr/local/opt/llvm/bin/clang++" "-DCMAKE_BUILD_TYPE=RelWithDebInfo" "-DCMAKE_PREFIX_PATH=/Users/clausklein/Workspace/cpp/beman-project/execution26/stagedir"
Directory: /Users/clausklein/Workspace/cpp/beman-project/execution26/build/release/src/beman/execution26/tests
"find-package-test" start time: Oct 31 10:41 CET
Output:
----------------------------------------------------------
<end of output>
Test time =   0.02 sec
----------------------------------------------------------
Test Failed.
"find-package-test" end time: Oct 31 10:41 CET
"find-package-test" time elapsed: 00:00:00
----------------------------------------------------------

End testing: Oct 31 10:41 CET
4 
if(NOT PROJECT_IS_TOP_LEVEL)
    # test if the targets are findable from the build directory
    # cmake-format: off
    add_test(NAME find-package-test
      COMMAND ${CMAKE_CTEST_COMMAND}
        # --verbose
        --output-on-failure
        -C $<CONFIG>
        --build-and-test
            "${CMAKE_CURRENT_SOURCE_DIR}"
            "${CMAKE_CURRENT_BINARY_DIR}/find-package-test"
        --build-generator ${CMAKE_GENERATOR}
        --build-makeprogram ${CMAKE_MAKE_PROGRAM}
        --build-options
            "-DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER}"
            "-DCMAKE_BUILD_TYPE=$<CONFIG>"
            "-DCMAKE_PREFIX_PATH=${CMAKE_PREFIX_PATH}"
            # TODO(CK): Needed too? "--config $<CONFIG>"
    )
    # cmake-format: on
endif()

This works with Ninja:

      Start 80: find-package-test

80: Test command: /usr/local/bin/ctest "--output-on-failure" "-C" "RelWithDebInfo" "--build-and-test" "/Users/clausklein/Workspace/cpp/beman-project/execution26/src/beman/execution26/tests" "/Users/clausklein/Workspace/cpp/beman-project/execution26/build/release/src/beman/execution26/tests/find-package-test" "--build-generator" "Ninja" "--build-makeprogram" "/usr/local/bin/ninja" "--build-options" "-DCMAKE_CXX_COMPILER=/usr/local/opt/llvm/bin/clang++" "-DCMAKE_BUILD_TYPE=RelWithDebInfo" "-DCMAKE_PREFIX_PATH=/Users/clausklein/Workspace/cpp/beman-project/execution26/stagedir"
80: Working Directory: /Users/clausklein/Workspace/cpp/beman-project/execution26/build/release/src/beman/execution26/tests
80: Test timeout computed to be: 10000000
80: Internal cmake changing into directory: /Users/clausklein/Workspace/cpp/beman-project/execution26/build/release/src/beman/execution26/tests/find-package-test
80: ======== CMake output     ======
80: Configuring done (0.0s)
80: Generating done (0.0s)
80: Build files have been written to: /Users/clausklein/Workspace/cpp/beman-project/execution26/build/release/src/beman/execution26/tests/find-package-test
80: ======== End CMake output ======
80: Change Dir: '/Users/clausklein/Workspace/cpp/beman-project/execution26/build/release/src/beman/execution26/tests/find-package-test'
80: 
80: Run Clean Command: /usr/local/bin/ninja clean # <<<<<<< NOTE! <<<<<<<
80: [1/1] Cleaning all built files...
80: Cleaning... 7 files.
80: 
80: Run Build Command(s): /usr/local/bin/ninja
80: [1/4] Scanning /Users/clausklein/Workspace/cpp/beman-project/execution26/src/beman/execution26/tests/exec-awaitable.pass.cpp for CXX dependencies
80: [2/4] Generating CXX dyndep file CMakeFiles/.exec-awaitable.pass.dir/CXX.dd
80: [3/4] Building CXX object CMakeFiles/.exec-awaitable.pass.dir/exec-awaitable.pass.cpp.o
80: [4/4] Linking CXX executable .exec-awaitable.pass
80: 
80/80 Test #80: find-package-test .......................................   Passed    1.29 sec

100% tests passed, 0 tests failed out of 80

Total Test time (real) =   2.04 sec