From d21eb992e1bf8db19dd908f01ae70b6ed067d835 Mon Sep 17 00:00:00 2001 From: Steve Gerbino Date: Mon, 16 Nov 2026 17:06:48 +0100 Subject: [PATCH 1/2] fix: list test suites on stdout so discovery ignores stderr diagnostics The runner wrote the --list-tests names to stderr, and the discovery script merged stderr into the listing and split it on whitespace. Any diagnostic a test binary printed during static initialization was registered as one test per word, each passing trivially. The listing now goes to stdout, and discovery reads stdout alone, one suite per line. --- .../DiscoverAndWriteTestsScripts.cmake | 22 ++++++++++++++++--- extra/test_suite/test_suite.cpp | 5 ++++- extra/test_suite/test_suite.hpp | 2 ++ 3 files changed, 25 insertions(+), 4 deletions(-) diff --git a/extra/test_suite/DiscoverAndWriteTestsScripts.cmake b/extra/test_suite/DiscoverAndWriteTestsScripts.cmake index 00e072813..996b7878a 100644 --- a/extra/test_suite/DiscoverAndWriteTestsScripts.cmake +++ b/extra/test_suite/DiscoverAndWriteTestsScripts.cmake @@ -80,7 +80,9 @@ function(boost_capy_test_suite_write_tests_file) execute_process( COMMAND "${TEST_SUITE_TEST_EXECUTABLE}" ${TEST_SUITE_TEST_SPEC} --list-tests OUTPUT_VARIABLE TEST_SUITE_LIST_TESTS_OUTPUT - ERROR_VARIABLE TEST_SUITE_LIST_TESTS_OUTPUT + # Kept apart from the listing: a diagnostic a static initializer + # prints would otherwise be registered as test names + ERROR_VARIABLE TEST_SUITE_LIST_TESTS_ERROR RESULT_VARIABLE TEST_SUITE_RESULT WORKING_DIRECTORY "${TEST_SUITE_TEST_WORKING_DIR}" ) @@ -89,14 +91,28 @@ function(boost_capy_test_suite_write_tests_file) "Error listing tests from executable '${TEST_SUITE_TEST_EXECUTABLE}':\n" " Result: ${TEST_SUITE_RESULT}\n" " Output: ${TEST_SUITE_LIST_TESTS_OUTPUT}\n" + " Error: ${TEST_SUITE_LIST_TESTS_ERROR}\n" ) endif () + if (TEST_SUITE_LIST_TESTS_ERROR) + debug_message("Test executable wrote to stderr: ${TEST_SUITE_LIST_TESTS_ERROR}") + endif () - # Split lines into a list + # One suite name per line # Normalize line endings to \n string(REPLACE "\r\n" "\n" TEST_SUITE_LIST_TESTS_OUTPUT "${TEST_SUITE_LIST_TESTS_OUTPUT}") string(REPLACE "\r" "\n" TEST_SUITE_LIST_TESTS_OUTPUT "${TEST_SUITE_LIST_TESTS_OUTPUT}") - separate_arguments(TEST_SUITE_LIST UNIX_COMMAND "${TEST_SUITE_LIST_TESTS_OUTPUT}") + # A semicolon in a suite name would otherwise split it into list elements + string(REPLACE ";" "\\;" TEST_SUITE_LIST_TESTS_OUTPUT "${TEST_SUITE_LIST_TESTS_OUTPUT}") + string(REGEX MATCHALL "[^\n]+" TEST_SUITE_LIST_LINES "${TEST_SUITE_LIST_TESTS_OUTPUT}") + set(TEST_SUITE_LIST) + foreach (TEST_SUITE_LINE IN LISTS TEST_SUITE_LIST_LINES) + string(STRIP "${TEST_SUITE_LINE}" TEST_SUITE_LINE) + if (NOT TEST_SUITE_LINE STREQUAL "") + string(REPLACE ";" "\\;" TEST_SUITE_LINE "${TEST_SUITE_LINE}") + list(APPEND TEST_SUITE_LIST "${TEST_SUITE_LINE}") + endif () + endforeach () debug_message("Discovered tests: ${TEST_SUITE_LIST}") # Exit early if no tests are detected diff --git a/extra/test_suite/test_suite.cpp b/extra/test_suite/test_suite.cpp index 910db5caa..858232cff 100644 --- a/extra/test_suite/test_suite.cpp +++ b/extra/test_suite/test_suite.cpp @@ -626,10 +626,13 @@ run(std::ostream& out) // regardless of other arguments if (get_command_line_flag("l", "list-tests")) { + // Test discovery parses stdout; diagnostics a static initializer + // writes to stderr would otherwise be read as suite names for (any_suite const* e : suites::instance()) { - out << e->name() << "\n"; + std::cout << e->name() << "\n"; } + std::cout.flush(); any_runner.log_time_at_exit(false); return EXIT_SUCCESS; } diff --git a/extra/test_suite/test_suite.hpp b/extra/test_suite/test_suite.hpp index f048a8659..fe8e6ea89 100644 --- a/extra/test_suite/test_suite.hpp +++ b/extra/test_suite/test_suite.hpp @@ -446,6 +446,8 @@ get_command_line_short_flag(char const* short_name) - If no arguments are provided, all test suites are executed. - If arguments are provided, only test suites whose names match the arguments are executed. - The function also supports a `-h` or `--help` argument to display usage information. + - A `-l` or `--list-tests` argument writes the suite names to standard output, + one per line, regardless of `out`. @param out The output stream to log the results. @param argc The number of command-line arguments. From e4082dc529fefb49e3553d8c3d687db58a9547eb Mon Sep 17 00:00:00 2001 From: Steve Gerbino Date: Mon, 16 Nov 2026 17:07:33 +0100 Subject: [PATCH 2/2] fix: fail test discovery when the test executable cannot list its suites Both error checks in the discovery script went through debug_message, which prints only its first argument and only under TEST_SUITE_SCRIPT_DEBUG, so FATAL_ERROR was printed as text, never raised. A missing executable or a failed --list-tests let the build succeed and left ctest running a stale list, a partial one, or a placeholder test. The checks now raise the error, and the result is tested by name so a launch failure's message cannot break the condition. --- extra/test_suite/DiscoverAndWriteTestsScripts.cmake | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/extra/test_suite/DiscoverAndWriteTestsScripts.cmake b/extra/test_suite/DiscoverAndWriteTestsScripts.cmake index 996b7878a..5554c011d 100644 --- a/extra/test_suite/DiscoverAndWriteTestsScripts.cmake +++ b/extra/test_suite/DiscoverAndWriteTestsScripts.cmake @@ -71,7 +71,7 @@ function(boost_capy_test_suite_write_tests_file) # Variables for arguments if (NOT EXISTS "${TEST_SUITE_TEST_EXECUTABLE}") - debug_message(FATAL_ERROR "Specified test executable '${TEST_SUITE_TEST_EXECUTABLE}' does not exist") + message(FATAL_ERROR "Specified test executable '${TEST_SUITE_TEST_EXECUTABLE}' does not exist") endif () # Run the test executable to get a list of available tests @@ -86,8 +86,8 @@ function(boost_capy_test_suite_write_tests_file) RESULT_VARIABLE TEST_SUITE_RESULT WORKING_DIRECTORY "${TEST_SUITE_TEST_WORKING_DIR}" ) - if (NOT ${TEST_SUITE_RESULT} EQUAL 0) - debug_message(FATAL_ERROR + if (NOT TEST_SUITE_RESULT EQUAL 0) + message(FATAL_ERROR "Error listing tests from executable '${TEST_SUITE_TEST_EXECUTABLE}':\n" " Result: ${TEST_SUITE_RESULT}\n" " Output: ${TEST_SUITE_LIST_TESTS_OUTPUT}\n"