Repository navigation
Fix test discovery for binaries that write to stderr or fail to list #487
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from all commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -71,7 +71,7 @@ function(boost_capy_test_suite_write_tests_file) | |
|
|
||
| # Variables for arguments | ||
| if (NOT EXISTS "${TEST_SUITE_TEST_EXECUTABLE}") | ||
| debug_message(FATAL_ERROR "Specified test executable '${TEST_SUITE_TEST_EXECUTABLE}' does not exist") | ||
| message(FATAL_ERROR "Specified test executable '${TEST_SUITE_TEST_EXECUTABLE}' does not exist") | ||
| endif () | ||
|
|
||
| # Run the test executable to get a list of available tests | ||
|
|
@@ -80,23 +80,39 @@ function(boost_capy_test_suite_write_tests_file) | |
| execute_process( | ||
| COMMAND "${TEST_SUITE_TEST_EXECUTABLE}" ${TEST_SUITE_TEST_SPEC} --list-tests | ||
| OUTPUT_VARIABLE TEST_SUITE_LIST_TESTS_OUTPUT | ||
| ERROR_VARIABLE TEST_SUITE_LIST_TESTS_OUTPUT | ||
| # Kept apart from the listing: a diagnostic a static initializer | ||
| # prints would otherwise be registered as test names | ||
| ERROR_VARIABLE TEST_SUITE_LIST_TESTS_ERROR | ||
| RESULT_VARIABLE TEST_SUITE_RESULT | ||
| WORKING_DIRECTORY "${TEST_SUITE_TEST_WORKING_DIR}" | ||
| ) | ||
| if (NOT ${TEST_SUITE_RESULT} EQUAL 0) | ||
| debug_message(FATAL_ERROR | ||
| if (NOT TEST_SUITE_RESULT EQUAL 0) | ||
| message(FATAL_ERROR | ||
| "Error listing tests from executable '${TEST_SUITE_TEST_EXECUTABLE}':\n" | ||
| " Result: ${TEST_SUITE_RESULT}\n" | ||
| " Output: ${TEST_SUITE_LIST_TESTS_OUTPUT}\n" | ||
| " Error: ${TEST_SUITE_LIST_TESTS_ERROR}\n" | ||
| ) | ||
| endif () | ||
| if (TEST_SUITE_LIST_TESTS_ERROR) | ||
| debug_message("Test executable wrote to stderr: ${TEST_SUITE_LIST_TESTS_ERROR}") | ||
| endif () | ||
|
|
||
| # Split lines into a list | ||
| # One suite name per line | ||
| # Normalize line endings to \n | ||
| string(REPLACE "\r\n" "\n" TEST_SUITE_LIST_TESTS_OUTPUT "${TEST_SUITE_LIST_TESTS_OUTPUT}") | ||
| string(REPLACE "\r" "\n" TEST_SUITE_LIST_TESTS_OUTPUT "${TEST_SUITE_LIST_TESTS_OUTPUT}") | ||
| separate_arguments(TEST_SUITE_LIST UNIX_COMMAND "${TEST_SUITE_LIST_TESTS_OUTPUT}") | ||
| # A semicolon in a suite name would otherwise split it into list elements | ||
| string(REPLACE ";" "\\;" TEST_SUITE_LIST_TESTS_OUTPUT "${TEST_SUITE_LIST_TESTS_OUTPUT}") | ||
| string(REGEX MATCHALL "[^\n]+" TEST_SUITE_LIST_LINES "${TEST_SUITE_LIST_TESTS_OUTPUT}") | ||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Preserve semicolons inside suite names. For The generated tests match neither registered suite and return success without executing it. Preserve each complete name through both list-construction steps and command generation. Alternatively, fail discovery explicitly for unsupported names instead of generating passing tests.
|
||
| set(TEST_SUITE_LIST) | ||
| foreach (TEST_SUITE_LINE IN LISTS TEST_SUITE_LIST_LINES) | ||
| string(STRIP "${TEST_SUITE_LINE}" TEST_SUITE_LINE) | ||
| if (NOT TEST_SUITE_LINE STREQUAL "") | ||
| string(REPLACE ";" "\\;" TEST_SUITE_LINE "${TEST_SUITE_LINE}") | ||
| list(APPEND TEST_SUITE_LIST "${TEST_SUITE_LINE}") | ||
| endif () | ||
| endforeach () | ||
| debug_message("Discovered tests: ${TEST_SUITE_LIST}") | ||
|
|
||
| # Exit early if no tests are detected | ||
|
|
||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
No automated test checks that stderr warnings stay out of the registered names or that a failed
--list-testsstops discovery before writing a tests file. The manual checks described in the PR cover these cases, but they will not catch a later regression in CI. Add automated tests for both behaviors.Prompt To Fix With AI
Note: If this suggestion doesn't match your team's coding style, reply to this and let me know. I'll remember it for next time!