From 3cf0f8d2fbc485f841dd34ee7b77cdd0ebfd89f6 Mon Sep 17 00:00:00 2001 From: KeviM Date: Fri, 20 Nov 2026 11:54:05 -0800 Subject: [PATCH 1/9] fix(http): apply size limits to the final body and stop retrying permanent failures Fixes #45. CURLOPT_MAXFILESIZE_LARGE compared every Content-Length in the redirect chain, and encoded lengths, against the decoded-body limit. The write callback only sees the final body, so the early refusal moves there. Malformed URLs, disallowed schemes, and redirect loops were NetworkError, which with_retry repeats. Retry-After values past int64 were ignored, and statuses above 599 were ServerError. --- CHANGELOG.md | 15 ++++ include/spc/error.hpp | 10 +-- include/spc/http_client.hpp | 9 +-- include/spc/rate_limit.hpp | 5 +- include/spc/retry.hpp | 16 +++-- src/core/error.cpp | 2 +- src/core/rate_limit.cpp | 5 +- src/http/client.cpp | 51 +++++++++++--- tests/test_client.cpp | 6 ++ tests/test_http_client.cpp | 133 +++++++++++++++++++++++++++++++++++- tests/test_retry.cpp | 10 +++ 11 files changed, 233 insertions(+), 29 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8481373..5099992 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,21 @@ uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [Unreleased] +### Changed + +- Report a malformed URL or a scheme other than HTTP(S) as `InvalidRequest`, and a + redirect loop as `ServerError`. `with_retry` no longer repeats them. +- Map only HTTP 500-599 to `ServerError`; a status above 599 is `Unknown`. + +### Fixed + +- Apply `ClientConfig::max_response_bytes` to the final response's decoded body. A + redirect hop's `Content-Length`, or a compressed length over the limit, no longer + fails a request whose body fits. +- Cap a `Retry-After` too large for int64 at the longest wait instead of ignoring it. +- Reset the daily quota when the clock steps back across midnight. +- Initialize `HttpResponse::status_code` to 0. + ## [0.4.7] - 2026-11-04 ### Fixed diff --git a/include/spc/error.hpp b/include/spc/error.hpp index 421a87e..c51c9d1 100644 --- a/include/spc/error.hpp +++ b/include/spc/error.hpp @@ -17,7 +17,8 @@ enum class ErrorCode { NetworkError, /// HTTP 429 or 503, or the SDK's own rate limit. RateLimited, - /// HTTP 5xx other than 503, or an ArcGIS query that never finished paging. + /// HTTP 500-599 other than 503, a redirect loop, or an ArcGIS query that + /// never finished paging. ServerError, /// A URL or ArcGIS service that no longer exists: a bug to report. A bad /// layer id is `InvalidRequest`. @@ -27,9 +28,10 @@ enum class ErrorCode { /// so treat it as "no data", not as a failure. Only `StaticFeedClient` /// returns it. FeedUnavailable, - /// The request cannot succeed as asked: an unsupported day or hazard, - /// HTTP 400, an ArcGIS error such as an invalid parameter, or a response - /// larger than `ClientConfig::max_response_bytes`. + /// The request cannot succeed as asked: an unsupported day or hazard, a + /// malformed URL or one that is not HTTP(S), HTTP 400, an ArcGIS error such + /// as an invalid parameter, or a response larger than + /// `ClientConfig::max_response_bytes`. InvalidRequest, /// The body was not valid JSON. Valid JSON without the expected fields /// parses to an empty result. diff --git a/include/spc/http_client.hpp b/include/spc/http_client.hpp index 2e99f71..f2e64f1 100644 --- a/include/spc/http_client.hpp +++ b/include/spc/http_client.hpp @@ -20,7 +20,7 @@ namespace spc { /// An HTTP response. Any status code is a successful transfer; the clients /// decide what a non-200 status means. struct HttpResponse { - std::int16_t status_code; + std::int16_t status_code{0}; std::string body; /// Headers of the final response, after redirects. std::vector> headers; @@ -36,9 +36,10 @@ struct ClientConfig { /// supported libcurl versions. Invalid values fail before sending. std::chrono::seconds timeout{15}; bool verify_ssl{true}; - /// Largest response body accepted. Bodies are held in memory and parsed - /// whole, and an IEM date range can be arbitrarily large. A bigger - /// response fails with `ErrorCode::InvalidRequest` and is not retried. + /// Largest decoded body accepted from the final response; 0 means no + /// limit. Bodies are held in memory and parsed whole, and an IEM date + /// range can be arbitrarily large. A bigger response fails with + /// `ErrorCode::InvalidRequest` and is not retried. std::size_t max_response_bytes{64UL * 1024UL * 1024UL}; }; diff --git a/include/spc/rate_limit.hpp b/include/spc/rate_limit.hpp index 1b15bde..627a6eb 100644 --- a/include/spc/rate_limit.hpp +++ b/include/spc/rate_limit.hpp @@ -30,8 +30,9 @@ class RateLimiter { [[nodiscard]] bool try_acquire() noexcept; /// Wait for a token, up to `Config::max_wait` if set. Returns false when - /// no token can arrive in time or the daily quota is spent; it does not - /// sleep when waiting cannot help. + /// no token can arrive in time or the daily quota is spent. A lone caller + /// returns at once when waiting cannot help; with several waiters, one may + /// wait and still get nothing. [[nodiscard]] bool acquire(); /// `acquire()` with an explicit limit on the wait. diff --git a/include/spc/retry.hpp b/include/spc/retry.hpp index 4663d82..4f22b3b 100644 --- a/include/spc/retry.hpp +++ b/include/spc/retry.hpp @@ -104,15 +104,21 @@ struct RetryPolicy { } value.remove_prefix(start); value.remove_suffix(value.size() - value.find_last_not_of(" \t") - 1); + // Cap before converting so a huge value cannot overflow. + constexpr std::int64_t kMaxSeconds = std::int64_t{365} * 24 * 60 * 60; std::int64_t seconds = 0; const std::from_chars_result parsed = std::from_chars(value.data(), value.data() + value.size(), seconds); - if (parsed.ec == std::errc{} && parsed.ptr == value.data() + value.size() && seconds > 0) { - // Cap before converting so a huge value cannot overflow. - constexpr std::int64_t kMaxSeconds = std::int64_t{365} * 24 * 60 * 60; - return std::chrono::milliseconds{std::min(seconds, kMaxSeconds) * 1000}; + if (parsed.ptr != value.data() + value.size()) { + return std::chrono::milliseconds{0}; + } + // Digits too many for int64 still ask for the longest wait. + if (parsed.ec == std::errc::result_out_of_range && value.front() != '-') { + seconds = kMaxSeconds; + } else if (parsed.ec != std::errc{} || seconds <= 0) { + return std::chrono::milliseconds{0}; } - return std::chrono::milliseconds{0}; + return std::chrono::milliseconds{std::min(seconds, kMaxSeconds) * 1000}; } return std::chrono::milliseconds{0}; } diff --git a/src/core/error.cpp b/src/core/error.cpp index 93a0628..ded6c50 100644 --- a/src/core/error.cpp +++ b/src/core/error.cpp @@ -53,7 +53,7 @@ Error Error::from_response(int status, const std::string& body, Feed404 semantic err.code = ErrorCode::InvalidRequest; } else if (status == 429 || status == 503) { err.code = ErrorCode::RateLimited; - } else if (status >= 500) { + } else if (status >= 500 && status <= 599) { err.code = ErrorCode::ServerError; } else { err.code = ErrorCode::Unknown; diff --git a/src/core/rate_limit.cpp b/src/core/rate_limit.cpp index 5034f0b..1a52230 100644 --- a/src/core/rate_limit.cpp +++ b/src/core/rate_limit.cpp @@ -32,7 +32,8 @@ RateLimiter::RateLimiter(Config config) void RateLimiter::check_daily_reset() noexcept { const std::chrono::system_clock::time_point today = utc_day_start(); - if (today > day_start_) { + // Not `>`: a clock stepped back across midnight must not freeze the quota. + if (today != day_start_) { daily_requests_used_ = 0; day_start_ = today; } @@ -148,7 +149,7 @@ std::int32_t RateLimiter::daily_requests_remaining() const noexcept { return 0; } // A new UTC day resets the count even before the next acquire. - if (utc_day_start() > day_start_) { + if (utc_day_start() != day_start_) { return config_.daily_limit; } return config_.daily_limit - daily_requests_used_; diff --git a/src/http/client.cpp b/src/http/client.cpp index c929724..c3a0ea7 100644 --- a/src/http/client.cpp +++ b/src/http/client.cpp @@ -2,6 +2,7 @@ #include #include +#include #include #include #include @@ -28,16 +29,43 @@ const ClientConfig kMovedFromConfig{}; /// Collects the body up to a ceiling. Returning a short count from the write /// callback makes libcurl abort with CURLE_WRITE_ERROR. struct BodySink { + CURL* curl{nullptr}; std::string body; std::size_t limit{0}; + bool started{false}; bool overflowed{false}; bool failed{false}; }; -// libcurl is C, so no exception may leave this callback. +/// Whether the response being received announces an identity body over +/// `limit`. An encoded Content-Length says nothing about the decoded size, so +/// those bodies are left to the running count. +bool announces_more_than(CURL* curl, std::size_t limit) noexcept { + curl_off_t length = -1; + if (curl_easy_getinfo(curl, CURLINFO_CONTENT_LENGTH_DOWNLOAD_T, &length) != CURLE_OK || + length < 0) { + return false; + } + curl_header* encoding = nullptr; + if (curl_easy_header(curl, "Content-Encoding", 0, CURLH_HEADER, -1, &encoding) == CURLHE_OK) { + return false; + } + return static_cast(length) > static_cast(limit); +} + +// libcurl is C, so no exception may leave this callback. It sees only the +// final response: libcurl drops the bodies of redirects it follows, which is +// why the early size check lives here and not in CURLOPT_MAXFILESIZE. std::size_t write_body(char* data, std::size_t size, std::size_t count, void* user) noexcept { BodySink* sink = static_cast(user); const std::size_t chunk = size * count; + if (!sink->started) { + sink->started = true; + if (sink->limit > 0 && announces_more_than(sink->curl, sink->limit)) { + sink->overflowed = true; + return 0; + } + } if (sink->limit > 0 && chunk > sink->limit - sink->body.size()) { sink->overflowed = true; return 0; @@ -142,6 +170,7 @@ Result perform(Connection& connection, const std::string& url, } BodySink sink; + sink.curl = curl; sink.limit = config.max_response_bytes; curl_easy_setopt(curl, CURLOPT_ERRORBUFFER, connection.errors.data()); curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, &write_body); @@ -151,14 +180,10 @@ Result perform(Connection& connection, const std::string& url, curl_easy_setopt(curl, CURLOPT_NOSIGNAL, 1L); curl_easy_setopt(curl, CURLOPT_USERAGENT, config.user_agent.c_str()); curl_easy_setopt(curl, CURLOPT_ACCEPT_ENCODING, ""); - if (sink.limit > 0) { - // Lets libcurl refuse early when Content-Length is already too big. - curl_easy_setopt(curl, CURLOPT_MAXFILESIZE_LARGE, static_cast(sink.limit)); - } const CURLcode rc = curl_easy_perform(curl); // Not a NetworkError: retrying would download the same body again. - if (sink.overflowed || rc == CURLE_FILESIZE_EXCEEDED) { + if (sink.overflowed) { return std::unexpected( Error::invalid_request("response exceeded ClientConfig::max_response_bytes")); } @@ -166,9 +191,17 @@ Result perform(Connection& connection, const std::string& url, return std::unexpected(Error::network("out of memory while reading the response")); } if (rc != CURLE_OK) { - const bool detailed = connection.errors.front() != '\0'; - return std::unexpected( - Error::network(detailed ? connection.errors.data() : curl_easy_strerror(rc))); + std::string message = + connection.errors.front() != '\0' ? connection.errors.data() : curl_easy_strerror(rc); + // These fail the same way on every attempt, so they are not NetworkError, + // which with_retry repeats. + if (rc == CURLE_UNSUPPORTED_PROTOCOL || rc == CURLE_URL_MALFORMAT) { + return std::unexpected(Error::invalid_request(std::move(message))); + } + if (rc == CURLE_TOO_MANY_REDIRECTS) { + return std::unexpected(Error::server(std::move(message))); + } + return std::unexpected(Error::network(std::move(message))); } long status = 0; diff --git a/tests/test_client.cpp b/tests/test_client.cpp index fd4ba40..f99b8da 100644 --- a/tests/test_client.cpp +++ b/tests/test_client.cpp @@ -514,6 +514,12 @@ TEST(ArcGISClientRouting, RejectsNonJsonFormatsBeforeNetworkAccess) { EXPECT_TRUE(transport->requests.empty()); } +TEST(ClientErrors, OnlyStatuses500To599AreServerErrors) { + EXPECT_EQ(Error::from_response(500, "").code, ErrorCode::ServerError); + EXPECT_EQ(Error::from_response(599, "").code, ErrorCode::ServerError); + EXPECT_EQ(Error::from_response(600, "").code, ErrorCode::Unknown); +} + TEST(ClientErrors, AMalformedBodyIsAParseErrorNotAnException) { std::shared_ptr transport = std::make_shared(); transport->responses = {{200, "maintenance", {}}, diff --git a/tests/test_http_client.cpp b/tests/test_http_client.cpp index be2275c..5d0329c 100644 --- a/tests/test_http_client.cpp +++ b/tests/test_http_client.cpp @@ -3,6 +3,7 @@ /// headers, scheme restrictions, and concurrent use. #include "spc/http_client.hpp" +#include "spc/retry.hpp" #include "support/loopback_server.hpp" #include @@ -11,10 +12,12 @@ #include #include #include +#include #include #include #include #include +#include #include #include #include @@ -56,6 +59,40 @@ std::string chunked_response(std::size_t chunks, std::size_t chunk_size) { return response + "0\r\n\r\n"; } +void append_le32(std::string& out, std::uint32_t value) { + for (int shift = 0; shift < 32; shift += 8) { + out += static_cast((value >> shift) & 0xFFU); + } +} + +std::uint32_t crc32(std::string_view data) { + std::uint32_t crc = 0xFFFFFFFFU; + for (const char c : data) { + crc ^= static_cast(c); + for (int bit = 0; bit < 8; ++bit) { + crc = (crc >> 1) ^ (0xEDB88320U & (0U - (crc & 1U))); + } + } + return ~crc; +} + +/// `data` as gzip with one stored deflate block, so the encoded body is 23 +/// bytes longer than the decoded one. +std::string gzip_stored(std::string_view data) { + std::string out{"\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff", 10}; + const std::uint16_t size = static_cast(data.size()); + const std::uint16_t inverse = static_cast(~size); + out += '\x01'; // final block, stored + out += static_cast(size & 0xFFU); + out += static_cast(size >> 8); + out += static_cast(inverse & 0xFFU); + out += static_cast(inverse >> 8); + out += data; + append_le32(out, crc32(data)); + append_le32(out, static_cast(data.size())); + return out; +} + TEST(LoopbackServer, ShutdownIsBoundedWhenAClientStopsReading) { std::atomic responding{false}; std::unique_ptr server = @@ -215,6 +252,72 @@ TEST(HttpClient, RejectsAChunkedBodyThatGrowsPastTheLimit) { EXPECT_NE(response.error().message.find("max_response_bytes"), std::string::npos); } +TEST(HttpClient, IgnoresTheSizeOfARedirectBodyItDiscards) { + // The chunked final response has no length of its own, so the hop's must + // not carry over to it. + LoopbackServer server{[](const LoopbackRequest& request) { + if (request.target == "/start") { + return http_response("302 Found", "REDIRECT-BODY", "Location: /final\r\n"); + } + if (request.target == "/start-chunked") { + return http_response("302 Found", "REDIRECT-BODY", "Location: /final-chunked\r\n"); + } + if (request.target == "/final-chunked") { + return chunked_response(1, 2); + } + return http_response("200 OK", "OK"); + }}; + ClientConfig config; + config.max_response_bytes = 8; + const HttpClient client{config}; + + const Result sized = client.get(server.url("/start")); + const Result chunked = client.get(server.url("/start-chunked")); + + ASSERT_TRUE(sized) << sized.error().message; + EXPECT_EQ(sized->status_code, 200); + EXPECT_EQ(sized->body, "OK"); + ASSERT_TRUE(chunked) << chunked.error().message; + EXPECT_EQ(chunked->body, "xx"); +} + +TEST(HttpClient, RefusesAnAdvertisedOversizeBodyBeforeReadingIt) { + // The server stops after 10 bytes. Reading on would end in a partial-body + // NetworkError, which with_retry would retry. + LoopbackServer server{[](const LoopbackRequest&) { + return std::string{"HTTP/1.1 200 OK\r\nContent-Length: 1048576\r\n" + "Connection: close\r\n\r\n0123456789"}; + }}; + ClientConfig config; + config.max_response_bytes = 1024; + const HttpClient client{config}; + + const Result response = client.get(server.url("/big")); + + ASSERT_FALSE(response); + EXPECT_EQ(response.error().code, ErrorCode::InvalidRequest); + EXPECT_NE(response.error().message.find("max_response_bytes"), std::string::npos); +} + +TEST(HttpClient, LimitsTheDecodedBodyNotTheEncodedOne) { + if ((curl_version_info(CURLVERSION_NOW)->features & CURL_VERSION_LIBZ) == 0) { + GTEST_SKIP() << "libcurl was built without gzip support"; + } + const std::string body(64, 'x'); + const std::string encoded = gzip_stored(body); + LoopbackServer server{[&encoded](const LoopbackRequest&) { + return http_response("200 OK", encoded, "Content-Encoding: gzip\r\n"); + }}; + ClientConfig config; + config.max_response_bytes = 80; // between the decoded 64 and the encoded 87 + const HttpClient client{config}; + + const Result response = client.get(server.url("/gzip")); + + ASSERT_TRUE(response) << response.error().message; + EXPECT_EQ(response->body, body); +} + TEST(HttpClient, RefusesEverySchemeOtherThanHttpAndHttps) { // file:// would otherwise read the local file and return it as the body. const HttpClient client; @@ -222,7 +325,17 @@ TEST(HttpClient, RefusesEverySchemeOtherThanHttpAndHttps) { const Result result = client.get("file:///etc/hosts"); ASSERT_FALSE(result) << "file:// must not be fetched"; - EXPECT_EQ(result.error().code, ErrorCode::NetworkError); + // Not NetworkError, which with_retry would retry. + EXPECT_EQ(result.error().code, ErrorCode::InvalidRequest); +} + +TEST(HttpClient, AMalformedUrlIsAnInvalidRequest) { + const HttpClient client; + + const Result result = client.get("http://bad host/"); + + ASSERT_FALSE(result); + EXPECT_EQ(result.error().code, ErrorCode::InvalidRequest); } TEST(HttpClient, RefusesARedirectToAnotherScheme) { @@ -234,7 +347,21 @@ TEST(HttpClient, RefusesARedirectToAnotherScheme) { const Result response = client.get(server.url("/escape")); ASSERT_FALSE(response); - EXPECT_EQ(response.error().code, ErrorCode::NetworkError); + EXPECT_EQ(response.error().code, ErrorCode::InvalidRequest); +} + +TEST(HttpClient, ARedirectLoopIsAServerErrorThatIsNotRetried) { + LoopbackServer server{[](const LoopbackRequest&) { + return http_response("302 Found", "", "Location: /loop\r\n"); + }}; + const HttpClient client; + + const Result response = client.get(server.url("/loop")); + + ASSERT_FALSE(response); + EXPECT_EQ(response.error().code, ErrorCode::ServerError); + EXPECT_FALSE(should_retry(response.error(), RetryPolicy{})); + EXPECT_EQ(server.requests().size(), 11U); // the first request and 10 redirects } TEST(HttpClient, ReusedHandlesKeepTheSchemeRestriction) { @@ -286,6 +413,8 @@ TEST(HttpClient, AMovedFromClientReportsAnErrorInsteadOfCrashing) { ASSERT_FALSE(response); EXPECT_EQ(response.error().code, ErrorCode::InvalidRequest); EXPECT_TRUE(moved.config().verify_ssl); + // NOLINTNEXTLINE(bugprone-use-after-move,clang-analyzer-cplusplus.Move) + EXPECT_EQ(original.config().max_response_bytes, ClientConfig{}.max_response_bytes); } } // namespace diff --git a/tests/test_retry.cpp b/tests/test_retry.cpp index 195308c..3920d55 100644 --- a/tests/test_retry.cpp +++ b/tests/test_retry.cpp @@ -126,6 +126,16 @@ TEST(Retry, AHugeRetryAfterIsCappedInsteadOfOverflowing) { EXPECT_LE(delay, std::chrono::hours{24 * 365}); } +TEST(Retry, ARetryAfterBeyondInt64IsCappedLikeAnyHugeValue) { + const HttpResponse fits{429, "", {{"Retry-After", "9223372036854775807"}}}; + const HttpResponse too_big{429, "", {{"Retry-After", "99999999999999999999"}}}; + const HttpResponse negative{429, "", {{"Retry-After", "-99999999999999999999"}}}; + + EXPECT_EQ(retry_after(too_big), retry_after(fits)); + EXPECT_EQ(retry_after(fits), std::chrono::hours{24 * 365}); + EXPECT_EQ(retry_after(negative).count(), 0); +} + TEST(Retry, IgnoresAnUnparseableRetryAfterAndFallsBackToTheBackoff) { RetryPolicy policy; policy.max_attempts = 2; From 42767fa21655c9c583b9aaabe7079107ce8f1e81 Mon Sep 17 00:00:00 2001 From: KeviM Date: Fri, 20 Nov 2026 11:59:32 -0800 Subject: [PATCH 2/9] fix(models): skip fieldless features, non-finite strings, and null references Found in the audit: conditional-intensity and day 1-2 fire-weather layers kept features with null or missing fields, numeric strings "nan" and "inf" reached public doubles, and a null latestWkid or geometry spatialReference hid a valid reference. The locale test now drops dn so it can fail. --- CHANGELOG.md | 5 +++ include/spc/models/convective.hpp | 3 +- include/spc/models/fire_weather.hpp | 4 +- src/models/convective.cpp | 3 ++ src/models/fire_weather.cpp | 3 ++ src/models/json.cpp | 19 +++++---- src/models/json.hpp | 8 ++-- src/models/mesoscale.cpp | 2 +- tests/fixtures/README.md | 10 +++-- tests/test_arcgis.cpp | 3 +- tests/test_corpus.cpp | 6 +-- tests/test_locale.cpp | 19 ++++++--- tests/test_parser.cpp | 64 +++++++++++++++++++++++++++++ 13 files changed, 123 insertions(+), 26 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5099992..99313e1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,11 @@ uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html). - Cap a `Retry-After` too large for int64 at the longest wait instead of ignoring it. - Reset the daily quota when the clock steps back across midnight. - Initialize `HttpResponse::status_code` to 0. +- Skip features whose fields are not an object, conditional-intensity polygons + without a label, and day 1-2 fire-weather polygons with no known `dn` band. +- Read the numeric strings `"nan"` and `"inf"` as 0 instead of passing them on. +- Fall back to `wkid` when `latestWkid` is null or not a number, and ignore a null + geometry `spatialReference`. ## [0.4.7] - 2026-11-04 diff --git a/include/spc/models/convective.hpp b/include/spc/models/convective.hpp index 1222d43..ad0180e 100644 --- a/include/spc/models/convective.hpp +++ b/include/spc/models/convective.hpp @@ -52,7 +52,8 @@ struct ConditionalIntensityPayload { /// @throws std::runtime_error for invalid JSON or Esri geometry that cannot be converted. [[nodiscard]] Day48OutlookPayload parse_day4_8(std::string_view body, std::int32_t day); -/// Parse a conditional-intensity layer (GeoJSON or Esri). +/// Parse a conditional-intensity layer (GeoJSON or Esri). Polygons without a +/// label are skipped. /// @throws std::runtime_error for invalid JSON or Esri geometry that cannot be converted. [[nodiscard]] ConditionalIntensityPayload parse_conditional_intensity(std::string_view body, std::int32_t day, std::string hazard); diff --git a/include/spc/models/fire_weather.hpp b/include/spc/models/fire_weather.hpp index c456b20..9a1ae0c 100644 --- a/include/spc/models/fire_weather.hpp +++ b/include/spc/models/fire_weather.hpp @@ -39,8 +39,8 @@ struct FireWeatherPayload { /// ELEV 1, CRIT 2, EXTM 3. Dry-thunderstorm labels and anything else map to 0. [[nodiscard]] std::uint8_t fire_severity_from_label(std::string_view label) noexcept; -/// Parse one fire-weather layer (Esri or GeoJSON). No-risk polygons are -/// skipped. +/// Parse one fire-weather layer (Esri or GeoJSON). No-risk polygons, and +/// day 1-2 polygons with neither a label nor a known `dn` band, are skipped. /// /// `layer` is required because days 1 and 2 carry only a numeric `dn` band, /// and the outlook (5 ELEV, 8 CRIT, 10 EXTM) and dry-thunderstorm (5 IDRT, diff --git a/src/models/convective.cpp b/src/models/convective.cpp index 438a7cc..a77c291 100644 --- a/src/models/convective.cpp +++ b/src/models/convective.cpp @@ -77,6 +77,9 @@ detail::conditional_intensity_from_tree(const Json& root, std::int32_t day, std: } ConditionalIntensityFeature f; f.label = detail::first_string(*props, {"LABEL", "label"}); + if (f.label.empty()) { + continue; + } f.cig_level = cig_severity_from_label(f.label); f.issued_at = detail::first_timestamp(*props, {"ISSUE", "issue"}); f.valid_from = detail::first_timestamp(*props, {"VALID", "valid"}); diff --git a/src/models/fire_weather.cpp b/src/models/fire_weather.cpp index 8b16fa2..d4c6214 100644 --- a/src/models/fire_weather.cpp +++ b/src/models/fire_weather.cpp @@ -100,6 +100,9 @@ FireWeatherPayload detail::fire_weather_from_tree(const Json& root, std::int32_t f.probability = detail::normalized_probability(*props); } else if (f.label.empty()) { f.label = label_from_dn(*props, layer); + if (f.label.empty()) { + continue; // no label and no known dn band + } } f.severity = fire_severity_from_label(f.label); f.issued_at = detail::first_timestamp(*props, {"ISSUE", "issue"}); diff --git a/src/models/json.cpp b/src/models/json.cpp index 4726b5f..1105995 100644 --- a/src/models/json.cpp +++ b/src/models/json.cpp @@ -126,9 +126,10 @@ double json_number_or_numeric_string(const Json& obj, const char* key) { return v->get(); } if (v->is_string()) { - // Matches a C-locale std::stod for every value SPC publishes. + // Matches a C-locale std::stod for every value SPC publishes. No field + // means "nan" or "inf", so those read as 0 too. const ParsedNumber parsed = parse_double(v->get()); - return parsed.ok ? parsed.value : 0.0; + return parsed.ok && std::isfinite(parsed.value) ? parsed.value : 0.0; } return 0.0; } @@ -234,13 +235,14 @@ std::vector parse_esri_rings_with_budget(const Json& geom, const Json* if (rings_node == nullptr || !rings_node->is_array()) { return out; } - if (const Json* local = lookup(geom, "spatialReference")) { + if (const Json* local = lookup(geom, "spatialReference"); + local != nullptr && local->is_object()) { reference = local; } bool mercator = false; - if (reference != nullptr) { + if (reference != nullptr && reference->is_object()) { const Json* id = lookup(*reference, "latestWkid"); - if (id == nullptr) { + if (id == nullptr || !id->is_number()) { id = lookup(*reference, "wkid"); } const double wkid = id != nullptr && id->is_number() ? id->get() : 0.0; @@ -348,8 +350,11 @@ Json parse_root_or_throw(std::string_view body) { } const Json* feature_fields(const Json& feature) { - const Json* properties = lookup(feature, "properties"); - return properties != nullptr ? properties : lookup(feature, "attributes"); + const Json* fields = lookup(feature, "properties"); + if (fields == nullptr) { + fields = lookup(feature, "attributes"); + } + return fields != nullptr && fields->is_object() ? fields : nullptr; } PayloadGeometryParser::PayloadGeometryParser(const Json& root) diff --git a/src/models/json.hpp b/src/models/json.hpp index 07e4643..19e6341 100644 --- a/src/models/json.hpp +++ b/src/models/json.hpp @@ -38,11 +38,12 @@ struct ParsedNumber { /// Parse the leading decimal number of `text` regardless of the process /// locale. `std::stod` follows LC_NUMERIC, so on a comma-decimal host it reads -/// "0.15" as 0. Leading whitespace and '+' are rejected. +/// "0.15" as 0. Leading whitespace and '+' are rejected; "nan" and "inf" parse, +/// as with std::from_chars. ParsedNumber parse_double(std::string_view text); /// `obj[key]` as a number, accepting numeric strings such as "5" or "0.15". -/// Missing or non-numeric values yield 0. +/// Missing, non-numeric, and non-finite values yield 0. double json_number_or_numeric_string(const Json& obj, const char* key); /// The LABEL/label/dn probability normalized to [0, 1]. Values above 1 are @@ -83,7 +84,8 @@ glz::expected parse_owned_root(const std::string& body); /// public `parse_*` contract. Json parse_root_or_throw(std::string_view body); -/// A feature's fields: GeoJSON `properties` or Esri `attributes`. +/// A feature's fields: GeoJSON `properties` or Esri `attributes`, or null +/// when they are missing or not an object. const Json* feature_fields(const Json& feature); /// One per payload: shares the collection reference and projection budget diff --git a/src/models/mesoscale.cpp b/src/models/mesoscale.cpp index 081b527..49801e3 100644 --- a/src/models/mesoscale.cpp +++ b/src/models/mesoscale.cpp @@ -21,7 +21,7 @@ MesoscalePayload detail::mesoscale_from_tree(const Json& root) { detail::PayloadGeometryParser geometry_parser(root); for (const Json& feat : features_node->get_array()) { const Json* props = detail::feature_fields(feat); - if (props == nullptr || !props->is_object()) { + if (props == nullptr) { continue; } MesoscaleDiscussion md; diff --git a/tests/fixtures/README.md b/tests/fixtures/README.md index 4185bd8..ba5b41e 100644 --- a/tests/fixtures/README.md +++ b/tests/fixtures/README.md @@ -25,9 +25,9 @@ Pairs captured as Esri JSON (`.esri.json`, `f=json`) and GeoJSON (`.geojson`, Single captures: - `arcgis_day1_torn_conditional_intensity.esri.json`: `label` `"CIG1"`. -- `arcgis_day4_8_nonempty.synthetic.json`: a non-empty Day 4 response. -- `arcgis_day{1,2}_fire_weather.esri.json`: fire weather, with only a numeric - `dn` and no label. +- `arcgis_day{1,2}_fire_weather.esri.json`: fire weather in the layer's native + Web Mercator (wkid 102100), with only a numeric `dn`, no label, and no + `issue`. The client asks for `outSR=4326` instead; these test the conversion. - `arcgis_mesoscale_discussion.esri.json`: one active discussion. - `arcgis_mesoscale_discussion_noarea.esri.json`: the `NoArea` placeholder the layer returns when no discussion is active (captured 2026-09-25). @@ -35,6 +35,10 @@ Single captures: MapServers. `python3 tools/verify_arcgis_metadata.py` compares it with the live services. +Synthetic: + +- `arcgis_day4_8_nonempty.synthetic.json`: a non-empty Day 4 response. + ## IEM (`mesonet.agron.iastate.edu`) - `iem_storm_reports.json`: Local Storm Reports, Point geometry. diff --git a/tests/test_arcgis.cpp b/tests/test_arcgis.cpp index 9617d62..6e8d996 100644 --- a/tests/test_arcgis.cpp +++ b/tests/test_arcgis.cpp @@ -358,7 +358,8 @@ TEST(Models, FireWeatherLayerIsTheOnlyThingThatDisambiguatesDayOneAndTwoDn) { parse_fire_weather(body, day, FireWeatherLayer::DryThunderstorm); ASSERT_EQ(outlook.features.size(), 3u) << name; - ASSERT_EQ(dry.features.size(), 3u) << name; + // dn 10 has no dry-thunderstorm band, so that polygon is skipped. + ASSERT_EQ(dry.features.size(), 2u) << name; EXPECT_EQ(outlook.features[0].label, "ELEV") << name; EXPECT_EQ(outlook.features[1].label, "CRIT") << name; EXPECT_EQ(outlook.features[2].label, "EXTM") << name; diff --git a/tests/test_corpus.cpp b/tests/test_corpus.cpp index 08f4b1f..76bd5c1 100644 --- a/tests/test_corpus.cpp +++ b/tests/test_corpus.cpp @@ -1,7 +1,7 @@ /// @file test_corpus.cpp -/// @brief The captured SPC fixtures through the Day 1-3 parsers. spc-data -/// checks the same fixtures for identical output; this catches a regression -/// here first. +/// @brief The captured fixtures through every parser, including watches, +/// storm reports, and discussions. spc-data checks the Day 1-3 fixtures for +/// identical output; this catches a regression here first. #include "spc/models/mesoscale.hpp" #include "spc/models/outlook.hpp" diff --git a/tests/test_locale.cpp b/tests/test_locale.cpp index 245f40f..552cacd 100644 --- a/tests/test_locale.cpp +++ b/tests/test_locale.cpp @@ -10,6 +10,7 @@ #include "spc/models/outlook.hpp" #include "support/fixtures.hpp" +#include #include #include #include @@ -118,14 +119,22 @@ TEST(LocaleIndependence, ProbabilisticOutlookKeepsItsIsoplethsUnderACommaDecimal GTEST_SKIP() << "de_DE.UTF-8 is not installed on this host"; } - const ProbOutlookPayload payload = - parse_probabilistic(read_fixture("arcgis_day1_prob_tornado.geojson"), 1, "tornado"); + // Each feature's dn repeats its label, so drop dn: only the "0.02"-style + // labels can then supply the probabilities. + std::string body = read_fixture("arcgis_day1_prob_tornado.geojson"); + for (std::size_t at = body.find("\"dn\""); at != std::string::npos; + at = body.find("\"dn\"", at)) { + body.replace(at, 4, "\"xx\""); + } + const ProbOutlookPayload payload = parse_probabilistic(body, 1, "tornado"); - ASSERT_GT(payload.features.size(), 0u); + ASSERT_EQ(payload.features.size(), 4u); + std::vector probabilities; for (const ProbOutlookFeature& f : payload.features) { - EXPECT_GE(f.probability, 0.01); - EXPECT_LE(f.probability, 1.0); + probabilities.push_back(f.probability); } + std::ranges::sort(probabilities); + EXPECT_EQ(probabilities, (std::vector{0.02, 0.05, 0.10, 0.15})); } TEST(LocaleIndependence, FireWeatherNoRiskSentinelIsStillDroppedUnderACommaDecimalLocale) { diff --git a/tests/test_parser.cpp b/tests/test_parser.cpp index e209ca5..dac44b4 100644 --- a/tests/test_parser.cpp +++ b/tests/test_parser.cpp @@ -2,9 +2,15 @@ /// @brief Day 1-3 parser behavior shared with spc-data. The assertions match /// spc-data's own parser tests. +#include "spc/models/convective.hpp" +#include "spc/models/fire_weather.hpp" #include "spc/models/outlook.hpp" +#include "spc/models/storm_report.hpp" +#include "spc/models/watch.hpp" +#include #include +#include namespace { @@ -118,4 +124,62 @@ TEST(Parser, UnknownLabelSkipped) { EXPECT_TRUE(p.features.empty()); } +// A ring inside a feature whose fields are replaced per case. +std::string esri_feature(std::string_view fields) { + return R"({"features":[{"attributes":)" + std::string{fields} + + R"(,"geometry":{"rings":[[[0,1],[1,1],[1,0],[0,0],[0,1]]]}}]})"; +} + +TEST(Parser, ConditionalIntensitySkipsFeaturesWithoutALabel) { + for (const char* fields : {"null", "42", "{}", R"({"dn":null})"}) { + EXPECT_TRUE( + parse_conditional_intensity(esri_feature(fields), 1, "tornado").features.empty()) + << fields; + } + EXPECT_EQ(parse_conditional_intensity(esri_feature(R"({"label":"CIG1"})"), 1, "tornado") + .features.size(), + 1u); +} + +TEST(Parser, FireWeatherSkipsDay1And2FeaturesWithNoBand) { + for (const char* fields : {"null", "42", "{}", R"({"dn":null})", R"({"dn":6})"}) { + EXPECT_TRUE( + parse_fire_weather(esri_feature(fields), 1, FireWeatherLayer::Outlook).features.empty()) + << fields; + } + const FireWeatherPayload kept = + parse_fire_weather(esri_feature(R"({"dn":8})"), 1, FireWeatherLayer::Outlook); + ASSERT_EQ(kept.features.size(), 1u); + EXPECT_EQ(kept.features[0].label, "CRIT"); +} + +TEST(Parser, NonFiniteNumericStringsReadAsZero) { + const StormReportPayload reports = parse_storm_reports( + R"({"features":[{"properties":{"magnitude":"nan","lon":"inf","lat":"-inf"},"geometry":null}]})"); + ASSERT_EQ(reports.reports.size(), 1u); + EXPECT_EQ(reports.reports[0].magnitude, 0.0); + EXPECT_TRUE(std::isfinite(reports.reports[0].location.lon)); + EXPECT_TRUE(std::isfinite(reports.reports[0].location.lat)); + + const WatchPayload watches = parse_watches( + R"({"features":[{"properties":{"max_hail_size":"inf","max_wind_gust_knots":"nan"},"geometry":null}]})"); + ASSERT_EQ(watches.watches.size(), 1u); + EXPECT_EQ(watches.watches[0].max_hail_size, 0.0); + EXPECT_EQ(watches.watches[0].max_wind_gust_knots, 0.0); +} + +TEST(Parser, ANullOrNonNumericLatestWkidFallsBackToWkid) { + const std::string ring = R"("geometry":{"rings":[[[0,1],[1,1],[1,0],[0,0],[0,1]]]})"; + for (const char* reference : + {R"({"wkid":4326,"latestWkid":null})", R"({"wkid":4326,"latestWkid":"4326"})"}) { + const std::string body = std::string{R"({"spatialReference":)"} + reference + + R"(,"features":[{"attributes":{"label":"CIG1"},)" + ring + "}]}"; + EXPECT_EQ(parse_conditional_intensity(body, 1, "tornado").features.size(), 1u) << reference; + } + // A geometry's null reference leaves the payload's in force. + const std::string body = + R"({"spatialReference":{"wkid":4326},"features":[{"attributes":{"label":"CIG1"},"geometry":{"spatialReference":null,"rings":[[[0,1],[1,1],[1,0],[0,0],[0,1]]]}}]})"; + EXPECT_EQ(parse_conditional_intensity(body, 1, "tornado").features.size(), 1u); +} + } // namespace From d4e2c809f6e395ec82ead95bc2da30a36c70b560 Mon Sep 17 00:00:00 2001 From: KeviM Date: Fri, 20 Nov 2026 12:06:45 -0800 Subject: [PATCH 3/9] build: link Glaze as a build-local target; test client URLs Glaze came in through glaze_SOURCE_DIR, which is empty when CMake finds an installed Glaze, so that configuration failed. Linking the target behind BUILD_LOCAL_INTERFACE keeps it out of the export set and brings Glaze's MSVC options; its headers stay system headers. Also test the MapServer and static-feed URL of each typed query, skip the benchmark self-check's library probe on MSVC, and fix stale docs. --- .coderabbit.yaml | 2 -- .github/pull_request_template.md | 3 +- CHANGELOG.md | 4 +++ CLAUDE.md | 6 ++-- CMakeLists.txt | 19 ++++++++---- benchmarks/CMakeLists.txt | 2 +- benchmarks/benchmarks.cpp | 10 +++++-- src/CMakeLists.txt | 15 +++++----- tests/CMakeLists.txt | 2 +- tests/support/allocation_counter.hpp | 3 +- tests/test_client.cpp | 43 ++++++++++++++++++++++++++++ 11 files changed, 85 insertions(+), 24 deletions(-) diff --git a/.coderabbit.yaml b/.coderabbit.yaml index df05ba4..37e75b2 100644 --- a/.coderabbit.yaml +++ b/.coderabbit.yaml @@ -18,10 +18,8 @@ knowledge_base: code_guidelines: enabled: true filePatterns: - - AGENTS.md - CLAUDE.md - CONTRIBUTING.md - - .github/review-policy.md chat: auto_reply: false art: false diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index f5da791..55cf174 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -6,6 +6,7 @@ -- [ ] `make test` and `make lint` pass +- [ ] `make test`, `make lint`, `make lint-md`, `make fixtures-check`, and + `make test-consumers` pass - [ ] `CHANGELOG.md` has an `[Unreleased]` entry for user-visible changes - [ ] Public API changes are called out above diff --git a/CHANGELOG.md b/CHANGELOG.md index 99313e1..ef49da0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,6 +25,10 @@ uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html). - Read the numeric strings `"nan"` and `"inf"` as 0 instead of passing them on. - Fall back to `wkid` when `latestWkid` is null or not a number, and ignore a null geometry `spatialReference`. +- Link Glaze through its target, so a build that finds an installed Glaze + (`FETCHCONTENT_TRY_FIND_PACKAGE_MODE`) configures and installs, and MSVC gets the + options Glaze requires. +- The benchmarks' allocation-counter self-check no longer fails on MSVC. ## [0.4.7] - 2026-11-04 diff --git a/CLAUDE.md b/CLAUDE.md index 200f86c..596cdb5 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -21,9 +21,9 @@ when you touch ArcGIS layer ids; it needs the network. - Public failures return `Result` (`std::expected`). The standalone `parse_*` functions throw `std::runtime_error` on malformed JSON; the clients turn that into `ErrorCode::ParseError`. -- Glaze is private. It lives behind `src/models/json.hpp`, and no header under - `include/` may include it; `tools/test_consumers.sh` fails if an install - ships Glaze headers. +- Glaze is private: only `src/` includes it, mostly through + `src/models/json.hpp`. No header under `include/` may include it; + `tools/test_consumers.sh` fails if an install ships Glaze headers. - `HttpClient` is thread-safe through a pool of libcurl handles. Custom transports implement `HttpTransport::get`, which must be safe to call concurrently. diff --git a/CMakeLists.txt b/CMakeLists.txt index df981f6..a4c932f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -114,6 +114,12 @@ FetchContent_Declare( GIT_SHALLOW TRUE ) FetchContent_MakeAvailable(glaze) +# Glaze's headers are system headers wherever it is linked, so its warnings +# never reach -Werror or clang-tidy. An installed (imported) Glaze already is. +get_target_property(SPC_GLAZE_TARGET glaze::glaze ALIASED_TARGET) +if(SPC_GLAZE_TARGET) + set_property(TARGET ${SPC_GLAZE_TARGET} PROPERTY SYSTEM TRUE) +endif() add_subdirectory(src) @@ -162,12 +168,15 @@ if(SPC_INSTALL) INCLUDES DESTINATION ${CMAKE_INSTALL_INCLUDEDIR} ) install(DIRECTORY include/spc DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}) - # The static libraries contain compiled Glaze code, so ship its license. install(FILES LICENSE DESTINATION ${CMAKE_INSTALL_DATADIR}/licenses/spc-cpp) - install(FILES ${glaze_SOURCE_DIR}/LICENSE - DESTINATION ${CMAKE_INSTALL_DATADIR}/licenses/spc-cpp - RENAME glaze-LICENSE - ) + # The static libraries contain compiled Glaze code, so ship its license. + # A packaged Glaze, found instead of fetched, ships its own. + if(glaze_SOURCE_DIR) + install(FILES ${glaze_SOURCE_DIR}/LICENSE + DESTINATION ${CMAKE_INSTALL_DATADIR}/licenses/spc-cpp + RENAME glaze-LICENSE + ) + endif() configure_package_config_file( ${PROJECT_SOURCE_DIR}/cmake/spcConfig.cmake.in diff --git a/benchmarks/CMakeLists.txt b/benchmarks/CMakeLists.txt index 3f47a10..f86ed4d 100644 --- a/benchmarks/CMakeLists.txt +++ b/benchmarks/CMakeLists.txt @@ -10,7 +10,7 @@ target_include_directories(spc_benchmarks PRIVATE ${PROJECT_SOURCE_DIR}/src ${PROJECT_SOURCE_DIR}/tests ) -target_include_directories(spc_benchmarks SYSTEM PRIVATE ${glaze_SOURCE_DIR}/include) +target_link_libraries(spc_benchmarks PRIVATE glaze::glaze) target_compile_definitions(spc_benchmarks PRIVATE SPC_FIXTURES_DIR="${PROJECT_SOURCE_DIR}/tests/fixtures" ) diff --git a/benchmarks/benchmarks.cpp b/benchmarks/benchmarks.cpp index ac72601..6ac860c 100644 --- a/benchmarks/benchmarks.cpp +++ b/benchmarks/benchmarks.cpp @@ -307,21 +307,25 @@ bool allocation_counter_works() { benchmark::DoNotOptimize(block); ::operator delete(block); const std::uint64_t direct = probe.stats().allocations; - const std::runtime_error error("a message the standard library copies to the heap"); - benchmark::DoNotOptimize(error.what()); - const std::uint64_t library = probe.stats().allocations - direct; if (direct != 1) { std::fputs("The allocation counter misses operator new calls. Under Valgrind, run with " "--soname-synonyms=somalloc=nouserintercepts.\n", stderr); return false; } +#if !defined(_MSVC_STL_VERSION) + // MSVC keeps exception messages in CRT memory, not operator new, so there + // it has no library allocation to observe. + const std::runtime_error error("a message the standard library copies to the heap"); + benchmark::DoNotOptimize(error.what()); + const std::uint64_t library = probe.stats().allocations - direct; if (library == 0) { std::fputs("The allocation counter misses allocations made inside the standard library, " "which happens when LTO internalizes the replacement operator new.\n", stderr); return false; } +#endif return true; } diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index e7bd876..41be59a 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -1,11 +1,12 @@ # Static libraries: spc_core -> spc_http -> spc_models -> spc_api, and the # spc umbrella that links them all. # -# Glaze is added as a SYSTEM PRIVATE include directory rather than linked: -# no installed header uses it, and a linked glaze::glaze would have to join -# the export set. +# Glaze is linked privately and only inside this build: no installed header +# uses it, and BUILD_LOCAL_INTERFACE keeps it out of the export set. Linking +# the target, not its include path, also works with a packaged Glaze and +# carries the MSVC options Glaze requires. -set(SPC_GLAZE_INCLUDE_DIR ${glaze_SOURCE_DIR}/include) +set(SPC_GLAZE $) add_library(spc_core STATIC core/error.cpp @@ -20,7 +21,7 @@ target_include_directories(spc_core PUBLIC $ $ ) -target_include_directories(spc_core SYSTEM PRIVATE ${SPC_GLAZE_INCLUDE_DIR}) +target_link_libraries(spc_core PRIVATE ${SPC_GLAZE}) add_library(spc_http STATIC http/client.cpp @@ -40,7 +41,7 @@ add_library(spc_models STATIC spc_configure_target(spc_models) target_link_libraries(spc_models PUBLIC spc_core) target_include_directories(spc_models PRIVATE ${PROJECT_SOURCE_DIR}/src) -target_include_directories(spc_models SYSTEM PRIVATE ${SPC_GLAZE_INCLUDE_DIR}) +target_link_libraries(spc_models PRIVATE ${SPC_GLAZE}) add_library(spc_api STATIC api/client.cpp @@ -48,7 +49,7 @@ add_library(spc_api STATIC spc_configure_target(spc_api) target_link_libraries(spc_api PUBLIC spc_core spc_http spc_models) target_include_directories(spc_api PRIVATE ${PROJECT_SOURCE_DIR}/src) -target_include_directories(spc_api SYSTEM PRIVATE ${SPC_GLAZE_INCLUDE_DIR}) +target_link_libraries(spc_api PRIVATE ${SPC_GLAZE}) add_library(spc INTERFACE) target_link_libraries(spc INTERFACE spc_core spc_http spc_models spc_api) diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 1cb2ac9..eb0d3e5 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -26,7 +26,7 @@ target_include_directories(spc_tests PRIVATE ${PROJECT_SOURCE_DIR}/src ${CMAKE_CURRENT_SOURCE_DIR} ) -target_include_directories(spc_tests SYSTEM PRIVATE ${glaze_SOURCE_DIR}/include) +target_link_libraries(spc_tests PRIVATE glaze::glaze) target_compile_definitions(spc_tests PRIVATE SPC_FIXTURES_DIR="${CMAKE_CURRENT_SOURCE_DIR}/fixtures" SPC_GOLDEN_DIR="${CMAKE_CURRENT_SOURCE_DIR}/golden" diff --git a/tests/support/allocation_counter.hpp b/tests/support/allocation_counter.hpp index ec96026..c12bd99 100644 --- a/tests/support/allocation_counter.hpp +++ b/tests/support/allocation_counter.hpp @@ -17,7 +17,8 @@ struct AllocationStats { std::uint64_t allocations = 0; ///< operator new calls std::uint64_t bytes = 0; ///< bytes those calls asked for /// Most memory live at once. Live sizes are the allocator's block sizes, - /// because an unsized delete does not say how much it frees. + /// because an unsized delete does not say how much it frees. Freeing memory + /// allocated before the probe lowers the count, so this is then a lower bound. std::uint64_t peak_bytes = 0; /// Memory still live: what the measured code kept. Negative when it /// freed memory allocated before the probe. diff --git a/tests/test_client.cpp b/tests/test_client.cpp index f99b8da..f481afe 100644 --- a/tests/test_client.cpp +++ b/tests/test_client.cpp @@ -171,6 +171,49 @@ TEST(ArcGISClientRouting, CombinesBothPublishedFireWeatherLayers) { EXPECT_NE(transport->requests[1].find("outSR=4326"), std::string::npos); } +TEST(ArcGISClientRouting, TypedQueriesUseTheirOwnMapServers) { + // Layer ids alone still match when a query goes to the wrong MapServer. + const std::string services = "https://mapservices.weather.noaa.gov/vector/rest/services/"; + std::shared_ptr transport = std::make_shared(); + transport->responses.assign(5, empty_feature_collection()); + ArcGISClient client{transport}; + + ASSERT_TRUE(client.query_categorical(1)); + ASSERT_TRUE(client.query_fire_weather(1)); + ASSERT_TRUE(client.query_active_md()); + ASSERT_TRUE(client.query_day4_8(4)); + + ASSERT_EQ(transport->requests.size(), 5u); + EXPECT_TRUE(transport->requests[0].starts_with(services + "outlooks/SPC_wx_outlks/MapServer/")); + EXPECT_TRUE( + transport->requests[1].starts_with(services + "fire_weather/SPC_firewx/MapServer/")); + EXPECT_TRUE( + transport->requests[2].starts_with(services + "fire_weather/SPC_firewx/MapServer/")); + EXPECT_TRUE(transport->requests[3].starts_with( + services + "outlooks/spc_mesoscale_discussion/MapServer/0/query?")); + EXPECT_TRUE(transport->requests[4].starts_with(services + "outlooks/SPC_wx_outlks/MapServer/")); +} + +TEST(StaticFeedRouting, EachProductUsesItsPublishedFile) { + std::shared_ptr transport = std::make_shared(); + transport->responses.assign(4, {200, R"({"type":"FeatureCollection","features":[]})", {}}); + StaticFeedClient client{transport}; + + ASSERT_TRUE(client.day_categorical(2)); + ASSERT_TRUE(client.day_probabilistic(1, "tornado")); + ASSERT_TRUE(client.day_probabilistic(3, "severe")); + ASSERT_TRUE(client.day4_8(5)); + + const std::string outlook = "https://www.spc.noaa.gov/products/outlook/"; + EXPECT_EQ(transport->requests, (std::vector{ + outlook + "day2otlk_cat.nolyr.geojson", + outlook + "day1otlk_torn.nolyr.geojson", + outlook + "day3otlk_prob.nolyr.geojson", + "https://www.spc.noaa.gov/products/exper/day4-8/" + "day5prob.nolyr.geojson", + })); +} + TEST(ArcGISClientRouting, PreservesLabelsFromBothDayOneFireProducts) { std::shared_ptr transport = std::make_shared(); const std::string body = R"({"features":[ From 2fa8fc9d4268ebc004eb6e90375f41aa64e3d209 Mon Sep 17 00:00:00 2001 From: KeviM Date: Fri, 20 Nov 2026 12:17:47 -0800 Subject: [PATCH 4/9] chore: point security reports at private reporting; pin dev tools Private vulnerability reporting was off while SECURITY.md and the issue templates sent reporters to it; it is now enabled and the docs say how it works. GoogleTest and Benchmark are pinned by commit, the Makefile's markdownlint-cli2 matches the CI action, and .gitignore covers more secret files. --- .github/ISSUE_TEMPLATE/bug_report.yml | 8 ++++---- .github/ISSUE_TEMPLATE/config.yml | 2 +- .github/workflows/ci.yml | 2 ++ .gitignore | 14 ++++++++++++++ CMakeLists.txt | 8 ++++---- CONTRIBUTING.md | 16 ++++++++++++++-- Makefile | 7 +++++-- SECURITY.md | 5 +++-- 8 files changed, 47 insertions(+), 15 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml index a324396..086867d 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.yml +++ b/.github/ISSUE_TEMPLATE/bug_report.yml @@ -2,6 +2,10 @@ name: Bug report description: A crash, a wrong result, or a build failure. labels: [bug] body: + - type: markdown + attributes: + value: | + Report security vulnerabilities through GitHub's private vulnerability reporting, as described in the [security policy](https://github.com/Reddimus/spc-cpp/security/policy), not here. - type: textarea id: what attributes: @@ -31,7 +35,3 @@ body: - libcurl: 8.5.0 validations: required: true - - type: markdown - attributes: - value: | - Report security vulnerabilities privately, as described in the [security policy](https://github.com/Reddimus/spc-cpp/security/policy). diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml index 4572a9f..9a39c60 100644 --- a/.github/ISSUE_TEMPLATE/config.yml +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -2,4 +2,4 @@ blank_issues_enabled: false contact_links: - name: Security vulnerability url: https://github.com/Reddimus/spc-cpp/security/policy - about: Report it privately instead of opening an issue. + about: Use GitHub's private vulnerability reporting, as the security policy describes, not an issue. diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7ac2de8..1a25fbd 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -197,6 +197,8 @@ jobs: timeout-minutes: 10 steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + # The Makefile pins the markdownlint-cli2 version this action bundles. + # Update MARKDOWNLINT there when this pin changes. - uses: DavidAnson/markdownlint-cli2-action@696eb3f2509270d7b882d2aa68347bc676e7af34 # v25.0.0 sanitizer: diff --git a/.gitignore b/.gitignore index d93b912..43cc42f 100644 --- a/.gitignore +++ b/.gitignore @@ -25,10 +25,24 @@ Thumbs.db # Secrets .env +.env.* +!.env.example +.envrc +.netrc +.npmrc +.pypirc *.pem *.key *.p12 *.pfx +id_rsa +id_ecdsa +id_ecdsa_sk +id_ed25519 +id_ed25519_sk credentials*.json service-account*.json +token.json secrets/ +**/.claude/settings.local.json +.claude/worktrees/ diff --git a/CMakeLists.txt b/CMakeLists.txt index a4c932f..3ed8b52 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -123,12 +123,13 @@ endif() add_subdirectory(src) +# The test and benchmark frameworks are pinned by commit, so a moved tag cannot +# change what runs. GIT_SHALLOW does not work with a hash; both clones are small. if(SPC_BUILD_TESTS) FetchContent_Declare( googletest GIT_REPOSITORY https://github.com/google/googletest.git - GIT_TAG v1.18.0 - GIT_SHALLOW TRUE + GIT_TAG 063de7e9578f82b369302001269680b4b1553359 # v1.18.0 ) set(gtest_force_shared_crt ON CACHE BOOL "" FORCE) set(INSTALL_GTEST OFF CACHE BOOL "" FORCE) @@ -141,8 +142,7 @@ if(SPC_BUILD_BENCHMARKS) FetchContent_Declare( benchmark GIT_REPOSITORY https://github.com/google/benchmark.git - GIT_TAG v1.9.5 - GIT_SHALLOW TRUE + GIT_TAG 192ef10025eb2c4cdd392bc502f0c852196baa48 # v1.9.5 SYSTEM EXCLUDE_FROM_ALL ) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 31abf2c..47d0840 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,7 +1,8 @@ # Contributing -Thanks for helping. Report security problems privately through -[SECURITY.md](SECURITY.md), not in a public issue. +Thanks for helping. Report security problems through GitHub's private +vulnerability reporting, as [SECURITY.md](SECURITY.md) describes, not in a +public issue. ## Set up @@ -162,6 +163,17 @@ with an error. the tests and consumer checks, then publishes the release with the changelog section as its notes. +## Security reports + +SECURITY.md and the issue templates send reporters to GitHub's private +vulnerability reporting. It is a repository setting, and with it off reporters +have no private channel. Keep it on (Settings, Advanced Security) and check it +after changing repository settings: + +```bash +gh api repos/Reddimus/spc-cpp/private-vulnerability-reporting # {"enabled":true} +``` + ## Repository tools `.mcp.json` (Claude) and `.codex/config.toml` (Codex) configure diff --git a/Makefile b/Makefile index 97ec8d2..3bbbf90 100644 --- a/Makefile +++ b/Makefile @@ -4,6 +4,9 @@ BUILD_DIR := build CMAKE := cmake NPROC := $(shell nproc 2>/dev/null || sysctl -n hw.ncpu 2>/dev/null || echo 4) SOURCES = find src include tests examples benchmarks \( -name '*.cpp' -o -name '*.hpp' \) -print0 +# The version bundled by the markdownlint-cli2-action pin in ci.yml, so local +# and CI lint agree. Change both together. +MARKDOWNLINT := npx --yes markdownlint-cli2@0.24.0 .PHONY: all build debug configure configure-debug test test-consumers fixtures-check lint \ lint-md format format-md tidy bench coverage pre-commit install-hooks clean help @@ -39,14 +42,14 @@ lint: @python3 tools/test_cpp_auto_audit.py lint-md: - npx markdownlint-cli2 "**/*.md" + $(MARKDOWNLINT) "**/*.md" format: @command -v clang-format >/dev/null || { echo "clang-format is not installed"; exit 1; } @$(SOURCES) | xargs -0 clang-format -i format-md: - npx markdownlint-cli2 --fix "**/*.md" + $(MARKDOWNLINT) --fix "**/*.md" # Compile the libraries with clang-tidy. On macOS, see CONTRIBUTING.md. tidy: diff --git a/SECURITY.md b/SECURITY.md index 7389ff9..a867c02 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -12,8 +12,9 @@ Fixes land in the next release. Older tags are not patched, so upgrade your ## Reporting a vulnerability Use GitHub's [private vulnerability -reporting](https://github.com/Reddimus/spc-cpp/security/advisories/new). Do not -open a public issue. +reporting](https://github.com/Reddimus/spc-cpp/security/advisories/new). It +needs a GitHub account, and the report stays private until an advisory is +published. Do not open a public issue. Include the affected version or commit, a minimal reproduction, and the impact you see, such as memory corruption or a denial of service. From e1cc55fc88bb05e1383795709486ae92bb7abedc Mon Sep 17 00:00:00 2001 From: KeviM Date: Fri, 20 Nov 2026 12:23:30 -0800 Subject: [PATCH 5/9] fix(tools): audit every auto, check nested fixtures, match release tags literally The explicit-type audit only saw auto at line start and treated any iterator-looking substring as an exemption; it now checks each auto token and exempts only an initializer that is one iterator call. The fixture checker recurses and rejects duplicate entries, release notes match the tag literally, and the tools run on Python 3.9. --- .github/workflows/release.yml | 5 +- CONTRIBUTING.md | 9 +- tools/cpp_auto_audit.py | 428 +++++++++++++++++++----------- tools/test_cpp_auto_audit.py | 92 +++++++ tools/verify_fixture_checksums.py | 58 ++-- 5 files changed, 406 insertions(+), 186 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 003891b..4ebbb77 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -53,11 +53,12 @@ jobs: - name: Extract the release notes # Write to a file: echoing the notes through the shell would run their - # `code` spans as command substitutions. + # `code` spans as command substitutions. index() matches the heading + # literally; a regex would read the version's "." and "+" as operators. run: | version="${GITHUB_REF_NAME#v}" awk -v v="$version" ' - $0 ~ "^## \\[" v "\\]" { capture = 1; next } + index($0, "## [" v "]") == 1 { capture = 1; next } capture && /^## / { exit } capture && /^\[[^]]+\]: / { exit } capture { print } diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 47d0840..7656f9b 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -91,12 +91,13 @@ make tidy CMAKE_ARGS="-DCMAKE_CXX_COMPILER=$llvm/bin/clang++ \ ## Fixtures `tests/fixtures/` holds captured NOAA and IEM responses. After adding or -changing one, update the checksums: +changing one, update the checksums. Files in subdirectories are fixtures too; +dotfiles are not: ```bash -for f in $(ls tests/fixtures | grep -v -e README.md -e SHA256SUMS); do - shasum -a 256 "tests/fixtures/$f" -done > tests/fixtures/SHA256SUMS +find tests/fixtures -type f ! -path '*/.*' ! -path tests/fixtures/README.md \ + ! -path tests/fixtures/SHA256SUMS -exec shasum -a 256 {} + | + LC_ALL=C sort -k 2 > tests/fixtures/SHA256SUMS make fixtures-check ``` diff --git a/tools/cpp_auto_audit.py b/tools/cpp_auto_audit.py index 4a8e46e..f25b9fd 100644 --- a/tools/cpp_auto_audit.py +++ b/tools/cpp_auto_audit.py @@ -1,15 +1,19 @@ #!/usr/bin/env python3 """Fail when C++ sources use `auto` outside the allowed cases. -Allowed: structured bindings, lambda objects, iterator-like values, and any -line with an `auto-ok` comment. Everything else spells out its type. The allowlist file -takes one `relative/path.cpp|line snippet` rule per line, matched as a -substring of the stripped source line. +Allowed: structured bindings, lambda objects, iterators, and any line with an +`auto-ok` comment. Everything else spells out its type. Without type +information, an iterator is recognized only when the whole initializer is one +iterator-producing call, such as `v.begin()`, `m.find(k)` or `std::next(it)`. +Function and template parameters are not checked. The allowlist file takes +one `relative/path.cpp|line snippet` rule per line, matched as a substring of +the stripped source line. """ from __future__ import annotations import argparse +from collections.abc import Iterator import subprocess import sys from dataclasses import dataclass @@ -20,55 +24,36 @@ CPP_SUFFIXES = {".cpp", ".hpp", ".cc", ".cxx", ".hh", ".hxx"} DEFAULT_ALLOWLIST = "tools/cpp_auto_allowlist.txt" WORD_AUTO = re.compile(r"\bauto\b") +IDENTIFIER = re.compile(r"[A-Za-z_]\w*") DECL_SPECIFIERS = { "const", "constexpr", "static", "volatile", "mutable", "constinit", "thread_local", "extern", "inline", } -DECL_SPECIFIER = "(?:" + "|".join(sorted(DECL_SPECIFIERS)) + ")" -AUTO_DECLARATOR = rf"\bauto\b(?:\s*(?:[\*&]|{DECL_SPECIFIER}\b))*\s*" -STRUCTURED_BINDING = re.compile(AUTO_DECLARATOR + r"\[") -DECLARATION_AUTO = re.compile( - r"^\s*(?:}\s*)?(?:(?:else\s+)?if(?:\s+constexpr)?\s*\(\s*|(?:for|while|switch)\s*\(\s*)?" - + rf"(?:{DECL_SPECIFIER}\s+)*" - + AUTO_DECLARATOR - + rf"(?:(?!{DECL_SPECIFIER}\b)\w+\b(?!\s*[,)])|\[)" -) - -ITERATOR_HINTS = ( - ".begin(", - ".end(", - ".cbegin(", - ".cend(", - ".rbegin(", - ".rend(", - ".crbegin(", - ".crend(", - ".find(", - ".lower_bound(", - ".upper_bound(", - ".equal_range(", - "std::begin(", - "std::end(", - "std::cbegin(", - "std::cend(", - "std::find(", - "std::lower_bound(", - "std::upper_bound(", - "std::equal_range(", - "std::max_element(", - "std::min_element(", - "std::prev(", - "std::next(", -) +# Words that may precede `auto` without being a type-constraint (concept) name. +NOT_CONSTRAINTS = DECL_SPECIFIERS | { + "case", "co_await", "co_return", "co_yield", "consteval", "do", "else", "explicit", + "friend", "new", "operator", "requires", "return", "template", "throw", "typedef", + "typename", "using", "virtual", +} +CONDITION_KEYWORDS = {"for", "if", "switch", "while"} +PAIRS = {"(": ")", "[": "]", "{": "}"} +ITERATOR_MEMBERS = { + "begin", "end", "cbegin", "cend", "rbegin", "rend", "crbegin", "crend", + "find", "lower_bound", "upper_bound", "equal_range", +} +ITERATOR_FUNCTIONS = { # in namespace std + "begin", "end", "cbegin", "cend", "find", "lower_bound", "upper_bound", + "equal_range", "max_element", "min_element", "prev", "next", +} -@dataclass(frozen=True, slots=True) +@dataclass(frozen=True) class AllowRule: relative_path: str snippet: str -@dataclass(frozen=True, slots=True) +@dataclass(frozen=True) class Violation: relative_path: str line_number: int @@ -123,59 +108,224 @@ def read_allowlist(path: Path) -> list[AllowRule]: ) -def declaration_code(line: str) -> str: - """Hide declaration groups while preserving initializer character offsets.""" - tokens = list(CPP_TOKEN.finditer(line)) - code = list(line) - - def hide_group(start: int, opener: str, closer: str) -> int: - depth = 0 - for end in range(start, len(tokens)): - text = tokens[end][0] - if text == opener: - depth += 1 - elif text == closer: - depth -= 1 - if depth == 0: - code[tokens[start].start():tokens[end].end()] = " " * ( - tokens[end].end() - tokens[start].start() - ) - return end + 1 - return len(tokens) - - index = 0 - while index < len(tokens): - text = tokens[index][0] - if text == "[" and index + 1 < len(tokens) and tokens[index + 1][0] == "[": - index = hide_group(index, "[", "]") - elif text == "alignas" and index + 1 < len(tokens) and tokens[index + 1][0] == "(": - code[tokens[index].start():tokens[index].end()] = " " * len(text) - index = hide_group(index + 1, "(", ")") - elif text == "auto": - index += 1 +def group_end(tokens: list[str], start: int) -> int | None: + """Index of the bracket that closes tokens[start].""" + opener, closer = tokens[start], PAIRS[tokens[start]] + depth = 0 + for index in range(start, len(tokens)): + if tokens[index] == opener: + depth += 1 + elif tokens[index] == closer: + depth -= 1 + if depth == 0: + return index + return None + + +def group_start(tokens: list[str], end: int, opener: str) -> int | None: + """Index of the `opener` that tokens[end] closes.""" + closer = tokens[end] + depth = 0 + for index in range(end, -1, -1): + if tokens[index] == closer: + depth += 1 + elif tokens[index] == opener: + depth -= 1 + if depth == 0: + return index + return None + + +def code_tokens(source: str, start: int) -> Iterator[str]: + # Keep literals: find()'s arguments show whether it is a string search. + for token in CPP_TOKEN.finditer(source, start): + if not token[0].startswith(("//", "/*")): + yield token[0] + + +def statement_tokens(source: str, start: int) -> list[str]: + """Tokens through the `;` or unmatched closer that ends the statement.""" + tokens: list[str] = [] + depth = 0 + for text in code_tokens(source, start): + tokens.append(text) + if text in PAIRS: + depth += 1 + elif text in PAIRS.values(): + depth -= 1 + if depth < 0: + break + elif text == ";" and depth == 0: break - else: - index += 1 - parentheses = 0 + return tokens + + +def leading_expression(tokens: list[str]) -> list[str]: + """Tokens before the first ungrouped `,` or `;`, or an unmatched closer.""" + depth = 0 + for index, text in enumerate(tokens): + if text in PAIRS: + depth += 1 + elif text in PAIRS.values(): + if depth == 0: + return tokens[:index] + depth -= 1 + elif text in {",", ";"} and depth == 0: + return tokens[:index] + return tokens + + +def is_text_literal(token: str) -> bool: + return '"' in token or token.startswith("'") + + +def iterator_call(tokens: list[str]) -> bool: + """Whether the whole expression is one call that yields an iterator. + + The call may end a member chain, as in `a.items().begin()`, but not be an + operand or an argument. A type-free check cannot tell a container's find() + from std::string's, so find() counts only with one argument and no string + or character literal: string searches usually take a literal or a position. + """ + while tokens and tokens[0] == "(" and group_end(tokens, 0) == len(tokens) - 1: + tokens = tokens[1:-1] + if not tokens or not IDENTIFIER.fullmatch(tokens[0]): + return False + callee = [tokens[0]] + index = 1 + while ( + tokens[index:index + 2] == [":", ":"] + and index + 2 < len(tokens) + and IDENTIFIER.fullmatch(tokens[index + 2]) + ): + callee.append(tokens[index + 2]) + index += 3 + member = False + call: tuple[bool, list[str], list[str]] | None = None while index < len(tokens): - text = tokens[index][0] - if text in {"*", "&"} or text in DECL_SPECIFIERS: - index += 1 - elif text == "(": - code[tokens[index].start()] = " " - parentheses += 1 + if tokens[index] in {"(", "["}: + end = group_end(tokens, index) + if end is None: + return False + named_call = tokens[index] == "(" and callee + call = (member, callee, tokens[index + 1:end]) if named_call else None + callee = [] + index = end + 1 + elif tokens[index] == "." or tokens[index:index + 2] == ["-", ">"]: + index += 1 if tokens[index] == "." else 2 + if index == len(tokens) or not IDENTIFIER.fullmatch(tokens[index]): + return False + member, callee, call = True, [tokens[index]], None index += 1 + else: + return False + if call is None: + return False + member, callee, arguments = call + if not member: + return len(callee) == 2 and callee[0] == "std" and callee[1] in ITERATOR_FUNCTIONS + if callee != ["find"]: + return callee[0] in ITERATOR_MEMBERS + return ( + bool(arguments) + and leading_expression(arguments) == arguments + and not any(is_text_literal(token) for token in arguments) + ) + + +def placeholder_context(tokens: list[str], start: int) -> str: + """Say whether the `auto` at tokens[start] begins a declaration statement. + + Returns "statement", "new" for a new-expression, or "other" for + parameters, template arguments, trailing return types, and expressions. + Only the current line is visible, so a line start counts as a statement. + """ + index = start - 1 + # A type-constraint such as std::integral or std::convertible_to. + if index >= 1 and tokens[index] == ">" and tokens[index - 1] != "-": + opening = group_start(tokens, index, "<") + if ( + opening is not None + and opening >= 1 + and IDENTIFIER.fullmatch(tokens[opening - 1]) + and tokens[opening - 1] not in NOT_CONSTRAINTS + ): + index = opening - 1 + if index >= 0 and IDENTIFIER.fullmatch(tokens[index]) and tokens[index] not in NOT_CONSTRAINTS: + index -= 1 + while index >= 1 and tokens[index] == tokens[index - 1] == ":": + index -= 2 + if index < 0 or not IDENTIFIER.fullmatch(tokens[index]): + break + index -= 1 + while index >= 0: + if tokens[index] in DECL_SPECIFIERS: + index -= 1 + elif tokens[index] == "]" and index >= 1 and tokens[index - 1] == "]": + opening = group_start(tokens, index, "[") + if opening is None: + break + index = opening - 1 + elif tokens[index] == ")": + opening = group_start(tokens, index, "(") + if opening is None or opening == 0 or tokens[opening - 1] != "alignas": + break + index = opening - 2 else: break - if index < len(tokens) and re.fullmatch(r"[A-Za-z_]\w*", tokens[index][0]): + if index < 0 or tokens[index] in {";", "{", "}", "else", "do"}: + return "statement" + previous = tokens[index] + if previous == ":" and (index == 0 or tokens[index - 1] != ":"): + return "statement" # a label or access specifier + if previous == "new": + return "new" + if previous == ")": + opening = group_start(tokens, index, "(") + if opening is not None and opening >= 1 and tokens[opening - 1] == "new": + return "new" # placement new + if previous == "(": + keyword = index - 1 + if keyword >= 0 and tokens[keyword] == "constexpr": + keyword -= 1 + if keyword >= 0 and tokens[keyword] in CONDITION_KEYWORDS: + return "statement" + return "other" + + +def declaration_allowed(tokens: list[str]) -> bool: + """Apply the rules to the tokens that follow a declaration's `auto`.""" + index = 0 + parentheses = 0 + while index < len(tokens) and (tokens[index] in {"*", "&", "("} or tokens[index] in DECL_SPECIFIERS): + parentheses += tokens[index] == "(" index += 1 - while index < len(tokens) and tokens[index][0] == ")" and parentheses: - code[tokens[index].start()] = " " - parentheses -= 1 - index += 1 - while index + 1 < len(tokens) and tokens[index][0] == tokens[index + 1][0] == "[": - index = hide_group(index, "[", "]") - return "".join(code) + if index == len(tokens) or not IDENTIFIER.fullmatch(tokens[index]): + return True # a structured binding, or an expression such as auto{x} + index += 1 + while index < len(tokens) and tokens[index] == ")" and parentheses: + parentheses -= 1 + index += 1 + while tokens[index:index + 2] == ["[", "["]: + end = group_end(tokens, index) + if end is None: + return False + index = end + 1 + following = tokens[index] if index < len(tokens) else None + if following in {",", ")"}: + return True # a parameter continued from an earlier line + if following == "=": + initializer = tokens[index + 1:] + elif following in {"(", "{"}: + initializer = tokens[index + 1:group_end(tokens, index)] + else: + return False + lambda_allowed = lambda_initializer_allowed(initializer) + if lambda_allowed is not None: + return lambda_allowed + # Every declarator of one auto declaration deduces the same type, so the + # first initializer decides. + return iterator_call(leading_expression(initializer)) def lexical_context(source: str) -> tuple[str, set[int]]: @@ -203,54 +353,15 @@ def lexical_context(source: str) -> tuple[str, set[int]]: return "".join(pieces), opt_out_lines -def direct_iterator_return(body: list[str], declaration: str) -> bool: +def direct_iterator_return(body: list[str]) -> bool: # Recognize a single returned iterator call, not incidental calls elsewhere # in a body or an iterator comparison whose result is a bool. - if not body or body[0] != "return" or body[-1] != ";": - return False - expression = body[1:-1] - - def closing_paren(tokens: list[str], start: int) -> int | None: - depth = 0 - for index in range(start, len(tokens)): - if tokens[index] == "(": - depth += 1 - elif tokens[index] == ")": - depth -= 1 - if depth == 0: - return index - return None + return len(body) > 2 and body[0] == "return" and body[-1] == ";" and iterator_call(body[1:-1]) - while ( - expression - and expression[0] == "(" - and closing_paren(expression, 0) == len(expression) - 1 - ): - expression = expression[1:-1] - if "(" not in expression: - return False - opening = expression.index("(") - callee = "".join(expression[:opening]) - if not re.fullmatch(r"(?:[A-Za-z_]\w*(?:::|\.|->))*[A-Za-z_]\w*", callee): - return False - return ( - closing_paren(expression, opening) == len(expression) - 1 - and is_iterator_like(f"{declaration} = {callee}(") - ) - -def lambda_initializer_allowed( - line: str, source: str | None = None, offset: int = 0 -) -> bool | None: +def lambda_initializer_allowed(initializer: list[str]) -> bool | None: """Return None for another initializer, otherwise apply lambda/iterator rules.""" - equals = line.find("=") - if equals < 0: - return None - tokens = ( - token[0] - for token in CPP_TOKEN.finditer(source if source is not None else line, offset + equals + 1) - if token.lastgroup != "skip" - ) + tokens = iter(initializer) first = next(tokens, None) parentheses = 0 while first == "(": @@ -259,20 +370,18 @@ def lambda_initializer_allowed( if first != "[": return None - pairs = {"[": "]", "(": ")", "{": "}"} - def consume_group(opener: str, contents: list[str] | None = None) -> bool: - stack = [">" if opener == "<" else pairs[opener]] + stack = [">" if opener == "<" else PAIRS[opener]] for token in tokens: - if token in pairs: - stack.append(pairs[token]) + if token in PAIRS: + stack.append(PAIRS[token]) elif token == "<" and stack[-1] == ">": stack.append(">") elif token == ">" and stack[-1] == ">": stack.pop() if not stack: return True - elif token in pairs.values(): + elif token in PAIRS.values(): if token != stack.pop(): return False if not stack: @@ -316,7 +425,7 @@ def consume_group(opener: str, contents: list[str] | None = None) -> bool: break if parentheses or following not in {None, ";", ",", ")", "}"}: return False - return not invoked or direct_iterator_return(body, line.partition("=")[0]) + return not invoked or direct_iterator_return(body) if token in {"(", "["} or (first_header_token and token == "<"): if not consume_group(token): return False @@ -327,27 +436,24 @@ def consume_group(opener: str, contents: list[str] | None = None) -> bool: return False -def is_iterator_like(line: str) -> bool: - return any(hint in line for hint in ITERATOR_HINTS) - - -def is_allowed_auto( - line: str, source: str | None = None, offset: int = 0, *, auto_ok: bool = False -) -> bool: - line = declaration_code(line) - stripped = line.strip() - if auto_ok: - return True - if not DECLARATION_AUTO.search(stripped): - return True - if STRUCTURED_BINDING.search(stripped.partition("=")[0]): - return True - lambda_allowed = lambda_initializer_allowed(line, source, offset) - if lambda_allowed is not None: - return lambda_allowed - if is_iterator_like(stripped): - return True - return False +def line_allowed(code_line: str, source: str, line_offset: int) -> bool: + """Check every `auto` on a line whose literals and comments are blanked.""" + matches = list(CPP_TOKEN.finditer(code_line)) + tokens = [match[0] for match in matches] + for index, token in enumerate(tokens): + if token != "auto": + continue + start = end = index + if index >= 2 and tokens[index - 2:index] == ["decltype", "("] and tokens[index + 1:index + 2] == [")"]: + start, end = index - 2, index + 1 + context = placeholder_context(tokens, start) + if context == "new": + return False + if context == "statement" and not declaration_allowed( + statement_tokens(source, line_offset + matches[end].end()) + ): + return False + return True def matches_allowlist(relative_path: str, line: str, rules: list[AllowRule]) -> bool: @@ -371,7 +477,7 @@ def collect_violations(repo_root: Path, allowlist: list[AllowRule]) -> list[Viol code_line = code[line_offset:line_offset + len(line)] if not WORD_AUTO.search(code_line): continue - if is_allowed_auto(code_line, source, line_offset, auto_ok=line_number in opt_out_lines): + if line_number in opt_out_lines or line_allowed(code_line, source, line_offset): continue if matches_allowlist(relative_path, line, allowlist): continue diff --git a/tools/test_cpp_auto_audit.py b/tools/test_cpp_auto_audit.py index c9387a6..67579b9 100644 --- a/tools/test_cpp_auto_audit.py +++ b/tools/test_cpp_auto_audit.py @@ -132,6 +132,98 @@ def test_direct_iterator_returns_from_invoked_lambdas_are_allowed(self): result = self.audit() self.assertEqual(result.returncode, 0, result.stderr) + def test_placeholder_forms_and_mid_line_declarations_require_types(self): + sources = [ + "decltype(auto) value = compute();", + "static decltype(auto) value = compute();", + "std::integral auto value = 42;", + "const std::convertible_to> auto value = 42;", + "int a = 1; auto b = 2;", + "int f() { auto x = g(); return x; }", + "struct S { static constexpr auto value = 42; };", + "auto fn = [] { auto x = 1; return x; };", + "auto it = values.begin(); auto n = 5;", + "for (int i = 0; auto& value : values) {}", + "int* p = new auto(42);", + "int* p = new const auto{42};", + "int* p = new (buffer) auto(42);", + ] + for source in sources: + with self.subTest(source=source): + (self.root / "forms.cpp").write_text(source + "\n") + result = self.audit() + self.assertEqual(result.returncode, 1, result.stdout) + self.assertIn("forms.cpp:1", result.stderr) + + def test_iterator_call_must_be_the_whole_initializer(self): + declarations = [ + "auto pos = text.find(',');", + 'auto pos = text.find("ab");', + "auto pos = text.find(separator, 2);", + "auto pos = [&] { return text.find(','); }();", + "auto size = std::distance(v.begin(), v.end());", + "auto found = table.find(key) != table.end();", + "auto first = *values.begin();", + "auto value = lookup(values.begin());", + "auto next = values.begin() + 1;", + "auto value = values.begin()->second;", + "auto value = std::ranges::find(values, 42);", + "auto it(values.size());", + "auto value = values.begin()();", + "auto value =\n lookup(values.begin());", + ] + for declaration in declarations: + with self.subTest(declaration=declaration): + (self.root / "iterator.cpp").write_text(declaration + "\n") + result = self.audit() + self.assertEqual(result.returncode, 1, result.stdout) + self.assertIn("iterator.cpp:1", result.stderr) + + def test_whole_iterator_initializers_remain_allowed(self): + declarations = [ + "auto it = m.find(k);", + "auto it = m.find(make_key(a, b));", + "auto it = (m.find(k));", + "auto it{values.begin()};", + "auto it(values.cend());", + "auto it = config.layers().find(id);", + "auto it = this->items_.lower_bound(key);", + "auto range = table.equal_range(key);", + "auto it = std::find(values.begin(), values.end(), 42);", + "auto it = std::next(values.begin(), 2);", + "auto it = std::max_element(values.begin(), values.end());", + "auto a = values.begin(), b = values.end();", + "for (auto it = values.begin(); it != values.end(); ++it) {}", + "if (auto it = m.find(k); it != m.end()) {}", + "auto it = m.find(k); auto other = m.find(j);", + "auto it =\n m.upper_bound(k);", + ] + for declaration in declarations: + with self.subTest(declaration=declaration): + (self.root / "iterator.cpp").write_text(declaration + "\n") + result = self.audit() + self.assertEqual(result.returncode, 0, result.stderr) + + def test_parameters_bindings_and_lambdas_remain_allowed(self): + sources = [ + "for (auto& [k, v] : m) {}", + "auto f = [](int x) { return x; };", + "const auto& [a, b] = pair;", + "int f() { auto [a, b] = pair; return a; }", + "auto f = [](auto x, const auto& y) { return x + y; };", + "auto f = [](std::integral auto x) { return x; };", + "auto f = []() -> decltype(auto) { return 42; };", + "auto f = []() -> const auto& { return value; };", + "void g(std::integral auto value, auto&& other);", + "template int h();", + "template int h();", + ] + for source in sources: + with self.subTest(source=source): + (self.root / "allowed.cpp").write_text(source + "\n") + result = self.audit() + self.assertEqual(result.returncode, 0, result.stderr) + def test_structured_binding_declarations_remain_allowed(self): declarations = [ "auto [a, b] = pair;", diff --git a/tools/verify_fixture_checksums.py b/tools/verify_fixture_checksums.py index 02035d8..8976288 100755 --- a/tools/verify_fixture_checksums.py +++ b/tools/verify_fixture_checksums.py @@ -4,6 +4,7 @@ from __future__ import annotations import hashlib +import os from pathlib import Path import sys @@ -11,37 +12,56 @@ ROOT = Path(__file__).resolve().parents[1] FIXTURES = ROOT / "tests" / "fixtures" MANIFEST = FIXTURES / "SHA256SUMS" +PREFIX = "tests/fixtures/" # manifest paths are repository-relative, as shasum prints them +NOT_FIXTURES = {"README.md", "SHA256SUMS"} + + +def fixture_files() -> set[str]: + """Paths relative to tests/fixtures of every fixture, in any subdirectory.""" + found: set[str] = set() + for directory, subdirectories, names in os.walk(FIXTURES): + # Dotfiles such as a Finder .DS_Store are not fixtures. + subdirectories[:] = [name for name in subdirectories if not name.startswith(".")] + for name in names: + relative = (Path(directory) / name).relative_to(FIXTURES).as_posix() + if not name.startswith(".") and relative not in NOT_FIXTURES: + found.add(relative) + return found def main() -> int: - expected: dict[Path, str] = {} + expected: dict[str, str] = {} + first_lines: dict[str, int] = {} for line_number, line in enumerate(MANIFEST.read_text(encoding="utf-8").splitlines(), 1): - digest, separator, relative_name = line.partition(" ") - if not separator or len(digest) != 64: + digest, separator, name = line.partition(" ") + if not separator or len(digest) != 64 or not name.startswith(PREFIX): print(f"SHA256SUMS:{line_number}: malformed entry", file=sys.stderr) return 1 - expected[ROOT / relative_name] = digest + relative = name[len(PREFIX):] + # A later duplicate would otherwise hide a stale earlier digest. + if relative in expected: + print( + f"SHA256SUMS:{line_number}: duplicate entry for {name}" + f" (first on line {first_lines[relative]})", + file=sys.stderr, + ) + return 1 + expected[relative] = digest + first_lines[relative] = line_number - actual = { - path - for path in FIXTURES.iterdir() - # Dotfiles such as a Finder .DS_Store are not fixtures. - if path.is_file() - and not path.name.startswith(".") - and path.name not in {"README.md", "SHA256SUMS"} - } + actual = fixture_files() if actual != set(expected): - missing = sorted(str(path.relative_to(ROOT)) for path in actual - set(expected)) - stale = sorted(str(path.relative_to(ROOT)) for path in set(expected) - actual) - print(f"unlisted fixtures: {missing}", file=sys.stderr) - print(f"missing fixtures: {stale}", file=sys.stderr) + unlisted = sorted(PREFIX + path for path in actual - set(expected)) + missing = sorted(PREFIX + path for path in set(expected) - actual) + print(f"unlisted fixtures: {unlisted}", file=sys.stderr) + print(f"missing fixtures: {missing}", file=sys.stderr) return 1 failures: list[str] = [] - for path, wanted in expected.items(): - found = hashlib.sha256(path.read_bytes()).hexdigest() + for relative, wanted in expected.items(): + found = hashlib.sha256((FIXTURES / relative).read_bytes()).hexdigest() if found != wanted: - failures.append(f"{path.relative_to(ROOT)}: {found} != {wanted}") + failures.append(f"{PREFIX}{relative}: {found} != {wanted}") if failures: print("\n".join(failures), file=sys.stderr) return 1 From b7f863f8dca008bbc25bec3deff6b71b72c5d4c2 Mon Sep 17 00:00:00 2001 From: KeviM Date: Fri, 20 Nov 2026 12:37:57 -0800 Subject: [PATCH 6/9] fix(http): refuse HTTPS-to-HTTP redirects; require libcurl 7.86 An https request followed a redirect to plain http and returned that body as a normal response. Redirects from https now allow only https, and the refusal is InvalidRequest. A TLS loopback server (optional OpenSSL 3, test only) covers it; Linux CI installs libssl-dev so those tests run. libcurl 7.85's curl/header.h lacks extern "C", so curl_easy_nextheader never linked there; 7.86 is the real floor. --- .github/workflows/ci.yml | 10 +- CHANGELOG.md | 8 ++ CMakeLists.txt | 5 +- README.md | 2 +- cmake/spcConfig.cmake.in | 2 +- include/spc/error.hpp | 5 +- include/spc/http_client.hpp | 6 +- src/http/client.cpp | 17 ++- tests/CMakeLists.txt | 29 +++++ tests/support/loopback_server.hpp | 130 +++++++++++--------- tests/support/tls_loopback_server.hpp | 164 ++++++++++++++++++++++++++ tests/test_http_client.cpp | 159 +++++++++++++++++++++++++ 12 files changed, 462 insertions(+), 75 deletions(-) create mode 100644 tests/support/tls_loopback_server.hpp diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1a25fbd..7ea0c4a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -27,7 +27,7 @@ jobs: run: | sudo apt-get update sudo apt-get install -y --no-install-recommends \ - build-essential clang-format libcurl4-openssl-dev locales + build-essential clang-format libcurl4-openssl-dev libssl-dev locales # The locale tests skip without a comma-decimal locale. sudo locale-gen de_DE.UTF-8 - name: Build @@ -56,7 +56,7 @@ jobs: sudo apt-get update sudo apt-get install -y --no-install-recommends \ build-essential clang-18 libc++-18-dev libc++abi-18-dev \ - libcurl4-openssl-dev locales + libcurl4-openssl-dev libssl-dev locales sudo locale-gen de_DE.UTF-8 - name: Build run: >- @@ -75,7 +75,7 @@ jobs: run: | sudo apt-get update sudo apt-get install -y --no-install-recommends \ - build-essential clang-19 libcurl4-openssl-dev + build-essential clang-19 libcurl4-openssl-dev libssl-dev - name: Build run: >- make build CMAKE_ARGS="-DCMAKE_CXX_COMPILER=clang++-19 @@ -210,7 +210,7 @@ jobs: run: | sudo apt-get update sudo apt-get install -y --no-install-recommends \ - build-essential libcurl4-openssl-dev locales + build-essential libcurl4-openssl-dev libssl-dev locales sudo locale-gen de_DE.UTF-8 - name: Build with AddressSanitizer and UndefinedBehaviorSanitizer run: >- @@ -230,7 +230,7 @@ jobs: run: | sudo apt-get update sudo apt-get install -y --no-install-recommends \ - build-essential libcurl4-openssl-dev locales + build-essential libcurl4-openssl-dev libssl-dev locales sudo locale-gen de_DE.UTF-8 - name: Build with ThreadSanitizer run: >- diff --git a/CHANGELOG.md b/CHANGELOG.md index ef49da0..dbdfe6b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,8 +6,16 @@ uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [Unreleased] +### Security + +- An HTTPS request no longer follows a redirect to plain HTTP; it fails with + `InvalidRequest`. + ### Changed +- libcurl 7.86 is the minimum. 7.85's `curl/header.h` lacks `extern "C"`, so the + library never linked against it. + - Report a malformed URL or a scheme other than HTTP(S) as `InvalidRequest`, and a redirect loop as `ServerError`. `with_retry` no longer repeats them. - Map only HTTP 500-599 to `ServerError`; a status above 599 is `Unknown`. diff --git a/CMakeLists.txt b/CMakeLists.txt index 3ed8b52..a4f025d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -99,8 +99,9 @@ if(SPC_ENABLE_COVERAGE) add_link_options(--coverage) endif() -# CURLOPT_PROTOCOLS_STR, which keeps the transport on http and https, needs 7.85. -find_package(CURL 7.85 REQUIRED) +# 7.85 added CURLOPT_PROTOCOLS_STR, which keeps the transport on http and https, +# and 7.86 made curl/header.h's functions extern "C", so C++ can link them. +find_package(CURL 7.86 REQUIRED) find_package(Threads REQUIRED) include(FetchContent) diff --git a/README.md b/README.md index 5bce8d0..4c4220a 100644 --- a/README.md +++ b/README.md @@ -38,7 +38,7 @@ int main() { - CMake 3.31 or newer. Ubuntu 24.04 ships 3.28, so run `sudo snap install cmake --classic`. - GCC 13+, Clang 18+ with libc++ or 19+ with libstdc++, or Apple Clang 17+. -- libcurl 7.85 or newer with its headers: `libcurl4-openssl-dev` on Ubuntu, +- libcurl 7.86 or newer with its headers: `libcurl4-openssl-dev` on Ubuntu, built into macOS. CI builds on Linux, macOS for Apple silicon and Intel, and Windows with MSVC. diff --git a/cmake/spcConfig.cmake.in b/cmake/spcConfig.cmake.in index 1887646..d7dcf8d 100644 --- a/cmake/spcConfig.cmake.in +++ b/cmake/spcConfig.cmake.in @@ -1,7 +1,7 @@ @PACKAGE_INIT@ include(CMakeFindDependencyMacro) -find_dependency(CURL 7.85) +find_dependency(CURL 7.86) find_dependency(Threads) include("${CMAKE_CURRENT_LIST_DIR}/spcTargets.cmake") diff --git a/include/spc/error.hpp b/include/spc/error.hpp index c51c9d1..2555781 100644 --- a/include/spc/error.hpp +++ b/include/spc/error.hpp @@ -29,8 +29,9 @@ enum class ErrorCode { /// returns it. FeedUnavailable, /// The request cannot succeed as asked: an unsupported day or hazard, a - /// malformed URL or one that is not HTTP(S), HTTP 400, an ArcGIS error such - /// as an invalid parameter, or a response larger than + /// malformed URL or one that is not HTTP(S), a refused redirect (to another + /// scheme, or from HTTPS to HTTP), HTTP 400, an ArcGIS error such as an + /// invalid parameter, or a response larger than /// `ClientConfig::max_response_bytes`. InvalidRequest, /// The body was not valid JSON. Valid JSON without the expected fields diff --git a/include/spc/http_client.hpp b/include/spc/http_client.hpp index f2e64f1..45ce19a 100644 --- a/include/spc/http_client.hpp +++ b/include/spc/http_client.hpp @@ -61,7 +61,8 @@ class HttpTransport { }; /// libcurl-backed transport. Follows redirects, accepts compressed bodies, -/// and speaks only http and https. +/// and speaks only http and https. An https request follows only https +/// redirects, so its response never arrives over plaintext. /// /// Thread-safe: concurrent `get` calls each use their own pooled libcurl /// handle, and handles are reused so connections stay open between requests. @@ -77,7 +78,8 @@ class HttpClient final : public HttpTransport { /// Absolute HTTP and HTTPS URLs use case-insensitive schemes and bypass /// `config().base_url`; other inputs are appended to it. Transfers and - /// redirects permit only HTTP and HTTPS. + /// redirects permit only HTTP and HTTPS, and an HTTPS request refuses a + /// redirect to HTTP. A refused redirect is `ErrorCode::InvalidRequest`. [[nodiscard]] Result get(std::string_view path) const override; /// The settings this client was built with; defaults on a moved-from client. diff --git a/src/http/client.cpp b/src/http/client.cpp index c3a0ea7..c1fb5cb 100644 --- a/src/http/client.cpp +++ b/src/http/client.cpp @@ -11,8 +11,9 @@ #include #include -#if !CURL_AT_LEAST_VERSION(7, 85, 0) -#error "spc-cpp needs libcurl 7.85.0 or newer for CURLOPT_PROTOCOLS_STR" +// 7.85 added CURLOPT_PROTOCOLS_STR; before 7.86, curl/header.h lacked extern "C". +#if !CURL_AT_LEAST_VERSION(7, 86, 0) +#error "spc-cpp needs libcurl 7.86.0 or newer" #endif namespace spc { @@ -84,6 +85,12 @@ bool is_absolute_url(std::string_view path) { (path.size() >= 8 && curl_strnequal(path.data(), "https://", 8) != 0); } +/// Whether libcurl fetches `url` over TLS. It ignores the scheme's case and +/// takes one to three slashes after it, so this matches the first slash only. +bool is_https_url(std::string_view url) { + return url.size() >= 7 && curl_strnequal(url.data(), "https:/", 7) != 0; +} + /// libcurl's process-wide state. Each client holds a shared reference, so the /// global cleanup runs only after the last handle is gone, whatever order /// static objects are destroyed in. @@ -155,6 +162,10 @@ Result perform(Connection& connection, const std::string& url, Error::invalid_request("ClientConfig::timeout is outside libcurl's supported range")); } + // An https request stays on https: anyone on the path could alter a + // response fetched over an http redirect. + const char* const redirects = is_https_url(url) ? "https" : "http,https"; + // Refuse to send anything unless the restrictions actually took effect; // an older runtime libcurl rejects CURLOPT_PROTOCOLS_STR and would then // happily read file:// URLs. @@ -162,7 +173,7 @@ Result perform(Connection& connection, const std::string& url, const bool configured = curl_easy_setopt(curl, CURLOPT_URL, url.c_str()) == CURLE_OK && curl_easy_setopt(curl, CURLOPT_PROTOCOLS_STR, "http,https") == CURLE_OK && - curl_easy_setopt(curl, CURLOPT_REDIR_PROTOCOLS_STR, "http,https") == CURLE_OK && + curl_easy_setopt(curl, CURLOPT_REDIR_PROTOCOLS_STR, redirects) == CURLE_OK && curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, verify) == CURLE_OK && curl_easy_setopt(curl, CURLOPT_SSL_VERIFYHOST, verify * 2L) == CURLE_OK; if (!configured) { diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index eb0d3e5..ac617d8 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -19,6 +19,35 @@ if(WIN32) _CRT_SECURE_NO_WARNINGS) endif() +# The https redirect tests need a TLS server, which tests/support builds on +# OpenSSL 3. It is optional and those tests skip without it. The link check +# turns it off where the copy found cannot serve the build, such as a +# universal macOS binary against Homebrew's single-architecture OpenSSL; it +# is not cached, because the architectures can change between configures. +# Windows is left out: its tests would need OpenSSL's DLLs on PATH to start. +unset(SPC_TEST_OPENSSL_LINKS CACHE) +if(NOT WIN32) + find_package(OpenSSL 3.0 QUIET) + if(OpenSSL_FOUND) + include(CheckCXXSourceCompiles) + set(CMAKE_REQUIRED_LIBRARIES OpenSSL::SSL) + set(CMAKE_REQUIRED_QUIET ON) + check_cxx_source_compiles(" + #include + int main() { return SSL_CTX_new(TLS_server_method()) == nullptr ? 1 : 0; }" + SPC_TEST_OPENSSL_LINKS) + unset(CMAKE_REQUIRED_LIBRARIES) + unset(CMAKE_REQUIRED_QUIET) + if(SPC_TEST_OPENSSL_LINKS) + target_link_libraries(spc_tests PRIVATE OpenSSL::SSL) + target_compile_definitions(spc_tests PRIVATE SPC_TEST_HAS_TLS_SERVER) + endif() + endif() +endif() +if(NOT SPC_TEST_OPENSSL_LINKS) + message(STATUS "No usable OpenSSL 3: the https redirect tests will skip") +endif() + spc_set_warnings(spc_tests) target_link_libraries(spc_tests PRIVATE spc::spc GTest::gtest_main Threads::Threads) # Tests reach the private JSON helpers in src/models/json.hpp. diff --git a/tests/support/loopback_server.hpp b/tests/support/loopback_server.hpp index 6c488cb..9b05ead 100644 --- a/tests/support/loopback_server.hpp +++ b/tests/support/loopback_server.hpp @@ -79,6 +79,45 @@ inline void initialize() {} inline void set_option(Socket socket, int option, const void* value, int length) { (void)::setsockopt(socket, SOL_SOCKET, option, static_cast(value), length); } + +/// A listening socket on 127.0.0.1; `port` receives the free port it got. +inline Socket listen_on_loopback(std::uint16_t& port) { + initialize(); + const Socket listener = ::socket(AF_INET, SOCK_STREAM, 0); + if (listener == kInvalidSocket) { + throw std::runtime_error("socket() failed"); + } + const int reuse = 1; + set_option(listener, SO_REUSEADDR, &reuse, sizeof(reuse)); + sockaddr_in address{}; + address.sin_family = AF_INET; + address.sin_addr.s_addr = htonl(INADDR_LOOPBACK); + address.sin_port = 0; // any free port + if (::bind(listener, reinterpret_cast(&address), sizeof(address)) != 0 || + ::listen(listener, 64) != 0) { + close_socket(listener); + throw std::runtime_error("bind() or listen() failed"); + } + AddressLength length = sizeof(address); + (void)::getsockname(listener, reinterpret_cast(&address), &length); + port = ntohs(address.sin_port); + return listener; +} + +/// Bound how long a stalled client can hold the serving thread. +inline void limit_io_time(Socket client) { +#ifdef _WIN32 + const DWORD timeout = 2000; +#else + const timeval timeout{2, 0}; +#endif + set_option(client, SO_RCVTIMEO, &timeout, sizeof(timeout)); + set_option(client, SO_SNDTIMEO, &timeout, sizeof(timeout)); +#ifdef SO_NOSIGPIPE + const int no_sigpipe = 1; + set_option(client, SO_NOSIGPIPE, &no_sigpipe, sizeof(no_sigpipe)); +#endif +} } // namespace socket_detail struct LoopbackRequest { @@ -95,30 +134,41 @@ struct LoopbackRequest { } }; +/// The request line's target and the headers of a raw request. +inline LoopbackRequest parse_request(const std::string& raw) { + LoopbackRequest request; + const std::size_t line_end = raw.find("\r\n"); + const std::string request_line = raw.substr(0, line_end); + const std::size_t first_space = request_line.find(' '); + const std::size_t second_space = request_line.find(' ', first_space + 1); + if (first_space != std::string::npos && second_space != std::string::npos) { + request.target = request_line.substr(first_space + 1, second_space - first_space - 1); + } + std::size_t start = line_end + 2; + while (start < raw.size()) { + const std::size_t end = raw.find("\r\n", start); + if (end == std::string::npos || end == start) { + break; + } + const std::string line = raw.substr(start, end - start); + const std::size_t colon = line.find(':'); + if (colon != std::string::npos) { + const std::size_t value_start = line.find_first_not_of(' ', colon + 1); + request.headers.emplace_back(line.substr(0, colon), value_start == std::string::npos + ? std::string{} + : line.substr(value_start)); + } + start = end + 2; + } + return request; +} + class LoopbackServer { public: using Handler = std::function; explicit LoopbackServer(Handler handler) : handler_(std::move(handler)) { - socket_detail::initialize(); - listener_ = ::socket(AF_INET, SOCK_STREAM, 0); - if (listener_ == socket_detail::kInvalidSocket) { - throw std::runtime_error("socket() failed"); - } - const int reuse = 1; - socket_detail::set_option(listener_, SO_REUSEADDR, &reuse, sizeof(reuse)); - sockaddr_in address{}; - address.sin_family = AF_INET; - address.sin_addr.s_addr = htonl(INADDR_LOOPBACK); - address.sin_port = 0; // any free port - if (::bind(listener_, reinterpret_cast(&address), sizeof(address)) != 0 || - ::listen(listener_, 64) != 0) { - socket_detail::close_socket(listener_); - throw std::runtime_error("bind() or listen() failed"); - } - socket_detail::AddressLength length = sizeof(address); - (void)::getsockname(listener_, reinterpret_cast(&address), &length); - port_ = ntohs(address.sin_port); + listener_ = socket_detail::listen_on_loopback(port_); thread_ = std::thread([this] { serve(); }); } @@ -162,17 +212,7 @@ class LoopbackServer { } void handle(socket_detail::Socket client) { -#ifdef _WIN32 - const DWORD timeout = 2000; -#else - const timeval timeout{2, 0}; -#endif - socket_detail::set_option(client, SO_RCVTIMEO, &timeout, sizeof(timeout)); - socket_detail::set_option(client, SO_SNDTIMEO, &timeout, sizeof(timeout)); -#ifdef SO_NOSIGPIPE - const int no_sigpipe = 1; - socket_detail::set_option(client, SO_NOSIGPIPE, &no_sigpipe, sizeof(no_sigpipe)); -#endif + socket_detail::limit_io_time(client); std::string raw; std::vector buffer(4096); while (raw.find("\r\n\r\n") == std::string::npos && raw.size() < 65536) { @@ -183,7 +223,7 @@ class LoopbackServer { } raw.append(buffer.data(), static_cast(received)); } - const LoopbackRequest request = parse(raw); + const LoopbackRequest request = parse_request(raw); { const std::lock_guard lock(mutex_); requests_.push_back(request); @@ -192,34 +232,6 @@ class LoopbackServer { (void)::shutdown(client, socket_detail::kShutdownWrite); } - static LoopbackRequest parse(const std::string& raw) { - LoopbackRequest request; - const std::size_t line_end = raw.find("\r\n"); - const std::string request_line = raw.substr(0, line_end); - const std::size_t first_space = request_line.find(' '); - const std::size_t second_space = request_line.find(' ', first_space + 1); - if (first_space != std::string::npos && second_space != std::string::npos) { - request.target = request_line.substr(first_space + 1, second_space - first_space - 1); - } - std::size_t start = line_end + 2; - while (start < raw.size()) { - const std::size_t end = raw.find("\r\n", start); - if (end == std::string::npos || end == start) { - break; - } - const std::string line = raw.substr(start, end - start); - const std::size_t colon = line.find(':'); - if (colon != std::string::npos) { - const std::size_t value_start = line.find_first_not_of(' ', colon + 1); - request.headers.emplace_back(line.substr(0, colon), value_start == std::string::npos - ? std::string{} - : line.substr(value_start)); - } - start = end + 2; - } - return request; - } - void send_all(socket_detail::Socket client, const std::string& response) { #ifdef MSG_NOSIGNAL constexpr int kFlags = MSG_NOSIGNAL; diff --git a/tests/support/tls_loopback_server.hpp b/tests/support/tls_loopback_server.hpp new file mode 100644 index 0000000..6412221 --- /dev/null +++ b/tests/support/tls_loopback_server.hpp @@ -0,0 +1,164 @@ +/// @file tls_loopback_server.hpp +/// @brief LoopbackServer over TLS, for tests where the response has to arrive +/// on an https request. +/// +/// It needs OpenSSL 3 and POSIX sockets. tests/CMakeLists.txt treats that as +/// optional and defines SPC_TEST_HAS_TLS_SERVER when it is there. Each server +/// signs its own certificate at startup, so no private key is checked in, and +/// clients need `verify_ssl = false`. + +#pragma once + +#ifdef SPC_TEST_HAS_TLS_SERVER + +#include "support/loopback_server.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace spc::test { + +class TlsLoopbackServer { +public: + explicit TlsLoopbackServer(LoopbackServer::Handler handler) + : handler_(std::move(handler)), context_(self_signed_context()) { + listener_ = socket_detail::listen_on_loopback(port_); + thread_ = std::thread([this] { serve(); }); + } + + ~TlsLoopbackServer() { + stop_.store(true); + thread_.join(); + socket_detail::close_socket(listener_); + } + + TlsLoopbackServer(const TlsLoopbackServer&) = delete; + TlsLoopbackServer& operator=(const TlsLoopbackServer&) = delete; + TlsLoopbackServer(TlsLoopbackServer&&) = delete; + TlsLoopbackServer& operator=(TlsLoopbackServer&&) = delete; + + /// "127.0.0.1:port", for tests that spell the scheme themselves. + [[nodiscard]] std::string authority() const { return "127.0.0.1:" + std::to_string(port_); } + + [[nodiscard]] std::string url(std::string_view target) const { + return "https://" + authority() + std::string{target}; + } + + [[nodiscard]] std::vector requests() const { + const std::lock_guard lock(mutex_); + return requests_; + } + +private: + struct Free { + void operator()(SSL_CTX* context) const { SSL_CTX_free(context); } + void operator()(SSL* ssl) const { SSL_free(ssl); } + void operator()(EVP_PKEY* key) const { EVP_PKEY_free(key); } + void operator()(X509* certificate) const { X509_free(certificate); } + void operator()(X509_NAME* name) const { X509_NAME_free(name); } + }; + + static std::unique_ptr self_signed_context() { + const std::unique_ptr key{ + EVP_PKEY_Q_keygen(nullptr, nullptr, "EC", "P-256")}; + const std::unique_ptr certificate{X509_new()}; + const std::unique_ptr name{X509_NAME_new()}; + std::unique_ptr context{SSL_CTX_new(TLS_server_method())}; + if (key == nullptr || certificate == nullptr || name == nullptr || context == nullptr) { + throw std::runtime_error("OpenSSL could not allocate the test certificate"); + } + constexpr unsigned char kHost[] = "127.0.0.1"; + const bool ready = + X509_NAME_add_entry_by_txt(name.get(), "CN", MBSTRING_ASC, kHost, -1, -1, 0) == 1 && + X509_set_version(certificate.get(), X509_VERSION_3) == 1 && + ASN1_INTEGER_set(X509_get_serialNumber(certificate.get()), 1) == 1 && + X509_gmtime_adj(X509_getm_notBefore(certificate.get()), -3600) != nullptr && + X509_gmtime_adj(X509_getm_notAfter(certificate.get()), 3600) != nullptr && + X509_set_subject_name(certificate.get(), name.get()) == 1 && + X509_set_issuer_name(certificate.get(), name.get()) == 1 && + X509_set_pubkey(certificate.get(), key.get()) == 1 && + X509_sign(certificate.get(), key.get(), EVP_sha256()) > 0 && + SSL_CTX_use_certificate(context.get(), certificate.get()) == 1 && + SSL_CTX_use_PrivateKey(context.get(), key.get()) == 1; + if (!ready) { + throw std::runtime_error("OpenSSL could not build the test certificate"); + } + return context; + } + + void serve() { + // OpenSSL writes with write(), which raises SIGPIPE once the client has + // hung up. Blocked, the signal becomes an error return on this thread. + sigset_t broken_pipe; + (void)sigemptyset(&broken_pipe); + (void)sigaddset(&broken_pipe, SIGPIPE); + (void)::pthread_sigmask(SIG_BLOCK, &broken_pipe, nullptr); + while (!stop_.load()) { + if (socket_detail::wait_for_connection(listener_) <= 0) { + continue; + } + const socket_detail::Socket client = ::accept(listener_, nullptr, nullptr); + if (client == socket_detail::kInvalidSocket) { + continue; + } + handle(client); + socket_detail::close_socket(client); + } + } + + void handle(socket_detail::Socket client) { + socket_detail::limit_io_time(client); + const std::unique_ptr ssl{SSL_new(context_.get())}; + if (ssl == nullptr || SSL_set_fd(ssl.get(), client) != 1 || SSL_accept(ssl.get()) != 1) { + return; // e.g. a client that verifies the certificate + } + std::string raw; + std::array buffer{}; + while (raw.find("\r\n\r\n") == std::string::npos && raw.size() < 65536) { + const int received = + SSL_read(ssl.get(), buffer.data(), static_cast(buffer.size())); + if (received <= 0) { + return; + } + raw.append(buffer.data(), static_cast(received)); + } + const LoopbackRequest request = parse_request(raw); + { + const std::lock_guard lock(mutex_); + requests_.push_back(request); + } + const std::string response = handler_(request); + if (!response.empty() && + SSL_write(ssl.get(), response.data(), static_cast(response.size())) > 0) { + (void)SSL_shutdown(ssl.get()); + } + } + + LoopbackServer::Handler handler_; + std::unique_ptr context_; + socket_detail::Socket listener_{socket_detail::kInvalidSocket}; + std::uint16_t port_{0}; + std::atomic stop_{false}; + mutable std::mutex mutex_; + std::vector requests_; + std::thread thread_; +}; + +} // namespace spc::test + +#endif // SPC_TEST_HAS_TLS_SERVER diff --git a/tests/test_http_client.cpp b/tests/test_http_client.cpp index 5d0329c..65c0c79 100644 --- a/tests/test_http_client.cpp +++ b/tests/test_http_client.cpp @@ -5,6 +5,7 @@ #include "spc/http_client.hpp" #include "spc/retry.hpp" #include "support/loopback_server.hpp" +#include "support/tls_loopback_server.hpp" #include #include @@ -350,6 +351,164 @@ TEST(HttpClient, RefusesARedirectToAnotherScheme) { EXPECT_EQ(response.error().code, ErrorCode::InvalidRequest); } +TEST(HttpClient, AFailedHttpsRequestLeavesHttpRedirectsAllowedOnItsHandle) { + LoopbackServer target{[](const LoopbackRequest&) { return http_response("200 OK", "ok"); }}; + LoopbackServer origin{[&target](const LoopbackRequest&) { + return http_response("302 Found", "", "Location: " + target.url("/final") + "\r\n"); + }}; + ClientConfig config; + config.timeout = std::chrono::seconds{1}; + const HttpClient client{config}; + + // Port zero cannot host a server; the request only configures the handle. + EXPECT_FALSE(client.get("https://127.0.0.1:0/")); + const Result response = client.get(origin.url("/start")); + + ASSERT_TRUE(response) << response.error().message; + EXPECT_EQ(response->body, "ok"); +} + +#ifdef SPC_TEST_HAS_TLS_SERVER + +using test::TlsLoopbackServer; + +/// TlsLoopbackServer's certificate is self-signed. +ClientConfig unverified_tls() { + ClientConfig config; + config.verify_ssl = false; + return config; +} + +bool curl_has_tls() { + return (curl_version_info(CURLVERSION_NOW)->features & CURL_VERSION_SSL) != 0; +} + +TEST(HttpClient, AnHttpsRequestRefusesARedirectToHttp) { + if (!curl_has_tls()) { + GTEST_SKIP() << "libcurl was built without TLS"; + } + // Anyone on the path can alter the plaintext hop, so its body must not + // come back as the answer to an https request. + LoopbackServer plain{ + [](const LoopbackRequest&) { return http_response("200 OK", "plaintext"); }}; + TlsLoopbackServer secure{[&plain](const LoopbackRequest& request) { + if (request.target == "/two-hops") { + return http_response("302 Found", "", "Location: /start\r\n"); + } + return http_response("302 Found", "", "Location: " + plain.url("/final") + "\r\n"); + }}; + const HttpClient client{unverified_tls()}; + ClientConfig based = unverified_tls(); + based.base_url = "HTTPS://" + secure.authority(); + const HttpClient based_client{based}; + + // libcurl ignores the scheme's case and takes one to three slashes after it. + std::vector>> responses; + for (const std::string scheme : {"https://", "HTTPS://", "hTtPs:/", "https:///"}) { + responses.emplace_back(scheme, client.get(scheme + secure.authority() + "/start")); + } + responses.emplace_back("base_url", based_client.get("/start")); + responses.emplace_back("https, https, http", client.get(secure.url("/two-hops"))); + + for (const std::pair>& entry : responses) { + const Result& response = entry.second; + if (response) { + ADD_FAILURE() << entry.first << " followed the redirect and returned \"" + << response->body << "\" with status " << response->status_code; + continue; + } + // Not NetworkError, which with_retry would retry. + EXPECT_EQ(response.error().code, ErrorCode::InvalidRequest) + << entry.first << ": " << response.error().message; + } + // Every request got its redirect: none failed earlier, in the handshake. + EXPECT_EQ(secure.requests().size(), responses.size() + 1); + EXPECT_TRUE(plain.requests().empty()) << "the plaintext server was contacted"; +} + +TEST(HttpClient, AnHttpsRequestFollowsHttpsRedirects) { + if (!curl_has_tls()) { + GTEST_SKIP() << "libcurl was built without TLS"; + } + TlsLoopbackServer target{[](const LoopbackRequest& request) { + if (request.target == "/relative") { + return http_response("302 Found", "", "Location: /final\r\n"); + } + return http_response("200 OK", "secure"); + }}; + TlsLoopbackServer origin{[&target](const LoopbackRequest&) { + return http_response("302 Found", "", "Location: " + target.url("/relative") + "\r\n"); + }}; + const HttpClient client{unverified_tls()}; + + const Result response = client.get(origin.url("/start")); + + ASSERT_TRUE(response) << response.error().message; + EXPECT_EQ(response->body, "secure"); + EXPECT_EQ(target.requests().size(), 2U); +} + +TEST(HttpClient, AnHttpRequestFollowsRedirectsToHttpAndHttps) { + if (!curl_has_tls()) { + GTEST_SKIP() << "libcurl was built without TLS"; + } + LoopbackServer plain{ + [](const LoopbackRequest&) { return http_response("200 OK", "plaintext"); }}; + TlsLoopbackServer secure{ + [](const LoopbackRequest&) { return http_response("200 OK", "secure"); }}; + LoopbackServer origin{[&plain, &secure](const LoopbackRequest& request) { + const std::string location = + request.target == "/to-https" ? secure.url("/final") : plain.url("/final"); + return http_response("302 Found", "", "Location: " + location + "\r\n"); + }}; + const HttpClient client{unverified_tls()}; + + const Result to_http = client.get(origin.url("/to-http")); + const Result to_https = client.get(origin.url("/to-https")); + + ASSERT_TRUE(to_http) << to_http.error().message; + EXPECT_EQ(to_http->body, "plaintext"); + ASSERT_TRUE(to_https) << to_https.error().message; + EXPECT_EQ(to_https->body, "secure"); +} + +TEST(HttpClient, ReusedHandlesTakeTheRedirectRuleOfEachRequest) { + if (!curl_has_tls()) { + GTEST_SKIP() << "libcurl was built without TLS"; + } + // Sequential requests share one pooled handle, so neither rule may outlive + // the request that set it. + LoopbackServer plain{[](const LoopbackRequest& request) { + if (request.target == "/start") { + return http_response("302 Found", "", "Location: /final\r\n"); + } + return http_response("200 OK", "plaintext"); + }}; + TlsLoopbackServer secure{[&plain](const LoopbackRequest&) { + return http_response("302 Found", "", "Location: " + plain.url("/final") + "\r\n"); + }}; + const HttpClient client{unverified_tls()}; + + const Result first_https = client.get(secure.url("/start")); + const Result http = client.get(plain.url("/start")); + const Result second_https = client.get(secure.url("/start")); + + EXPECT_FALSE(first_https); + ASSERT_TRUE(http) << http.error().message; + EXPECT_EQ(http->body, "plaintext"); + EXPECT_FALSE(second_https); + EXPECT_EQ(secure.requests().size(), 2U); // both https requests got their redirect + EXPECT_EQ(plain.requests().size(), 2U); // only the http request's two hops +} + +#else + +TEST(HttpClient, HttpsRedirectTestsNeedATlsServer) { + GTEST_SKIP() << "built without OpenSSL 3, so the https redirect tests did not run"; +} + +#endif // SPC_TEST_HAS_TLS_SERVER + TEST(HttpClient, ARedirectLoopIsAServerErrorThatIsNotRetried) { LoopbackServer server{[](const LoopbackRequest&) { return http_response("302 Found", "", "Location: /loop\r\n"); From 1a2b8a80f17e2ae76c8a3e02e8fb73840d348bcd Mon Sep 17 00:00:00 2001 From: KeviM Date: Fri, 20 Nov 2026 13:03:24 -0800 Subject: [PATCH 7/9] fix(api): bound memory across ArcGIS pages; retry ArcGIS errors in 200 bodies Each page got a fresh Web Mercator expansion budget and its own response cap, so a server that kept paging could make one call hold gigabytes. One budget now covers the call, both fire-weather layers included, and the pages together may not exceed max_response_bytes. ArcGIS 429 and 5xx errors sent with HTTP 200 are retried like the same status, one page at a time; a missing code is no longer reported as 400; and a std::exception such as bad_alloc in any client call is a ParseError instead of escaping. --- CHANGELOG.md | 10 + README.md | 15 +- include/spc/api.hpp | 22 +- include/spc/error.hpp | 14 +- include/spc/http_client.hpp | 5 +- include/spc/pagination.hpp | 3 + src/api/call_budget.hpp | 53 +++++ src/api/client.cpp | 247 +++++++++++++++------ src/models/convective.cpp | 10 +- src/models/fire_weather.cpp | 5 +- src/models/from_tree.hpp | 14 +- src/models/json.cpp | 12 +- src/models/json.hpp | 23 +- src/models/mesoscale.cpp | 4 +- tests/test_client.cpp | 413 +++++++++++++++++++++++++++++++++++- 15 files changed, 734 insertions(+), 116 deletions(-) create mode 100644 src/api/call_budget.hpp diff --git a/CHANGELOG.md b/CHANGELOG.md index dbdfe6b..d9d53d0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,10 @@ uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html). - An HTTPS request no longer follows a redirect to plain HTTP; it fails with `InvalidRequest`. +- Bound memory across all pages of an ArcGIS call. One Web Mercator expansion + budget covers the whole call, both fire-weather layers included, and the pages + together may not exceed `ClientConfig::max_response_bytes` (`InvalidRequest`). + Before, a paging server could make one call hold gigabytes. ### Changed @@ -19,6 +23,10 @@ uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html). - Report a malformed URL or a scheme other than HTTP(S) as `InvalidRequest`, and a redirect loop as `ServerError`. `with_retry` no longer repeats them. - Map only HTTP 500-599 to `ServerError`; a status above 599 is `Unknown`. +- Retry ArcGIS 429 and 5xx errors sent in HTTP 200 bodies like the same HTTP + status, one page at a time. +- An ArcGIS error without a usable code reports `http_status` 0 and + `arcgisCode=0` instead of a made-up 400. ### Fixed @@ -28,6 +36,8 @@ uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html). - Cap a `Retry-After` too large for int64 at the longest wait instead of ignoring it. - Reset the daily quota when the clock steps back across midnight. - Initialize `HttpResponse::status_code` to 0. +- A `std::exception` inside any client call, `std::bad_alloc` included, returns + `ParseError` instead of escaping from ArcGIS calls. - Skip features whose fields are not an object, conditional-intensity polygons without a label, and day 1-2 fire-weather polygons with no known `dn` band. - Read the numeric strings `"nan"` and `"inf"` as 0 instead of passing them on. diff --git a/README.md b/README.md index 4c4220a..2c559bd 100644 --- a/README.md +++ b/README.md @@ -133,12 +133,16 @@ problems. `Error::code` says what went wrong: | `NetworkError` | Connection, TLS, or timeout failure, after retries. | | `RateLimited` | HTTP 429 or 503 after retries, or `ArchiveClient`'s own limit would wait more than 5 s. | | `ServerError` | HTTP 5xx other than 503, after retries, or an ArcGIS query that never finished paging. | -| `InvalidRequest` | An unsupported day or hazard, HTTP 400, an ArcGIS error such as a bad layer ID, or a response over 64 MiB. | +| `InvalidRequest` | An unsupported day or hazard, HTTP 400, an ArcGIS error such as a bad layer ID, a response over 64 MiB, or an ArcGIS query whose pages total more than that. | | `NotFound` | A URL or ArcGIS service that no longer exists. Report it as a bug. | -| `ParseError` | Malformed JSON or Esri geometry that cannot be converted. | +| `ParseError` | Malformed JSON, Esri geometry that cannot be converted, or not enough memory to build the result. | | `FeedUnavailable` | Only `StaticFeedClient` returns it. SPC has not issued the product, as with day 1 probabilities overnight. Treat it as no data. `ArcGISClient` returns an empty result instead. | | `Unknown` | Any other HTTP status, such as 422 for a malformed `ArchiveClient` timestamp. `Error::http_status` holds it. | +ArcGIS reports its own failures with HTTP 200 and an error code in the body. +`ArcGISClient` retries a 429 or 5xx code as it would the same HTTP status, +repeating only the page that failed. + The offline `parse_*` functions, such as `spc::parse_categorical(body, day)`, throw `std::runtime_error` on malformed JSON or Esri geometry that cannot be converted. Coordinates are longitude/latitude in degrees. Esri Web Mercator (3857, 102100, @@ -148,12 +152,15 @@ Projected edges are subdivided to preserve their boundaries within one metre. Malformed Esri points and invalid projected coordinates, including discarded holes, raise a parse error. A payload needing more than one million added points also raises a parse error. +An `ArcGISClient` call counts every page and layer it reads as one payload. ## Settings and threads `spc::ClientConfig` sets the user agent, the 15 s timeout, and the 64 MiB -response limit. The default user agent carries spc-cpp's contact address, so -set one with yours: +response limit. The server decides how many pages an ArcGIS query has, so that +limit also caps the pages of one `ArcGISClient` call together; with a transport +of your own the cap is 64 MiB. The default user agent carries spc-cpp's contact +address, so set one with yours: ```cpp const spc::ArcGISClient client{spc::ClientConfig{.user_agent = "myapp/1.0 (you@example.com)"}}; diff --git a/include/spc/api.hpp b/include/spc/api.hpp index ad8b57d..401730c 100644 --- a/include/spc/api.hpp +++ b/include/spc/api.hpp @@ -3,9 +3,11 @@ /// IEM archive. /// /// Every method returns `Result` and never throws for network or data -/// errors. Clients are move-only; a moved-from client may only be assigned -/// to or destroyed. A client is safe to use from several threads when its -/// transport is; the default `HttpClient` is. +/// errors. A std::exception raised inside a call, std::bad_alloc included, +/// comes back as `ErrorCode::ParseError`; any other exception, which only a +/// custom transport can throw, propagates. Clients are move-only; a moved-from +/// client may only be assigned to or destroyed. A client is safe to use from +/// several threads when its transport is; the default `HttpClient` is. #pragma once @@ -80,7 +82,16 @@ enum class ArcGISService : std::uint8_t { /// SPC products from NOAA's ArcGIS MapServers (mapservices.weather.noaa.gov). /// /// An empty product is a successful, empty payload. Queries page through -/// results automatically. +/// results automatically, and a transient ArcGIS error sent with HTTP 200 is +/// retried like the same HTTP status, one page at a time. +/// +/// The server decides how many pages a query has, so a call is bounded as a +/// whole. Its pages, across every layer it reads, may total +/// `ClientConfig::max_response_bytes` (64 MiB when the transport is not an +/// `HttpClient`) and add one million projected points; past either the call +/// fails with `InvalidRequest` or `ParseError`. The first response counts +/// toward the total but is never refused by it: one response is the +/// transport's to bound. class ArcGISClient { public: explicit ArcGISClient(ClientConfig config = {}); @@ -116,7 +127,8 @@ class ArcGISClient { /// Active mesoscale discussions; empty when none are active. [[nodiscard]] Result query_active_md() const; - /// Any layer on one of the three MapServers. Returns each page's raw body. + /// Any layer on one of the three MapServers. Returns each page's raw body; + /// the call's byte limit bounds what they hold together. [[nodiscard]] Result> query_layer(ArcGISService service, std::int32_t layer_id, const QueryParams& params) const; diff --git a/include/spc/error.hpp b/include/spc/error.hpp index 2555781..abfca0e 100644 --- a/include/spc/error.hpp +++ b/include/spc/error.hpp @@ -31,11 +31,12 @@ enum class ErrorCode { /// The request cannot succeed as asked: an unsupported day or hazard, a /// malformed URL or one that is not HTTP(S), a refused redirect (to another /// scheme, or from HTTPS to HTTP), HTTP 400, an ArcGIS error such as an - /// invalid parameter, or a response larger than - /// `ClientConfig::max_response_bytes`. + /// invalid parameter, or a response, or the pages of one ArcGIS call + /// together, larger than `ClientConfig::max_response_bytes`. InvalidRequest, - /// The body was not valid JSON. Valid JSON without the expected fields - /// parses to an empty result. + /// The response could not be turned into a result: invalid JSON, Esri + /// geometry that cannot be converted, or an allocation failure inside the + /// call. Valid JSON without the expected fields parses to an empty result. ParseError, Unknown }; @@ -79,9 +80,10 @@ struct Error { ErrorCode code{ErrorCode::Unknown}; std::string message; /// HTTP status, or 0. For an ArcGIS error (sent with HTTP 200) this is the - /// ArcGIS code when it lies in 100..599. + /// ArcGIS code when it lies in 100..599, and 0 otherwise. int http_status{0}; /// Extra context, such as ArcGIS error details or "arcgisCode=1000". + /// "arcgisCode=0" means the error carried no usable code. std::string detail; [[nodiscard]] constexpr bool is_ok() const noexcept { return code == ErrorCode::Ok; } @@ -128,7 +130,7 @@ struct Error { /// An Error for an ArcGIS `{"error":{"code":...}}` body, which the /// MapServer sends with HTTP 200. Code 404 (a retired service) is - /// `NotFound`. + /// `NotFound`. Pass 0 when the error has no usable code. [[nodiscard]] static Error from_arcgis(int arcgis_code, const std::string& body); }; diff --git a/include/spc/http_client.hpp b/include/spc/http_client.hpp index 45ce19a..a3e891d 100644 --- a/include/spc/http_client.hpp +++ b/include/spc/http_client.hpp @@ -39,13 +39,16 @@ struct ClientConfig { /// Largest decoded body accepted from the final response; 0 means no /// limit. Bodies are held in memory and parsed whole, and an IEM date /// range can be arbitrarily large. A bigger response fails with - /// `ErrorCode::InvalidRequest` and is not retried. + /// `ErrorCode::InvalidRequest` and is not retried. `ArcGISClient` applies + /// the same limit to the pages of one call together. std::size_t max_response_bytes{64UL * 1024UL * 1024UL}; }; /// The GET interface the clients use. Implement it to supply your own /// network stack or canned test responses. One transport may be shared by /// several clients and threads, so `get` must be safe to call concurrently. +/// Report failures as an `Error`: a std::exception thrown from `get` reaches +/// the client's caller as `ErrorCode::ParseError`, and any other propagates. class HttpTransport { public: virtual ~HttpTransport() = default; diff --git a/include/spc/pagination.hpp b/include/spc/pagination.hpp index bf50187..8fb1413 100644 --- a/include/spc/pagination.hpp +++ b/include/spc/pagination.hpp @@ -13,6 +13,9 @@ namespace spc { /// `exceededTransferLimit` when more remain. The server may also return fewer /// records than `resultRecordCount`, so `advance()` moves the offset by the /// number actually returned. SPC layers rarely need a second page. +/// +/// `max_pages` bounds the pages, not the memory they fill: `ArcGISClient` +/// separately caps the bytes and projected points of a whole call. class ArcGISPager { public: /// `page_size` is the `resultRecordCount` sent with each request. diff --git a/src/api/call_budget.hpp b/src/api/call_budget.hpp new file mode 100644 index 0000000..2004531 --- /dev/null +++ b/src/api/call_budget.hpp @@ -0,0 +1,53 @@ +/// @file call_budget.hpp +/// @brief Private: what one ArcGIS client call may take from the server. + +#pragma once + +#include "models/json.hpp" +#include "spc/http_client.hpp" + +#include + +namespace spc::detail { + +/// The bytes one call's pages may total: the response limit of an HttpClient, +/// so paging cannot multiply it. Another transport has no limit the client +/// can read, so it gets the default one. +[[nodiscard]] inline std::size_t call_byte_limit(const HttpTransport& transport) { + const HttpClient* client = dynamic_cast(&transport); + return client != nullptr ? client->config().max_response_bytes + : ClientConfig{}.max_response_bytes; +} + +/// What one client call may take from the server, across every page and layer +/// it reads. A response is bounded on its own, by the transport's size limit +/// and the parser's projection budget; charging every page to one budget keeps +/// a server that pages from multiplying those bounds. +class CallBudget { +public: + /// `max_bytes` caps the bodies of all pages together; 0 means no limit. + explicit CallBudget(std::size_t max_bytes) noexcept : max_bytes_(max_bytes) {} + + /// Whether the call can take a page of `body_bytes`. Its first page always + /// fits: the size of a single response is the transport's to bound. + [[nodiscard]] bool admits(std::size_t body_bytes) const noexcept { + return max_bytes_ == 0 || !charged_ || + (bytes_ <= max_bytes_ && body_bytes <= max_bytes_ - bytes_); + } + + /// Count an accepted page, the first included. + void charge(std::size_t body_bytes) noexcept { + bytes_ += body_bytes; + charged_ = true; + } + + [[nodiscard]] ProjectionBudget& projection() noexcept { return projection_; } + +private: + std::size_t max_bytes_; + std::size_t bytes_{0}; + bool charged_{false}; + ProjectionBudget projection_; +}; + +} // namespace spc::detail diff --git a/src/api/client.cpp b/src/api/client.cpp index e1e3d5e..309d5eb 100644 --- a/src/api/client.cpp +++ b/src/api/client.cpp @@ -1,6 +1,7 @@ /// @file client.cpp /// @brief StaticFeedClient, ArcGISClient, and ArchiveClient. +#include "api/call_budget.hpp" #include "models/from_tree.hpp" #include "models/json.hpp" #include "spc/api.hpp" @@ -10,11 +11,13 @@ #include #include +#include #include #include #include #include #include +#include #include #include #include @@ -26,6 +29,7 @@ namespace spc { namespace { +using detail::CallBudget; using detail::Json; constexpr std::string_view kStaticOutlookBase = "https://www.spc.noaa.gov/products/outlook/"; @@ -186,13 +190,30 @@ Result body_or_error(Result response, Feed404 semanti return std::unexpected(Error::from_response(response->status_code, response->body, semantics)); } -/// Run `parse` and turn its std::runtime_error into a ParseError. +/// The Error for the std::exception being handled; call it only inside a +/// catch block for one. ParseError, as for a parser's std::runtime_error: the +/// response could not be turned into a result. Other exceptions propagate: +/// catching everything would also swallow a thread cancellation. +Error current_failure() noexcept { + try { + try { + throw; + } catch (const std::exception& e) { + return Error::parse(e.what()); + } + } catch (const std::exception&) { + // No memory for the message either; an empty string needs none. + return {ErrorCode::ParseError, {}, 0, {}}; + } +} + +/// Run `parse` and turn the std::exception it throws into a ParseError. template Result> parsed(const Parse& parse) { try { return parse(); - } catch (const std::exception& e) { - return std::unexpected(Error::parse(e.what())); + } catch (const std::exception&) { + return std::unexpected(current_failure()); } } @@ -236,10 +257,12 @@ bool envelope_flag(const Json& root, const char* key) { Result inspect_arcgis_envelope(const Json& root, const std::string& body) { const Json* error = detail::lookup(root, "error"); if (error != nullptr && error->is_object()) { - // ArcGIS reports failures with HTTP 200 and an error object. + // ArcGIS reports failures with HTTP 200 and an error object. The code + // is passed on as sent; a missing or non-numeric one is 0, not a + // made-up status. const std::int32_t code = detail::to_int32(detail::json_number_or_numeric_string(*error, "code")).value_or(0); - return std::unexpected(Error::from_arcgis(code > 0 ? code : 400, body)); + return std::unexpected(Error::from_arcgis(code, body)); } ArcGISEnvelope envelope; // f=json puts the flag at the root; f=geojson puts it at the root and @@ -276,6 +299,12 @@ Result parse_arcgis_page(const std::string& body) { return ArcGISPage{std::move(*root), *envelope}; } +/// One accepted page: its body and what was parsed from it. +struct FetchedPage { + std::string body; + ArcGISPage page; +}; + bool is_json_format(std::string_view format) { return format == "json" || format == "pjson" || format == "geojson"; } @@ -346,7 +375,7 @@ StaticFeedClient::~StaticFeedClient() = default; StaticFeedClient::StaticFeedClient(StaticFeedClient&&) noexcept = default; StaticFeedClient& StaticFeedClient::operator=(StaticFeedClient&&) noexcept = default; -Result StaticFeedClient::day_categorical(std::int32_t day) const { +Result StaticFeedClient::day_categorical(std::int32_t day) const try { if (day < 1 || day > 3) { return std::unexpected( Error::invalid_request("categorical outlook day must be 1, 2, or 3")); @@ -354,10 +383,12 @@ Result StaticFeedClient::day_categorical(std::int32_t return built_from( impl_->fetch(std::format("{}day{}otlk_cat.nolyr.geojson", kStaticOutlookBase, day)), [day](const Json& root) { return detail::categorical_from_tree(root, day); }); +} catch (const std::exception&) { + return std::unexpected(current_failure()); } Result StaticFeedClient::day_probabilistic(std::int32_t day, - std::string_view hazard) const { + std::string_view hazard) const try { const std::string_view normalized = normalized_hazard(day, hazard); if (day1_3_probability_layer(day, normalized) == nullptr) { return std::unexpected(unsupported_probability()); @@ -370,15 +401,19 @@ Result StaticFeedClient::day_probabilistic(std::int32_t day, return built_from(impl_->fetch(url), [day, normalized](const Json& root) { return detail::probabilistic_from_tree(root, day, normalized); }); +} catch (const std::exception&) { + return std::unexpected(current_failure()); } -Result StaticFeedClient::day4_8(std::int32_t day) const { +Result StaticFeedClient::day4_8(std::int32_t day) const try { if (day < 4 || day > 8) { return std::unexpected( Error::invalid_request("extended outlook day must be between 4 and 8")); } return built_from(impl_->fetch(std::format("{}day{}prob.nolyr.geojson", kStaticDay48Base, day)), [day](const Json& root) { return detail::day4_8_from_tree(root, day); }); +} catch (const std::exception&) { + return std::unexpected(current_failure()); } // ===================== ArcGISClient ===================== @@ -386,36 +421,77 @@ Result StaticFeedClient::day4_8(std::int32_t day) const { struct ArcGISClient::Impl { std::shared_ptr http; RetryPolicy retry; + /// Total page bytes one call accepts. + std::size_t max_call_bytes; explicit Impl(std::shared_ptr transport) - : http(usable_transport(std::move(transport))) {} + : http(usable_transport(std::move(transport))), + max_call_bytes(detail::call_byte_limit(*http)) {} + + [[nodiscard]] CallBudget call_budget() const noexcept { return CallBudget{max_call_bytes}; } + + /// GET one page and parse it, retrying under `retry`. ArcGIS sends its own + /// failures with HTTP 200, so the body is parsed inside the retry loop and + /// a transient ArcGIS code is retried like the same HTTP status. A 404 is + /// NotFound: ArcGIS reports an empty product as HTTP 200 with no features. + [[nodiscard]] Result fetch_page(const std::string& url, + const CallBudget& budget) const { + // What the latest attempt's HTTP 200 body parsed to. + std::optional> page; + Result response = with_retry( + [&]() -> Result { + page.reset(); + Result attempt = http->get(url); + if (!attempt || attempt->status_code != 200) { + return attempt; + } + if (!budget.admits(attempt->body.size())) { + // Refused before parsing, which would hold several times the body. + return std::unexpected(Error::invalid_request( + "ArcGIS query pages exceeded ClientConfig::max_response_bytes in total")); + } + page.emplace(parse_arcgis_page(attempt->body)); + if (!page->has_value() && page->error().http_status != 0) { + // An ArcGIS error whose code is an HTTP status: let + // with_retry judge that status. + attempt->status_code = static_cast(page->error().http_status); + } + return attempt; + }, + retry); + if (!response) { + return std::unexpected(std::move(response.error())); + } + if (!page) { + return std::unexpected( + Error::from_response(response->status_code, response->body, Feed404::NotFound)); + } + if (!page->has_value()) { + return std::unexpected(std::move(page->error())); + } + return FetchedPage{std::move(response->body), std::move(**page)}; + } - /// Fetch every page of `query`, parse each once, and hand `on_page` the - /// page's body and tree as it arrives. `on_page` may take the body. A 404 - /// is NotFound: ArcGIS reports an empty product as HTTP 200 with no - /// features. + /// Fetch every page of `query`, charge each to `budget`, and hand `on_page` + /// the page's body and tree as it arrives. `on_page` may take the body. template requires std::is_invocable_r_v, const OnPage&, std::string&, const Json&> - [[nodiscard]] Result paged(const LayerQuery& query, const OnPage& on_page) const { + [[nodiscard]] Result paged(const LayerQuery& query, CallBudget& budget, + const OnPage& on_page) const { ArcGISPager pager; while (pager.has_more()) { - const std::string url = page_url(query, pager); - Result body = - body_or_error(with_retry([&] { return http->get(url); }, retry), Feed404::NotFound); - if (!body) { - return std::unexpected(body.error()); + Result fetched = fetch_page(page_url(query, pager), budget); + if (!fetched) { + return std::unexpected(std::move(fetched.error())); } - const Result page = parse_arcgis_page(*body); - if (!page) { - return std::unexpected(page.error()); - } - const ArcGISEnvelope envelope = page->envelope; + budget.charge(fetched->body.size()); + const ArcGISEnvelope envelope = fetched->page.envelope; if (envelope.exceeded_transfer_limit && envelope.feature_count == 0) { // The offset would never move. return std::unexpected( Error::server("ArcGIS reported a truncated page containing no records")); } - Result accepted = on_page(*body, page->root); + Result accepted = on_page(fetched->body, fetched->page.root); if (!accepted) { return accepted; } @@ -432,25 +508,26 @@ struct ArcGISClient::Impl { /// `build`, and move the page's `items` into `payload`. Each page's body is /// freed before its payload is built, so the two are never alive together. template - [[nodiscard]] Result paged_into(const LayerQuery& query, Payload payload, - std::vector Payload::*items, + [[nodiscard]] Result paged_into(const LayerQuery& query, CallBudget& budget, + Payload payload, std::vector Payload::*items, const Build& build) const { - Result done = paged(query, [&](std::string& body, const Json& root) -> Result { - release(body); - Result page = parsed([&] { return build(root); }); - if (!page) { - return std::unexpected(std::move(page.error())); - } - std::vector& target = payload.*items; - std::vector& source = (*page).*items; - if (target.empty()) { - target = std::move(source); - } else { - target.insert(target.end(), std::make_move_iterator(source.begin()), - std::make_move_iterator(source.end())); - } - return {}; - }); + Result done = + paged(query, budget, [&](std::string& body, const Json& root) -> Result { + release(body); + Result page = parsed([&] { return build(root); }); + if (!page) { + return std::unexpected(std::move(page.error())); + } + std::vector& target = payload.*items; + std::vector& source = (*page).*items; + if (target.empty()) { + target = std::move(source); + } else { + target.insert(target.end(), std::make_move_iterator(source.begin()), + std::make_move_iterator(source.end())); + } + return {}; + }); if (!done) { return std::unexpected(std::move(done.error())); } @@ -466,7 +543,7 @@ ArcGISClient::~ArcGISClient() = default; ArcGISClient::ArcGISClient(ArcGISClient&&) noexcept = default; ArcGISClient& ArcGISClient::operator=(ArcGISClient&&) noexcept = default; -Result ArcGISClient::query_categorical(std::int32_t day) const { +Result ArcGISClient::query_categorical(std::int32_t day) const try { const LayerDescriptor* descriptor = find_layer(LayerProduct::Categorical, day, ""); if (descriptor == nullptr) { return std::unexpected( @@ -474,15 +551,18 @@ Result ArcGISClient::query_categorical(std::int32_t d } CategoricalOutlookPayload seed; seed.day_offset = day; + CallBudget budget = impl_->call_budget(); // GeoJSON, the shape the spc-data categorical parser reads. return impl_->paged_into( - product_layer(kArcGisOutlooks, descriptor->id, "geojson"), std::move(seed), + product_layer(kArcGisOutlooks, descriptor->id, "geojson"), budget, std::move(seed), &CategoricalOutlookPayload::features, [day](const Json& root) { return detail::categorical_from_tree(root, day); }); +} catch (const std::exception&) { + return std::unexpected(current_failure()); } Result ArcGISClient::query_probabilistic(std::int32_t day, - std::string_view hazard) const { + std::string_view hazard) const try { const std::string_view normalized = normalized_hazard(day, hazard); const LayerDescriptor* descriptor = day1_3_probability_layer(day, normalized); if (descriptor == nullptr) { @@ -491,15 +571,18 @@ Result ArcGISClient::query_probabilistic(std::int32_t day, ProbOutlookPayload seed; seed.day_offset = day; seed.hazard = normalized; - return impl_->paged_into(product_layer(kArcGisOutlooks, descriptor->id, "geojson"), + CallBudget budget = impl_->call_budget(); + return impl_->paged_into(product_layer(kArcGisOutlooks, descriptor->id, "geojson"), budget, std::move(seed), &ProbOutlookPayload::features, [day, normalized](const Json& root) { return detail::probabilistic_from_tree(root, day, normalized); }); +} catch (const std::exception&) { + return std::unexpected(current_failure()); } Result -ArcGISClient::query_conditional_intensity(std::int32_t day, std::string_view hazard) const { +ArcGISClient::query_conditional_intensity(std::int32_t day, std::string_view hazard) const try { const std::string_view normalized = normalized_hazard(day, hazard); const LayerDescriptor* descriptor = find_layer(LayerProduct::ConditionalIntensity, day, normalized); @@ -511,14 +594,19 @@ ArcGISClient::query_conditional_intensity(std::int32_t day, std::string_view haz ConditionalIntensityPayload seed; seed.day = day; seed.hazard = normalized; - return impl_->paged_into(product_layer(kArcGisOutlooks, descriptor->id), std::move(seed), - &ConditionalIntensityPayload::features, - [day, hazard = std::string{normalized}](const Json& root) { - return detail::conditional_intensity_from_tree(root, day, hazard); + CallBudget budget = impl_->call_budget(); + return impl_->paged_into(product_layer(kArcGisOutlooks, descriptor->id), budget, + std::move(seed), &ConditionalIntensityPayload::features, + [day, hazard = std::string{normalized}, + projection = &budget.projection()](const Json& root) { + return detail::conditional_intensity_from_tree(root, day, hazard, + projection); }); +} catch (const std::exception&) { + return std::unexpected(current_failure()); } -Result ArcGISClient::query_day4_8(std::int32_t day) const { +Result ArcGISClient::query_day4_8(std::int32_t day) const try { const LayerDescriptor* descriptor = find_layer(LayerProduct::Probability, day, "severe"); if (descriptor == nullptr || day < 4) { return std::unexpected( @@ -526,13 +614,17 @@ Result ArcGISClient::query_day4_8(std::int32_t day) const { } Day48OutlookPayload seed; seed.day = day; - return impl_->paged_into(product_layer(kArcGisOutlooks, descriptor->id), std::move(seed), - &Day48OutlookPayload::features, [day](const Json& root) { - return detail::day4_8_from_tree(root, day); + CallBudget budget = impl_->call_budget(); + return impl_->paged_into(product_layer(kArcGisOutlooks, descriptor->id), budget, + std::move(seed), &Day48OutlookPayload::features, + [day, projection = &budget.projection()](const Json& root) { + return detail::day4_8_from_tree(root, day, projection); }); +} catch (const std::exception&) { + return std::unexpected(current_failure()); } -Result ArcGISClient::query_fire_weather(std::int32_t day) const { +Result ArcGISClient::query_fire_weather(std::int32_t day) const try { const std::vector layers = fire_weather_layers(day); if (layers.empty()) { return std::unexpected( @@ -540,33 +632,42 @@ Result ArcGISClient::query_fire_weather(std::int32_t day) co } FireWeatherPayload payload; payload.day = day; + // One budget for both layers: together they are the payload. + CallBudget budget = impl_->call_budget(); for (const LayerDescriptor* descriptor : layers) { const FireWeatherLayer layer = fire_weather_layer(descriptor->subtype); LayerQuery query = product_layer(kArcGisFireWeather, descriptor->id); // These layers default to Web Mercator; ask for lon/lat. query.out_spatial_reference = "4326"; - Result merged = - impl_->paged_into(query, std::move(payload), &FireWeatherPayload::features, - [day, layer](const Json& root) { - return detail::fire_weather_from_tree(root, day, layer); - }); + Result merged = impl_->paged_into( + query, budget, std::move(payload), &FireWeatherPayload::features, + [day, layer, projection = &budget.projection()](const Json& root) { + return detail::fire_weather_from_tree(root, day, layer, projection); + }); if (!merged) { return merged; } payload = std::move(*merged); } return payload; +} catch (const std::exception&) { + return std::unexpected(current_failure()); } -Result ArcGISClient::query_active_md() const { - return impl_->paged_into(product_layer(kArcGisMesoscale, 0), MesoscalePayload{}, +Result ArcGISClient::query_active_md() const try { + CallBudget budget = impl_->call_budget(); + return impl_->paged_into(product_layer(kArcGisMesoscale, 0), budget, MesoscalePayload{}, &MesoscalePayload::discussions, - [](const Json& root) { return detail::mesoscale_from_tree(root); }); + [projection = &budget.projection()](const Json& root) { + return detail::mesoscale_from_tree(root, projection); + }); +} catch (const std::exception&) { + return std::unexpected(current_failure()); } Result> ArcGISClient::query_layer(ArcGISService service, std::int32_t layer_id, - const QueryParams& params) const { + const QueryParams& params) const try { if (layer_id < 0) { return std::unexpected(Error::invalid_request("ArcGIS layer id must be non-negative")); } @@ -585,9 +686,11 @@ Result> ArcGISClient::query_layer(ArcGISService service base = kArcGisMesoscale; break; } + // The budget's byte limit bounds what `pages` retains. + CallBudget budget = impl_->call_budget(); std::vector pages; Result done = - impl_->paged(LayerQuery{base, layer_id, params, {}}, + impl_->paged(LayerQuery{base, layer_id, params, {}}, budget, [&pages](std::string& body, const Json& /*root*/) -> Result { pages.push_back(std::move(body)); return {}; @@ -596,6 +699,8 @@ Result> ArcGISClient::query_layer(ArcGISService service return std::unexpected(std::move(done.error())); } return pages; +} catch (const std::exception&) { + return std::unexpected(current_failure()); } // ===================== ArchiveClient ===================== @@ -652,20 +757,22 @@ ArchiveClient::~ArchiveClient() = default; ArchiveClient::ArchiveClient(ArchiveClient&&) noexcept = default; ArchiveClient& ArchiveClient::operator=(ArchiveClient&&) noexcept = default; -Result ArchiveClient::watches(std::string_view timestamp) const { +Result ArchiveClient::watches(std::string_view timestamp) const try { std::string url = std::format("{}json/spcwatch.py", kIemBase); if (!timestamp.empty()) { url += "?ts=" + percent_encode(timestamp); } return built_from(impl_->fetch(url), [](const Json& root) { return detail::watches_from_tree(root); }); +} catch (const std::exception&) { + return std::unexpected(current_failure()); } // A start/end pair cannot be told apart by type. // NOLINTBEGIN(bugprone-easily-swappable-parameters) Result ArchiveClient::storm_reports(std::string_view start_iso, std::string_view end_iso, - std::string_view wfo) const { + std::string_view wfo) const try { // NOLINTEND(bugprone-easily-swappable-parameters) // Encode every value: an ISO offset's '+' would otherwise decode as a space. std::string url = std::format("{}geojson/lsr.geojson?sts={}&ets={}", kIemBase, @@ -676,6 +783,8 @@ Result ArchiveClient::storm_reports(std::string_view start_i } return built_from(impl_->fetch(url), [](const Json& root) { return detail::storm_reports_from_tree(root); }); +} catch (const std::exception&) { + return std::unexpected(current_failure()); } } // namespace spc diff --git a/src/models/convective.cpp b/src/models/convective.cpp index a77c291..4a16504 100644 --- a/src/models/convective.cpp +++ b/src/models/convective.cpp @@ -26,14 +26,15 @@ std::uint8_t cig_severity_from_label(std::string_view label) noexcept { return 0; } -Day48OutlookPayload detail::day4_8_from_tree(const Json& root, std::int32_t day) { +Day48OutlookPayload detail::day4_8_from_tree(const Json& root, std::int32_t day, + detail::ProjectionBudget* budget) { Day48OutlookPayload payload; payload.day = day; const Json* features_node = detail::lookup(root, "features"); if (features_node == nullptr || !features_node->is_array()) { return payload; } - detail::PayloadGeometryParser geometry_parser(root); + detail::PayloadGeometryParser geometry_parser(root, budget); for (const Json& feat : features_node->get_array()) { const Json* props = detail::feature_fields(feat); const Json* geometry = detail::lookup(feat, "geometry"); @@ -60,7 +61,8 @@ Day48OutlookPayload parse_day4_8(std::string_view body, std::int32_t day) { } ConditionalIntensityPayload -detail::conditional_intensity_from_tree(const Json& root, std::int32_t day, std::string hazard) { +detail::conditional_intensity_from_tree(const Json& root, std::int32_t day, std::string hazard, + detail::ProjectionBudget* budget) { ConditionalIntensityPayload payload; payload.day = day; payload.hazard = std::move(hazard); @@ -68,7 +70,7 @@ detail::conditional_intensity_from_tree(const Json& root, std::int32_t day, std: if (features_node == nullptr || !features_node->is_array()) { return payload; } - detail::PayloadGeometryParser geometry_parser(root); + detail::PayloadGeometryParser geometry_parser(root, budget); for (const Json& feat : features_node->get_array()) { const Json* props = detail::feature_fields(feat); const Json* geometry = detail::lookup(feat, "geometry"); diff --git a/src/models/fire_weather.cpp b/src/models/fire_weather.cpp index d4c6214..769ae56 100644 --- a/src/models/fire_weather.cpp +++ b/src/models/fire_weather.cpp @@ -74,14 +74,15 @@ std::uint8_t fire_severity_from_label(std::string_view label) noexcept { } FireWeatherPayload detail::fire_weather_from_tree(const Json& root, std::int32_t day, - FireWeatherLayer layer) { + FireWeatherLayer layer, + detail::ProjectionBudget* budget) { FireWeatherPayload payload; payload.day = day; const Json* features_node = detail::lookup(root, "features"); if (features_node == nullptr || !features_node->is_array()) { return payload; } - detail::PayloadGeometryParser geometry_parser(root); + detail::PayloadGeometryParser geometry_parser(root, budget); for (const Json& feat : features_node->get_array()) { const Json* props = detail::feature_fields(feat); const Json* geometry = detail::lookup(feat, "geometry"); diff --git a/src/models/from_tree.hpp b/src/models/from_tree.hpp index 5978774..bddfb21 100644 --- a/src/models/from_tree.hpp +++ b/src/models/from_tree.hpp @@ -5,6 +5,9 @@ /// client that parsed a response once builds its payload from that tree. The /// names are distinct because glz::generic converts implicitly from /// std::string, which would make overloads of `parse_*` ambiguous. +/// +/// The parsers that project Esri rings take the payload's `ProjectionBudget` +/// when `root` is only part of it, such as one page of a paged query. #pragma once @@ -27,15 +30,18 @@ CategoricalOutlookPayload categorical_from_tree(const Json& root, std::int32_t d ProbOutlookPayload probabilistic_from_tree(const Json& root, std::int32_t day_offset, std::string_view hazard); -Day48OutlookPayload day4_8_from_tree(const Json& root, std::int32_t day); +Day48OutlookPayload day4_8_from_tree(const Json& root, std::int32_t day, + ProjectionBudget* budget = nullptr); ConditionalIntensityPayload conditional_intensity_from_tree(const Json& root, std::int32_t day, - std::string hazard); + std::string hazard, + ProjectionBudget* budget = nullptr); FireWeatherPayload fire_weather_from_tree(const Json& root, std::int32_t day, - FireWeatherLayer layer); + FireWeatherLayer layer, + ProjectionBudget* budget = nullptr); -MesoscalePayload mesoscale_from_tree(const Json& root); +MesoscalePayload mesoscale_from_tree(const Json& root, ProjectionBudget* budget = nullptr); StormReportPayload storm_reports_from_tree(const Json& root); diff --git a/src/models/json.cpp b/src/models/json.cpp index 1105995..8363b19 100644 --- a/src/models/json.cpp +++ b/src/models/json.cpp @@ -20,7 +20,6 @@ struct JsonReadOptions : glz::opts { constexpr double kMercatorRadius = 6378137.0; constexpr double kDegrees = 180.0 / std::numbers::pi; -constexpr std::size_t kMaxAddedProjectionPoints = 1000000; void validate_mercator(const Polygon& ring) { for (const LonLat point : ring) { @@ -300,8 +299,8 @@ std::vector parse_esri_rings_with_budget(const Json& geom, const Json* } // namespace std::vector parse_esri_rings(const Json& geom, const Json* reference) { - std::size_t remaining = kMaxAddedProjectionPoints; - return parse_esri_rings_with_budget(geom, reference, remaining); + ProjectionBudget budget; + return parse_esri_rings_with_budget(geom, reference, budget.remaining_added_points); } glz::expected parse_root(std::string_view body) { @@ -357,13 +356,12 @@ const Json* feature_fields(const Json& feature) { return fields != nullptr && fields->is_object() ? fields : nullptr; } -PayloadGeometryParser::PayloadGeometryParser(const Json& root) - : reference_(lookup(root, "spatialReference")), - remaining_added_points_(kMaxAddedProjectionPoints) {} +PayloadGeometryParser::PayloadGeometryParser(const Json& root, ProjectionBudget* shared) + : reference_(lookup(root, "spatialReference")), budget_(shared != nullptr ? shared : &own_) {} std::vector PayloadGeometryParser::parse(const Json& geometry) { return lookup(geometry, "rings") != nullptr - ? parse_esri_rings_with_budget(geometry, reference_, remaining_added_points_) + ? parse_esri_rings_with_budget(geometry, reference_, budget_->remaining_added_points) : parse_rings(geometry); } diff --git a/src/models/json.hpp b/src/models/json.hpp index 19e6341..860e45b 100644 --- a/src/models/json.hpp +++ b/src/models/json.hpp @@ -63,8 +63,8 @@ std::vector parse_rings(const Json& geom); /// rings (clockwise) and holes (counter-clockwise) in one list; the holes are /// dropped so the result matches `parse_rings` for the same layer. Web Mercator /// is converted to degrees with at most one metre of boundary error. A geometry -/// needing over one million added points fails. A geometry reference overrides -/// the response reference. +/// needing more added points than its payload's `ProjectionBudget` has left +/// fails. A geometry reference overrides the response reference. /// Malformed points fail for every supported spatial reference. std::vector parse_esri_rings(const Json& geom, const Json* reference = nullptr); @@ -88,11 +88,21 @@ Json parse_root_or_throw(std::string_view body); /// when they are missing or not an object. const Json* feature_fields(const Json& feature); -/// One per payload: shares the collection reference and projection budget -/// across all features, accepting Esri rings or GeoJSON coordinates. +/// The points Web Mercator projection may still add to one payload: one +/// million, 16 MB, at the start. An edge is subdivided every 10 km, so 20 +/// bytes of JSON can add 4,000 points. +struct ProjectionBudget { + std::size_t remaining_added_points{1000000}; +}; + +/// One per response: shares the collection reference and projection budget +/// across all features, accepting Esri rings or GeoJSON coordinates. The +/// budget is the payload's. A caller that builds one payload from several +/// responses passes the same `shared` budget for each; otherwise the response +/// is the payload and the parser uses its own. class PayloadGeometryParser { public: - explicit PayloadGeometryParser(const Json& root); + explicit PayloadGeometryParser(const Json& root, ProjectionBudget* shared = nullptr); ~PayloadGeometryParser() = default; PayloadGeometryParser(const PayloadGeometryParser&) = delete; PayloadGeometryParser& operator=(const PayloadGeometryParser&) = delete; @@ -102,7 +112,8 @@ class PayloadGeometryParser { private: const Json* reference_; - std::size_t remaining_added_points_; + ProjectionBudget own_; + ProjectionBudget* budget_; }; /// The first non-empty string among `keys`, e.g. {"LABEL", "label"}. diff --git a/src/models/mesoscale.cpp b/src/models/mesoscale.cpp index 49801e3..c6c6538 100644 --- a/src/models/mesoscale.cpp +++ b/src/models/mesoscale.cpp @@ -12,13 +12,13 @@ namespace spc { using detail::Json; -MesoscalePayload detail::mesoscale_from_tree(const Json& root) { +MesoscalePayload detail::mesoscale_from_tree(const Json& root, detail::ProjectionBudget* budget) { MesoscalePayload payload; const Json* features_node = detail::lookup(root, "features"); if (features_node == nullptr || !features_node->is_array()) { return payload; } - detail::PayloadGeometryParser geometry_parser(root); + detail::PayloadGeometryParser geometry_parser(root, budget); for (const Json& feat : features_node->get_array()) { const Json* props = detail::feature_fields(feat); if (props == nullptr) { diff --git a/tests/test_client.cpp b/tests/test_client.cpp index f481afe..848f056 100644 --- a/tests/test_client.cpp +++ b/tests/test_client.cpp @@ -2,17 +2,24 @@ /// @brief Client routing, paging, and error mapping, through a recording /// transport. No network access. +#include "api/call_budget.hpp" #include "spc/api.hpp" #include "spc/pagination.hpp" +#include "spc/retry.hpp" #include "spc/version.hpp" #include "support/fixtures.hpp" +#include #include +#include #include #include +#include #include +#include #include #include +#include #include #include @@ -63,6 +70,92 @@ class AlwaysTruncatingTransport final : public HttpTransport { } }; +/// Answers every request with a page padded to about `page_bytes`, truncated +/// or final, and counts what it served. Pages are built on demand so the +/// transport holds none of them. +class PaddedPagesTransport final : public HttpTransport { +public: + PaddedPagesTransport(std::size_t page_bytes, bool truncated) + : page_bytes_(page_bytes), truncated_(truncated) {} + + mutable std::int32_t calls = 0; + mutable std::size_t bytes_served = 0; + + [[nodiscard]] Result get(std::string_view /*path*/) const override { + HttpResponse response{200, {}, {}}; + response.body.reserve(page_bytes_ + 160); + response.body += R"({"type":"FeatureCollection","features":[)" + R"({"type":"Feature","properties":{},"geometry":null}],"pad":")"; + response.body.append(page_bytes_, 'x'); + response.body += std::format(R"(","exceededTransferLimit":{}}})", truncated_); + ++calls; + bytes_served += response.body.size(); + return response; + } + +private: + std::size_t page_bytes_; + bool truncated_; +}; + +/// Fails every request the way an allocation failure inside a call does. +class OutOfMemoryTransport final : public HttpTransport { +public: + [[nodiscard]] Result get(std::string_view /*path*/) const override { + throw std::bad_alloc{}; + } +}; + +/// Serves one truncated page, then throws something that is not a +/// std::exception, as a custom transport might. +class ThrowsAfterAPageTransport final : public HttpTransport { +public: + mutable std::int32_t calls = 0; + + [[nodiscard]] Result get(std::string_view /*path*/) const override { + if (calls++ == 0) { + return HttpResponse{200, truncated_page(1), {}}; + } + throw 42; + } +}; + +/// The code `call` failed with, `Ok` if it succeeded, or a test failure if it +/// threw. +template +ErrorCode outcome_of(const Call& call) { + try { + const std::invoke_result_t result = call(); + return result ? ErrorCode::Ok : result.error().code; + } catch (const std::exception& e) { + ADD_FAILURE() << "client call threw " << e.what(); + } catch (...) { + ADD_FAILURE() << "client call threw a non-std exception"; + } + return ErrorCode::Unknown; +} + +/// An Esri page of `count` Web Mercator features. Each ring has one diagonal +/// edge that projection subdivides into 3,800 steps, so a feature costs 3,799 +/// of the one million added points a payload may hold. +std::string mercator_page(std::size_t count, bool truncated) { + std::string features; + for (std::size_t i = 0; i < count; ++i) { + if (i != 0) { + features += ','; + } + features += + R"({"attributes":{"dn":15,"LABEL":"15","name":"Risk"},"geometry":{"rings":[[[-19000000,-19000000],[-19000000,19000000],[19000000,19000000],[-19000000,-19000000]]]}})"; + } + return std::format( + R"({{"spatialReference":{{"wkid":3857}},"features":[{}],"exceededTransferLimit":{}}})", + features, truncated); +} + +std::string arcgis_error_body(std::int32_t code) { + return std::format(R"({{"error":{{"code":{},"message":"Busy","details":[]}}}})", code); +} + TEST(StaticFeedClientRouting, UsesPublishedProbabilisticFilenames) { std::shared_ptr transport = std::make_shared(); transport->responses.assign(3, empty_feature_collection()); @@ -398,6 +491,8 @@ TEST(ArcGISClientPaging, ReturnsLogicalArcGISErrorsReportedWithHttp200) { EXPECT_EQ(result.error().code, ErrorCode::InvalidRequest); EXPECT_EQ(result.error().http_status, 400); EXPECT_EQ(result.error().message, "Invalid or missing input parameters."); + // Not transient, so it is asked once. + EXPECT_EQ(transport->requests.size(), 1u); } TEST(ArcGISClientPaging, KeepsTheArcGISErrorDetails) { @@ -419,15 +514,11 @@ TEST(ArcGISClientPaging, KeepsTheArcGISErrorDetails) { TEST(ArcGISClientPaging, ClassifiesArcGIS5xxCodesLikeHttpStatuses) { // HTTP 503 is RateLimited and other 5xx are ServerError, so the same codes - // in an ArcGIS error object mean the same. + // in an ArcGIS error object mean the same, and are retried the same. for (const auto& [code, expected] : {std::pair{503, ErrorCode::RateLimited}, std::pair{500, ErrorCode::ServerError}}) { std::shared_ptr transport = std::make_shared(); - transport->responses = { - {200, - std::format(R"({{"error":{{"code":{},"message":"Busy","details":[]}}}})", code), - {}}, - }; + transport->responses.assign(RetryPolicy{}.max_attempts, {200, arcgis_error_body(code), {}}); const ArcGISClient client{transport}; const Result result = client.query_categorical(1); @@ -435,9 +526,282 @@ TEST(ArcGISClientPaging, ClassifiesArcGIS5xxCodesLikeHttpStatuses) { ASSERT_FALSE(result) << code; EXPECT_EQ(result.error().code, expected) << code; EXPECT_EQ(result.error().http_status, code); + EXPECT_EQ(result.error().message, "Busy"); + // The ArcGIS error itself, not one rebuilt from the status. + EXPECT_EQ(result.error().detail, std::format("arcgisCode={}", code)); + EXPECT_EQ(transport->requests.size(), RetryPolicy{}.max_attempts) << code; + } +} + +TEST(ArcGISClientPaging, RetriesTransientArcGISErrorsSentWithHttp200) { + for (const std::int32_t code : {429, 500, 503}) { + std::shared_ptr transport = std::make_shared(); + transport->responses = {{200, arcgis_error_body(code), {}}, empty_feature_collection()}; + const ArcGISClient client{transport}; + + const Result> result = + client.query_layer(ArcGISService::Outlooks, 1, {}); + + ASSERT_TRUE(result) << code << ": " << result.error().message; + // The error body is not a page. + ASSERT_EQ(result->size(), 1u) << code; + EXPECT_EQ((*result)[0], empty_feature_collection().body); + EXPECT_EQ(transport->requests.size(), 2u) << code; + } +} + +TEST(ArcGISClientPaging, RetriesTheFailingPageWithoutRestartingTheQuery) { + const std::string feature = + R"({"type":"Feature","properties":{"label":"TSTM"},)" + R"("geometry":{"type":"Polygon","coordinates":[[[0,0],[1,0],[1,1],[0,0]]]}})"; + std::shared_ptr transport = std::make_shared(); + transport->responses = { + {200, R"({"features":[)" + feature + R"(],"exceededTransferLimit":true})", {}}, + {200, arcgis_error_body(500), {}}, + {200, R"({"features":[)" + feature + R"(]})", {}}, + }; + const ArcGISClient client{transport}; + + const Result result = client.query_categorical(1); + + ASSERT_TRUE(result) << result.error().message; + EXPECT_EQ(result->features.size(), 2u); + ASSERT_EQ(transport->requests.size(), 3u); + EXPECT_NE(transport->requests[1].find("resultOffset=1"), std::string::npos); + EXPECT_EQ(transport->requests[2], transport->requests[1]); +} + +TEST(ArcGISClientPaging, ARetriedPageReportsItsLastAttempt) { + std::shared_ptr transport = std::make_shared(); + transport->responses = {{200, arcgis_error_body(503), {}}, {404, "gone", {}}}; + const ArcGISClient client{transport}; + + const Result> result = + client.query_layer(ArcGISService::Outlooks, 1, {}); + + ASSERT_FALSE(result); + EXPECT_EQ(result.error().code, ErrorCode::NotFound); + EXPECT_EQ(result.error().http_status, 404); + EXPECT_EQ(result.error().message, "gone"); + EXPECT_EQ(result.error().detail, ""); + EXPECT_EQ(transport->requests.size(), 2u); +} + +TEST(ArcGISClientPaging, OtherArcGISCodesAreAskedOnce) { + for (const std::int32_t code : {403, 404, 498, 1000}) { + std::shared_ptr transport = std::make_shared(); + transport->responses = {{200, arcgis_error_body(code), {}}, empty_feature_collection()}; + const ArcGISClient client{transport}; + + EXPECT_FALSE(client.query_layer(ArcGISService::Outlooks, 1, {})) << code; + EXPECT_EQ(transport->requests.size(), 1u) << code; + } +} + +TEST(ArcGISClientPaging, AnArcGISErrorWithoutACodeDoesNotInventOne) { + for (const std::string_view error : + {R"({"message":"Query failed."})", R"({"code":null,"message":"Query failed."})", + R"({"code":"abc","message":"Query failed."})"}) { + std::shared_ptr transport = std::make_shared(); + transport->responses = {{200, std::format(R"({{"error":{}}})", error), {}}}; + const ArcGISClient client{transport}; + + const Result> result = + client.query_layer(ArcGISService::Outlooks, 1, {}); + + ASSERT_FALSE(result) << error; + EXPECT_EQ(result.error().code, ErrorCode::InvalidRequest) << error; + EXPECT_EQ(result.error().http_status, 0) << error; + EXPECT_EQ(result.error().message, "Query failed.") << error; + EXPECT_EQ(result.error().detail.find("400"), std::string::npos) << result.error().detail; + EXPECT_EQ(transport->requests.size(), 1u) << error; + } +} + +// ===== Bounds on one call ===== +// +// A response is bounded on its own: its size by the transport, its projection +// growth by the parser. A call that pages must not multiply either bound by +// the number of pages. + +TEST(ArcGISClientLimits, ProjectionBudgetSpansEveryPageOfOneCall) { + // 300 features need 1,139,700 added points: over the limit on one page, so + // over it when split across two. + struct Case { + std::string_view name; + ErrorCode (*run)(const ArcGISClient&); + }; + const std::vector cases = { + {"query_day4_8", + [](const ArcGISClient& c) { return outcome_of([&c] { return c.query_day4_8(4); }); }}, + {"query_conditional_intensity", + [](const ArcGISClient& c) { + return outcome_of([&c] { return c.query_conditional_intensity(1, "tornado"); }); + }}, + {"query_active_md", + [](const ArcGISClient& c) { return outcome_of([&c] { return c.query_active_md(); }); }}, + }; + for (const Case& test_case : cases) { + std::shared_ptr one_page = std::make_shared(); + one_page->responses = {{200, mercator_page(300, false), {}}}; + EXPECT_EQ(test_case.run(ArcGISClient{one_page}), ErrorCode::ParseError) << test_case.name; + + std::shared_ptr two_pages = std::make_shared(); + two_pages->responses = {{200, mercator_page(150, true), {}}, + {200, mercator_page(150, false), {}}}; + EXPECT_EQ(test_case.run(ArcGISClient{two_pages}), ErrorCode::ParseError) << test_case.name; + EXPECT_EQ(two_pages->requests.size(), 2u) << test_case.name; } } +TEST(ArcGISClientLimits, ProjectionBudgetSpansBothFireWeatherLayers) { + std::shared_ptr transport = std::make_shared(); + transport->responses = {{200, mercator_page(150, false), {}}, + {200, mercator_page(150, false), {}}}; + const ArcGISClient client{transport}; + + const Result result = client.query_fire_weather(1); + + ASSERT_FALSE(result) << result->features.size() << " features"; + EXPECT_EQ(result.error().code, ErrorCode::ParseError); + EXPECT_NE(result.error().message.find("added-point limit"), std::string::npos); + EXPECT_EQ(transport->requests.size(), 2u); +} + +TEST(ArcGISClientLimits, PagesWithinTheProjectionBudgetStillMerge) { + // 200 features need 759,800 added points. + std::shared_ptr transport = std::make_shared(); + transport->responses = {{200, mercator_page(100, true), {}}, + {200, mercator_page(100, false), {}}}; + const ArcGISClient client{transport}; + + const Result result = client.query_day4_8(4); + + ASSERT_TRUE(result) << result.error().message; + EXPECT_EQ(result->features.size(), 200u); +} + +TEST(ArcGISClientLimits, EachCallStartsWithAFreshProjectionBudget) { + std::shared_ptr transport = std::make_shared(); + transport->responses.assign(2, {200, mercator_page(200, false), {}}); + const ArcGISClient client{transport}; + + EXPECT_TRUE(client.query_day4_8(4)); + EXPECT_TRUE(client.query_day4_8(4)); +} + +TEST(ArcGISClientLimits, RawLayerQueryBoundsTheBytesItRetains) { + // A custom transport enforces no response limit of its own. + constexpr std::size_t kPageBytes = 1024UL * 1024UL; + const std::size_t limit = ClientConfig{}.max_response_bytes; + std::shared_ptr transport = + std::make_shared(kPageBytes, true); + const ArcGISClient client{transport}; + + const Result> result = + client.query_layer(ArcGISService::Outlooks, 1, {}); + + ASSERT_FALSE(result); + EXPECT_EQ(result.error().code, ErrorCode::InvalidRequest) << result.error().message; + EXPECT_LT(transport->calls, ArcGISPager{}.max_pages()); + EXPECT_LE(transport->bytes_served, limit + 2 * kPageBytes); +} + +TEST(ArcGISClientLimits, TypedQueriesBoundTheBytesOneCallAccepts) { + constexpr std::size_t kPageBytes = 1024UL * 1024UL; + const std::size_t limit = ClientConfig{}.max_response_bytes; + std::shared_ptr transport = + std::make_shared(kPageBytes, true); + const ArcGISClient client{transport}; + + const Result result = client.query_categorical(1); + + ASSERT_FALSE(result); + EXPECT_EQ(result.error().code, ErrorCode::InvalidRequest) << result.error().message; + EXPECT_LT(transport->calls, ArcGISPager{}.max_pages()); + EXPECT_LE(transport->bytes_served, limit + 2 * kPageBytes); +} + +TEST(ArcGISClientLimits, TheByteLimitSpansBothFireWeatherLayers) { + // Each layer's one page is under the limit; together they pass it. + const std::size_t limit = ClientConfig{}.max_response_bytes; + std::shared_ptr transport = + std::make_shared(limit / 2 + 1, false); + const ArcGISClient client{transport}; + + const Result result = client.query_fire_weather(1); + + ASSERT_FALSE(result); + EXPECT_EQ(result.error().code, ErrorCode::InvalidRequest) << result.error().message; + EXPECT_EQ(transport->calls, 2); +} + +TEST(ArcGISClientLimits, AnOversizedFirstPageLeavesNoRoomForASecond) { + const std::size_t limit = ClientConfig{}.max_response_bytes; + std::shared_ptr transport = + std::make_shared(limit + 1, true); + const ArcGISClient client{transport}; + + const Result> result = + client.query_layer(ArcGISService::Outlooks, 1, {}); + + ASSERT_FALSE(result); + EXPECT_EQ(result.error().code, ErrorCode::InvalidRequest) << result.error().message; + EXPECT_EQ(transport->calls, 2); +} + +TEST(CallBudget, CapsTheBytesOfEveryPageAfterTheFirst) { + detail::CallBudget budget{10}; + EXPECT_TRUE(budget.admits(4)); + budget.charge(4); + // An exact fit is allowed; one byte more is not. + EXPECT_TRUE(budget.admits(6)); + EXPECT_FALSE(budget.admits(7)); + budget.charge(6); + EXPECT_TRUE(budget.admits(0)); + EXPECT_FALSE(budget.admits(1)); +} + +TEST(CallBudget, AnOversizedFirstPageIsCountedButNotRefused) { + detail::CallBudget budget{10}; + EXPECT_TRUE(budget.admits(11)); + budget.charge(11); + EXPECT_FALSE(budget.admits(0)); + EXPECT_FALSE(budget.admits(1)); +} + +TEST(CallBudget, ZeroMeansNoByteLimit) { + // As ClientConfig::max_response_bytes = 0 means for one response. + detail::CallBudget budget{0}; + budget.charge(std::numeric_limits::max() / 2); + EXPECT_TRUE(budget.admits(std::numeric_limits::max() / 2)); +} + +TEST(CallBudget, TakesItsByteLimitFromAnHttpClientTransport) { + ClientConfig config; + config.max_response_bytes = 4096; + EXPECT_EQ(detail::call_byte_limit(HttpClient{config}), 4096u); + config.max_response_bytes = 0; + EXPECT_EQ(detail::call_byte_limit(HttpClient{config}), 0u); + // Any other transport has no limit to read. + EXPECT_EQ(detail::call_byte_limit(RecordingTransport{}), ClientConfig{}.max_response_bytes); +} + +TEST(ArcGISClientLimits, ASingleResponseIsTheTransportsToBound) { + // The limit is on what paging adds: one response over it is accepted. + const std::size_t limit = ClientConfig{}.max_response_bytes; + std::shared_ptr transport = + std::make_shared(limit + 1, false); + const ArcGISClient client{transport}; + + const Result> result = + client.query_layer(ArcGISService::Outlooks, 1, {}); + + ASSERT_TRUE(result) << result.error().message; + ASSERT_EQ(result->size(), 1u); + EXPECT_EQ((*result)[0].size(), transport->bytes_served); +} + // ===== What a 404 means ===== // // SPC's static products answer 404 when nothing is issued. Any other 404 is a @@ -579,6 +943,43 @@ TEST(ClientErrors, AMalformedBodyIsAParseErrorNotAnException) { EXPECT_EQ(from_arcgis.error().code, ErrorCode::ParseError); } +TEST(ClientErrors, AnAllocationFailureIsAnErrorNotAnException) { + std::shared_ptr transport = std::make_shared(); + const ArcGISClient arcgis{transport}; + const StaticFeedClient feeds{transport}; + const ArchiveClient archive{transport}; + + // One code for every client: the parsers' own. + EXPECT_EQ(outcome_of([&] { return arcgis.query_layer(ArcGISService::Outlooks, 1, {}); }), + ErrorCode::ParseError); + EXPECT_EQ(outcome_of([&] { return arcgis.query_categorical(1); }), ErrorCode::ParseError); + EXPECT_EQ(outcome_of([&] { return arcgis.query_probabilistic(1, "tornado"); }), + ErrorCode::ParseError); + EXPECT_EQ(outcome_of([&] { return arcgis.query_conditional_intensity(1, "tornado"); }), + ErrorCode::ParseError); + EXPECT_EQ(outcome_of([&] { return arcgis.query_day4_8(4); }), ErrorCode::ParseError); + EXPECT_EQ(outcome_of([&] { return arcgis.query_fire_weather(1); }), ErrorCode::ParseError); + EXPECT_EQ(outcome_of([&] { return arcgis.query_active_md(); }), ErrorCode::ParseError); + EXPECT_EQ(outcome_of([&] { return feeds.day_categorical(1); }), ErrorCode::ParseError); + EXPECT_EQ(outcome_of([&] { return feeds.day_probabilistic(1, "tornado"); }), + ErrorCode::ParseError); + EXPECT_EQ(outcome_of([&] { return feeds.day4_8(4); }), ErrorCode::ParseError); + EXPECT_EQ(outcome_of([&] { return archive.watches(); }), ErrorCode::ParseError); + EXPECT_EQ(outcome_of([&] { return archive.storm_reports("2026-05-19T12:00Z", "2026-05-20"); }), + ErrorCode::ParseError); +} + +TEST(ClientErrors, AnExceptionOutsideStdFromATransportPropagates) { + // Only a custom transport can throw one, and catching everything would also + // swallow a thread cancellation's forced unwind. + std::shared_ptr transport = + std::make_shared(); + const ArcGISClient client{transport}; + + EXPECT_THROW((void)client.query_categorical(1), int); + EXPECT_EQ(transport->calls, 2); +} + TEST(ClientErrors, UnsupportedEsriProjectionBecomesParseError) { std::shared_ptr transport = std::make_shared(); transport->responses = { From b74c4bafc216efe31a299fffecbbe4c1ea46c83f Mon Sep 17 00:00:00 2001 From: KeviM Date: Fri, 20 Nov 2026 13:29:12 -0800 Subject: [PATCH 8/9] fix: treat one-slash URLs as absolute; bound error-body parsing From independent review. libcurl fetches https:/host as absolute HTTPS, but is_absolute_url wanted two slashes, so with a base_url the request went to the plaintext base. Error bodies over 64 KiB are no longer parsed into a JSON tree, and golden coverage now walks fixture subdirectories like the checksum tool does. --- CHANGELOG.md | 4 ++++ src/core/error.cpp | 9 +++++++-- src/http/client.cpp | 12 +++++++----- tests/test_client.cpp | 9 +++++++++ tests/test_golden.cpp | 34 +++++++++++++++++++++++----------- tests/test_http_client.cpp | 18 ++++++++++++++++++ 6 files changed, 68 insertions(+), 18 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d9d53d0..53dc7ce 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,8 @@ uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html). - An HTTPS request no longer follows a redirect to plain HTTP; it fails with `InvalidRequest`. +- A one-slash URL such as `https:/host/x`, which libcurl fetches as absolute, is + no longer appended to `ClientConfig::base_url` and sent to that host instead. - Bound memory across all pages of an ArcGIS call. One Web Mercator expansion budget covers the whole call, both fire-weather layers included, and the pages together may not exceed `ClientConfig::max_response_bytes` (`InvalidRequest`). @@ -38,6 +40,8 @@ uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html). - Initialize `HttpResponse::status_code` to 0. - A `std::exception` inside any client call, `std::bad_alloc` included, returns `ParseError` instead of escaping from ArcGIS calls. +- Error bodies over 64 KiB are not parsed for their message; the message is the + start of the body. - Skip features whose fields are not an object, conditional-intensity polygons without a label, and day 1-2 fire-weather polygons with no known `dn` band. - Read the numeric strings `"nan"` and `"inf"` as 0 instead of passing them on. diff --git a/src/core/error.cpp b/src/core/error.cpp index ded6c50..fd22c48 100644 --- a/src/core/error.cpp +++ b/src/core/error.cpp @@ -1,5 +1,6 @@ #include "spc/error.hpp" +#include #include #include #include @@ -9,12 +10,16 @@ namespace spc { namespace { +/// Error bodies are small; a bigger one is not parsed, so an error never +/// holds a JSON tree several times the size of a body as large as the limit. +constexpr std::size_t kMaxParsedErrorBody = 64 * 1024; + /// Use the ArcGIS error message when the body is JSON; otherwise keep the start /// of the raw body, such as SPC's HTML 404 page. void fill_message_from_body(Error& err, const std::string& body) { glz::generic root{}; - const glz::error_ctx ec = glz::read_json(root, body); - if (!ec && root.is_object()) { + const bool parsed = body.size() <= kMaxParsedErrorBody && !glz::read_json(root, body); + if (parsed && root.is_object()) { const glz::generic::object_t& obj = root.get_object(); glz::generic::object_t::const_iterator it = obj.find("error"); if (it != obj.end() && it->second.is_object()) { diff --git a/src/http/client.cpp b/src/http/client.cpp index c1fb5cb..8393985 100644 --- a/src/http/client.cpp +++ b/src/http/client.cpp @@ -80,17 +80,19 @@ std::size_t write_body(char* data, std::size_t size, std::size_t count, void* us return chunk; } -bool is_absolute_url(std::string_view path) { - return (path.size() >= 7 && curl_strnequal(path.data(), "http://", 7) != 0) || - (path.size() >= 8 && curl_strnequal(path.data(), "https://", 8) != 0); -} - /// Whether libcurl fetches `url` over TLS. It ignores the scheme's case and /// takes one to three slashes after it, so this matches the first slash only. bool is_https_url(std::string_view url) { return url.size() >= 7 && curl_strnequal(url.data(), "https:/", 7) != 0; } +/// Whether `path` names its own HTTP(S) scheme, by the same rule as libcurl; +/// otherwise a one-slash "https:/host" would be appended to a plaintext base. +bool is_absolute_url(std::string_view path) { + return (path.size() >= 6 && curl_strnequal(path.data(), "http:/", 6) != 0) || + is_https_url(path); +} + /// libcurl's process-wide state. Each client holds a shared reference, so the /// global cleanup runs only after the last handle is gone, whatever order /// static objects are destroyed in. diff --git a/tests/test_client.cpp b/tests/test_client.cpp index 848f056..44ae5bd 100644 --- a/tests/test_client.cpp +++ b/tests/test_client.cpp @@ -921,6 +921,15 @@ TEST(ArcGISClientRouting, RejectsNonJsonFormatsBeforeNetworkAccess) { EXPECT_TRUE(transport->requests.empty()); } +TEST(ClientErrors, AnOversizedErrorBodyIsNotParsed) { + const std::string small = R"({"error":{"code":500,"message":"Busy"}})"; + std::string big = R"({"error":{"code":500,"message":"Busy","pad":")"; + big += std::string(70 * 1024, 'x') + R"("}})"; + + EXPECT_EQ(Error::from_response(500, small).message, "Busy"); + EXPECT_EQ(Error::from_response(500, big).message, big.substr(0, 256)); +} + TEST(ClientErrors, OnlyStatuses500To599AreServerErrors) { EXPECT_EQ(Error::from_response(500, "").code, ErrorCode::ServerError); EXPECT_EQ(Error::from_response(599, "").code, ErrorCode::ServerError); diff --git a/tests/test_golden.cpp b/tests/test_golden.cpp index 7c9dd48..8817f36 100644 --- a/tests/test_golden.cpp +++ b/tests/test_golden.cpp @@ -200,12 +200,29 @@ bool has_case(std::string_view fixture) { [fixture](const GoldenCase& c) { return c.fixture == fixture; }); } +/// Regular files under `root`, named by their path relative to it, as +/// tools/verify_fixture_checksums.py names them. Dotfiles such as a Finder +/// .DS_Store and dot-directories are skipped. +std::vector files_under(const std::filesystem::path& root) { + std::vector names; + for (std::filesystem::recursive_directory_iterator it{root}; + it != std::filesystem::recursive_directory_iterator{}; ++it) { + if (it->path().filename().string().starts_with('.')) { + if (it->is_directory()) { + it.disable_recursion_pending(); + } + continue; + } + if (it->is_regular_file()) { + names.push_back(std::filesystem::relative(it->path(), root).generic_string()); + } + } + return names; +} + TEST(GoldenCoverage, EveryFixtureIsParsedOrListedAsNotParsed) { - for (const std::filesystem::directory_entry& entry : - std::filesystem::directory_iterator(SPC_FIXTURES_DIR)) { - const std::string name = entry.path().filename().string(); - // Dotfiles such as a Finder .DS_Store are not fixtures. - if (name == "README.md" || name == "SHA256SUMS" || name.starts_with('.')) { + for (const std::string& name : files_under(SPC_FIXTURES_DIR)) { + if (name == "README.md" || name == "SHA256SUMS") { continue; } EXPECT_TRUE(has_case(name) || std::ranges::find(kNotParsed, name) != kNotParsed.end()) @@ -218,12 +235,7 @@ TEST(GoldenCoverage, EveryFixtureIsParsedOrListedAsNotParsed) { } TEST(GoldenCoverage, EveryGoldenFileHasACase) { - for (const std::filesystem::directory_entry& entry : - std::filesystem::directory_iterator(SPC_GOLDEN_DIR)) { - const std::string name = entry.path().filename().string(); - if (name.starts_with('.')) { - continue; // .DS_Store and editor files - } + for (const std::string& name : files_under(SPC_GOLDEN_DIR)) { EXPECT_TRUE(name.ends_with(".txt") && has_case(name.substr(0, name.size() - 4))) << name << " has no golden case; delete it"; } diff --git a/tests/test_http_client.cpp b/tests/test_http_client.cpp index 65c0c79..06f890a 100644 --- a/tests/test_http_client.cpp +++ b/tests/test_http_client.cpp @@ -183,9 +183,27 @@ TEST(HttpClient, AbsoluteHttpsSchemesNeverUseThePlaintextBaseUrl) { // plaintext base and succeeds, exposing an HTTPS request as an HTTP path. EXPECT_FALSE(client.get("HTTPS://127.0.0.1:0/absolute")); EXPECT_FALSE(client.get("hTtPs://127.0.0.1:0/absolute")); + // libcurl also fetches a one-slash https URL over TLS. + EXPECT_FALSE(client.get("hTtPs:/127.0.0.1:0/absolute")); EXPECT_TRUE(server.requests().empty()); } +TEST(HttpClient, AOneSlashHttpUrlBypassesTheBaseUrl) { + LoopbackServer base{[](const LoopbackRequest&) { return http_response("200 OK", "base"); }}; + LoopbackServer other{[](const LoopbackRequest&) { return http_response("200 OK", "other"); }}; + ClientConfig config; + config.base_url = base.url("/base/"); + const HttpClient client{config}; + const std::string one_slash = + "http:/" + other.url("/x").substr(std::string_view{"http://"}.size()); + + const Result response = client.get(one_slash); + + ASSERT_TRUE(response) << response.error().message; + EXPECT_EQ(response->body, "other"); + EXPECT_TRUE(base.requests().empty()); +} + TEST(HttpClient, RejectsInvalidTimeoutsBeforeSendingARequest) { LoopbackServer server{[](const LoopbackRequest&) { return http_response("200 OK", "{}"); }}; for (const std::chrono::seconds timeout : From 5420a1a752c5f4bde5296324527ce55da3dd2769 Mon Sep 17 00:00:00 2001 From: KeviM Date: Fri, 20 Nov 2026 13:44:11 -0800 Subject: [PATCH 9/9] fix(core): compute the error-body cap in size_t clang-tidy's bugprone-implicit-widening-of-multiplication-result flagged 64 * 1024 being computed in int. --- src/core/error.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/error.cpp b/src/core/error.cpp index fd22c48..cb46240 100644 --- a/src/core/error.cpp +++ b/src/core/error.cpp @@ -12,7 +12,7 @@ namespace { /// Error bodies are small; a bigger one is not parsed, so an error never /// holds a JSON tree several times the size of a body as large as the limit. -constexpr std::size_t kMaxParsedErrorBody = 64 * 1024; +constexpr std::size_t kMaxParsedErrorBody = std::size_t{64} * 1024; /// Use the ArcGIS error message when the body is JSON; otherwise keep the start /// of the raw body, such as SPC's HTML 404 page.