diff --git a/.coderabbit.yaml b/.coderabbit.yaml index df05ba4..37e75b2 100644 --- a/.coderabbit.yaml +++ b/.coderabbit.yaml @@ -18,10 +18,8 @@ knowledge_base: code_guidelines: enabled: true filePatterns: - - AGENTS.md - CLAUDE.md - CONTRIBUTING.md - - .github/review-policy.md chat: auto_reply: false art: false diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml index a324396..086867d 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.yml +++ b/.github/ISSUE_TEMPLATE/bug_report.yml @@ -2,6 +2,10 @@ name: Bug report description: A crash, a wrong result, or a build failure. labels: [bug] body: + - type: markdown + attributes: + value: | + Report security vulnerabilities through GitHub's private vulnerability reporting, as described in the [security policy](https://github.com/Reddimus/spc-cpp/security/policy), not here. - type: textarea id: what attributes: @@ -31,7 +35,3 @@ body: - libcurl: 8.5.0 validations: required: true - - type: markdown - attributes: - value: | - Report security vulnerabilities privately, as described in the [security policy](https://github.com/Reddimus/spc-cpp/security/policy). diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml index 4572a9f..9a39c60 100644 --- a/.github/ISSUE_TEMPLATE/config.yml +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -2,4 +2,4 @@ blank_issues_enabled: false contact_links: - name: Security vulnerability url: https://github.com/Reddimus/spc-cpp/security/policy - about: Report it privately instead of opening an issue. + about: Use GitHub's private vulnerability reporting, as the security policy describes, not an issue. diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index f5da791..55cf174 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -6,6 +6,7 @@ -- [ ] `make test` and `make lint` pass +- [ ] `make test`, `make lint`, `make lint-md`, `make fixtures-check`, and + `make test-consumers` pass - [ ] `CHANGELOG.md` has an `[Unreleased]` entry for user-visible changes - [ ] Public API changes are called out above diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7ac2de8..7ea0c4a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -27,7 +27,7 @@ jobs: run: | sudo apt-get update sudo apt-get install -y --no-install-recommends \ - build-essential clang-format libcurl4-openssl-dev locales + build-essential clang-format libcurl4-openssl-dev libssl-dev locales # The locale tests skip without a comma-decimal locale. sudo locale-gen de_DE.UTF-8 - name: Build @@ -56,7 +56,7 @@ jobs: sudo apt-get update sudo apt-get install -y --no-install-recommends \ build-essential clang-18 libc++-18-dev libc++abi-18-dev \ - libcurl4-openssl-dev locales + libcurl4-openssl-dev libssl-dev locales sudo locale-gen de_DE.UTF-8 - name: Build run: >- @@ -75,7 +75,7 @@ jobs: run: | sudo apt-get update sudo apt-get install -y --no-install-recommends \ - build-essential clang-19 libcurl4-openssl-dev + build-essential clang-19 libcurl4-openssl-dev libssl-dev - name: Build run: >- make build CMAKE_ARGS="-DCMAKE_CXX_COMPILER=clang++-19 @@ -197,6 +197,8 @@ jobs: timeout-minutes: 10 steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + # The Makefile pins the markdownlint-cli2 version this action bundles. + # Update MARKDOWNLINT there when this pin changes. - uses: DavidAnson/markdownlint-cli2-action@696eb3f2509270d7b882d2aa68347bc676e7af34 # v25.0.0 sanitizer: @@ -208,7 +210,7 @@ jobs: run: | sudo apt-get update sudo apt-get install -y --no-install-recommends \ - build-essential libcurl4-openssl-dev locales + build-essential libcurl4-openssl-dev libssl-dev locales sudo locale-gen de_DE.UTF-8 - name: Build with AddressSanitizer and UndefinedBehaviorSanitizer run: >- @@ -228,7 +230,7 @@ jobs: run: | sudo apt-get update sudo apt-get install -y --no-install-recommends \ - build-essential libcurl4-openssl-dev locales + build-essential libcurl4-openssl-dev libssl-dev locales sudo locale-gen de_DE.UTF-8 - name: Build with ThreadSanitizer run: >- diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 003891b..4ebbb77 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -53,11 +53,12 @@ jobs: - name: Extract the release notes # Write to a file: echoing the notes through the shell would run their - # `code` spans as command substitutions. + # `code` spans as command substitutions. index() matches the heading + # literally; a regex would read the version's "." and "+" as operators. run: | version="${GITHUB_REF_NAME#v}" awk -v v="$version" ' - $0 ~ "^## \\[" v "\\]" { capture = 1; next } + index($0, "## [" v "]") == 1 { capture = 1; next } capture && /^## / { exit } capture && /^\[[^]]+\]: / { exit } capture { print } diff --git a/.gitignore b/.gitignore index d93b912..43cc42f 100644 --- a/.gitignore +++ b/.gitignore @@ -25,10 +25,24 @@ Thumbs.db # Secrets .env +.env.* +!.env.example +.envrc +.netrc +.npmrc +.pypirc *.pem *.key *.p12 *.pfx +id_rsa +id_ecdsa +id_ecdsa_sk +id_ed25519 +id_ed25519_sk credentials*.json service-account*.json +token.json secrets/ +**/.claude/settings.local.json +.claude/worktrees/ diff --git a/CHANGELOG.md b/CHANGELOG.md index 8481373..53dc7ce 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,52 @@ uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [Unreleased] +### Security + +- An HTTPS request no longer follows a redirect to plain HTTP; it fails with + `InvalidRequest`. +- A one-slash URL such as `https:/host/x`, which libcurl fetches as absolute, is + no longer appended to `ClientConfig::base_url` and sent to that host instead. +- Bound memory across all pages of an ArcGIS call. One Web Mercator expansion + budget covers the whole call, both fire-weather layers included, and the pages + together may not exceed `ClientConfig::max_response_bytes` (`InvalidRequest`). + Before, a paging server could make one call hold gigabytes. + +### Changed + +- libcurl 7.86 is the minimum. 7.85's `curl/header.h` lacks `extern "C"`, so the + library never linked against it. + +- Report a malformed URL or a scheme other than HTTP(S) as `InvalidRequest`, and a + redirect loop as `ServerError`. `with_retry` no longer repeats them. +- Map only HTTP 500-599 to `ServerError`; a status above 599 is `Unknown`. +- Retry ArcGIS 429 and 5xx errors sent in HTTP 200 bodies like the same HTTP + status, one page at a time. +- An ArcGIS error without a usable code reports `http_status` 0 and + `arcgisCode=0` instead of a made-up 400. + +### Fixed + +- Apply `ClientConfig::max_response_bytes` to the final response's decoded body. A + redirect hop's `Content-Length`, or a compressed length over the limit, no longer + fails a request whose body fits. +- Cap a `Retry-After` too large for int64 at the longest wait instead of ignoring it. +- Reset the daily quota when the clock steps back across midnight. +- Initialize `HttpResponse::status_code` to 0. +- A `std::exception` inside any client call, `std::bad_alloc` included, returns + `ParseError` instead of escaping from ArcGIS calls. +- Error bodies over 64 KiB are not parsed for their message; the message is the + start of the body. +- Skip features whose fields are not an object, conditional-intensity polygons + without a label, and day 1-2 fire-weather polygons with no known `dn` band. +- Read the numeric strings `"nan"` and `"inf"` as 0 instead of passing them on. +- Fall back to `wkid` when `latestWkid` is null or not a number, and ignore a null + geometry `spatialReference`. +- Link Glaze through its target, so a build that finds an installed Glaze + (`FETCHCONTENT_TRY_FIND_PACKAGE_MODE`) configures and installs, and MSVC gets the + options Glaze requires. +- The benchmarks' allocation-counter self-check no longer fails on MSVC. + ## [0.4.7] - 2026-11-04 ### Fixed diff --git a/CLAUDE.md b/CLAUDE.md index 200f86c..596cdb5 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -21,9 +21,9 @@ when you touch ArcGIS layer ids; it needs the network. - Public failures return `Result` (`std::expected`). The standalone `parse_*` functions throw `std::runtime_error` on malformed JSON; the clients turn that into `ErrorCode::ParseError`. -- Glaze is private. It lives behind `src/models/json.hpp`, and no header under - `include/` may include it; `tools/test_consumers.sh` fails if an install - ships Glaze headers. +- Glaze is private: only `src/` includes it, mostly through + `src/models/json.hpp`. No header under `include/` may include it; + `tools/test_consumers.sh` fails if an install ships Glaze headers. - `HttpClient` is thread-safe through a pool of libcurl handles. Custom transports implement `HttpTransport::get`, which must be safe to call concurrently. diff --git a/CMakeLists.txt b/CMakeLists.txt index df981f6..a4f025d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -99,8 +99,9 @@ if(SPC_ENABLE_COVERAGE) add_link_options(--coverage) endif() -# CURLOPT_PROTOCOLS_STR, which keeps the transport on http and https, needs 7.85. -find_package(CURL 7.85 REQUIRED) +# 7.85 added CURLOPT_PROTOCOLS_STR, which keeps the transport on http and https, +# and 7.86 made curl/header.h's functions extern "C", so C++ can link them. +find_package(CURL 7.86 REQUIRED) find_package(Threads REQUIRED) include(FetchContent) @@ -114,15 +115,22 @@ FetchContent_Declare( GIT_SHALLOW TRUE ) FetchContent_MakeAvailable(glaze) +# Glaze's headers are system headers wherever it is linked, so its warnings +# never reach -Werror or clang-tidy. An installed (imported) Glaze already is. +get_target_property(SPC_GLAZE_TARGET glaze::glaze ALIASED_TARGET) +if(SPC_GLAZE_TARGET) + set_property(TARGET ${SPC_GLAZE_TARGET} PROPERTY SYSTEM TRUE) +endif() add_subdirectory(src) +# The test and benchmark frameworks are pinned by commit, so a moved tag cannot +# change what runs. GIT_SHALLOW does not work with a hash; both clones are small. if(SPC_BUILD_TESTS) FetchContent_Declare( googletest GIT_REPOSITORY https://github.com/google/googletest.git - GIT_TAG v1.18.0 - GIT_SHALLOW TRUE + GIT_TAG 063de7e9578f82b369302001269680b4b1553359 # v1.18.0 ) set(gtest_force_shared_crt ON CACHE BOOL "" FORCE) set(INSTALL_GTEST OFF CACHE BOOL "" FORCE) @@ -135,8 +143,7 @@ if(SPC_BUILD_BENCHMARKS) FetchContent_Declare( benchmark GIT_REPOSITORY https://github.com/google/benchmark.git - GIT_TAG v1.9.5 - GIT_SHALLOW TRUE + GIT_TAG 192ef10025eb2c4cdd392bc502f0c852196baa48 # v1.9.5 SYSTEM EXCLUDE_FROM_ALL ) @@ -162,12 +169,15 @@ if(SPC_INSTALL) INCLUDES DESTINATION ${CMAKE_INSTALL_INCLUDEDIR} ) install(DIRECTORY include/spc DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}) - # The static libraries contain compiled Glaze code, so ship its license. install(FILES LICENSE DESTINATION ${CMAKE_INSTALL_DATADIR}/licenses/spc-cpp) - install(FILES ${glaze_SOURCE_DIR}/LICENSE - DESTINATION ${CMAKE_INSTALL_DATADIR}/licenses/spc-cpp - RENAME glaze-LICENSE - ) + # The static libraries contain compiled Glaze code, so ship its license. + # A packaged Glaze, found instead of fetched, ships its own. + if(glaze_SOURCE_DIR) + install(FILES ${glaze_SOURCE_DIR}/LICENSE + DESTINATION ${CMAKE_INSTALL_DATADIR}/licenses/spc-cpp + RENAME glaze-LICENSE + ) + endif() configure_package_config_file( ${PROJECT_SOURCE_DIR}/cmake/spcConfig.cmake.in diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 31abf2c..7656f9b 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,7 +1,8 @@ # Contributing -Thanks for helping. Report security problems privately through -[SECURITY.md](SECURITY.md), not in a public issue. +Thanks for helping. Report security problems through GitHub's private +vulnerability reporting, as [SECURITY.md](SECURITY.md) describes, not in a +public issue. ## Set up @@ -90,12 +91,13 @@ make tidy CMAKE_ARGS="-DCMAKE_CXX_COMPILER=$llvm/bin/clang++ \ ## Fixtures `tests/fixtures/` holds captured NOAA and IEM responses. After adding or -changing one, update the checksums: +changing one, update the checksums. Files in subdirectories are fixtures too; +dotfiles are not: ```bash -for f in $(ls tests/fixtures | grep -v -e README.md -e SHA256SUMS); do - shasum -a 256 "tests/fixtures/$f" -done > tests/fixtures/SHA256SUMS +find tests/fixtures -type f ! -path '*/.*' ! -path tests/fixtures/README.md \ + ! -path tests/fixtures/SHA256SUMS -exec shasum -a 256 {} + | + LC_ALL=C sort -k 2 > tests/fixtures/SHA256SUMS make fixtures-check ``` @@ -162,6 +164,17 @@ with an error. the tests and consumer checks, then publishes the release with the changelog section as its notes. +## Security reports + +SECURITY.md and the issue templates send reporters to GitHub's private +vulnerability reporting. It is a repository setting, and with it off reporters +have no private channel. Keep it on (Settings, Advanced Security) and check it +after changing repository settings: + +```bash +gh api repos/Reddimus/spc-cpp/private-vulnerability-reporting # {"enabled":true} +``` + ## Repository tools `.mcp.json` (Claude) and `.codex/config.toml` (Codex) configure diff --git a/Makefile b/Makefile index 97ec8d2..3bbbf90 100644 --- a/Makefile +++ b/Makefile @@ -4,6 +4,9 @@ BUILD_DIR := build CMAKE := cmake NPROC := $(shell nproc 2>/dev/null || sysctl -n hw.ncpu 2>/dev/null || echo 4) SOURCES = find src include tests examples benchmarks \( -name '*.cpp' -o -name '*.hpp' \) -print0 +# The version bundled by the markdownlint-cli2-action pin in ci.yml, so local +# and CI lint agree. Change both together. +MARKDOWNLINT := npx --yes markdownlint-cli2@0.24.0 .PHONY: all build debug configure configure-debug test test-consumers fixtures-check lint \ lint-md format format-md tidy bench coverage pre-commit install-hooks clean help @@ -39,14 +42,14 @@ lint: @python3 tools/test_cpp_auto_audit.py lint-md: - npx markdownlint-cli2 "**/*.md" + $(MARKDOWNLINT) "**/*.md" format: @command -v clang-format >/dev/null || { echo "clang-format is not installed"; exit 1; } @$(SOURCES) | xargs -0 clang-format -i format-md: - npx markdownlint-cli2 --fix "**/*.md" + $(MARKDOWNLINT) --fix "**/*.md" # Compile the libraries with clang-tidy. On macOS, see CONTRIBUTING.md. tidy: diff --git a/README.md b/README.md index 5bce8d0..2c559bd 100644 --- a/README.md +++ b/README.md @@ -38,7 +38,7 @@ int main() { - CMake 3.31 or newer. Ubuntu 24.04 ships 3.28, so run `sudo snap install cmake --classic`. - GCC 13+, Clang 18+ with libc++ or 19+ with libstdc++, or Apple Clang 17+. -- libcurl 7.85 or newer with its headers: `libcurl4-openssl-dev` on Ubuntu, +- libcurl 7.86 or newer with its headers: `libcurl4-openssl-dev` on Ubuntu, built into macOS. CI builds on Linux, macOS for Apple silicon and Intel, and Windows with MSVC. @@ -133,12 +133,16 @@ problems. `Error::code` says what went wrong: | `NetworkError` | Connection, TLS, or timeout failure, after retries. | | `RateLimited` | HTTP 429 or 503 after retries, or `ArchiveClient`'s own limit would wait more than 5 s. | | `ServerError` | HTTP 5xx other than 503, after retries, or an ArcGIS query that never finished paging. | -| `InvalidRequest` | An unsupported day or hazard, HTTP 400, an ArcGIS error such as a bad layer ID, or a response over 64 MiB. | +| `InvalidRequest` | An unsupported day or hazard, HTTP 400, an ArcGIS error such as a bad layer ID, a response over 64 MiB, or an ArcGIS query whose pages total more than that. | | `NotFound` | A URL or ArcGIS service that no longer exists. Report it as a bug. | -| `ParseError` | Malformed JSON or Esri geometry that cannot be converted. | +| `ParseError` | Malformed JSON, Esri geometry that cannot be converted, or not enough memory to build the result. | | `FeedUnavailable` | Only `StaticFeedClient` returns it. SPC has not issued the product, as with day 1 probabilities overnight. Treat it as no data. `ArcGISClient` returns an empty result instead. | | `Unknown` | Any other HTTP status, such as 422 for a malformed `ArchiveClient` timestamp. `Error::http_status` holds it. | +ArcGIS reports its own failures with HTTP 200 and an error code in the body. +`ArcGISClient` retries a 429 or 5xx code as it would the same HTTP status, +repeating only the page that failed. + The offline `parse_*` functions, such as `spc::parse_categorical(body, day)`, throw `std::runtime_error` on malformed JSON or Esri geometry that cannot be converted. Coordinates are longitude/latitude in degrees. Esri Web Mercator (3857, 102100, @@ -148,12 +152,15 @@ Projected edges are subdivided to preserve their boundaries within one metre. Malformed Esri points and invalid projected coordinates, including discarded holes, raise a parse error. A payload needing more than one million added points also raises a parse error. +An `ArcGISClient` call counts every page and layer it reads as one payload. ## Settings and threads `spc::ClientConfig` sets the user agent, the 15 s timeout, and the 64 MiB -response limit. The default user agent carries spc-cpp's contact address, so -set one with yours: +response limit. The server decides how many pages an ArcGIS query has, so that +limit also caps the pages of one `ArcGISClient` call together; with a transport +of your own the cap is 64 MiB. The default user agent carries spc-cpp's contact +address, so set one with yours: ```cpp const spc::ArcGISClient client{spc::ClientConfig{.user_agent = "myapp/1.0 (you@example.com)"}}; diff --git a/SECURITY.md b/SECURITY.md index 7389ff9..a867c02 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -12,8 +12,9 @@ Fixes land in the next release. Older tags are not patched, so upgrade your ## Reporting a vulnerability Use GitHub's [private vulnerability -reporting](https://github.com/Reddimus/spc-cpp/security/advisories/new). Do not -open a public issue. +reporting](https://github.com/Reddimus/spc-cpp/security/advisories/new). It +needs a GitHub account, and the report stays private until an advisory is +published. Do not open a public issue. Include the affected version or commit, a minimal reproduction, and the impact you see, such as memory corruption or a denial of service. diff --git a/benchmarks/CMakeLists.txt b/benchmarks/CMakeLists.txt index 3f47a10..f86ed4d 100644 --- a/benchmarks/CMakeLists.txt +++ b/benchmarks/CMakeLists.txt @@ -10,7 +10,7 @@ target_include_directories(spc_benchmarks PRIVATE ${PROJECT_SOURCE_DIR}/src ${PROJECT_SOURCE_DIR}/tests ) -target_include_directories(spc_benchmarks SYSTEM PRIVATE ${glaze_SOURCE_DIR}/include) +target_link_libraries(spc_benchmarks PRIVATE glaze::glaze) target_compile_definitions(spc_benchmarks PRIVATE SPC_FIXTURES_DIR="${PROJECT_SOURCE_DIR}/tests/fixtures" ) diff --git a/benchmarks/benchmarks.cpp b/benchmarks/benchmarks.cpp index ac72601..6ac860c 100644 --- a/benchmarks/benchmarks.cpp +++ b/benchmarks/benchmarks.cpp @@ -307,21 +307,25 @@ bool allocation_counter_works() { benchmark::DoNotOptimize(block); ::operator delete(block); const std::uint64_t direct = probe.stats().allocations; - const std::runtime_error error("a message the standard library copies to the heap"); - benchmark::DoNotOptimize(error.what()); - const std::uint64_t library = probe.stats().allocations - direct; if (direct != 1) { std::fputs("The allocation counter misses operator new calls. Under Valgrind, run with " "--soname-synonyms=somalloc=nouserintercepts.\n", stderr); return false; } +#if !defined(_MSVC_STL_VERSION) + // MSVC keeps exception messages in CRT memory, not operator new, so there + // it has no library allocation to observe. + const std::runtime_error error("a message the standard library copies to the heap"); + benchmark::DoNotOptimize(error.what()); + const std::uint64_t library = probe.stats().allocations - direct; if (library == 0) { std::fputs("The allocation counter misses allocations made inside the standard library, " "which happens when LTO internalizes the replacement operator new.\n", stderr); return false; } +#endif return true; } diff --git a/cmake/spcConfig.cmake.in b/cmake/spcConfig.cmake.in index 1887646..d7dcf8d 100644 --- a/cmake/spcConfig.cmake.in +++ b/cmake/spcConfig.cmake.in @@ -1,7 +1,7 @@ @PACKAGE_INIT@ include(CMakeFindDependencyMacro) -find_dependency(CURL 7.85) +find_dependency(CURL 7.86) find_dependency(Threads) include("${CMAKE_CURRENT_LIST_DIR}/spcTargets.cmake") diff --git a/include/spc/api.hpp b/include/spc/api.hpp index ad8b57d..401730c 100644 --- a/include/spc/api.hpp +++ b/include/spc/api.hpp @@ -3,9 +3,11 @@ /// IEM archive. /// /// Every method returns `Result` and never throws for network or data -/// errors. Clients are move-only; a moved-from client may only be assigned -/// to or destroyed. A client is safe to use from several threads when its -/// transport is; the default `HttpClient` is. +/// errors. A std::exception raised inside a call, std::bad_alloc included, +/// comes back as `ErrorCode::ParseError`; any other exception, which only a +/// custom transport can throw, propagates. Clients are move-only; a moved-from +/// client may only be assigned to or destroyed. A client is safe to use from +/// several threads when its transport is; the default `HttpClient` is. #pragma once @@ -80,7 +82,16 @@ enum class ArcGISService : std::uint8_t { /// SPC products from NOAA's ArcGIS MapServers (mapservices.weather.noaa.gov). /// /// An empty product is a successful, empty payload. Queries page through -/// results automatically. +/// results automatically, and a transient ArcGIS error sent with HTTP 200 is +/// retried like the same HTTP status, one page at a time. +/// +/// The server decides how many pages a query has, so a call is bounded as a +/// whole. Its pages, across every layer it reads, may total +/// `ClientConfig::max_response_bytes` (64 MiB when the transport is not an +/// `HttpClient`) and add one million projected points; past either the call +/// fails with `InvalidRequest` or `ParseError`. The first response counts +/// toward the total but is never refused by it: one response is the +/// transport's to bound. class ArcGISClient { public: explicit ArcGISClient(ClientConfig config = {}); @@ -116,7 +127,8 @@ class ArcGISClient { /// Active mesoscale discussions; empty when none are active. [[nodiscard]] Result query_active_md() const; - /// Any layer on one of the three MapServers. Returns each page's raw body. + /// Any layer on one of the three MapServers. Returns each page's raw body; + /// the call's byte limit bounds what they hold together. [[nodiscard]] Result> query_layer(ArcGISService service, std::int32_t layer_id, const QueryParams& params) const; diff --git a/include/spc/error.hpp b/include/spc/error.hpp index 421a87e..abfca0e 100644 --- a/include/spc/error.hpp +++ b/include/spc/error.hpp @@ -17,7 +17,8 @@ enum class ErrorCode { NetworkError, /// HTTP 429 or 503, or the SDK's own rate limit. RateLimited, - /// HTTP 5xx other than 503, or an ArcGIS query that never finished paging. + /// HTTP 500-599 other than 503, a redirect loop, or an ArcGIS query that + /// never finished paging. ServerError, /// A URL or ArcGIS service that no longer exists: a bug to report. A bad /// layer id is `InvalidRequest`. @@ -27,12 +28,15 @@ enum class ErrorCode { /// so treat it as "no data", not as a failure. Only `StaticFeedClient` /// returns it. FeedUnavailable, - /// The request cannot succeed as asked: an unsupported day or hazard, - /// HTTP 400, an ArcGIS error such as an invalid parameter, or a response - /// larger than `ClientConfig::max_response_bytes`. + /// The request cannot succeed as asked: an unsupported day or hazard, a + /// malformed URL or one that is not HTTP(S), a refused redirect (to another + /// scheme, or from HTTPS to HTTP), HTTP 400, an ArcGIS error such as an + /// invalid parameter, or a response, or the pages of one ArcGIS call + /// together, larger than `ClientConfig::max_response_bytes`. InvalidRequest, - /// The body was not valid JSON. Valid JSON without the expected fields - /// parses to an empty result. + /// The response could not be turned into a result: invalid JSON, Esri + /// geometry that cannot be converted, or an allocation failure inside the + /// call. Valid JSON without the expected fields parses to an empty result. ParseError, Unknown }; @@ -76,9 +80,10 @@ struct Error { ErrorCode code{ErrorCode::Unknown}; std::string message; /// HTTP status, or 0. For an ArcGIS error (sent with HTTP 200) this is the - /// ArcGIS code when it lies in 100..599. + /// ArcGIS code when it lies in 100..599, and 0 otherwise. int http_status{0}; /// Extra context, such as ArcGIS error details or "arcgisCode=1000". + /// "arcgisCode=0" means the error carried no usable code. std::string detail; [[nodiscard]] constexpr bool is_ok() const noexcept { return code == ErrorCode::Ok; } @@ -125,7 +130,7 @@ struct Error { /// An Error for an ArcGIS `{"error":{"code":...}}` body, which the /// MapServer sends with HTTP 200. Code 404 (a retired service) is - /// `NotFound`. + /// `NotFound`. Pass 0 when the error has no usable code. [[nodiscard]] static Error from_arcgis(int arcgis_code, const std::string& body); }; diff --git a/include/spc/http_client.hpp b/include/spc/http_client.hpp index 2e99f71..a3e891d 100644 --- a/include/spc/http_client.hpp +++ b/include/spc/http_client.hpp @@ -20,7 +20,7 @@ namespace spc { /// An HTTP response. Any status code is a successful transfer; the clients /// decide what a non-200 status means. struct HttpResponse { - std::int16_t status_code; + std::int16_t status_code{0}; std::string body; /// Headers of the final response, after redirects. std::vector> headers; @@ -36,15 +36,19 @@ struct ClientConfig { /// supported libcurl versions. Invalid values fail before sending. std::chrono::seconds timeout{15}; bool verify_ssl{true}; - /// Largest response body accepted. Bodies are held in memory and parsed - /// whole, and an IEM date range can be arbitrarily large. A bigger - /// response fails with `ErrorCode::InvalidRequest` and is not retried. + /// Largest decoded body accepted from the final response; 0 means no + /// limit. Bodies are held in memory and parsed whole, and an IEM date + /// range can be arbitrarily large. A bigger response fails with + /// `ErrorCode::InvalidRequest` and is not retried. `ArcGISClient` applies + /// the same limit to the pages of one call together. std::size_t max_response_bytes{64UL * 1024UL * 1024UL}; }; /// The GET interface the clients use. Implement it to supply your own /// network stack or canned test responses. One transport may be shared by /// several clients and threads, so `get` must be safe to call concurrently. +/// Report failures as an `Error`: a std::exception thrown from `get` reaches +/// the client's caller as `ErrorCode::ParseError`, and any other propagates. class HttpTransport { public: virtual ~HttpTransport() = default; @@ -60,7 +64,8 @@ class HttpTransport { }; /// libcurl-backed transport. Follows redirects, accepts compressed bodies, -/// and speaks only http and https. +/// and speaks only http and https. An https request follows only https +/// redirects, so its response never arrives over plaintext. /// /// Thread-safe: concurrent `get` calls each use their own pooled libcurl /// handle, and handles are reused so connections stay open between requests. @@ -76,7 +81,8 @@ class HttpClient final : public HttpTransport { /// Absolute HTTP and HTTPS URLs use case-insensitive schemes and bypass /// `config().base_url`; other inputs are appended to it. Transfers and - /// redirects permit only HTTP and HTTPS. + /// redirects permit only HTTP and HTTPS, and an HTTPS request refuses a + /// redirect to HTTP. A refused redirect is `ErrorCode::InvalidRequest`. [[nodiscard]] Result get(std::string_view path) const override; /// The settings this client was built with; defaults on a moved-from client. diff --git a/include/spc/models/convective.hpp b/include/spc/models/convective.hpp index 1222d43..ad0180e 100644 --- a/include/spc/models/convective.hpp +++ b/include/spc/models/convective.hpp @@ -52,7 +52,8 @@ struct ConditionalIntensityPayload { /// @throws std::runtime_error for invalid JSON or Esri geometry that cannot be converted. [[nodiscard]] Day48OutlookPayload parse_day4_8(std::string_view body, std::int32_t day); -/// Parse a conditional-intensity layer (GeoJSON or Esri). +/// Parse a conditional-intensity layer (GeoJSON or Esri). Polygons without a +/// label are skipped. /// @throws std::runtime_error for invalid JSON or Esri geometry that cannot be converted. [[nodiscard]] ConditionalIntensityPayload parse_conditional_intensity(std::string_view body, std::int32_t day, std::string hazard); diff --git a/include/spc/models/fire_weather.hpp b/include/spc/models/fire_weather.hpp index c456b20..9a1ae0c 100644 --- a/include/spc/models/fire_weather.hpp +++ b/include/spc/models/fire_weather.hpp @@ -39,8 +39,8 @@ struct FireWeatherPayload { /// ELEV 1, CRIT 2, EXTM 3. Dry-thunderstorm labels and anything else map to 0. [[nodiscard]] std::uint8_t fire_severity_from_label(std::string_view label) noexcept; -/// Parse one fire-weather layer (Esri or GeoJSON). No-risk polygons are -/// skipped. +/// Parse one fire-weather layer (Esri or GeoJSON). No-risk polygons, and +/// day 1-2 polygons with neither a label nor a known `dn` band, are skipped. /// /// `layer` is required because days 1 and 2 carry only a numeric `dn` band, /// and the outlook (5 ELEV, 8 CRIT, 10 EXTM) and dry-thunderstorm (5 IDRT, diff --git a/include/spc/pagination.hpp b/include/spc/pagination.hpp index bf50187..8fb1413 100644 --- a/include/spc/pagination.hpp +++ b/include/spc/pagination.hpp @@ -13,6 +13,9 @@ namespace spc { /// `exceededTransferLimit` when more remain. The server may also return fewer /// records than `resultRecordCount`, so `advance()` moves the offset by the /// number actually returned. SPC layers rarely need a second page. +/// +/// `max_pages` bounds the pages, not the memory they fill: `ArcGISClient` +/// separately caps the bytes and projected points of a whole call. class ArcGISPager { public: /// `page_size` is the `resultRecordCount` sent with each request. diff --git a/include/spc/rate_limit.hpp b/include/spc/rate_limit.hpp index 1b15bde..627a6eb 100644 --- a/include/spc/rate_limit.hpp +++ b/include/spc/rate_limit.hpp @@ -30,8 +30,9 @@ class RateLimiter { [[nodiscard]] bool try_acquire() noexcept; /// Wait for a token, up to `Config::max_wait` if set. Returns false when - /// no token can arrive in time or the daily quota is spent; it does not - /// sleep when waiting cannot help. + /// no token can arrive in time or the daily quota is spent. A lone caller + /// returns at once when waiting cannot help; with several waiters, one may + /// wait and still get nothing. [[nodiscard]] bool acquire(); /// `acquire()` with an explicit limit on the wait. diff --git a/include/spc/retry.hpp b/include/spc/retry.hpp index 4663d82..4f22b3b 100644 --- a/include/spc/retry.hpp +++ b/include/spc/retry.hpp @@ -104,15 +104,21 @@ struct RetryPolicy { } value.remove_prefix(start); value.remove_suffix(value.size() - value.find_last_not_of(" \t") - 1); + // Cap before converting so a huge value cannot overflow. + constexpr std::int64_t kMaxSeconds = std::int64_t{365} * 24 * 60 * 60; std::int64_t seconds = 0; const std::from_chars_result parsed = std::from_chars(value.data(), value.data() + value.size(), seconds); - if (parsed.ec == std::errc{} && parsed.ptr == value.data() + value.size() && seconds > 0) { - // Cap before converting so a huge value cannot overflow. - constexpr std::int64_t kMaxSeconds = std::int64_t{365} * 24 * 60 * 60; - return std::chrono::milliseconds{std::min(seconds, kMaxSeconds) * 1000}; + if (parsed.ptr != value.data() + value.size()) { + return std::chrono::milliseconds{0}; + } + // Digits too many for int64 still ask for the longest wait. + if (parsed.ec == std::errc::result_out_of_range && value.front() != '-') { + seconds = kMaxSeconds; + } else if (parsed.ec != std::errc{} || seconds <= 0) { + return std::chrono::milliseconds{0}; } - return std::chrono::milliseconds{0}; + return std::chrono::milliseconds{std::min(seconds, kMaxSeconds) * 1000}; } return std::chrono::milliseconds{0}; } diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index e7bd876..41be59a 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -1,11 +1,12 @@ # Static libraries: spc_core -> spc_http -> spc_models -> spc_api, and the # spc umbrella that links them all. # -# Glaze is added as a SYSTEM PRIVATE include directory rather than linked: -# no installed header uses it, and a linked glaze::glaze would have to join -# the export set. +# Glaze is linked privately and only inside this build: no installed header +# uses it, and BUILD_LOCAL_INTERFACE keeps it out of the export set. Linking +# the target, not its include path, also works with a packaged Glaze and +# carries the MSVC options Glaze requires. -set(SPC_GLAZE_INCLUDE_DIR ${glaze_SOURCE_DIR}/include) +set(SPC_GLAZE $) add_library(spc_core STATIC core/error.cpp @@ -20,7 +21,7 @@ target_include_directories(spc_core PUBLIC $ $ ) -target_include_directories(spc_core SYSTEM PRIVATE ${SPC_GLAZE_INCLUDE_DIR}) +target_link_libraries(spc_core PRIVATE ${SPC_GLAZE}) add_library(spc_http STATIC http/client.cpp @@ -40,7 +41,7 @@ add_library(spc_models STATIC spc_configure_target(spc_models) target_link_libraries(spc_models PUBLIC spc_core) target_include_directories(spc_models PRIVATE ${PROJECT_SOURCE_DIR}/src) -target_include_directories(spc_models SYSTEM PRIVATE ${SPC_GLAZE_INCLUDE_DIR}) +target_link_libraries(spc_models PRIVATE ${SPC_GLAZE}) add_library(spc_api STATIC api/client.cpp @@ -48,7 +49,7 @@ add_library(spc_api STATIC spc_configure_target(spc_api) target_link_libraries(spc_api PUBLIC spc_core spc_http spc_models) target_include_directories(spc_api PRIVATE ${PROJECT_SOURCE_DIR}/src) -target_include_directories(spc_api SYSTEM PRIVATE ${SPC_GLAZE_INCLUDE_DIR}) +target_link_libraries(spc_api PRIVATE ${SPC_GLAZE}) add_library(spc INTERFACE) target_link_libraries(spc INTERFACE spc_core spc_http spc_models spc_api) diff --git a/src/api/call_budget.hpp b/src/api/call_budget.hpp new file mode 100644 index 0000000..2004531 --- /dev/null +++ b/src/api/call_budget.hpp @@ -0,0 +1,53 @@ +/// @file call_budget.hpp +/// @brief Private: what one ArcGIS client call may take from the server. + +#pragma once + +#include "models/json.hpp" +#include "spc/http_client.hpp" + +#include + +namespace spc::detail { + +/// The bytes one call's pages may total: the response limit of an HttpClient, +/// so paging cannot multiply it. Another transport has no limit the client +/// can read, so it gets the default one. +[[nodiscard]] inline std::size_t call_byte_limit(const HttpTransport& transport) { + const HttpClient* client = dynamic_cast(&transport); + return client != nullptr ? client->config().max_response_bytes + : ClientConfig{}.max_response_bytes; +} + +/// What one client call may take from the server, across every page and layer +/// it reads. A response is bounded on its own, by the transport's size limit +/// and the parser's projection budget; charging every page to one budget keeps +/// a server that pages from multiplying those bounds. +class CallBudget { +public: + /// `max_bytes` caps the bodies of all pages together; 0 means no limit. + explicit CallBudget(std::size_t max_bytes) noexcept : max_bytes_(max_bytes) {} + + /// Whether the call can take a page of `body_bytes`. Its first page always + /// fits: the size of a single response is the transport's to bound. + [[nodiscard]] bool admits(std::size_t body_bytes) const noexcept { + return max_bytes_ == 0 || !charged_ || + (bytes_ <= max_bytes_ && body_bytes <= max_bytes_ - bytes_); + } + + /// Count an accepted page, the first included. + void charge(std::size_t body_bytes) noexcept { + bytes_ += body_bytes; + charged_ = true; + } + + [[nodiscard]] ProjectionBudget& projection() noexcept { return projection_; } + +private: + std::size_t max_bytes_; + std::size_t bytes_{0}; + bool charged_{false}; + ProjectionBudget projection_; +}; + +} // namespace spc::detail diff --git a/src/api/client.cpp b/src/api/client.cpp index e1e3d5e..309d5eb 100644 --- a/src/api/client.cpp +++ b/src/api/client.cpp @@ -1,6 +1,7 @@ /// @file client.cpp /// @brief StaticFeedClient, ArcGISClient, and ArchiveClient. +#include "api/call_budget.hpp" #include "models/from_tree.hpp" #include "models/json.hpp" #include "spc/api.hpp" @@ -10,11 +11,13 @@ #include #include +#include #include #include #include #include #include +#include #include #include #include @@ -26,6 +29,7 @@ namespace spc { namespace { +using detail::CallBudget; using detail::Json; constexpr std::string_view kStaticOutlookBase = "https://www.spc.noaa.gov/products/outlook/"; @@ -186,13 +190,30 @@ Result body_or_error(Result response, Feed404 semanti return std::unexpected(Error::from_response(response->status_code, response->body, semantics)); } -/// Run `parse` and turn its std::runtime_error into a ParseError. +/// The Error for the std::exception being handled; call it only inside a +/// catch block for one. ParseError, as for a parser's std::runtime_error: the +/// response could not be turned into a result. Other exceptions propagate: +/// catching everything would also swallow a thread cancellation. +Error current_failure() noexcept { + try { + try { + throw; + } catch (const std::exception& e) { + return Error::parse(e.what()); + } + } catch (const std::exception&) { + // No memory for the message either; an empty string needs none. + return {ErrorCode::ParseError, {}, 0, {}}; + } +} + +/// Run `parse` and turn the std::exception it throws into a ParseError. template Result> parsed(const Parse& parse) { try { return parse(); - } catch (const std::exception& e) { - return std::unexpected(Error::parse(e.what())); + } catch (const std::exception&) { + return std::unexpected(current_failure()); } } @@ -236,10 +257,12 @@ bool envelope_flag(const Json& root, const char* key) { Result inspect_arcgis_envelope(const Json& root, const std::string& body) { const Json* error = detail::lookup(root, "error"); if (error != nullptr && error->is_object()) { - // ArcGIS reports failures with HTTP 200 and an error object. + // ArcGIS reports failures with HTTP 200 and an error object. The code + // is passed on as sent; a missing or non-numeric one is 0, not a + // made-up status. const std::int32_t code = detail::to_int32(detail::json_number_or_numeric_string(*error, "code")).value_or(0); - return std::unexpected(Error::from_arcgis(code > 0 ? code : 400, body)); + return std::unexpected(Error::from_arcgis(code, body)); } ArcGISEnvelope envelope; // f=json puts the flag at the root; f=geojson puts it at the root and @@ -276,6 +299,12 @@ Result parse_arcgis_page(const std::string& body) { return ArcGISPage{std::move(*root), *envelope}; } +/// One accepted page: its body and what was parsed from it. +struct FetchedPage { + std::string body; + ArcGISPage page; +}; + bool is_json_format(std::string_view format) { return format == "json" || format == "pjson" || format == "geojson"; } @@ -346,7 +375,7 @@ StaticFeedClient::~StaticFeedClient() = default; StaticFeedClient::StaticFeedClient(StaticFeedClient&&) noexcept = default; StaticFeedClient& StaticFeedClient::operator=(StaticFeedClient&&) noexcept = default; -Result StaticFeedClient::day_categorical(std::int32_t day) const { +Result StaticFeedClient::day_categorical(std::int32_t day) const try { if (day < 1 || day > 3) { return std::unexpected( Error::invalid_request("categorical outlook day must be 1, 2, or 3")); @@ -354,10 +383,12 @@ Result StaticFeedClient::day_categorical(std::int32_t return built_from( impl_->fetch(std::format("{}day{}otlk_cat.nolyr.geojson", kStaticOutlookBase, day)), [day](const Json& root) { return detail::categorical_from_tree(root, day); }); +} catch (const std::exception&) { + return std::unexpected(current_failure()); } Result StaticFeedClient::day_probabilistic(std::int32_t day, - std::string_view hazard) const { + std::string_view hazard) const try { const std::string_view normalized = normalized_hazard(day, hazard); if (day1_3_probability_layer(day, normalized) == nullptr) { return std::unexpected(unsupported_probability()); @@ -370,15 +401,19 @@ Result StaticFeedClient::day_probabilistic(std::int32_t day, return built_from(impl_->fetch(url), [day, normalized](const Json& root) { return detail::probabilistic_from_tree(root, day, normalized); }); +} catch (const std::exception&) { + return std::unexpected(current_failure()); } -Result StaticFeedClient::day4_8(std::int32_t day) const { +Result StaticFeedClient::day4_8(std::int32_t day) const try { if (day < 4 || day > 8) { return std::unexpected( Error::invalid_request("extended outlook day must be between 4 and 8")); } return built_from(impl_->fetch(std::format("{}day{}prob.nolyr.geojson", kStaticDay48Base, day)), [day](const Json& root) { return detail::day4_8_from_tree(root, day); }); +} catch (const std::exception&) { + return std::unexpected(current_failure()); } // ===================== ArcGISClient ===================== @@ -386,36 +421,77 @@ Result StaticFeedClient::day4_8(std::int32_t day) const { struct ArcGISClient::Impl { std::shared_ptr http; RetryPolicy retry; + /// Total page bytes one call accepts. + std::size_t max_call_bytes; explicit Impl(std::shared_ptr transport) - : http(usable_transport(std::move(transport))) {} + : http(usable_transport(std::move(transport))), + max_call_bytes(detail::call_byte_limit(*http)) {} + + [[nodiscard]] CallBudget call_budget() const noexcept { return CallBudget{max_call_bytes}; } + + /// GET one page and parse it, retrying under `retry`. ArcGIS sends its own + /// failures with HTTP 200, so the body is parsed inside the retry loop and + /// a transient ArcGIS code is retried like the same HTTP status. A 404 is + /// NotFound: ArcGIS reports an empty product as HTTP 200 with no features. + [[nodiscard]] Result fetch_page(const std::string& url, + const CallBudget& budget) const { + // What the latest attempt's HTTP 200 body parsed to. + std::optional> page; + Result response = with_retry( + [&]() -> Result { + page.reset(); + Result attempt = http->get(url); + if (!attempt || attempt->status_code != 200) { + return attempt; + } + if (!budget.admits(attempt->body.size())) { + // Refused before parsing, which would hold several times the body. + return std::unexpected(Error::invalid_request( + "ArcGIS query pages exceeded ClientConfig::max_response_bytes in total")); + } + page.emplace(parse_arcgis_page(attempt->body)); + if (!page->has_value() && page->error().http_status != 0) { + // An ArcGIS error whose code is an HTTP status: let + // with_retry judge that status. + attempt->status_code = static_cast(page->error().http_status); + } + return attempt; + }, + retry); + if (!response) { + return std::unexpected(std::move(response.error())); + } + if (!page) { + return std::unexpected( + Error::from_response(response->status_code, response->body, Feed404::NotFound)); + } + if (!page->has_value()) { + return std::unexpected(std::move(page->error())); + } + return FetchedPage{std::move(response->body), std::move(**page)}; + } - /// Fetch every page of `query`, parse each once, and hand `on_page` the - /// page's body and tree as it arrives. `on_page` may take the body. A 404 - /// is NotFound: ArcGIS reports an empty product as HTTP 200 with no - /// features. + /// Fetch every page of `query`, charge each to `budget`, and hand `on_page` + /// the page's body and tree as it arrives. `on_page` may take the body. template requires std::is_invocable_r_v, const OnPage&, std::string&, const Json&> - [[nodiscard]] Result paged(const LayerQuery& query, const OnPage& on_page) const { + [[nodiscard]] Result paged(const LayerQuery& query, CallBudget& budget, + const OnPage& on_page) const { ArcGISPager pager; while (pager.has_more()) { - const std::string url = page_url(query, pager); - Result body = - body_or_error(with_retry([&] { return http->get(url); }, retry), Feed404::NotFound); - if (!body) { - return std::unexpected(body.error()); + Result fetched = fetch_page(page_url(query, pager), budget); + if (!fetched) { + return std::unexpected(std::move(fetched.error())); } - const Result page = parse_arcgis_page(*body); - if (!page) { - return std::unexpected(page.error()); - } - const ArcGISEnvelope envelope = page->envelope; + budget.charge(fetched->body.size()); + const ArcGISEnvelope envelope = fetched->page.envelope; if (envelope.exceeded_transfer_limit && envelope.feature_count == 0) { // The offset would never move. return std::unexpected( Error::server("ArcGIS reported a truncated page containing no records")); } - Result accepted = on_page(*body, page->root); + Result accepted = on_page(fetched->body, fetched->page.root); if (!accepted) { return accepted; } @@ -432,25 +508,26 @@ struct ArcGISClient::Impl { /// `build`, and move the page's `items` into `payload`. Each page's body is /// freed before its payload is built, so the two are never alive together. template - [[nodiscard]] Result paged_into(const LayerQuery& query, Payload payload, - std::vector Payload::*items, + [[nodiscard]] Result paged_into(const LayerQuery& query, CallBudget& budget, + Payload payload, std::vector Payload::*items, const Build& build) const { - Result done = paged(query, [&](std::string& body, const Json& root) -> Result { - release(body); - Result page = parsed([&] { return build(root); }); - if (!page) { - return std::unexpected(std::move(page.error())); - } - std::vector& target = payload.*items; - std::vector& source = (*page).*items; - if (target.empty()) { - target = std::move(source); - } else { - target.insert(target.end(), std::make_move_iterator(source.begin()), - std::make_move_iterator(source.end())); - } - return {}; - }); + Result done = + paged(query, budget, [&](std::string& body, const Json& root) -> Result { + release(body); + Result page = parsed([&] { return build(root); }); + if (!page) { + return std::unexpected(std::move(page.error())); + } + std::vector& target = payload.*items; + std::vector& source = (*page).*items; + if (target.empty()) { + target = std::move(source); + } else { + target.insert(target.end(), std::make_move_iterator(source.begin()), + std::make_move_iterator(source.end())); + } + return {}; + }); if (!done) { return std::unexpected(std::move(done.error())); } @@ -466,7 +543,7 @@ ArcGISClient::~ArcGISClient() = default; ArcGISClient::ArcGISClient(ArcGISClient&&) noexcept = default; ArcGISClient& ArcGISClient::operator=(ArcGISClient&&) noexcept = default; -Result ArcGISClient::query_categorical(std::int32_t day) const { +Result ArcGISClient::query_categorical(std::int32_t day) const try { const LayerDescriptor* descriptor = find_layer(LayerProduct::Categorical, day, ""); if (descriptor == nullptr) { return std::unexpected( @@ -474,15 +551,18 @@ Result ArcGISClient::query_categorical(std::int32_t d } CategoricalOutlookPayload seed; seed.day_offset = day; + CallBudget budget = impl_->call_budget(); // GeoJSON, the shape the spc-data categorical parser reads. return impl_->paged_into( - product_layer(kArcGisOutlooks, descriptor->id, "geojson"), std::move(seed), + product_layer(kArcGisOutlooks, descriptor->id, "geojson"), budget, std::move(seed), &CategoricalOutlookPayload::features, [day](const Json& root) { return detail::categorical_from_tree(root, day); }); +} catch (const std::exception&) { + return std::unexpected(current_failure()); } Result ArcGISClient::query_probabilistic(std::int32_t day, - std::string_view hazard) const { + std::string_view hazard) const try { const std::string_view normalized = normalized_hazard(day, hazard); const LayerDescriptor* descriptor = day1_3_probability_layer(day, normalized); if (descriptor == nullptr) { @@ -491,15 +571,18 @@ Result ArcGISClient::query_probabilistic(std::int32_t day, ProbOutlookPayload seed; seed.day_offset = day; seed.hazard = normalized; - return impl_->paged_into(product_layer(kArcGisOutlooks, descriptor->id, "geojson"), + CallBudget budget = impl_->call_budget(); + return impl_->paged_into(product_layer(kArcGisOutlooks, descriptor->id, "geojson"), budget, std::move(seed), &ProbOutlookPayload::features, [day, normalized](const Json& root) { return detail::probabilistic_from_tree(root, day, normalized); }); +} catch (const std::exception&) { + return std::unexpected(current_failure()); } Result -ArcGISClient::query_conditional_intensity(std::int32_t day, std::string_view hazard) const { +ArcGISClient::query_conditional_intensity(std::int32_t day, std::string_view hazard) const try { const std::string_view normalized = normalized_hazard(day, hazard); const LayerDescriptor* descriptor = find_layer(LayerProduct::ConditionalIntensity, day, normalized); @@ -511,14 +594,19 @@ ArcGISClient::query_conditional_intensity(std::int32_t day, std::string_view haz ConditionalIntensityPayload seed; seed.day = day; seed.hazard = normalized; - return impl_->paged_into(product_layer(kArcGisOutlooks, descriptor->id), std::move(seed), - &ConditionalIntensityPayload::features, - [day, hazard = std::string{normalized}](const Json& root) { - return detail::conditional_intensity_from_tree(root, day, hazard); + CallBudget budget = impl_->call_budget(); + return impl_->paged_into(product_layer(kArcGisOutlooks, descriptor->id), budget, + std::move(seed), &ConditionalIntensityPayload::features, + [day, hazard = std::string{normalized}, + projection = &budget.projection()](const Json& root) { + return detail::conditional_intensity_from_tree(root, day, hazard, + projection); }); +} catch (const std::exception&) { + return std::unexpected(current_failure()); } -Result ArcGISClient::query_day4_8(std::int32_t day) const { +Result ArcGISClient::query_day4_8(std::int32_t day) const try { const LayerDescriptor* descriptor = find_layer(LayerProduct::Probability, day, "severe"); if (descriptor == nullptr || day < 4) { return std::unexpected( @@ -526,13 +614,17 @@ Result ArcGISClient::query_day4_8(std::int32_t day) const { } Day48OutlookPayload seed; seed.day = day; - return impl_->paged_into(product_layer(kArcGisOutlooks, descriptor->id), std::move(seed), - &Day48OutlookPayload::features, [day](const Json& root) { - return detail::day4_8_from_tree(root, day); + CallBudget budget = impl_->call_budget(); + return impl_->paged_into(product_layer(kArcGisOutlooks, descriptor->id), budget, + std::move(seed), &Day48OutlookPayload::features, + [day, projection = &budget.projection()](const Json& root) { + return detail::day4_8_from_tree(root, day, projection); }); +} catch (const std::exception&) { + return std::unexpected(current_failure()); } -Result ArcGISClient::query_fire_weather(std::int32_t day) const { +Result ArcGISClient::query_fire_weather(std::int32_t day) const try { const std::vector layers = fire_weather_layers(day); if (layers.empty()) { return std::unexpected( @@ -540,33 +632,42 @@ Result ArcGISClient::query_fire_weather(std::int32_t day) co } FireWeatherPayload payload; payload.day = day; + // One budget for both layers: together they are the payload. + CallBudget budget = impl_->call_budget(); for (const LayerDescriptor* descriptor : layers) { const FireWeatherLayer layer = fire_weather_layer(descriptor->subtype); LayerQuery query = product_layer(kArcGisFireWeather, descriptor->id); // These layers default to Web Mercator; ask for lon/lat. query.out_spatial_reference = "4326"; - Result merged = - impl_->paged_into(query, std::move(payload), &FireWeatherPayload::features, - [day, layer](const Json& root) { - return detail::fire_weather_from_tree(root, day, layer); - }); + Result merged = impl_->paged_into( + query, budget, std::move(payload), &FireWeatherPayload::features, + [day, layer, projection = &budget.projection()](const Json& root) { + return detail::fire_weather_from_tree(root, day, layer, projection); + }); if (!merged) { return merged; } payload = std::move(*merged); } return payload; +} catch (const std::exception&) { + return std::unexpected(current_failure()); } -Result ArcGISClient::query_active_md() const { - return impl_->paged_into(product_layer(kArcGisMesoscale, 0), MesoscalePayload{}, +Result ArcGISClient::query_active_md() const try { + CallBudget budget = impl_->call_budget(); + return impl_->paged_into(product_layer(kArcGisMesoscale, 0), budget, MesoscalePayload{}, &MesoscalePayload::discussions, - [](const Json& root) { return detail::mesoscale_from_tree(root); }); + [projection = &budget.projection()](const Json& root) { + return detail::mesoscale_from_tree(root, projection); + }); +} catch (const std::exception&) { + return std::unexpected(current_failure()); } Result> ArcGISClient::query_layer(ArcGISService service, std::int32_t layer_id, - const QueryParams& params) const { + const QueryParams& params) const try { if (layer_id < 0) { return std::unexpected(Error::invalid_request("ArcGIS layer id must be non-negative")); } @@ -585,9 +686,11 @@ Result> ArcGISClient::query_layer(ArcGISService service base = kArcGisMesoscale; break; } + // The budget's byte limit bounds what `pages` retains. + CallBudget budget = impl_->call_budget(); std::vector pages; Result done = - impl_->paged(LayerQuery{base, layer_id, params, {}}, + impl_->paged(LayerQuery{base, layer_id, params, {}}, budget, [&pages](std::string& body, const Json& /*root*/) -> Result { pages.push_back(std::move(body)); return {}; @@ -596,6 +699,8 @@ Result> ArcGISClient::query_layer(ArcGISService service return std::unexpected(std::move(done.error())); } return pages; +} catch (const std::exception&) { + return std::unexpected(current_failure()); } // ===================== ArchiveClient ===================== @@ -652,20 +757,22 @@ ArchiveClient::~ArchiveClient() = default; ArchiveClient::ArchiveClient(ArchiveClient&&) noexcept = default; ArchiveClient& ArchiveClient::operator=(ArchiveClient&&) noexcept = default; -Result ArchiveClient::watches(std::string_view timestamp) const { +Result ArchiveClient::watches(std::string_view timestamp) const try { std::string url = std::format("{}json/spcwatch.py", kIemBase); if (!timestamp.empty()) { url += "?ts=" + percent_encode(timestamp); } return built_from(impl_->fetch(url), [](const Json& root) { return detail::watches_from_tree(root); }); +} catch (const std::exception&) { + return std::unexpected(current_failure()); } // A start/end pair cannot be told apart by type. // NOLINTBEGIN(bugprone-easily-swappable-parameters) Result ArchiveClient::storm_reports(std::string_view start_iso, std::string_view end_iso, - std::string_view wfo) const { + std::string_view wfo) const try { // NOLINTEND(bugprone-easily-swappable-parameters) // Encode every value: an ISO offset's '+' would otherwise decode as a space. std::string url = std::format("{}geojson/lsr.geojson?sts={}&ets={}", kIemBase, @@ -676,6 +783,8 @@ Result ArchiveClient::storm_reports(std::string_view start_i } return built_from(impl_->fetch(url), [](const Json& root) { return detail::storm_reports_from_tree(root); }); +} catch (const std::exception&) { + return std::unexpected(current_failure()); } } // namespace spc diff --git a/src/core/error.cpp b/src/core/error.cpp index 93a0628..cb46240 100644 --- a/src/core/error.cpp +++ b/src/core/error.cpp @@ -1,5 +1,6 @@ #include "spc/error.hpp" +#include #include #include #include @@ -9,12 +10,16 @@ namespace spc { namespace { +/// Error bodies are small; a bigger one is not parsed, so an error never +/// holds a JSON tree several times the size of a body as large as the limit. +constexpr std::size_t kMaxParsedErrorBody = std::size_t{64} * 1024; + /// Use the ArcGIS error message when the body is JSON; otherwise keep the start /// of the raw body, such as SPC's HTML 404 page. void fill_message_from_body(Error& err, const std::string& body) { glz::generic root{}; - const glz::error_ctx ec = glz::read_json(root, body); - if (!ec && root.is_object()) { + const bool parsed = body.size() <= kMaxParsedErrorBody && !glz::read_json(root, body); + if (parsed && root.is_object()) { const glz::generic::object_t& obj = root.get_object(); glz::generic::object_t::const_iterator it = obj.find("error"); if (it != obj.end() && it->second.is_object()) { @@ -53,7 +58,7 @@ Error Error::from_response(int status, const std::string& body, Feed404 semantic err.code = ErrorCode::InvalidRequest; } else if (status == 429 || status == 503) { err.code = ErrorCode::RateLimited; - } else if (status >= 500) { + } else if (status >= 500 && status <= 599) { err.code = ErrorCode::ServerError; } else { err.code = ErrorCode::Unknown; diff --git a/src/core/rate_limit.cpp b/src/core/rate_limit.cpp index 5034f0b..1a52230 100644 --- a/src/core/rate_limit.cpp +++ b/src/core/rate_limit.cpp @@ -32,7 +32,8 @@ RateLimiter::RateLimiter(Config config) void RateLimiter::check_daily_reset() noexcept { const std::chrono::system_clock::time_point today = utc_day_start(); - if (today > day_start_) { + // Not `>`: a clock stepped back across midnight must not freeze the quota. + if (today != day_start_) { daily_requests_used_ = 0; day_start_ = today; } @@ -148,7 +149,7 @@ std::int32_t RateLimiter::daily_requests_remaining() const noexcept { return 0; } // A new UTC day resets the count even before the next acquire. - if (utc_day_start() > day_start_) { + if (utc_day_start() != day_start_) { return config_.daily_limit; } return config_.daily_limit - daily_requests_used_; diff --git a/src/http/client.cpp b/src/http/client.cpp index c929724..8393985 100644 --- a/src/http/client.cpp +++ b/src/http/client.cpp @@ -2,6 +2,7 @@ #include #include +#include #include #include #include @@ -10,8 +11,9 @@ #include #include -#if !CURL_AT_LEAST_VERSION(7, 85, 0) -#error "spc-cpp needs libcurl 7.85.0 or newer for CURLOPT_PROTOCOLS_STR" +// 7.85 added CURLOPT_PROTOCOLS_STR; before 7.86, curl/header.h lacked extern "C". +#if !CURL_AT_LEAST_VERSION(7, 86, 0) +#error "spc-cpp needs libcurl 7.86.0 or newer" #endif namespace spc { @@ -28,16 +30,43 @@ const ClientConfig kMovedFromConfig{}; /// Collects the body up to a ceiling. Returning a short count from the write /// callback makes libcurl abort with CURLE_WRITE_ERROR. struct BodySink { + CURL* curl{nullptr}; std::string body; std::size_t limit{0}; + bool started{false}; bool overflowed{false}; bool failed{false}; }; -// libcurl is C, so no exception may leave this callback. +/// Whether the response being received announces an identity body over +/// `limit`. An encoded Content-Length says nothing about the decoded size, so +/// those bodies are left to the running count. +bool announces_more_than(CURL* curl, std::size_t limit) noexcept { + curl_off_t length = -1; + if (curl_easy_getinfo(curl, CURLINFO_CONTENT_LENGTH_DOWNLOAD_T, &length) != CURLE_OK || + length < 0) { + return false; + } + curl_header* encoding = nullptr; + if (curl_easy_header(curl, "Content-Encoding", 0, CURLH_HEADER, -1, &encoding) == CURLHE_OK) { + return false; + } + return static_cast(length) > static_cast(limit); +} + +// libcurl is C, so no exception may leave this callback. It sees only the +// final response: libcurl drops the bodies of redirects it follows, which is +// why the early size check lives here and not in CURLOPT_MAXFILESIZE. std::size_t write_body(char* data, std::size_t size, std::size_t count, void* user) noexcept { BodySink* sink = static_cast(user); const std::size_t chunk = size * count; + if (!sink->started) { + sink->started = true; + if (sink->limit > 0 && announces_more_than(sink->curl, sink->limit)) { + sink->overflowed = true; + return 0; + } + } if (sink->limit > 0 && chunk > sink->limit - sink->body.size()) { sink->overflowed = true; return 0; @@ -51,9 +80,17 @@ std::size_t write_body(char* data, std::size_t size, std::size_t count, void* us return chunk; } +/// Whether libcurl fetches `url` over TLS. It ignores the scheme's case and +/// takes one to three slashes after it, so this matches the first slash only. +bool is_https_url(std::string_view url) { + return url.size() >= 7 && curl_strnequal(url.data(), "https:/", 7) != 0; +} + +/// Whether `path` names its own HTTP(S) scheme, by the same rule as libcurl; +/// otherwise a one-slash "https:/host" would be appended to a plaintext base. bool is_absolute_url(std::string_view path) { - return (path.size() >= 7 && curl_strnequal(path.data(), "http://", 7) != 0) || - (path.size() >= 8 && curl_strnequal(path.data(), "https://", 8) != 0); + return (path.size() >= 6 && curl_strnequal(path.data(), "http:/", 6) != 0) || + is_https_url(path); } /// libcurl's process-wide state. Each client holds a shared reference, so the @@ -127,6 +164,10 @@ Result perform(Connection& connection, const std::string& url, Error::invalid_request("ClientConfig::timeout is outside libcurl's supported range")); } + // An https request stays on https: anyone on the path could alter a + // response fetched over an http redirect. + const char* const redirects = is_https_url(url) ? "https" : "http,https"; + // Refuse to send anything unless the restrictions actually took effect; // an older runtime libcurl rejects CURLOPT_PROTOCOLS_STR and would then // happily read file:// URLs. @@ -134,7 +175,7 @@ Result perform(Connection& connection, const std::string& url, const bool configured = curl_easy_setopt(curl, CURLOPT_URL, url.c_str()) == CURLE_OK && curl_easy_setopt(curl, CURLOPT_PROTOCOLS_STR, "http,https") == CURLE_OK && - curl_easy_setopt(curl, CURLOPT_REDIR_PROTOCOLS_STR, "http,https") == CURLE_OK && + curl_easy_setopt(curl, CURLOPT_REDIR_PROTOCOLS_STR, redirects) == CURLE_OK && curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, verify) == CURLE_OK && curl_easy_setopt(curl, CURLOPT_SSL_VERIFYHOST, verify * 2L) == CURLE_OK; if (!configured) { @@ -142,6 +183,7 @@ Result perform(Connection& connection, const std::string& url, } BodySink sink; + sink.curl = curl; sink.limit = config.max_response_bytes; curl_easy_setopt(curl, CURLOPT_ERRORBUFFER, connection.errors.data()); curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, &write_body); @@ -151,14 +193,10 @@ Result perform(Connection& connection, const std::string& url, curl_easy_setopt(curl, CURLOPT_NOSIGNAL, 1L); curl_easy_setopt(curl, CURLOPT_USERAGENT, config.user_agent.c_str()); curl_easy_setopt(curl, CURLOPT_ACCEPT_ENCODING, ""); - if (sink.limit > 0) { - // Lets libcurl refuse early when Content-Length is already too big. - curl_easy_setopt(curl, CURLOPT_MAXFILESIZE_LARGE, static_cast(sink.limit)); - } const CURLcode rc = curl_easy_perform(curl); // Not a NetworkError: retrying would download the same body again. - if (sink.overflowed || rc == CURLE_FILESIZE_EXCEEDED) { + if (sink.overflowed) { return std::unexpected( Error::invalid_request("response exceeded ClientConfig::max_response_bytes")); } @@ -166,9 +204,17 @@ Result perform(Connection& connection, const std::string& url, return std::unexpected(Error::network("out of memory while reading the response")); } if (rc != CURLE_OK) { - const bool detailed = connection.errors.front() != '\0'; - return std::unexpected( - Error::network(detailed ? connection.errors.data() : curl_easy_strerror(rc))); + std::string message = + connection.errors.front() != '\0' ? connection.errors.data() : curl_easy_strerror(rc); + // These fail the same way on every attempt, so they are not NetworkError, + // which with_retry repeats. + if (rc == CURLE_UNSUPPORTED_PROTOCOL || rc == CURLE_URL_MALFORMAT) { + return std::unexpected(Error::invalid_request(std::move(message))); + } + if (rc == CURLE_TOO_MANY_REDIRECTS) { + return std::unexpected(Error::server(std::move(message))); + } + return std::unexpected(Error::network(std::move(message))); } long status = 0; diff --git a/src/models/convective.cpp b/src/models/convective.cpp index 438a7cc..4a16504 100644 --- a/src/models/convective.cpp +++ b/src/models/convective.cpp @@ -26,14 +26,15 @@ std::uint8_t cig_severity_from_label(std::string_view label) noexcept { return 0; } -Day48OutlookPayload detail::day4_8_from_tree(const Json& root, std::int32_t day) { +Day48OutlookPayload detail::day4_8_from_tree(const Json& root, std::int32_t day, + detail::ProjectionBudget* budget) { Day48OutlookPayload payload; payload.day = day; const Json* features_node = detail::lookup(root, "features"); if (features_node == nullptr || !features_node->is_array()) { return payload; } - detail::PayloadGeometryParser geometry_parser(root); + detail::PayloadGeometryParser geometry_parser(root, budget); for (const Json& feat : features_node->get_array()) { const Json* props = detail::feature_fields(feat); const Json* geometry = detail::lookup(feat, "geometry"); @@ -60,7 +61,8 @@ Day48OutlookPayload parse_day4_8(std::string_view body, std::int32_t day) { } ConditionalIntensityPayload -detail::conditional_intensity_from_tree(const Json& root, std::int32_t day, std::string hazard) { +detail::conditional_intensity_from_tree(const Json& root, std::int32_t day, std::string hazard, + detail::ProjectionBudget* budget) { ConditionalIntensityPayload payload; payload.day = day; payload.hazard = std::move(hazard); @@ -68,7 +70,7 @@ detail::conditional_intensity_from_tree(const Json& root, std::int32_t day, std: if (features_node == nullptr || !features_node->is_array()) { return payload; } - detail::PayloadGeometryParser geometry_parser(root); + detail::PayloadGeometryParser geometry_parser(root, budget); for (const Json& feat : features_node->get_array()) { const Json* props = detail::feature_fields(feat); const Json* geometry = detail::lookup(feat, "geometry"); @@ -77,6 +79,9 @@ detail::conditional_intensity_from_tree(const Json& root, std::int32_t day, std: } ConditionalIntensityFeature f; f.label = detail::first_string(*props, {"LABEL", "label"}); + if (f.label.empty()) { + continue; + } f.cig_level = cig_severity_from_label(f.label); f.issued_at = detail::first_timestamp(*props, {"ISSUE", "issue"}); f.valid_from = detail::first_timestamp(*props, {"VALID", "valid"}); diff --git a/src/models/fire_weather.cpp b/src/models/fire_weather.cpp index 8b16fa2..769ae56 100644 --- a/src/models/fire_weather.cpp +++ b/src/models/fire_weather.cpp @@ -74,14 +74,15 @@ std::uint8_t fire_severity_from_label(std::string_view label) noexcept { } FireWeatherPayload detail::fire_weather_from_tree(const Json& root, std::int32_t day, - FireWeatherLayer layer) { + FireWeatherLayer layer, + detail::ProjectionBudget* budget) { FireWeatherPayload payload; payload.day = day; const Json* features_node = detail::lookup(root, "features"); if (features_node == nullptr || !features_node->is_array()) { return payload; } - detail::PayloadGeometryParser geometry_parser(root); + detail::PayloadGeometryParser geometry_parser(root, budget); for (const Json& feat : features_node->get_array()) { const Json* props = detail::feature_fields(feat); const Json* geometry = detail::lookup(feat, "geometry"); @@ -100,6 +101,9 @@ FireWeatherPayload detail::fire_weather_from_tree(const Json& root, std::int32_t f.probability = detail::normalized_probability(*props); } else if (f.label.empty()) { f.label = label_from_dn(*props, layer); + if (f.label.empty()) { + continue; // no label and no known dn band + } } f.severity = fire_severity_from_label(f.label); f.issued_at = detail::first_timestamp(*props, {"ISSUE", "issue"}); diff --git a/src/models/from_tree.hpp b/src/models/from_tree.hpp index 5978774..bddfb21 100644 --- a/src/models/from_tree.hpp +++ b/src/models/from_tree.hpp @@ -5,6 +5,9 @@ /// client that parsed a response once builds its payload from that tree. The /// names are distinct because glz::generic converts implicitly from /// std::string, which would make overloads of `parse_*` ambiguous. +/// +/// The parsers that project Esri rings take the payload's `ProjectionBudget` +/// when `root` is only part of it, such as one page of a paged query. #pragma once @@ -27,15 +30,18 @@ CategoricalOutlookPayload categorical_from_tree(const Json& root, std::int32_t d ProbOutlookPayload probabilistic_from_tree(const Json& root, std::int32_t day_offset, std::string_view hazard); -Day48OutlookPayload day4_8_from_tree(const Json& root, std::int32_t day); +Day48OutlookPayload day4_8_from_tree(const Json& root, std::int32_t day, + ProjectionBudget* budget = nullptr); ConditionalIntensityPayload conditional_intensity_from_tree(const Json& root, std::int32_t day, - std::string hazard); + std::string hazard, + ProjectionBudget* budget = nullptr); FireWeatherPayload fire_weather_from_tree(const Json& root, std::int32_t day, - FireWeatherLayer layer); + FireWeatherLayer layer, + ProjectionBudget* budget = nullptr); -MesoscalePayload mesoscale_from_tree(const Json& root); +MesoscalePayload mesoscale_from_tree(const Json& root, ProjectionBudget* budget = nullptr); StormReportPayload storm_reports_from_tree(const Json& root); diff --git a/src/models/json.cpp b/src/models/json.cpp index 4726b5f..8363b19 100644 --- a/src/models/json.cpp +++ b/src/models/json.cpp @@ -20,7 +20,6 @@ struct JsonReadOptions : glz::opts { constexpr double kMercatorRadius = 6378137.0; constexpr double kDegrees = 180.0 / std::numbers::pi; -constexpr std::size_t kMaxAddedProjectionPoints = 1000000; void validate_mercator(const Polygon& ring) { for (const LonLat point : ring) { @@ -126,9 +125,10 @@ double json_number_or_numeric_string(const Json& obj, const char* key) { return v->get(); } if (v->is_string()) { - // Matches a C-locale std::stod for every value SPC publishes. + // Matches a C-locale std::stod for every value SPC publishes. No field + // means "nan" or "inf", so those read as 0 too. const ParsedNumber parsed = parse_double(v->get()); - return parsed.ok ? parsed.value : 0.0; + return parsed.ok && std::isfinite(parsed.value) ? parsed.value : 0.0; } return 0.0; } @@ -234,13 +234,14 @@ std::vector parse_esri_rings_with_budget(const Json& geom, const Json* if (rings_node == nullptr || !rings_node->is_array()) { return out; } - if (const Json* local = lookup(geom, "spatialReference")) { + if (const Json* local = lookup(geom, "spatialReference"); + local != nullptr && local->is_object()) { reference = local; } bool mercator = false; - if (reference != nullptr) { + if (reference != nullptr && reference->is_object()) { const Json* id = lookup(*reference, "latestWkid"); - if (id == nullptr) { + if (id == nullptr || !id->is_number()) { id = lookup(*reference, "wkid"); } const double wkid = id != nullptr && id->is_number() ? id->get() : 0.0; @@ -298,8 +299,8 @@ std::vector parse_esri_rings_with_budget(const Json& geom, const Json* } // namespace std::vector parse_esri_rings(const Json& geom, const Json* reference) { - std::size_t remaining = kMaxAddedProjectionPoints; - return parse_esri_rings_with_budget(geom, reference, remaining); + ProjectionBudget budget; + return parse_esri_rings_with_budget(geom, reference, budget.remaining_added_points); } glz::expected parse_root(std::string_view body) { @@ -348,17 +349,19 @@ Json parse_root_or_throw(std::string_view body) { } const Json* feature_fields(const Json& feature) { - const Json* properties = lookup(feature, "properties"); - return properties != nullptr ? properties : lookup(feature, "attributes"); + const Json* fields = lookup(feature, "properties"); + if (fields == nullptr) { + fields = lookup(feature, "attributes"); + } + return fields != nullptr && fields->is_object() ? fields : nullptr; } -PayloadGeometryParser::PayloadGeometryParser(const Json& root) - : reference_(lookup(root, "spatialReference")), - remaining_added_points_(kMaxAddedProjectionPoints) {} +PayloadGeometryParser::PayloadGeometryParser(const Json& root, ProjectionBudget* shared) + : reference_(lookup(root, "spatialReference")), budget_(shared != nullptr ? shared : &own_) {} std::vector PayloadGeometryParser::parse(const Json& geometry) { return lookup(geometry, "rings") != nullptr - ? parse_esri_rings_with_budget(geometry, reference_, remaining_added_points_) + ? parse_esri_rings_with_budget(geometry, reference_, budget_->remaining_added_points) : parse_rings(geometry); } diff --git a/src/models/json.hpp b/src/models/json.hpp index 07e4643..860e45b 100644 --- a/src/models/json.hpp +++ b/src/models/json.hpp @@ -38,11 +38,12 @@ struct ParsedNumber { /// Parse the leading decimal number of `text` regardless of the process /// locale. `std::stod` follows LC_NUMERIC, so on a comma-decimal host it reads -/// "0.15" as 0. Leading whitespace and '+' are rejected. +/// "0.15" as 0. Leading whitespace and '+' are rejected; "nan" and "inf" parse, +/// as with std::from_chars. ParsedNumber parse_double(std::string_view text); /// `obj[key]` as a number, accepting numeric strings such as "5" or "0.15". -/// Missing or non-numeric values yield 0. +/// Missing, non-numeric, and non-finite values yield 0. double json_number_or_numeric_string(const Json& obj, const char* key); /// The LABEL/label/dn probability normalized to [0, 1]. Values above 1 are @@ -62,8 +63,8 @@ std::vector parse_rings(const Json& geom); /// rings (clockwise) and holes (counter-clockwise) in one list; the holes are /// dropped so the result matches `parse_rings` for the same layer. Web Mercator /// is converted to degrees with at most one metre of boundary error. A geometry -/// needing over one million added points fails. A geometry reference overrides -/// the response reference. +/// needing more added points than its payload's `ProjectionBudget` has left +/// fails. A geometry reference overrides the response reference. /// Malformed points fail for every supported spatial reference. std::vector parse_esri_rings(const Json& geom, const Json* reference = nullptr); @@ -83,14 +84,25 @@ glz::expected parse_owned_root(const std::string& body); /// public `parse_*` contract. Json parse_root_or_throw(std::string_view body); -/// A feature's fields: GeoJSON `properties` or Esri `attributes`. +/// A feature's fields: GeoJSON `properties` or Esri `attributes`, or null +/// when they are missing or not an object. const Json* feature_fields(const Json& feature); -/// One per payload: shares the collection reference and projection budget -/// across all features, accepting Esri rings or GeoJSON coordinates. +/// The points Web Mercator projection may still add to one payload: one +/// million, 16 MB, at the start. An edge is subdivided every 10 km, so 20 +/// bytes of JSON can add 4,000 points. +struct ProjectionBudget { + std::size_t remaining_added_points{1000000}; +}; + +/// One per response: shares the collection reference and projection budget +/// across all features, accepting Esri rings or GeoJSON coordinates. The +/// budget is the payload's. A caller that builds one payload from several +/// responses passes the same `shared` budget for each; otherwise the response +/// is the payload and the parser uses its own. class PayloadGeometryParser { public: - explicit PayloadGeometryParser(const Json& root); + explicit PayloadGeometryParser(const Json& root, ProjectionBudget* shared = nullptr); ~PayloadGeometryParser() = default; PayloadGeometryParser(const PayloadGeometryParser&) = delete; PayloadGeometryParser& operator=(const PayloadGeometryParser&) = delete; @@ -100,7 +112,8 @@ class PayloadGeometryParser { private: const Json* reference_; - std::size_t remaining_added_points_; + ProjectionBudget own_; + ProjectionBudget* budget_; }; /// The first non-empty string among `keys`, e.g. {"LABEL", "label"}. diff --git a/src/models/mesoscale.cpp b/src/models/mesoscale.cpp index 081b527..c6c6538 100644 --- a/src/models/mesoscale.cpp +++ b/src/models/mesoscale.cpp @@ -12,16 +12,16 @@ namespace spc { using detail::Json; -MesoscalePayload detail::mesoscale_from_tree(const Json& root) { +MesoscalePayload detail::mesoscale_from_tree(const Json& root, detail::ProjectionBudget* budget) { MesoscalePayload payload; const Json* features_node = detail::lookup(root, "features"); if (features_node == nullptr || !features_node->is_array()) { return payload; } - detail::PayloadGeometryParser geometry_parser(root); + detail::PayloadGeometryParser geometry_parser(root, budget); for (const Json& feat : features_node->get_array()) { const Json* props = detail::feature_fields(feat); - if (props == nullptr || !props->is_object()) { + if (props == nullptr) { continue; } MesoscaleDiscussion md; diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 1cb2ac9..ac617d8 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -19,6 +19,35 @@ if(WIN32) _CRT_SECURE_NO_WARNINGS) endif() +# The https redirect tests need a TLS server, which tests/support builds on +# OpenSSL 3. It is optional and those tests skip without it. The link check +# turns it off where the copy found cannot serve the build, such as a +# universal macOS binary against Homebrew's single-architecture OpenSSL; it +# is not cached, because the architectures can change between configures. +# Windows is left out: its tests would need OpenSSL's DLLs on PATH to start. +unset(SPC_TEST_OPENSSL_LINKS CACHE) +if(NOT WIN32) + find_package(OpenSSL 3.0 QUIET) + if(OpenSSL_FOUND) + include(CheckCXXSourceCompiles) + set(CMAKE_REQUIRED_LIBRARIES OpenSSL::SSL) + set(CMAKE_REQUIRED_QUIET ON) + check_cxx_source_compiles(" + #include + int main() { return SSL_CTX_new(TLS_server_method()) == nullptr ? 1 : 0; }" + SPC_TEST_OPENSSL_LINKS) + unset(CMAKE_REQUIRED_LIBRARIES) + unset(CMAKE_REQUIRED_QUIET) + if(SPC_TEST_OPENSSL_LINKS) + target_link_libraries(spc_tests PRIVATE OpenSSL::SSL) + target_compile_definitions(spc_tests PRIVATE SPC_TEST_HAS_TLS_SERVER) + endif() + endif() +endif() +if(NOT SPC_TEST_OPENSSL_LINKS) + message(STATUS "No usable OpenSSL 3: the https redirect tests will skip") +endif() + spc_set_warnings(spc_tests) target_link_libraries(spc_tests PRIVATE spc::spc GTest::gtest_main Threads::Threads) # Tests reach the private JSON helpers in src/models/json.hpp. @@ -26,7 +55,7 @@ target_include_directories(spc_tests PRIVATE ${PROJECT_SOURCE_DIR}/src ${CMAKE_CURRENT_SOURCE_DIR} ) -target_include_directories(spc_tests SYSTEM PRIVATE ${glaze_SOURCE_DIR}/include) +target_link_libraries(spc_tests PRIVATE glaze::glaze) target_compile_definitions(spc_tests PRIVATE SPC_FIXTURES_DIR="${CMAKE_CURRENT_SOURCE_DIR}/fixtures" SPC_GOLDEN_DIR="${CMAKE_CURRENT_SOURCE_DIR}/golden" diff --git a/tests/fixtures/README.md b/tests/fixtures/README.md index 4185bd8..ba5b41e 100644 --- a/tests/fixtures/README.md +++ b/tests/fixtures/README.md @@ -25,9 +25,9 @@ Pairs captured as Esri JSON (`.esri.json`, `f=json`) and GeoJSON (`.geojson`, Single captures: - `arcgis_day1_torn_conditional_intensity.esri.json`: `label` `"CIG1"`. -- `arcgis_day4_8_nonempty.synthetic.json`: a non-empty Day 4 response. -- `arcgis_day{1,2}_fire_weather.esri.json`: fire weather, with only a numeric - `dn` and no label. +- `arcgis_day{1,2}_fire_weather.esri.json`: fire weather in the layer's native + Web Mercator (wkid 102100), with only a numeric `dn`, no label, and no + `issue`. The client asks for `outSR=4326` instead; these test the conversion. - `arcgis_mesoscale_discussion.esri.json`: one active discussion. - `arcgis_mesoscale_discussion_noarea.esri.json`: the `NoArea` placeholder the layer returns when no discussion is active (captured 2026-09-25). @@ -35,6 +35,10 @@ Single captures: MapServers. `python3 tools/verify_arcgis_metadata.py` compares it with the live services. +Synthetic: + +- `arcgis_day4_8_nonempty.synthetic.json`: a non-empty Day 4 response. + ## IEM (`mesonet.agron.iastate.edu`) - `iem_storm_reports.json`: Local Storm Reports, Point geometry. diff --git a/tests/support/allocation_counter.hpp b/tests/support/allocation_counter.hpp index ec96026..c12bd99 100644 --- a/tests/support/allocation_counter.hpp +++ b/tests/support/allocation_counter.hpp @@ -17,7 +17,8 @@ struct AllocationStats { std::uint64_t allocations = 0; ///< operator new calls std::uint64_t bytes = 0; ///< bytes those calls asked for /// Most memory live at once. Live sizes are the allocator's block sizes, - /// because an unsized delete does not say how much it frees. + /// because an unsized delete does not say how much it frees. Freeing memory + /// allocated before the probe lowers the count, so this is then a lower bound. std::uint64_t peak_bytes = 0; /// Memory still live: what the measured code kept. Negative when it /// freed memory allocated before the probe. diff --git a/tests/support/loopback_server.hpp b/tests/support/loopback_server.hpp index 6c488cb..9b05ead 100644 --- a/tests/support/loopback_server.hpp +++ b/tests/support/loopback_server.hpp @@ -79,6 +79,45 @@ inline void initialize() {} inline void set_option(Socket socket, int option, const void* value, int length) { (void)::setsockopt(socket, SOL_SOCKET, option, static_cast(value), length); } + +/// A listening socket on 127.0.0.1; `port` receives the free port it got. +inline Socket listen_on_loopback(std::uint16_t& port) { + initialize(); + const Socket listener = ::socket(AF_INET, SOCK_STREAM, 0); + if (listener == kInvalidSocket) { + throw std::runtime_error("socket() failed"); + } + const int reuse = 1; + set_option(listener, SO_REUSEADDR, &reuse, sizeof(reuse)); + sockaddr_in address{}; + address.sin_family = AF_INET; + address.sin_addr.s_addr = htonl(INADDR_LOOPBACK); + address.sin_port = 0; // any free port + if (::bind(listener, reinterpret_cast(&address), sizeof(address)) != 0 || + ::listen(listener, 64) != 0) { + close_socket(listener); + throw std::runtime_error("bind() or listen() failed"); + } + AddressLength length = sizeof(address); + (void)::getsockname(listener, reinterpret_cast(&address), &length); + port = ntohs(address.sin_port); + return listener; +} + +/// Bound how long a stalled client can hold the serving thread. +inline void limit_io_time(Socket client) { +#ifdef _WIN32 + const DWORD timeout = 2000; +#else + const timeval timeout{2, 0}; +#endif + set_option(client, SO_RCVTIMEO, &timeout, sizeof(timeout)); + set_option(client, SO_SNDTIMEO, &timeout, sizeof(timeout)); +#ifdef SO_NOSIGPIPE + const int no_sigpipe = 1; + set_option(client, SO_NOSIGPIPE, &no_sigpipe, sizeof(no_sigpipe)); +#endif +} } // namespace socket_detail struct LoopbackRequest { @@ -95,30 +134,41 @@ struct LoopbackRequest { } }; +/// The request line's target and the headers of a raw request. +inline LoopbackRequest parse_request(const std::string& raw) { + LoopbackRequest request; + const std::size_t line_end = raw.find("\r\n"); + const std::string request_line = raw.substr(0, line_end); + const std::size_t first_space = request_line.find(' '); + const std::size_t second_space = request_line.find(' ', first_space + 1); + if (first_space != std::string::npos && second_space != std::string::npos) { + request.target = request_line.substr(first_space + 1, second_space - first_space - 1); + } + std::size_t start = line_end + 2; + while (start < raw.size()) { + const std::size_t end = raw.find("\r\n", start); + if (end == std::string::npos || end == start) { + break; + } + const std::string line = raw.substr(start, end - start); + const std::size_t colon = line.find(':'); + if (colon != std::string::npos) { + const std::size_t value_start = line.find_first_not_of(' ', colon + 1); + request.headers.emplace_back(line.substr(0, colon), value_start == std::string::npos + ? std::string{} + : line.substr(value_start)); + } + start = end + 2; + } + return request; +} + class LoopbackServer { public: using Handler = std::function; explicit LoopbackServer(Handler handler) : handler_(std::move(handler)) { - socket_detail::initialize(); - listener_ = ::socket(AF_INET, SOCK_STREAM, 0); - if (listener_ == socket_detail::kInvalidSocket) { - throw std::runtime_error("socket() failed"); - } - const int reuse = 1; - socket_detail::set_option(listener_, SO_REUSEADDR, &reuse, sizeof(reuse)); - sockaddr_in address{}; - address.sin_family = AF_INET; - address.sin_addr.s_addr = htonl(INADDR_LOOPBACK); - address.sin_port = 0; // any free port - if (::bind(listener_, reinterpret_cast(&address), sizeof(address)) != 0 || - ::listen(listener_, 64) != 0) { - socket_detail::close_socket(listener_); - throw std::runtime_error("bind() or listen() failed"); - } - socket_detail::AddressLength length = sizeof(address); - (void)::getsockname(listener_, reinterpret_cast(&address), &length); - port_ = ntohs(address.sin_port); + listener_ = socket_detail::listen_on_loopback(port_); thread_ = std::thread([this] { serve(); }); } @@ -162,17 +212,7 @@ class LoopbackServer { } void handle(socket_detail::Socket client) { -#ifdef _WIN32 - const DWORD timeout = 2000; -#else - const timeval timeout{2, 0}; -#endif - socket_detail::set_option(client, SO_RCVTIMEO, &timeout, sizeof(timeout)); - socket_detail::set_option(client, SO_SNDTIMEO, &timeout, sizeof(timeout)); -#ifdef SO_NOSIGPIPE - const int no_sigpipe = 1; - socket_detail::set_option(client, SO_NOSIGPIPE, &no_sigpipe, sizeof(no_sigpipe)); -#endif + socket_detail::limit_io_time(client); std::string raw; std::vector buffer(4096); while (raw.find("\r\n\r\n") == std::string::npos && raw.size() < 65536) { @@ -183,7 +223,7 @@ class LoopbackServer { } raw.append(buffer.data(), static_cast(received)); } - const LoopbackRequest request = parse(raw); + const LoopbackRequest request = parse_request(raw); { const std::lock_guard lock(mutex_); requests_.push_back(request); @@ -192,34 +232,6 @@ class LoopbackServer { (void)::shutdown(client, socket_detail::kShutdownWrite); } - static LoopbackRequest parse(const std::string& raw) { - LoopbackRequest request; - const std::size_t line_end = raw.find("\r\n"); - const std::string request_line = raw.substr(0, line_end); - const std::size_t first_space = request_line.find(' '); - const std::size_t second_space = request_line.find(' ', first_space + 1); - if (first_space != std::string::npos && second_space != std::string::npos) { - request.target = request_line.substr(first_space + 1, second_space - first_space - 1); - } - std::size_t start = line_end + 2; - while (start < raw.size()) { - const std::size_t end = raw.find("\r\n", start); - if (end == std::string::npos || end == start) { - break; - } - const std::string line = raw.substr(start, end - start); - const std::size_t colon = line.find(':'); - if (colon != std::string::npos) { - const std::size_t value_start = line.find_first_not_of(' ', colon + 1); - request.headers.emplace_back(line.substr(0, colon), value_start == std::string::npos - ? std::string{} - : line.substr(value_start)); - } - start = end + 2; - } - return request; - } - void send_all(socket_detail::Socket client, const std::string& response) { #ifdef MSG_NOSIGNAL constexpr int kFlags = MSG_NOSIGNAL; diff --git a/tests/support/tls_loopback_server.hpp b/tests/support/tls_loopback_server.hpp new file mode 100644 index 0000000..6412221 --- /dev/null +++ b/tests/support/tls_loopback_server.hpp @@ -0,0 +1,164 @@ +/// @file tls_loopback_server.hpp +/// @brief LoopbackServer over TLS, for tests where the response has to arrive +/// on an https request. +/// +/// It needs OpenSSL 3 and POSIX sockets. tests/CMakeLists.txt treats that as +/// optional and defines SPC_TEST_HAS_TLS_SERVER when it is there. Each server +/// signs its own certificate at startup, so no private key is checked in, and +/// clients need `verify_ssl = false`. + +#pragma once + +#ifdef SPC_TEST_HAS_TLS_SERVER + +#include "support/loopback_server.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace spc::test { + +class TlsLoopbackServer { +public: + explicit TlsLoopbackServer(LoopbackServer::Handler handler) + : handler_(std::move(handler)), context_(self_signed_context()) { + listener_ = socket_detail::listen_on_loopback(port_); + thread_ = std::thread([this] { serve(); }); + } + + ~TlsLoopbackServer() { + stop_.store(true); + thread_.join(); + socket_detail::close_socket(listener_); + } + + TlsLoopbackServer(const TlsLoopbackServer&) = delete; + TlsLoopbackServer& operator=(const TlsLoopbackServer&) = delete; + TlsLoopbackServer(TlsLoopbackServer&&) = delete; + TlsLoopbackServer& operator=(TlsLoopbackServer&&) = delete; + + /// "127.0.0.1:port", for tests that spell the scheme themselves. + [[nodiscard]] std::string authority() const { return "127.0.0.1:" + std::to_string(port_); } + + [[nodiscard]] std::string url(std::string_view target) const { + return "https://" + authority() + std::string{target}; + } + + [[nodiscard]] std::vector requests() const { + const std::lock_guard lock(mutex_); + return requests_; + } + +private: + struct Free { + void operator()(SSL_CTX* context) const { SSL_CTX_free(context); } + void operator()(SSL* ssl) const { SSL_free(ssl); } + void operator()(EVP_PKEY* key) const { EVP_PKEY_free(key); } + void operator()(X509* certificate) const { X509_free(certificate); } + void operator()(X509_NAME* name) const { X509_NAME_free(name); } + }; + + static std::unique_ptr self_signed_context() { + const std::unique_ptr key{ + EVP_PKEY_Q_keygen(nullptr, nullptr, "EC", "P-256")}; + const std::unique_ptr certificate{X509_new()}; + const std::unique_ptr name{X509_NAME_new()}; + std::unique_ptr context{SSL_CTX_new(TLS_server_method())}; + if (key == nullptr || certificate == nullptr || name == nullptr || context == nullptr) { + throw std::runtime_error("OpenSSL could not allocate the test certificate"); + } + constexpr unsigned char kHost[] = "127.0.0.1"; + const bool ready = + X509_NAME_add_entry_by_txt(name.get(), "CN", MBSTRING_ASC, kHost, -1, -1, 0) == 1 && + X509_set_version(certificate.get(), X509_VERSION_3) == 1 && + ASN1_INTEGER_set(X509_get_serialNumber(certificate.get()), 1) == 1 && + X509_gmtime_adj(X509_getm_notBefore(certificate.get()), -3600) != nullptr && + X509_gmtime_adj(X509_getm_notAfter(certificate.get()), 3600) != nullptr && + X509_set_subject_name(certificate.get(), name.get()) == 1 && + X509_set_issuer_name(certificate.get(), name.get()) == 1 && + X509_set_pubkey(certificate.get(), key.get()) == 1 && + X509_sign(certificate.get(), key.get(), EVP_sha256()) > 0 && + SSL_CTX_use_certificate(context.get(), certificate.get()) == 1 && + SSL_CTX_use_PrivateKey(context.get(), key.get()) == 1; + if (!ready) { + throw std::runtime_error("OpenSSL could not build the test certificate"); + } + return context; + } + + void serve() { + // OpenSSL writes with write(), which raises SIGPIPE once the client has + // hung up. Blocked, the signal becomes an error return on this thread. + sigset_t broken_pipe; + (void)sigemptyset(&broken_pipe); + (void)sigaddset(&broken_pipe, SIGPIPE); + (void)::pthread_sigmask(SIG_BLOCK, &broken_pipe, nullptr); + while (!stop_.load()) { + if (socket_detail::wait_for_connection(listener_) <= 0) { + continue; + } + const socket_detail::Socket client = ::accept(listener_, nullptr, nullptr); + if (client == socket_detail::kInvalidSocket) { + continue; + } + handle(client); + socket_detail::close_socket(client); + } + } + + void handle(socket_detail::Socket client) { + socket_detail::limit_io_time(client); + const std::unique_ptr ssl{SSL_new(context_.get())}; + if (ssl == nullptr || SSL_set_fd(ssl.get(), client) != 1 || SSL_accept(ssl.get()) != 1) { + return; // e.g. a client that verifies the certificate + } + std::string raw; + std::array buffer{}; + while (raw.find("\r\n\r\n") == std::string::npos && raw.size() < 65536) { + const int received = + SSL_read(ssl.get(), buffer.data(), static_cast(buffer.size())); + if (received <= 0) { + return; + } + raw.append(buffer.data(), static_cast(received)); + } + const LoopbackRequest request = parse_request(raw); + { + const std::lock_guard lock(mutex_); + requests_.push_back(request); + } + const std::string response = handler_(request); + if (!response.empty() && + SSL_write(ssl.get(), response.data(), static_cast(response.size())) > 0) { + (void)SSL_shutdown(ssl.get()); + } + } + + LoopbackServer::Handler handler_; + std::unique_ptr context_; + socket_detail::Socket listener_{socket_detail::kInvalidSocket}; + std::uint16_t port_{0}; + std::atomic stop_{false}; + mutable std::mutex mutex_; + std::vector requests_; + std::thread thread_; +}; + +} // namespace spc::test + +#endif // SPC_TEST_HAS_TLS_SERVER diff --git a/tests/test_arcgis.cpp b/tests/test_arcgis.cpp index 9617d62..6e8d996 100644 --- a/tests/test_arcgis.cpp +++ b/tests/test_arcgis.cpp @@ -358,7 +358,8 @@ TEST(Models, FireWeatherLayerIsTheOnlyThingThatDisambiguatesDayOneAndTwoDn) { parse_fire_weather(body, day, FireWeatherLayer::DryThunderstorm); ASSERT_EQ(outlook.features.size(), 3u) << name; - ASSERT_EQ(dry.features.size(), 3u) << name; + // dn 10 has no dry-thunderstorm band, so that polygon is skipped. + ASSERT_EQ(dry.features.size(), 2u) << name; EXPECT_EQ(outlook.features[0].label, "ELEV") << name; EXPECT_EQ(outlook.features[1].label, "CRIT") << name; EXPECT_EQ(outlook.features[2].label, "EXTM") << name; diff --git a/tests/test_client.cpp b/tests/test_client.cpp index fd4ba40..44ae5bd 100644 --- a/tests/test_client.cpp +++ b/tests/test_client.cpp @@ -2,17 +2,24 @@ /// @brief Client routing, paging, and error mapping, through a recording /// transport. No network access. +#include "api/call_budget.hpp" #include "spc/api.hpp" #include "spc/pagination.hpp" +#include "spc/retry.hpp" #include "spc/version.hpp" #include "support/fixtures.hpp" +#include #include +#include #include #include +#include #include +#include #include #include +#include #include #include @@ -63,6 +70,92 @@ class AlwaysTruncatingTransport final : public HttpTransport { } }; +/// Answers every request with a page padded to about `page_bytes`, truncated +/// or final, and counts what it served. Pages are built on demand so the +/// transport holds none of them. +class PaddedPagesTransport final : public HttpTransport { +public: + PaddedPagesTransport(std::size_t page_bytes, bool truncated) + : page_bytes_(page_bytes), truncated_(truncated) {} + + mutable std::int32_t calls = 0; + mutable std::size_t bytes_served = 0; + + [[nodiscard]] Result get(std::string_view /*path*/) const override { + HttpResponse response{200, {}, {}}; + response.body.reserve(page_bytes_ + 160); + response.body += R"({"type":"FeatureCollection","features":[)" + R"({"type":"Feature","properties":{},"geometry":null}],"pad":")"; + response.body.append(page_bytes_, 'x'); + response.body += std::format(R"(","exceededTransferLimit":{}}})", truncated_); + ++calls; + bytes_served += response.body.size(); + return response; + } + +private: + std::size_t page_bytes_; + bool truncated_; +}; + +/// Fails every request the way an allocation failure inside a call does. +class OutOfMemoryTransport final : public HttpTransport { +public: + [[nodiscard]] Result get(std::string_view /*path*/) const override { + throw std::bad_alloc{}; + } +}; + +/// Serves one truncated page, then throws something that is not a +/// std::exception, as a custom transport might. +class ThrowsAfterAPageTransport final : public HttpTransport { +public: + mutable std::int32_t calls = 0; + + [[nodiscard]] Result get(std::string_view /*path*/) const override { + if (calls++ == 0) { + return HttpResponse{200, truncated_page(1), {}}; + } + throw 42; + } +}; + +/// The code `call` failed with, `Ok` if it succeeded, or a test failure if it +/// threw. +template +ErrorCode outcome_of(const Call& call) { + try { + const std::invoke_result_t result = call(); + return result ? ErrorCode::Ok : result.error().code; + } catch (const std::exception& e) { + ADD_FAILURE() << "client call threw " << e.what(); + } catch (...) { + ADD_FAILURE() << "client call threw a non-std exception"; + } + return ErrorCode::Unknown; +} + +/// An Esri page of `count` Web Mercator features. Each ring has one diagonal +/// edge that projection subdivides into 3,800 steps, so a feature costs 3,799 +/// of the one million added points a payload may hold. +std::string mercator_page(std::size_t count, bool truncated) { + std::string features; + for (std::size_t i = 0; i < count; ++i) { + if (i != 0) { + features += ','; + } + features += + R"({"attributes":{"dn":15,"LABEL":"15","name":"Risk"},"geometry":{"rings":[[[-19000000,-19000000],[-19000000,19000000],[19000000,19000000],[-19000000,-19000000]]]}})"; + } + return std::format( + R"({{"spatialReference":{{"wkid":3857}},"features":[{}],"exceededTransferLimit":{}}})", + features, truncated); +} + +std::string arcgis_error_body(std::int32_t code) { + return std::format(R"({{"error":{{"code":{},"message":"Busy","details":[]}}}})", code); +} + TEST(StaticFeedClientRouting, UsesPublishedProbabilisticFilenames) { std::shared_ptr transport = std::make_shared(); transport->responses.assign(3, empty_feature_collection()); @@ -171,6 +264,49 @@ TEST(ArcGISClientRouting, CombinesBothPublishedFireWeatherLayers) { EXPECT_NE(transport->requests[1].find("outSR=4326"), std::string::npos); } +TEST(ArcGISClientRouting, TypedQueriesUseTheirOwnMapServers) { + // Layer ids alone still match when a query goes to the wrong MapServer. + const std::string services = "https://mapservices.weather.noaa.gov/vector/rest/services/"; + std::shared_ptr transport = std::make_shared(); + transport->responses.assign(5, empty_feature_collection()); + ArcGISClient client{transport}; + + ASSERT_TRUE(client.query_categorical(1)); + ASSERT_TRUE(client.query_fire_weather(1)); + ASSERT_TRUE(client.query_active_md()); + ASSERT_TRUE(client.query_day4_8(4)); + + ASSERT_EQ(transport->requests.size(), 5u); + EXPECT_TRUE(transport->requests[0].starts_with(services + "outlooks/SPC_wx_outlks/MapServer/")); + EXPECT_TRUE( + transport->requests[1].starts_with(services + "fire_weather/SPC_firewx/MapServer/")); + EXPECT_TRUE( + transport->requests[2].starts_with(services + "fire_weather/SPC_firewx/MapServer/")); + EXPECT_TRUE(transport->requests[3].starts_with( + services + "outlooks/spc_mesoscale_discussion/MapServer/0/query?")); + EXPECT_TRUE(transport->requests[4].starts_with(services + "outlooks/SPC_wx_outlks/MapServer/")); +} + +TEST(StaticFeedRouting, EachProductUsesItsPublishedFile) { + std::shared_ptr transport = std::make_shared(); + transport->responses.assign(4, {200, R"({"type":"FeatureCollection","features":[]})", {}}); + StaticFeedClient client{transport}; + + ASSERT_TRUE(client.day_categorical(2)); + ASSERT_TRUE(client.day_probabilistic(1, "tornado")); + ASSERT_TRUE(client.day_probabilistic(3, "severe")); + ASSERT_TRUE(client.day4_8(5)); + + const std::string outlook = "https://www.spc.noaa.gov/products/outlook/"; + EXPECT_EQ(transport->requests, (std::vector{ + outlook + "day2otlk_cat.nolyr.geojson", + outlook + "day1otlk_torn.nolyr.geojson", + outlook + "day3otlk_prob.nolyr.geojson", + "https://www.spc.noaa.gov/products/exper/day4-8/" + "day5prob.nolyr.geojson", + })); +} + TEST(ArcGISClientRouting, PreservesLabelsFromBothDayOneFireProducts) { std::shared_ptr transport = std::make_shared(); const std::string body = R"({"features":[ @@ -355,6 +491,8 @@ TEST(ArcGISClientPaging, ReturnsLogicalArcGISErrorsReportedWithHttp200) { EXPECT_EQ(result.error().code, ErrorCode::InvalidRequest); EXPECT_EQ(result.error().http_status, 400); EXPECT_EQ(result.error().message, "Invalid or missing input parameters."); + // Not transient, so it is asked once. + EXPECT_EQ(transport->requests.size(), 1u); } TEST(ArcGISClientPaging, KeepsTheArcGISErrorDetails) { @@ -376,15 +514,11 @@ TEST(ArcGISClientPaging, KeepsTheArcGISErrorDetails) { TEST(ArcGISClientPaging, ClassifiesArcGIS5xxCodesLikeHttpStatuses) { // HTTP 503 is RateLimited and other 5xx are ServerError, so the same codes - // in an ArcGIS error object mean the same. + // in an ArcGIS error object mean the same, and are retried the same. for (const auto& [code, expected] : {std::pair{503, ErrorCode::RateLimited}, std::pair{500, ErrorCode::ServerError}}) { std::shared_ptr transport = std::make_shared(); - transport->responses = { - {200, - std::format(R"({{"error":{{"code":{},"message":"Busy","details":[]}}}})", code), - {}}, - }; + transport->responses.assign(RetryPolicy{}.max_attempts, {200, arcgis_error_body(code), {}}); const ArcGISClient client{transport}; const Result result = client.query_categorical(1); @@ -392,9 +526,282 @@ TEST(ArcGISClientPaging, ClassifiesArcGIS5xxCodesLikeHttpStatuses) { ASSERT_FALSE(result) << code; EXPECT_EQ(result.error().code, expected) << code; EXPECT_EQ(result.error().http_status, code); + EXPECT_EQ(result.error().message, "Busy"); + // The ArcGIS error itself, not one rebuilt from the status. + EXPECT_EQ(result.error().detail, std::format("arcgisCode={}", code)); + EXPECT_EQ(transport->requests.size(), RetryPolicy{}.max_attempts) << code; + } +} + +TEST(ArcGISClientPaging, RetriesTransientArcGISErrorsSentWithHttp200) { + for (const std::int32_t code : {429, 500, 503}) { + std::shared_ptr transport = std::make_shared(); + transport->responses = {{200, arcgis_error_body(code), {}}, empty_feature_collection()}; + const ArcGISClient client{transport}; + + const Result> result = + client.query_layer(ArcGISService::Outlooks, 1, {}); + + ASSERT_TRUE(result) << code << ": " << result.error().message; + // The error body is not a page. + ASSERT_EQ(result->size(), 1u) << code; + EXPECT_EQ((*result)[0], empty_feature_collection().body); + EXPECT_EQ(transport->requests.size(), 2u) << code; + } +} + +TEST(ArcGISClientPaging, RetriesTheFailingPageWithoutRestartingTheQuery) { + const std::string feature = + R"({"type":"Feature","properties":{"label":"TSTM"},)" + R"("geometry":{"type":"Polygon","coordinates":[[[0,0],[1,0],[1,1],[0,0]]]}})"; + std::shared_ptr transport = std::make_shared(); + transport->responses = { + {200, R"({"features":[)" + feature + R"(],"exceededTransferLimit":true})", {}}, + {200, arcgis_error_body(500), {}}, + {200, R"({"features":[)" + feature + R"(]})", {}}, + }; + const ArcGISClient client{transport}; + + const Result result = client.query_categorical(1); + + ASSERT_TRUE(result) << result.error().message; + EXPECT_EQ(result->features.size(), 2u); + ASSERT_EQ(transport->requests.size(), 3u); + EXPECT_NE(transport->requests[1].find("resultOffset=1"), std::string::npos); + EXPECT_EQ(transport->requests[2], transport->requests[1]); +} + +TEST(ArcGISClientPaging, ARetriedPageReportsItsLastAttempt) { + std::shared_ptr transport = std::make_shared(); + transport->responses = {{200, arcgis_error_body(503), {}}, {404, "gone", {}}}; + const ArcGISClient client{transport}; + + const Result> result = + client.query_layer(ArcGISService::Outlooks, 1, {}); + + ASSERT_FALSE(result); + EXPECT_EQ(result.error().code, ErrorCode::NotFound); + EXPECT_EQ(result.error().http_status, 404); + EXPECT_EQ(result.error().message, "gone"); + EXPECT_EQ(result.error().detail, ""); + EXPECT_EQ(transport->requests.size(), 2u); +} + +TEST(ArcGISClientPaging, OtherArcGISCodesAreAskedOnce) { + for (const std::int32_t code : {403, 404, 498, 1000}) { + std::shared_ptr transport = std::make_shared(); + transport->responses = {{200, arcgis_error_body(code), {}}, empty_feature_collection()}; + const ArcGISClient client{transport}; + + EXPECT_FALSE(client.query_layer(ArcGISService::Outlooks, 1, {})) << code; + EXPECT_EQ(transport->requests.size(), 1u) << code; + } +} + +TEST(ArcGISClientPaging, AnArcGISErrorWithoutACodeDoesNotInventOne) { + for (const std::string_view error : + {R"({"message":"Query failed."})", R"({"code":null,"message":"Query failed."})", + R"({"code":"abc","message":"Query failed."})"}) { + std::shared_ptr transport = std::make_shared(); + transport->responses = {{200, std::format(R"({{"error":{}}})", error), {}}}; + const ArcGISClient client{transport}; + + const Result> result = + client.query_layer(ArcGISService::Outlooks, 1, {}); + + ASSERT_FALSE(result) << error; + EXPECT_EQ(result.error().code, ErrorCode::InvalidRequest) << error; + EXPECT_EQ(result.error().http_status, 0) << error; + EXPECT_EQ(result.error().message, "Query failed.") << error; + EXPECT_EQ(result.error().detail.find("400"), std::string::npos) << result.error().detail; + EXPECT_EQ(transport->requests.size(), 1u) << error; + } +} + +// ===== Bounds on one call ===== +// +// A response is bounded on its own: its size by the transport, its projection +// growth by the parser. A call that pages must not multiply either bound by +// the number of pages. + +TEST(ArcGISClientLimits, ProjectionBudgetSpansEveryPageOfOneCall) { + // 300 features need 1,139,700 added points: over the limit on one page, so + // over it when split across two. + struct Case { + std::string_view name; + ErrorCode (*run)(const ArcGISClient&); + }; + const std::vector cases = { + {"query_day4_8", + [](const ArcGISClient& c) { return outcome_of([&c] { return c.query_day4_8(4); }); }}, + {"query_conditional_intensity", + [](const ArcGISClient& c) { + return outcome_of([&c] { return c.query_conditional_intensity(1, "tornado"); }); + }}, + {"query_active_md", + [](const ArcGISClient& c) { return outcome_of([&c] { return c.query_active_md(); }); }}, + }; + for (const Case& test_case : cases) { + std::shared_ptr one_page = std::make_shared(); + one_page->responses = {{200, mercator_page(300, false), {}}}; + EXPECT_EQ(test_case.run(ArcGISClient{one_page}), ErrorCode::ParseError) << test_case.name; + + std::shared_ptr two_pages = std::make_shared(); + two_pages->responses = {{200, mercator_page(150, true), {}}, + {200, mercator_page(150, false), {}}}; + EXPECT_EQ(test_case.run(ArcGISClient{two_pages}), ErrorCode::ParseError) << test_case.name; + EXPECT_EQ(two_pages->requests.size(), 2u) << test_case.name; } } +TEST(ArcGISClientLimits, ProjectionBudgetSpansBothFireWeatherLayers) { + std::shared_ptr transport = std::make_shared(); + transport->responses = {{200, mercator_page(150, false), {}}, + {200, mercator_page(150, false), {}}}; + const ArcGISClient client{transport}; + + const Result result = client.query_fire_weather(1); + + ASSERT_FALSE(result) << result->features.size() << " features"; + EXPECT_EQ(result.error().code, ErrorCode::ParseError); + EXPECT_NE(result.error().message.find("added-point limit"), std::string::npos); + EXPECT_EQ(transport->requests.size(), 2u); +} + +TEST(ArcGISClientLimits, PagesWithinTheProjectionBudgetStillMerge) { + // 200 features need 759,800 added points. + std::shared_ptr transport = std::make_shared(); + transport->responses = {{200, mercator_page(100, true), {}}, + {200, mercator_page(100, false), {}}}; + const ArcGISClient client{transport}; + + const Result result = client.query_day4_8(4); + + ASSERT_TRUE(result) << result.error().message; + EXPECT_EQ(result->features.size(), 200u); +} + +TEST(ArcGISClientLimits, EachCallStartsWithAFreshProjectionBudget) { + std::shared_ptr transport = std::make_shared(); + transport->responses.assign(2, {200, mercator_page(200, false), {}}); + const ArcGISClient client{transport}; + + EXPECT_TRUE(client.query_day4_8(4)); + EXPECT_TRUE(client.query_day4_8(4)); +} + +TEST(ArcGISClientLimits, RawLayerQueryBoundsTheBytesItRetains) { + // A custom transport enforces no response limit of its own. + constexpr std::size_t kPageBytes = 1024UL * 1024UL; + const std::size_t limit = ClientConfig{}.max_response_bytes; + std::shared_ptr transport = + std::make_shared(kPageBytes, true); + const ArcGISClient client{transport}; + + const Result> result = + client.query_layer(ArcGISService::Outlooks, 1, {}); + + ASSERT_FALSE(result); + EXPECT_EQ(result.error().code, ErrorCode::InvalidRequest) << result.error().message; + EXPECT_LT(transport->calls, ArcGISPager{}.max_pages()); + EXPECT_LE(transport->bytes_served, limit + 2 * kPageBytes); +} + +TEST(ArcGISClientLimits, TypedQueriesBoundTheBytesOneCallAccepts) { + constexpr std::size_t kPageBytes = 1024UL * 1024UL; + const std::size_t limit = ClientConfig{}.max_response_bytes; + std::shared_ptr transport = + std::make_shared(kPageBytes, true); + const ArcGISClient client{transport}; + + const Result result = client.query_categorical(1); + + ASSERT_FALSE(result); + EXPECT_EQ(result.error().code, ErrorCode::InvalidRequest) << result.error().message; + EXPECT_LT(transport->calls, ArcGISPager{}.max_pages()); + EXPECT_LE(transport->bytes_served, limit + 2 * kPageBytes); +} + +TEST(ArcGISClientLimits, TheByteLimitSpansBothFireWeatherLayers) { + // Each layer's one page is under the limit; together they pass it. + const std::size_t limit = ClientConfig{}.max_response_bytes; + std::shared_ptr transport = + std::make_shared(limit / 2 + 1, false); + const ArcGISClient client{transport}; + + const Result result = client.query_fire_weather(1); + + ASSERT_FALSE(result); + EXPECT_EQ(result.error().code, ErrorCode::InvalidRequest) << result.error().message; + EXPECT_EQ(transport->calls, 2); +} + +TEST(ArcGISClientLimits, AnOversizedFirstPageLeavesNoRoomForASecond) { + const std::size_t limit = ClientConfig{}.max_response_bytes; + std::shared_ptr transport = + std::make_shared(limit + 1, true); + const ArcGISClient client{transport}; + + const Result> result = + client.query_layer(ArcGISService::Outlooks, 1, {}); + + ASSERT_FALSE(result); + EXPECT_EQ(result.error().code, ErrorCode::InvalidRequest) << result.error().message; + EXPECT_EQ(transport->calls, 2); +} + +TEST(CallBudget, CapsTheBytesOfEveryPageAfterTheFirst) { + detail::CallBudget budget{10}; + EXPECT_TRUE(budget.admits(4)); + budget.charge(4); + // An exact fit is allowed; one byte more is not. + EXPECT_TRUE(budget.admits(6)); + EXPECT_FALSE(budget.admits(7)); + budget.charge(6); + EXPECT_TRUE(budget.admits(0)); + EXPECT_FALSE(budget.admits(1)); +} + +TEST(CallBudget, AnOversizedFirstPageIsCountedButNotRefused) { + detail::CallBudget budget{10}; + EXPECT_TRUE(budget.admits(11)); + budget.charge(11); + EXPECT_FALSE(budget.admits(0)); + EXPECT_FALSE(budget.admits(1)); +} + +TEST(CallBudget, ZeroMeansNoByteLimit) { + // As ClientConfig::max_response_bytes = 0 means for one response. + detail::CallBudget budget{0}; + budget.charge(std::numeric_limits::max() / 2); + EXPECT_TRUE(budget.admits(std::numeric_limits::max() / 2)); +} + +TEST(CallBudget, TakesItsByteLimitFromAnHttpClientTransport) { + ClientConfig config; + config.max_response_bytes = 4096; + EXPECT_EQ(detail::call_byte_limit(HttpClient{config}), 4096u); + config.max_response_bytes = 0; + EXPECT_EQ(detail::call_byte_limit(HttpClient{config}), 0u); + // Any other transport has no limit to read. + EXPECT_EQ(detail::call_byte_limit(RecordingTransport{}), ClientConfig{}.max_response_bytes); +} + +TEST(ArcGISClientLimits, ASingleResponseIsTheTransportsToBound) { + // The limit is on what paging adds: one response over it is accepted. + const std::size_t limit = ClientConfig{}.max_response_bytes; + std::shared_ptr transport = + std::make_shared(limit + 1, false); + const ArcGISClient client{transport}; + + const Result> result = + client.query_layer(ArcGISService::Outlooks, 1, {}); + + ASSERT_TRUE(result) << result.error().message; + ASSERT_EQ(result->size(), 1u); + EXPECT_EQ((*result)[0].size(), transport->bytes_served); +} + // ===== What a 404 means ===== // // SPC's static products answer 404 when nothing is issued. Any other 404 is a @@ -514,6 +921,21 @@ TEST(ArcGISClientRouting, RejectsNonJsonFormatsBeforeNetworkAccess) { EXPECT_TRUE(transport->requests.empty()); } +TEST(ClientErrors, AnOversizedErrorBodyIsNotParsed) { + const std::string small = R"({"error":{"code":500,"message":"Busy"}})"; + std::string big = R"({"error":{"code":500,"message":"Busy","pad":")"; + big += std::string(70 * 1024, 'x') + R"("}})"; + + EXPECT_EQ(Error::from_response(500, small).message, "Busy"); + EXPECT_EQ(Error::from_response(500, big).message, big.substr(0, 256)); +} + +TEST(ClientErrors, OnlyStatuses500To599AreServerErrors) { + EXPECT_EQ(Error::from_response(500, "").code, ErrorCode::ServerError); + EXPECT_EQ(Error::from_response(599, "").code, ErrorCode::ServerError); + EXPECT_EQ(Error::from_response(600, "").code, ErrorCode::Unknown); +} + TEST(ClientErrors, AMalformedBodyIsAParseErrorNotAnException) { std::shared_ptr transport = std::make_shared(); transport->responses = {{200, "maintenance", {}}, @@ -530,6 +952,43 @@ TEST(ClientErrors, AMalformedBodyIsAParseErrorNotAnException) { EXPECT_EQ(from_arcgis.error().code, ErrorCode::ParseError); } +TEST(ClientErrors, AnAllocationFailureIsAnErrorNotAnException) { + std::shared_ptr transport = std::make_shared(); + const ArcGISClient arcgis{transport}; + const StaticFeedClient feeds{transport}; + const ArchiveClient archive{transport}; + + // One code for every client: the parsers' own. + EXPECT_EQ(outcome_of([&] { return arcgis.query_layer(ArcGISService::Outlooks, 1, {}); }), + ErrorCode::ParseError); + EXPECT_EQ(outcome_of([&] { return arcgis.query_categorical(1); }), ErrorCode::ParseError); + EXPECT_EQ(outcome_of([&] { return arcgis.query_probabilistic(1, "tornado"); }), + ErrorCode::ParseError); + EXPECT_EQ(outcome_of([&] { return arcgis.query_conditional_intensity(1, "tornado"); }), + ErrorCode::ParseError); + EXPECT_EQ(outcome_of([&] { return arcgis.query_day4_8(4); }), ErrorCode::ParseError); + EXPECT_EQ(outcome_of([&] { return arcgis.query_fire_weather(1); }), ErrorCode::ParseError); + EXPECT_EQ(outcome_of([&] { return arcgis.query_active_md(); }), ErrorCode::ParseError); + EXPECT_EQ(outcome_of([&] { return feeds.day_categorical(1); }), ErrorCode::ParseError); + EXPECT_EQ(outcome_of([&] { return feeds.day_probabilistic(1, "tornado"); }), + ErrorCode::ParseError); + EXPECT_EQ(outcome_of([&] { return feeds.day4_8(4); }), ErrorCode::ParseError); + EXPECT_EQ(outcome_of([&] { return archive.watches(); }), ErrorCode::ParseError); + EXPECT_EQ(outcome_of([&] { return archive.storm_reports("2026-05-19T12:00Z", "2026-05-20"); }), + ErrorCode::ParseError); +} + +TEST(ClientErrors, AnExceptionOutsideStdFromATransportPropagates) { + // Only a custom transport can throw one, and catching everything would also + // swallow a thread cancellation's forced unwind. + std::shared_ptr transport = + std::make_shared(); + const ArcGISClient client{transport}; + + EXPECT_THROW((void)client.query_categorical(1), int); + EXPECT_EQ(transport->calls, 2); +} + TEST(ClientErrors, UnsupportedEsriProjectionBecomesParseError) { std::shared_ptr transport = std::make_shared(); transport->responses = { diff --git a/tests/test_corpus.cpp b/tests/test_corpus.cpp index 08f4b1f..76bd5c1 100644 --- a/tests/test_corpus.cpp +++ b/tests/test_corpus.cpp @@ -1,7 +1,7 @@ /// @file test_corpus.cpp -/// @brief The captured SPC fixtures through the Day 1-3 parsers. spc-data -/// checks the same fixtures for identical output; this catches a regression -/// here first. +/// @brief The captured fixtures through every parser, including watches, +/// storm reports, and discussions. spc-data checks the Day 1-3 fixtures for +/// identical output; this catches a regression here first. #include "spc/models/mesoscale.hpp" #include "spc/models/outlook.hpp" diff --git a/tests/test_golden.cpp b/tests/test_golden.cpp index 7c9dd48..8817f36 100644 --- a/tests/test_golden.cpp +++ b/tests/test_golden.cpp @@ -200,12 +200,29 @@ bool has_case(std::string_view fixture) { [fixture](const GoldenCase& c) { return c.fixture == fixture; }); } +/// Regular files under `root`, named by their path relative to it, as +/// tools/verify_fixture_checksums.py names them. Dotfiles such as a Finder +/// .DS_Store and dot-directories are skipped. +std::vector files_under(const std::filesystem::path& root) { + std::vector names; + for (std::filesystem::recursive_directory_iterator it{root}; + it != std::filesystem::recursive_directory_iterator{}; ++it) { + if (it->path().filename().string().starts_with('.')) { + if (it->is_directory()) { + it.disable_recursion_pending(); + } + continue; + } + if (it->is_regular_file()) { + names.push_back(std::filesystem::relative(it->path(), root).generic_string()); + } + } + return names; +} + TEST(GoldenCoverage, EveryFixtureIsParsedOrListedAsNotParsed) { - for (const std::filesystem::directory_entry& entry : - std::filesystem::directory_iterator(SPC_FIXTURES_DIR)) { - const std::string name = entry.path().filename().string(); - // Dotfiles such as a Finder .DS_Store are not fixtures. - if (name == "README.md" || name == "SHA256SUMS" || name.starts_with('.')) { + for (const std::string& name : files_under(SPC_FIXTURES_DIR)) { + if (name == "README.md" || name == "SHA256SUMS") { continue; } EXPECT_TRUE(has_case(name) || std::ranges::find(kNotParsed, name) != kNotParsed.end()) @@ -218,12 +235,7 @@ TEST(GoldenCoverage, EveryFixtureIsParsedOrListedAsNotParsed) { } TEST(GoldenCoverage, EveryGoldenFileHasACase) { - for (const std::filesystem::directory_entry& entry : - std::filesystem::directory_iterator(SPC_GOLDEN_DIR)) { - const std::string name = entry.path().filename().string(); - if (name.starts_with('.')) { - continue; // .DS_Store and editor files - } + for (const std::string& name : files_under(SPC_GOLDEN_DIR)) { EXPECT_TRUE(name.ends_with(".txt") && has_case(name.substr(0, name.size() - 4))) << name << " has no golden case; delete it"; } diff --git a/tests/test_http_client.cpp b/tests/test_http_client.cpp index be2275c..06f890a 100644 --- a/tests/test_http_client.cpp +++ b/tests/test_http_client.cpp @@ -3,7 +3,9 @@ /// headers, scheme restrictions, and concurrent use. #include "spc/http_client.hpp" +#include "spc/retry.hpp" #include "support/loopback_server.hpp" +#include "support/tls_loopback_server.hpp" #include #include @@ -11,10 +13,12 @@ #include #include #include +#include #include #include #include #include +#include #include #include #include @@ -56,6 +60,40 @@ std::string chunked_response(std::size_t chunks, std::size_t chunk_size) { return response + "0\r\n\r\n"; } +void append_le32(std::string& out, std::uint32_t value) { + for (int shift = 0; shift < 32; shift += 8) { + out += static_cast((value >> shift) & 0xFFU); + } +} + +std::uint32_t crc32(std::string_view data) { + std::uint32_t crc = 0xFFFFFFFFU; + for (const char c : data) { + crc ^= static_cast(c); + for (int bit = 0; bit < 8; ++bit) { + crc = (crc >> 1) ^ (0xEDB88320U & (0U - (crc & 1U))); + } + } + return ~crc; +} + +/// `data` as gzip with one stored deflate block, so the encoded body is 23 +/// bytes longer than the decoded one. +std::string gzip_stored(std::string_view data) { + std::string out{"\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff", 10}; + const std::uint16_t size = static_cast(data.size()); + const std::uint16_t inverse = static_cast(~size); + out += '\x01'; // final block, stored + out += static_cast(size & 0xFFU); + out += static_cast(size >> 8); + out += static_cast(inverse & 0xFFU); + out += static_cast(inverse >> 8); + out += data; + append_le32(out, crc32(data)); + append_le32(out, static_cast(data.size())); + return out; +} + TEST(LoopbackServer, ShutdownIsBoundedWhenAClientStopsReading) { std::atomic responding{false}; std::unique_ptr server = @@ -145,9 +183,27 @@ TEST(HttpClient, AbsoluteHttpsSchemesNeverUseThePlaintextBaseUrl) { // plaintext base and succeeds, exposing an HTTPS request as an HTTP path. EXPECT_FALSE(client.get("HTTPS://127.0.0.1:0/absolute")); EXPECT_FALSE(client.get("hTtPs://127.0.0.1:0/absolute")); + // libcurl also fetches a one-slash https URL over TLS. + EXPECT_FALSE(client.get("hTtPs:/127.0.0.1:0/absolute")); EXPECT_TRUE(server.requests().empty()); } +TEST(HttpClient, AOneSlashHttpUrlBypassesTheBaseUrl) { + LoopbackServer base{[](const LoopbackRequest&) { return http_response("200 OK", "base"); }}; + LoopbackServer other{[](const LoopbackRequest&) { return http_response("200 OK", "other"); }}; + ClientConfig config; + config.base_url = base.url("/base/"); + const HttpClient client{config}; + const std::string one_slash = + "http:/" + other.url("/x").substr(std::string_view{"http://"}.size()); + + const Result response = client.get(one_slash); + + ASSERT_TRUE(response) << response.error().message; + EXPECT_EQ(response->body, "other"); + EXPECT_TRUE(base.requests().empty()); +} + TEST(HttpClient, RejectsInvalidTimeoutsBeforeSendingARequest) { LoopbackServer server{[](const LoopbackRequest&) { return http_response("200 OK", "{}"); }}; for (const std::chrono::seconds timeout : @@ -215,6 +271,72 @@ TEST(HttpClient, RejectsAChunkedBodyThatGrowsPastTheLimit) { EXPECT_NE(response.error().message.find("max_response_bytes"), std::string::npos); } +TEST(HttpClient, IgnoresTheSizeOfARedirectBodyItDiscards) { + // The chunked final response has no length of its own, so the hop's must + // not carry over to it. + LoopbackServer server{[](const LoopbackRequest& request) { + if (request.target == "/start") { + return http_response("302 Found", "REDIRECT-BODY", "Location: /final\r\n"); + } + if (request.target == "/start-chunked") { + return http_response("302 Found", "REDIRECT-BODY", "Location: /final-chunked\r\n"); + } + if (request.target == "/final-chunked") { + return chunked_response(1, 2); + } + return http_response("200 OK", "OK"); + }}; + ClientConfig config; + config.max_response_bytes = 8; + const HttpClient client{config}; + + const Result sized = client.get(server.url("/start")); + const Result chunked = client.get(server.url("/start-chunked")); + + ASSERT_TRUE(sized) << sized.error().message; + EXPECT_EQ(sized->status_code, 200); + EXPECT_EQ(sized->body, "OK"); + ASSERT_TRUE(chunked) << chunked.error().message; + EXPECT_EQ(chunked->body, "xx"); +} + +TEST(HttpClient, RefusesAnAdvertisedOversizeBodyBeforeReadingIt) { + // The server stops after 10 bytes. Reading on would end in a partial-body + // NetworkError, which with_retry would retry. + LoopbackServer server{[](const LoopbackRequest&) { + return std::string{"HTTP/1.1 200 OK\r\nContent-Length: 1048576\r\n" + "Connection: close\r\n\r\n0123456789"}; + }}; + ClientConfig config; + config.max_response_bytes = 1024; + const HttpClient client{config}; + + const Result response = client.get(server.url("/big")); + + ASSERT_FALSE(response); + EXPECT_EQ(response.error().code, ErrorCode::InvalidRequest); + EXPECT_NE(response.error().message.find("max_response_bytes"), std::string::npos); +} + +TEST(HttpClient, LimitsTheDecodedBodyNotTheEncodedOne) { + if ((curl_version_info(CURLVERSION_NOW)->features & CURL_VERSION_LIBZ) == 0) { + GTEST_SKIP() << "libcurl was built without gzip support"; + } + const std::string body(64, 'x'); + const std::string encoded = gzip_stored(body); + LoopbackServer server{[&encoded](const LoopbackRequest&) { + return http_response("200 OK", encoded, "Content-Encoding: gzip\r\n"); + }}; + ClientConfig config; + config.max_response_bytes = 80; // between the decoded 64 and the encoded 87 + const HttpClient client{config}; + + const Result response = client.get(server.url("/gzip")); + + ASSERT_TRUE(response) << response.error().message; + EXPECT_EQ(response->body, body); +} + TEST(HttpClient, RefusesEverySchemeOtherThanHttpAndHttps) { // file:// would otherwise read the local file and return it as the body. const HttpClient client; @@ -222,7 +344,17 @@ TEST(HttpClient, RefusesEverySchemeOtherThanHttpAndHttps) { const Result result = client.get("file:///etc/hosts"); ASSERT_FALSE(result) << "file:// must not be fetched"; - EXPECT_EQ(result.error().code, ErrorCode::NetworkError); + // Not NetworkError, which with_retry would retry. + EXPECT_EQ(result.error().code, ErrorCode::InvalidRequest); +} + +TEST(HttpClient, AMalformedUrlIsAnInvalidRequest) { + const HttpClient client; + + const Result result = client.get("http://bad host/"); + + ASSERT_FALSE(result); + EXPECT_EQ(result.error().code, ErrorCode::InvalidRequest); } TEST(HttpClient, RefusesARedirectToAnotherScheme) { @@ -234,7 +366,179 @@ TEST(HttpClient, RefusesARedirectToAnotherScheme) { const Result response = client.get(server.url("/escape")); ASSERT_FALSE(response); - EXPECT_EQ(response.error().code, ErrorCode::NetworkError); + EXPECT_EQ(response.error().code, ErrorCode::InvalidRequest); +} + +TEST(HttpClient, AFailedHttpsRequestLeavesHttpRedirectsAllowedOnItsHandle) { + LoopbackServer target{[](const LoopbackRequest&) { return http_response("200 OK", "ok"); }}; + LoopbackServer origin{[&target](const LoopbackRequest&) { + return http_response("302 Found", "", "Location: " + target.url("/final") + "\r\n"); + }}; + ClientConfig config; + config.timeout = std::chrono::seconds{1}; + const HttpClient client{config}; + + // Port zero cannot host a server; the request only configures the handle. + EXPECT_FALSE(client.get("https://127.0.0.1:0/")); + const Result response = client.get(origin.url("/start")); + + ASSERT_TRUE(response) << response.error().message; + EXPECT_EQ(response->body, "ok"); +} + +#ifdef SPC_TEST_HAS_TLS_SERVER + +using test::TlsLoopbackServer; + +/// TlsLoopbackServer's certificate is self-signed. +ClientConfig unverified_tls() { + ClientConfig config; + config.verify_ssl = false; + return config; +} + +bool curl_has_tls() { + return (curl_version_info(CURLVERSION_NOW)->features & CURL_VERSION_SSL) != 0; +} + +TEST(HttpClient, AnHttpsRequestRefusesARedirectToHttp) { + if (!curl_has_tls()) { + GTEST_SKIP() << "libcurl was built without TLS"; + } + // Anyone on the path can alter the plaintext hop, so its body must not + // come back as the answer to an https request. + LoopbackServer plain{ + [](const LoopbackRequest&) { return http_response("200 OK", "plaintext"); }}; + TlsLoopbackServer secure{[&plain](const LoopbackRequest& request) { + if (request.target == "/two-hops") { + return http_response("302 Found", "", "Location: /start\r\n"); + } + return http_response("302 Found", "", "Location: " + plain.url("/final") + "\r\n"); + }}; + const HttpClient client{unverified_tls()}; + ClientConfig based = unverified_tls(); + based.base_url = "HTTPS://" + secure.authority(); + const HttpClient based_client{based}; + + // libcurl ignores the scheme's case and takes one to three slashes after it. + std::vector>> responses; + for (const std::string scheme : {"https://", "HTTPS://", "hTtPs:/", "https:///"}) { + responses.emplace_back(scheme, client.get(scheme + secure.authority() + "/start")); + } + responses.emplace_back("base_url", based_client.get("/start")); + responses.emplace_back("https, https, http", client.get(secure.url("/two-hops"))); + + for (const std::pair>& entry : responses) { + const Result& response = entry.second; + if (response) { + ADD_FAILURE() << entry.first << " followed the redirect and returned \"" + << response->body << "\" with status " << response->status_code; + continue; + } + // Not NetworkError, which with_retry would retry. + EXPECT_EQ(response.error().code, ErrorCode::InvalidRequest) + << entry.first << ": " << response.error().message; + } + // Every request got its redirect: none failed earlier, in the handshake. + EXPECT_EQ(secure.requests().size(), responses.size() + 1); + EXPECT_TRUE(plain.requests().empty()) << "the plaintext server was contacted"; +} + +TEST(HttpClient, AnHttpsRequestFollowsHttpsRedirects) { + if (!curl_has_tls()) { + GTEST_SKIP() << "libcurl was built without TLS"; + } + TlsLoopbackServer target{[](const LoopbackRequest& request) { + if (request.target == "/relative") { + return http_response("302 Found", "", "Location: /final\r\n"); + } + return http_response("200 OK", "secure"); + }}; + TlsLoopbackServer origin{[&target](const LoopbackRequest&) { + return http_response("302 Found", "", "Location: " + target.url("/relative") + "\r\n"); + }}; + const HttpClient client{unverified_tls()}; + + const Result response = client.get(origin.url("/start")); + + ASSERT_TRUE(response) << response.error().message; + EXPECT_EQ(response->body, "secure"); + EXPECT_EQ(target.requests().size(), 2U); +} + +TEST(HttpClient, AnHttpRequestFollowsRedirectsToHttpAndHttps) { + if (!curl_has_tls()) { + GTEST_SKIP() << "libcurl was built without TLS"; + } + LoopbackServer plain{ + [](const LoopbackRequest&) { return http_response("200 OK", "plaintext"); }}; + TlsLoopbackServer secure{ + [](const LoopbackRequest&) { return http_response("200 OK", "secure"); }}; + LoopbackServer origin{[&plain, &secure](const LoopbackRequest& request) { + const std::string location = + request.target == "/to-https" ? secure.url("/final") : plain.url("/final"); + return http_response("302 Found", "", "Location: " + location + "\r\n"); + }}; + const HttpClient client{unverified_tls()}; + + const Result to_http = client.get(origin.url("/to-http")); + const Result to_https = client.get(origin.url("/to-https")); + + ASSERT_TRUE(to_http) << to_http.error().message; + EXPECT_EQ(to_http->body, "plaintext"); + ASSERT_TRUE(to_https) << to_https.error().message; + EXPECT_EQ(to_https->body, "secure"); +} + +TEST(HttpClient, ReusedHandlesTakeTheRedirectRuleOfEachRequest) { + if (!curl_has_tls()) { + GTEST_SKIP() << "libcurl was built without TLS"; + } + // Sequential requests share one pooled handle, so neither rule may outlive + // the request that set it. + LoopbackServer plain{[](const LoopbackRequest& request) { + if (request.target == "/start") { + return http_response("302 Found", "", "Location: /final\r\n"); + } + return http_response("200 OK", "plaintext"); + }}; + TlsLoopbackServer secure{[&plain](const LoopbackRequest&) { + return http_response("302 Found", "", "Location: " + plain.url("/final") + "\r\n"); + }}; + const HttpClient client{unverified_tls()}; + + const Result first_https = client.get(secure.url("/start")); + const Result http = client.get(plain.url("/start")); + const Result second_https = client.get(secure.url("/start")); + + EXPECT_FALSE(first_https); + ASSERT_TRUE(http) << http.error().message; + EXPECT_EQ(http->body, "plaintext"); + EXPECT_FALSE(second_https); + EXPECT_EQ(secure.requests().size(), 2U); // both https requests got their redirect + EXPECT_EQ(plain.requests().size(), 2U); // only the http request's two hops +} + +#else + +TEST(HttpClient, HttpsRedirectTestsNeedATlsServer) { + GTEST_SKIP() << "built without OpenSSL 3, so the https redirect tests did not run"; +} + +#endif // SPC_TEST_HAS_TLS_SERVER + +TEST(HttpClient, ARedirectLoopIsAServerErrorThatIsNotRetried) { + LoopbackServer server{[](const LoopbackRequest&) { + return http_response("302 Found", "", "Location: /loop\r\n"); + }}; + const HttpClient client; + + const Result response = client.get(server.url("/loop")); + + ASSERT_FALSE(response); + EXPECT_EQ(response.error().code, ErrorCode::ServerError); + EXPECT_FALSE(should_retry(response.error(), RetryPolicy{})); + EXPECT_EQ(server.requests().size(), 11U); // the first request and 10 redirects } TEST(HttpClient, ReusedHandlesKeepTheSchemeRestriction) { @@ -286,6 +590,8 @@ TEST(HttpClient, AMovedFromClientReportsAnErrorInsteadOfCrashing) { ASSERT_FALSE(response); EXPECT_EQ(response.error().code, ErrorCode::InvalidRequest); EXPECT_TRUE(moved.config().verify_ssl); + // NOLINTNEXTLINE(bugprone-use-after-move,clang-analyzer-cplusplus.Move) + EXPECT_EQ(original.config().max_response_bytes, ClientConfig{}.max_response_bytes); } } // namespace diff --git a/tests/test_locale.cpp b/tests/test_locale.cpp index 245f40f..552cacd 100644 --- a/tests/test_locale.cpp +++ b/tests/test_locale.cpp @@ -10,6 +10,7 @@ #include "spc/models/outlook.hpp" #include "support/fixtures.hpp" +#include #include #include #include @@ -118,14 +119,22 @@ TEST(LocaleIndependence, ProbabilisticOutlookKeepsItsIsoplethsUnderACommaDecimal GTEST_SKIP() << "de_DE.UTF-8 is not installed on this host"; } - const ProbOutlookPayload payload = - parse_probabilistic(read_fixture("arcgis_day1_prob_tornado.geojson"), 1, "tornado"); + // Each feature's dn repeats its label, so drop dn: only the "0.02"-style + // labels can then supply the probabilities. + std::string body = read_fixture("arcgis_day1_prob_tornado.geojson"); + for (std::size_t at = body.find("\"dn\""); at != std::string::npos; + at = body.find("\"dn\"", at)) { + body.replace(at, 4, "\"xx\""); + } + const ProbOutlookPayload payload = parse_probabilistic(body, 1, "tornado"); - ASSERT_GT(payload.features.size(), 0u); + ASSERT_EQ(payload.features.size(), 4u); + std::vector probabilities; for (const ProbOutlookFeature& f : payload.features) { - EXPECT_GE(f.probability, 0.01); - EXPECT_LE(f.probability, 1.0); + probabilities.push_back(f.probability); } + std::ranges::sort(probabilities); + EXPECT_EQ(probabilities, (std::vector{0.02, 0.05, 0.10, 0.15})); } TEST(LocaleIndependence, FireWeatherNoRiskSentinelIsStillDroppedUnderACommaDecimalLocale) { diff --git a/tests/test_parser.cpp b/tests/test_parser.cpp index e209ca5..dac44b4 100644 --- a/tests/test_parser.cpp +++ b/tests/test_parser.cpp @@ -2,9 +2,15 @@ /// @brief Day 1-3 parser behavior shared with spc-data. The assertions match /// spc-data's own parser tests. +#include "spc/models/convective.hpp" +#include "spc/models/fire_weather.hpp" #include "spc/models/outlook.hpp" +#include "spc/models/storm_report.hpp" +#include "spc/models/watch.hpp" +#include #include +#include namespace { @@ -118,4 +124,62 @@ TEST(Parser, UnknownLabelSkipped) { EXPECT_TRUE(p.features.empty()); } +// A ring inside a feature whose fields are replaced per case. +std::string esri_feature(std::string_view fields) { + return R"({"features":[{"attributes":)" + std::string{fields} + + R"(,"geometry":{"rings":[[[0,1],[1,1],[1,0],[0,0],[0,1]]]}}]})"; +} + +TEST(Parser, ConditionalIntensitySkipsFeaturesWithoutALabel) { + for (const char* fields : {"null", "42", "{}", R"({"dn":null})"}) { + EXPECT_TRUE( + parse_conditional_intensity(esri_feature(fields), 1, "tornado").features.empty()) + << fields; + } + EXPECT_EQ(parse_conditional_intensity(esri_feature(R"({"label":"CIG1"})"), 1, "tornado") + .features.size(), + 1u); +} + +TEST(Parser, FireWeatherSkipsDay1And2FeaturesWithNoBand) { + for (const char* fields : {"null", "42", "{}", R"({"dn":null})", R"({"dn":6})"}) { + EXPECT_TRUE( + parse_fire_weather(esri_feature(fields), 1, FireWeatherLayer::Outlook).features.empty()) + << fields; + } + const FireWeatherPayload kept = + parse_fire_weather(esri_feature(R"({"dn":8})"), 1, FireWeatherLayer::Outlook); + ASSERT_EQ(kept.features.size(), 1u); + EXPECT_EQ(kept.features[0].label, "CRIT"); +} + +TEST(Parser, NonFiniteNumericStringsReadAsZero) { + const StormReportPayload reports = parse_storm_reports( + R"({"features":[{"properties":{"magnitude":"nan","lon":"inf","lat":"-inf"},"geometry":null}]})"); + ASSERT_EQ(reports.reports.size(), 1u); + EXPECT_EQ(reports.reports[0].magnitude, 0.0); + EXPECT_TRUE(std::isfinite(reports.reports[0].location.lon)); + EXPECT_TRUE(std::isfinite(reports.reports[0].location.lat)); + + const WatchPayload watches = parse_watches( + R"({"features":[{"properties":{"max_hail_size":"inf","max_wind_gust_knots":"nan"},"geometry":null}]})"); + ASSERT_EQ(watches.watches.size(), 1u); + EXPECT_EQ(watches.watches[0].max_hail_size, 0.0); + EXPECT_EQ(watches.watches[0].max_wind_gust_knots, 0.0); +} + +TEST(Parser, ANullOrNonNumericLatestWkidFallsBackToWkid) { + const std::string ring = R"("geometry":{"rings":[[[0,1],[1,1],[1,0],[0,0],[0,1]]]})"; + for (const char* reference : + {R"({"wkid":4326,"latestWkid":null})", R"({"wkid":4326,"latestWkid":"4326"})"}) { + const std::string body = std::string{R"({"spatialReference":)"} + reference + + R"(,"features":[{"attributes":{"label":"CIG1"},)" + ring + "}]}"; + EXPECT_EQ(parse_conditional_intensity(body, 1, "tornado").features.size(), 1u) << reference; + } + // A geometry's null reference leaves the payload's in force. + const std::string body = + R"({"spatialReference":{"wkid":4326},"features":[{"attributes":{"label":"CIG1"},"geometry":{"spatialReference":null,"rings":[[[0,1],[1,1],[1,0],[0,0],[0,1]]]}}]})"; + EXPECT_EQ(parse_conditional_intensity(body, 1, "tornado").features.size(), 1u); +} + } // namespace diff --git a/tests/test_retry.cpp b/tests/test_retry.cpp index 195308c..3920d55 100644 --- a/tests/test_retry.cpp +++ b/tests/test_retry.cpp @@ -126,6 +126,16 @@ TEST(Retry, AHugeRetryAfterIsCappedInsteadOfOverflowing) { EXPECT_LE(delay, std::chrono::hours{24 * 365}); } +TEST(Retry, ARetryAfterBeyondInt64IsCappedLikeAnyHugeValue) { + const HttpResponse fits{429, "", {{"Retry-After", "9223372036854775807"}}}; + const HttpResponse too_big{429, "", {{"Retry-After", "99999999999999999999"}}}; + const HttpResponse negative{429, "", {{"Retry-After", "-99999999999999999999"}}}; + + EXPECT_EQ(retry_after(too_big), retry_after(fits)); + EXPECT_EQ(retry_after(fits), std::chrono::hours{24 * 365}); + EXPECT_EQ(retry_after(negative).count(), 0); +} + TEST(Retry, IgnoresAnUnparseableRetryAfterAndFallsBackToTheBackoff) { RetryPolicy policy; policy.max_attempts = 2; diff --git a/tools/cpp_auto_audit.py b/tools/cpp_auto_audit.py index 4a8e46e..f25b9fd 100644 --- a/tools/cpp_auto_audit.py +++ b/tools/cpp_auto_audit.py @@ -1,15 +1,19 @@ #!/usr/bin/env python3 """Fail when C++ sources use `auto` outside the allowed cases. -Allowed: structured bindings, lambda objects, iterator-like values, and any -line with an `auto-ok` comment. Everything else spells out its type. The allowlist file -takes one `relative/path.cpp|line snippet` rule per line, matched as a -substring of the stripped source line. +Allowed: structured bindings, lambda objects, iterators, and any line with an +`auto-ok` comment. Everything else spells out its type. Without type +information, an iterator is recognized only when the whole initializer is one +iterator-producing call, such as `v.begin()`, `m.find(k)` or `std::next(it)`. +Function and template parameters are not checked. The allowlist file takes +one `relative/path.cpp|line snippet` rule per line, matched as a substring of +the stripped source line. """ from __future__ import annotations import argparse +from collections.abc import Iterator import subprocess import sys from dataclasses import dataclass @@ -20,55 +24,36 @@ CPP_SUFFIXES = {".cpp", ".hpp", ".cc", ".cxx", ".hh", ".hxx"} DEFAULT_ALLOWLIST = "tools/cpp_auto_allowlist.txt" WORD_AUTO = re.compile(r"\bauto\b") +IDENTIFIER = re.compile(r"[A-Za-z_]\w*") DECL_SPECIFIERS = { "const", "constexpr", "static", "volatile", "mutable", "constinit", "thread_local", "extern", "inline", } -DECL_SPECIFIER = "(?:" + "|".join(sorted(DECL_SPECIFIERS)) + ")" -AUTO_DECLARATOR = rf"\bauto\b(?:\s*(?:[\*&]|{DECL_SPECIFIER}\b))*\s*" -STRUCTURED_BINDING = re.compile(AUTO_DECLARATOR + r"\[") -DECLARATION_AUTO = re.compile( - r"^\s*(?:}\s*)?(?:(?:else\s+)?if(?:\s+constexpr)?\s*\(\s*|(?:for|while|switch)\s*\(\s*)?" - + rf"(?:{DECL_SPECIFIER}\s+)*" - + AUTO_DECLARATOR - + rf"(?:(?!{DECL_SPECIFIER}\b)\w+\b(?!\s*[,)])|\[)" -) - -ITERATOR_HINTS = ( - ".begin(", - ".end(", - ".cbegin(", - ".cend(", - ".rbegin(", - ".rend(", - ".crbegin(", - ".crend(", - ".find(", - ".lower_bound(", - ".upper_bound(", - ".equal_range(", - "std::begin(", - "std::end(", - "std::cbegin(", - "std::cend(", - "std::find(", - "std::lower_bound(", - "std::upper_bound(", - "std::equal_range(", - "std::max_element(", - "std::min_element(", - "std::prev(", - "std::next(", -) +# Words that may precede `auto` without being a type-constraint (concept) name. +NOT_CONSTRAINTS = DECL_SPECIFIERS | { + "case", "co_await", "co_return", "co_yield", "consteval", "do", "else", "explicit", + "friend", "new", "operator", "requires", "return", "template", "throw", "typedef", + "typename", "using", "virtual", +} +CONDITION_KEYWORDS = {"for", "if", "switch", "while"} +PAIRS = {"(": ")", "[": "]", "{": "}"} +ITERATOR_MEMBERS = { + "begin", "end", "cbegin", "cend", "rbegin", "rend", "crbegin", "crend", + "find", "lower_bound", "upper_bound", "equal_range", +} +ITERATOR_FUNCTIONS = { # in namespace std + "begin", "end", "cbegin", "cend", "find", "lower_bound", "upper_bound", + "equal_range", "max_element", "min_element", "prev", "next", +} -@dataclass(frozen=True, slots=True) +@dataclass(frozen=True) class AllowRule: relative_path: str snippet: str -@dataclass(frozen=True, slots=True) +@dataclass(frozen=True) class Violation: relative_path: str line_number: int @@ -123,59 +108,224 @@ def read_allowlist(path: Path) -> list[AllowRule]: ) -def declaration_code(line: str) -> str: - """Hide declaration groups while preserving initializer character offsets.""" - tokens = list(CPP_TOKEN.finditer(line)) - code = list(line) - - def hide_group(start: int, opener: str, closer: str) -> int: - depth = 0 - for end in range(start, len(tokens)): - text = tokens[end][0] - if text == opener: - depth += 1 - elif text == closer: - depth -= 1 - if depth == 0: - code[tokens[start].start():tokens[end].end()] = " " * ( - tokens[end].end() - tokens[start].start() - ) - return end + 1 - return len(tokens) - - index = 0 - while index < len(tokens): - text = tokens[index][0] - if text == "[" and index + 1 < len(tokens) and tokens[index + 1][0] == "[": - index = hide_group(index, "[", "]") - elif text == "alignas" and index + 1 < len(tokens) and tokens[index + 1][0] == "(": - code[tokens[index].start():tokens[index].end()] = " " * len(text) - index = hide_group(index + 1, "(", ")") - elif text == "auto": - index += 1 +def group_end(tokens: list[str], start: int) -> int | None: + """Index of the bracket that closes tokens[start].""" + opener, closer = tokens[start], PAIRS[tokens[start]] + depth = 0 + for index in range(start, len(tokens)): + if tokens[index] == opener: + depth += 1 + elif tokens[index] == closer: + depth -= 1 + if depth == 0: + return index + return None + + +def group_start(tokens: list[str], end: int, opener: str) -> int | None: + """Index of the `opener` that tokens[end] closes.""" + closer = tokens[end] + depth = 0 + for index in range(end, -1, -1): + if tokens[index] == closer: + depth += 1 + elif tokens[index] == opener: + depth -= 1 + if depth == 0: + return index + return None + + +def code_tokens(source: str, start: int) -> Iterator[str]: + # Keep literals: find()'s arguments show whether it is a string search. + for token in CPP_TOKEN.finditer(source, start): + if not token[0].startswith(("//", "/*")): + yield token[0] + + +def statement_tokens(source: str, start: int) -> list[str]: + """Tokens through the `;` or unmatched closer that ends the statement.""" + tokens: list[str] = [] + depth = 0 + for text in code_tokens(source, start): + tokens.append(text) + if text in PAIRS: + depth += 1 + elif text in PAIRS.values(): + depth -= 1 + if depth < 0: + break + elif text == ";" and depth == 0: break - else: - index += 1 - parentheses = 0 + return tokens + + +def leading_expression(tokens: list[str]) -> list[str]: + """Tokens before the first ungrouped `,` or `;`, or an unmatched closer.""" + depth = 0 + for index, text in enumerate(tokens): + if text in PAIRS: + depth += 1 + elif text in PAIRS.values(): + if depth == 0: + return tokens[:index] + depth -= 1 + elif text in {",", ";"} and depth == 0: + return tokens[:index] + return tokens + + +def is_text_literal(token: str) -> bool: + return '"' in token or token.startswith("'") + + +def iterator_call(tokens: list[str]) -> bool: + """Whether the whole expression is one call that yields an iterator. + + The call may end a member chain, as in `a.items().begin()`, but not be an + operand or an argument. A type-free check cannot tell a container's find() + from std::string's, so find() counts only with one argument and no string + or character literal: string searches usually take a literal or a position. + """ + while tokens and tokens[0] == "(" and group_end(tokens, 0) == len(tokens) - 1: + tokens = tokens[1:-1] + if not tokens or not IDENTIFIER.fullmatch(tokens[0]): + return False + callee = [tokens[0]] + index = 1 + while ( + tokens[index:index + 2] == [":", ":"] + and index + 2 < len(tokens) + and IDENTIFIER.fullmatch(tokens[index + 2]) + ): + callee.append(tokens[index + 2]) + index += 3 + member = False + call: tuple[bool, list[str], list[str]] | None = None while index < len(tokens): - text = tokens[index][0] - if text in {"*", "&"} or text in DECL_SPECIFIERS: - index += 1 - elif text == "(": - code[tokens[index].start()] = " " - parentheses += 1 + if tokens[index] in {"(", "["}: + end = group_end(tokens, index) + if end is None: + return False + named_call = tokens[index] == "(" and callee + call = (member, callee, tokens[index + 1:end]) if named_call else None + callee = [] + index = end + 1 + elif tokens[index] == "." or tokens[index:index + 2] == ["-", ">"]: + index += 1 if tokens[index] == "." else 2 + if index == len(tokens) or not IDENTIFIER.fullmatch(tokens[index]): + return False + member, callee, call = True, [tokens[index]], None index += 1 + else: + return False + if call is None: + return False + member, callee, arguments = call + if not member: + return len(callee) == 2 and callee[0] == "std" and callee[1] in ITERATOR_FUNCTIONS + if callee != ["find"]: + return callee[0] in ITERATOR_MEMBERS + return ( + bool(arguments) + and leading_expression(arguments) == arguments + and not any(is_text_literal(token) for token in arguments) + ) + + +def placeholder_context(tokens: list[str], start: int) -> str: + """Say whether the `auto` at tokens[start] begins a declaration statement. + + Returns "statement", "new" for a new-expression, or "other" for + parameters, template arguments, trailing return types, and expressions. + Only the current line is visible, so a line start counts as a statement. + """ + index = start - 1 + # A type-constraint such as std::integral or std::convertible_to. + if index >= 1 and tokens[index] == ">" and tokens[index - 1] != "-": + opening = group_start(tokens, index, "<") + if ( + opening is not None + and opening >= 1 + and IDENTIFIER.fullmatch(tokens[opening - 1]) + and tokens[opening - 1] not in NOT_CONSTRAINTS + ): + index = opening - 1 + if index >= 0 and IDENTIFIER.fullmatch(tokens[index]) and tokens[index] not in NOT_CONSTRAINTS: + index -= 1 + while index >= 1 and tokens[index] == tokens[index - 1] == ":": + index -= 2 + if index < 0 or not IDENTIFIER.fullmatch(tokens[index]): + break + index -= 1 + while index >= 0: + if tokens[index] in DECL_SPECIFIERS: + index -= 1 + elif tokens[index] == "]" and index >= 1 and tokens[index - 1] == "]": + opening = group_start(tokens, index, "[") + if opening is None: + break + index = opening - 1 + elif tokens[index] == ")": + opening = group_start(tokens, index, "(") + if opening is None or opening == 0 or tokens[opening - 1] != "alignas": + break + index = opening - 2 else: break - if index < len(tokens) and re.fullmatch(r"[A-Za-z_]\w*", tokens[index][0]): + if index < 0 or tokens[index] in {";", "{", "}", "else", "do"}: + return "statement" + previous = tokens[index] + if previous == ":" and (index == 0 or tokens[index - 1] != ":"): + return "statement" # a label or access specifier + if previous == "new": + return "new" + if previous == ")": + opening = group_start(tokens, index, "(") + if opening is not None and opening >= 1 and tokens[opening - 1] == "new": + return "new" # placement new + if previous == "(": + keyword = index - 1 + if keyword >= 0 and tokens[keyword] == "constexpr": + keyword -= 1 + if keyword >= 0 and tokens[keyword] in CONDITION_KEYWORDS: + return "statement" + return "other" + + +def declaration_allowed(tokens: list[str]) -> bool: + """Apply the rules to the tokens that follow a declaration's `auto`.""" + index = 0 + parentheses = 0 + while index < len(tokens) and (tokens[index] in {"*", "&", "("} or tokens[index] in DECL_SPECIFIERS): + parentheses += tokens[index] == "(" index += 1 - while index < len(tokens) and tokens[index][0] == ")" and parentheses: - code[tokens[index].start()] = " " - parentheses -= 1 - index += 1 - while index + 1 < len(tokens) and tokens[index][0] == tokens[index + 1][0] == "[": - index = hide_group(index, "[", "]") - return "".join(code) + if index == len(tokens) or not IDENTIFIER.fullmatch(tokens[index]): + return True # a structured binding, or an expression such as auto{x} + index += 1 + while index < len(tokens) and tokens[index] == ")" and parentheses: + parentheses -= 1 + index += 1 + while tokens[index:index + 2] == ["[", "["]: + end = group_end(tokens, index) + if end is None: + return False + index = end + 1 + following = tokens[index] if index < len(tokens) else None + if following in {",", ")"}: + return True # a parameter continued from an earlier line + if following == "=": + initializer = tokens[index + 1:] + elif following in {"(", "{"}: + initializer = tokens[index + 1:group_end(tokens, index)] + else: + return False + lambda_allowed = lambda_initializer_allowed(initializer) + if lambda_allowed is not None: + return lambda_allowed + # Every declarator of one auto declaration deduces the same type, so the + # first initializer decides. + return iterator_call(leading_expression(initializer)) def lexical_context(source: str) -> tuple[str, set[int]]: @@ -203,54 +353,15 @@ def lexical_context(source: str) -> tuple[str, set[int]]: return "".join(pieces), opt_out_lines -def direct_iterator_return(body: list[str], declaration: str) -> bool: +def direct_iterator_return(body: list[str]) -> bool: # Recognize a single returned iterator call, not incidental calls elsewhere # in a body or an iterator comparison whose result is a bool. - if not body or body[0] != "return" or body[-1] != ";": - return False - expression = body[1:-1] - - def closing_paren(tokens: list[str], start: int) -> int | None: - depth = 0 - for index in range(start, len(tokens)): - if tokens[index] == "(": - depth += 1 - elif tokens[index] == ")": - depth -= 1 - if depth == 0: - return index - return None + return len(body) > 2 and body[0] == "return" and body[-1] == ";" and iterator_call(body[1:-1]) - while ( - expression - and expression[0] == "(" - and closing_paren(expression, 0) == len(expression) - 1 - ): - expression = expression[1:-1] - if "(" not in expression: - return False - opening = expression.index("(") - callee = "".join(expression[:opening]) - if not re.fullmatch(r"(?:[A-Za-z_]\w*(?:::|\.|->))*[A-Za-z_]\w*", callee): - return False - return ( - closing_paren(expression, opening) == len(expression) - 1 - and is_iterator_like(f"{declaration} = {callee}(") - ) - -def lambda_initializer_allowed( - line: str, source: str | None = None, offset: int = 0 -) -> bool | None: +def lambda_initializer_allowed(initializer: list[str]) -> bool | None: """Return None for another initializer, otherwise apply lambda/iterator rules.""" - equals = line.find("=") - if equals < 0: - return None - tokens = ( - token[0] - for token in CPP_TOKEN.finditer(source if source is not None else line, offset + equals + 1) - if token.lastgroup != "skip" - ) + tokens = iter(initializer) first = next(tokens, None) parentheses = 0 while first == "(": @@ -259,20 +370,18 @@ def lambda_initializer_allowed( if first != "[": return None - pairs = {"[": "]", "(": ")", "{": "}"} - def consume_group(opener: str, contents: list[str] | None = None) -> bool: - stack = [">" if opener == "<" else pairs[opener]] + stack = [">" if opener == "<" else PAIRS[opener]] for token in tokens: - if token in pairs: - stack.append(pairs[token]) + if token in PAIRS: + stack.append(PAIRS[token]) elif token == "<" and stack[-1] == ">": stack.append(">") elif token == ">" and stack[-1] == ">": stack.pop() if not stack: return True - elif token in pairs.values(): + elif token in PAIRS.values(): if token != stack.pop(): return False if not stack: @@ -316,7 +425,7 @@ def consume_group(opener: str, contents: list[str] | None = None) -> bool: break if parentheses or following not in {None, ";", ",", ")", "}"}: return False - return not invoked or direct_iterator_return(body, line.partition("=")[0]) + return not invoked or direct_iterator_return(body) if token in {"(", "["} or (first_header_token and token == "<"): if not consume_group(token): return False @@ -327,27 +436,24 @@ def consume_group(opener: str, contents: list[str] | None = None) -> bool: return False -def is_iterator_like(line: str) -> bool: - return any(hint in line for hint in ITERATOR_HINTS) - - -def is_allowed_auto( - line: str, source: str | None = None, offset: int = 0, *, auto_ok: bool = False -) -> bool: - line = declaration_code(line) - stripped = line.strip() - if auto_ok: - return True - if not DECLARATION_AUTO.search(stripped): - return True - if STRUCTURED_BINDING.search(stripped.partition("=")[0]): - return True - lambda_allowed = lambda_initializer_allowed(line, source, offset) - if lambda_allowed is not None: - return lambda_allowed - if is_iterator_like(stripped): - return True - return False +def line_allowed(code_line: str, source: str, line_offset: int) -> bool: + """Check every `auto` on a line whose literals and comments are blanked.""" + matches = list(CPP_TOKEN.finditer(code_line)) + tokens = [match[0] for match in matches] + for index, token in enumerate(tokens): + if token != "auto": + continue + start = end = index + if index >= 2 and tokens[index - 2:index] == ["decltype", "("] and tokens[index + 1:index + 2] == [")"]: + start, end = index - 2, index + 1 + context = placeholder_context(tokens, start) + if context == "new": + return False + if context == "statement" and not declaration_allowed( + statement_tokens(source, line_offset + matches[end].end()) + ): + return False + return True def matches_allowlist(relative_path: str, line: str, rules: list[AllowRule]) -> bool: @@ -371,7 +477,7 @@ def collect_violations(repo_root: Path, allowlist: list[AllowRule]) -> list[Viol code_line = code[line_offset:line_offset + len(line)] if not WORD_AUTO.search(code_line): continue - if is_allowed_auto(code_line, source, line_offset, auto_ok=line_number in opt_out_lines): + if line_number in opt_out_lines or line_allowed(code_line, source, line_offset): continue if matches_allowlist(relative_path, line, allowlist): continue diff --git a/tools/test_cpp_auto_audit.py b/tools/test_cpp_auto_audit.py index c9387a6..67579b9 100644 --- a/tools/test_cpp_auto_audit.py +++ b/tools/test_cpp_auto_audit.py @@ -132,6 +132,98 @@ def test_direct_iterator_returns_from_invoked_lambdas_are_allowed(self): result = self.audit() self.assertEqual(result.returncode, 0, result.stderr) + def test_placeholder_forms_and_mid_line_declarations_require_types(self): + sources = [ + "decltype(auto) value = compute();", + "static decltype(auto) value = compute();", + "std::integral auto value = 42;", + "const std::convertible_to> auto value = 42;", + "int a = 1; auto b = 2;", + "int f() { auto x = g(); return x; }", + "struct S { static constexpr auto value = 42; };", + "auto fn = [] { auto x = 1; return x; };", + "auto it = values.begin(); auto n = 5;", + "for (int i = 0; auto& value : values) {}", + "int* p = new auto(42);", + "int* p = new const auto{42};", + "int* p = new (buffer) auto(42);", + ] + for source in sources: + with self.subTest(source=source): + (self.root / "forms.cpp").write_text(source + "\n") + result = self.audit() + self.assertEqual(result.returncode, 1, result.stdout) + self.assertIn("forms.cpp:1", result.stderr) + + def test_iterator_call_must_be_the_whole_initializer(self): + declarations = [ + "auto pos = text.find(',');", + 'auto pos = text.find("ab");', + "auto pos = text.find(separator, 2);", + "auto pos = [&] { return text.find(','); }();", + "auto size = std::distance(v.begin(), v.end());", + "auto found = table.find(key) != table.end();", + "auto first = *values.begin();", + "auto value = lookup(values.begin());", + "auto next = values.begin() + 1;", + "auto value = values.begin()->second;", + "auto value = std::ranges::find(values, 42);", + "auto it(values.size());", + "auto value = values.begin()();", + "auto value =\n lookup(values.begin());", + ] + for declaration in declarations: + with self.subTest(declaration=declaration): + (self.root / "iterator.cpp").write_text(declaration + "\n") + result = self.audit() + self.assertEqual(result.returncode, 1, result.stdout) + self.assertIn("iterator.cpp:1", result.stderr) + + def test_whole_iterator_initializers_remain_allowed(self): + declarations = [ + "auto it = m.find(k);", + "auto it = m.find(make_key(a, b));", + "auto it = (m.find(k));", + "auto it{values.begin()};", + "auto it(values.cend());", + "auto it = config.layers().find(id);", + "auto it = this->items_.lower_bound(key);", + "auto range = table.equal_range(key);", + "auto it = std::find(values.begin(), values.end(), 42);", + "auto it = std::next(values.begin(), 2);", + "auto it = std::max_element(values.begin(), values.end());", + "auto a = values.begin(), b = values.end();", + "for (auto it = values.begin(); it != values.end(); ++it) {}", + "if (auto it = m.find(k); it != m.end()) {}", + "auto it = m.find(k); auto other = m.find(j);", + "auto it =\n m.upper_bound(k);", + ] + for declaration in declarations: + with self.subTest(declaration=declaration): + (self.root / "iterator.cpp").write_text(declaration + "\n") + result = self.audit() + self.assertEqual(result.returncode, 0, result.stderr) + + def test_parameters_bindings_and_lambdas_remain_allowed(self): + sources = [ + "for (auto& [k, v] : m) {}", + "auto f = [](int x) { return x; };", + "const auto& [a, b] = pair;", + "int f() { auto [a, b] = pair; return a; }", + "auto f = [](auto x, const auto& y) { return x + y; };", + "auto f = [](std::integral auto x) { return x; };", + "auto f = []() -> decltype(auto) { return 42; };", + "auto f = []() -> const auto& { return value; };", + "void g(std::integral auto value, auto&& other);", + "template int h();", + "template int h();", + ] + for source in sources: + with self.subTest(source=source): + (self.root / "allowed.cpp").write_text(source + "\n") + result = self.audit() + self.assertEqual(result.returncode, 0, result.stderr) + def test_structured_binding_declarations_remain_allowed(self): declarations = [ "auto [a, b] = pair;", diff --git a/tools/verify_fixture_checksums.py b/tools/verify_fixture_checksums.py index 02035d8..8976288 100755 --- a/tools/verify_fixture_checksums.py +++ b/tools/verify_fixture_checksums.py @@ -4,6 +4,7 @@ from __future__ import annotations import hashlib +import os from pathlib import Path import sys @@ -11,37 +12,56 @@ ROOT = Path(__file__).resolve().parents[1] FIXTURES = ROOT / "tests" / "fixtures" MANIFEST = FIXTURES / "SHA256SUMS" +PREFIX = "tests/fixtures/" # manifest paths are repository-relative, as shasum prints them +NOT_FIXTURES = {"README.md", "SHA256SUMS"} + + +def fixture_files() -> set[str]: + """Paths relative to tests/fixtures of every fixture, in any subdirectory.""" + found: set[str] = set() + for directory, subdirectories, names in os.walk(FIXTURES): + # Dotfiles such as a Finder .DS_Store are not fixtures. + subdirectories[:] = [name for name in subdirectories if not name.startswith(".")] + for name in names: + relative = (Path(directory) / name).relative_to(FIXTURES).as_posix() + if not name.startswith(".") and relative not in NOT_FIXTURES: + found.add(relative) + return found def main() -> int: - expected: dict[Path, str] = {} + expected: dict[str, str] = {} + first_lines: dict[str, int] = {} for line_number, line in enumerate(MANIFEST.read_text(encoding="utf-8").splitlines(), 1): - digest, separator, relative_name = line.partition(" ") - if not separator or len(digest) != 64: + digest, separator, name = line.partition(" ") + if not separator or len(digest) != 64 or not name.startswith(PREFIX): print(f"SHA256SUMS:{line_number}: malformed entry", file=sys.stderr) return 1 - expected[ROOT / relative_name] = digest + relative = name[len(PREFIX):] + # A later duplicate would otherwise hide a stale earlier digest. + if relative in expected: + print( + f"SHA256SUMS:{line_number}: duplicate entry for {name}" + f" (first on line {first_lines[relative]})", + file=sys.stderr, + ) + return 1 + expected[relative] = digest + first_lines[relative] = line_number - actual = { - path - for path in FIXTURES.iterdir() - # Dotfiles such as a Finder .DS_Store are not fixtures. - if path.is_file() - and not path.name.startswith(".") - and path.name not in {"README.md", "SHA256SUMS"} - } + actual = fixture_files() if actual != set(expected): - missing = sorted(str(path.relative_to(ROOT)) for path in actual - set(expected)) - stale = sorted(str(path.relative_to(ROOT)) for path in set(expected) - actual) - print(f"unlisted fixtures: {missing}", file=sys.stderr) - print(f"missing fixtures: {stale}", file=sys.stderr) + unlisted = sorted(PREFIX + path for path in actual - set(expected)) + missing = sorted(PREFIX + path for path in set(expected) - actual) + print(f"unlisted fixtures: {unlisted}", file=sys.stderr) + print(f"missing fixtures: {missing}", file=sys.stderr) return 1 failures: list[str] = [] - for path, wanted in expected.items(): - found = hashlib.sha256(path.read_bytes()).hexdigest() + for relative, wanted in expected.items(): + found = hashlib.sha256((FIXTURES / relative).read_bytes()).hexdigest() if found != wanted: - failures.append(f"{path.relative_to(ROOT)}: {found} != {wanted}") + failures.append(f"{PREFIX}{relative}: {found} != {wanted}") if failures: print("\n".join(failures), file=sys.stderr) return 1