diff --git a/CLAUDE.md b/CLAUDE.md index 3b21945..600ecfb 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -41,7 +41,10 @@ RFC 7540). What `test/CMakeLists.txt` leaves out of that tree is the three tests nghttp2 session by hand -- `test_alt_svc`, `test_connection_end`, `test_h2c_upgrade` -- which are written against version 1's API and would not link. `build-h2v2-release/` is its Release tree, on OpenSSL so that `scripts/bench.sh -2` can put its -HTTP/2 beside the other two servers' with the HTTP/2 library as the only difference. +HTTP/2 beside the other two servers' with the HTTP/2 library as the only difference; +`build-corosio-h2v2-release/` is the same on COROSIO, and `build-corosio-h2v2-asan/` its ASAN tree: +the two runtimes resume a handler at different points of a connection's end, and some bugs of +`h2v2/` show on COROSIO only. `build-deploy/` is what gets installed: Release, on AWS-LC, without the tests and so without cfssl and the test PKI. `scripts/check_package.sh` installs it, moves the result and runs the programs diff --git a/CMakePresets.json b/CMakePresets.json index 8670697..84b4484 100644 --- a/CMakePresets.json +++ b/CMakePresets.json @@ -288,6 +288,32 @@ "CMAKE_BUILD_TYPE": "Release" } }, + { + "name": "build-corosio-h2v2-release", + "displayName": "COROSIO, nghttp2 v2, Release", + "description": "So that bench.sh -2 has nghttp2 version 2 on both I/O runtimes, beside build-h2v2-release/", + "inherits": [ + "clang", + "corosio", + "h2v2", + "base" + ], + "cacheVariables": { + "CMAKE_BUILD_TYPE": "Release" + } + }, + { + "name": "build-corosio-h2v2-asan", + "displayName": "COROSIO, nghttp2 v2, AddressSanitizer", + "description": "h2v2/ on the other runtime: its write path hands nghttp2 a pointer into the caller's buffer, and COROSIO resumes the writers differently", + "inherits": [ + "clang", + "corosio", + "h2v2", + "asan", + "base" + ] + }, { "name": "build-deploy", "displayName": "For cmake --install", @@ -372,6 +398,14 @@ "name": "build-h2v2-release", "configurePreset": "build-h2v2-release" }, + { + "name": "build-corosio-h2v2-release", + "configurePreset": "build-corosio-h2v2-release" + }, + { + "name": "build-corosio-h2v2-asan", + "configurePreset": "build-corosio-h2v2-asan" + }, { "name": "build-deploy", "configurePreset": "build-deploy" @@ -474,6 +508,16 @@ "name": "build-h2v2-release", "configurePreset": "build-h2v2-release", "inherits": "base" + }, + { + "name": "build-corosio-h2v2-release", + "configurePreset": "build-corosio-h2v2-release", + "inherits": "base" + }, + { + "name": "build-corosio-h2v2-asan", + "configurePreset": "build-corosio-h2v2-asan", + "inherits": "base" } ] } diff --git a/docs/benchmark.md b/docs/benchmark.md index 163e433..0ed3d6e 100644 --- a/docs/benchmark.md +++ b/docs/benchmark.md @@ -26,13 +26,17 @@ never against a number in this file. ### `scripts/bench.sh` ``` -scripts/bench.sh [-D seconds] [-c clients] [-m streams] [-t threads] [-i] [-p port] [-u path] [-P] [-n] [-N] [-v] +scripts/bench.sh [-D seconds] [-c clients] [-m streams] [-M depth] [-t threads] [-i] [-p port] [-u path] [-P] [-2] [-n] [-N] [-v] ``` It builds and starts the server of `build-openssl-release/` (ASIO) and `build-corosio-release/` (COROSIO) one after the other, configuring a missing tree with `scripts/configure.sh`, runs h2load over HTTP/1.1, HTTP/2 and HTTP/3 against each, and prints a table of req/s, MB/s, failures and -median/p99 latency, plus COROSIO/ASIO per protocol. +median/p99 latency, plus every row's req/s relative to the ASIO row of its protocol. +A row names what it measures in three columns: the server program (`anyhttp` or `nghttpd`), its +I/O runtime (`asio`, `corosio`; `libev` for nghttpd) and the protocol implementation (`beast` for +HTTP/1.1, `nghttp2` or `nghttp2v2` for HTTP/2, `ngtcp2+nghttp3` for HTTP/3), so that runtime and +implementation are never folded into one name. Both trees use OpenSSL, because COROSIO has to; `build-release/` is ASIO on AWS-LC, which is faster on its own (see [TLS library](#tls-library-aws-lc-vs-openssl)) and would skew the comparison. @@ -43,12 +47,20 @@ the server program on `/` ("Hello, World!") and `/test/*`; other paths are 404s as failed. With `-t N` it runs N workers, each an event loop to which the accepting thread hands connections round robin, which is closer to `-i` than to a shared context. +`-2` adds the trees over nghttp2 version 2 as two more HTTP/2 rows: `build-h2v2-release/` (ASIO) +and `build-corosio-h2v2-release/` (COROSIO), both on OpenSSL, implementation `nghttp2v2`, so that +the HTTP/2 library is the only difference from the `nghttp2` row of the same runtime. + +On a terminal the runtimes are in colour (asio blue, corosio orange, nghttpd grey), and so are the +implementations and the ratios that differ from ASIO by more than 5%; `NO_COLOR` turns that off. + | option | meaning | |---|---| | `-t N` | the servers run on N threads, h2load gets as many (up to the clients) | | `-i` | with `-t`: one I/O context and one server per thread instead of one shared context, see below | | `-M N` | h1 pipelining depth, default 1 (no pipelining), see below | | `-P` | plaintext HTTP/1.1 and HTTP/2 (prior knowledge), no HTTP/3 | +| `-2` | also the nghttp2 version 2 builds, for HTTP/2 | | `-u path` | request path, default `/` ("Hello, World!", nothing else) | | `-n` | don't build first | | `-N` | leave out nghttpd | diff --git a/include/anyhttp/h2/session.hpp b/include/anyhttp/h2/session.hpp index 9a05058..25ab569 100644 --- a/include/anyhttp/h2/session.hpp +++ b/include/anyhttp/h2/session.hpp @@ -78,6 +78,9 @@ class NGHttp2Session : public anyhttp::Session::Impl // has run out of data to send, whether nghttp2 still wants to read or not. bool recv_done_ = false; + // Set once a GOAWAY of ours has gone out, see recv_loop(). + bool goaway_sent_ = false; + // Set when writing to the stream has failed, for which a TLS "close_notify" comes too late. bool send_failed_ = false; @@ -87,9 +90,6 @@ class NGHttp2Session : public anyhttp::Session::Impl // Set once a client session's do_session() is over. Event session_done_; - // Set once a GOAWAY of ours has gone out, see recv_loop(). - bool goaway_sent_ = false; - void start_write(); // ---------------------------------------------------------------------------------------------- diff --git a/include/anyhttp/h2/stream.hpp b/include/anyhttp/h2/stream.hpp index f84fbfd..5547ddf 100644 --- a/include/anyhttp/h2/stream.hpp +++ b/include/anyhttp/h2/stream.hpp @@ -85,8 +85,6 @@ class NGHttp2Stream : public std::enable_shared_from_this public: size_t bytesRead = 0; size_t bytesWritten = 0; // TODO: not used, yet - size_t pending = 0; // TODO: not used, yet - size_t unhandled = 0; // TODO: not used, yet // ---------------------------------------------------------------------------------------------- diff --git a/scripts/bench.sh b/scripts/bench.sh index e707c57..7ce9dfd 100755 --- a/scripts/bench.sh +++ b/scripts/bench.sh @@ -1,7 +1,9 @@ #!/usr/bin/env bash # # Runs h2load against the standalone server of the ASIO and the COROSIO release builds, over -# HTTP/1.1, HTTP/2 and HTTP/3 (all TLS), and prints a summary table. Both builds use OpenSSL, as +# HTTP/1.1, HTTP/2 and HTTP/3 (all TLS), and prints a summary table. Each row names the server +# program, its I/O runtime and the protocol implementation: Beast for HTTP/1.1, nghttp2 (or +# nghttp2v2) for HTTP/2, ngtcp2+nghttp3 for HTTP/3. Both builds use OpenSSL, as # COROSIO has to, so that the TLS library is not part of the comparison: build-openssl-release/ # is the ASIO one (build-release/ is ASIO on AWS-LC). # @@ -11,9 +13,14 @@ # gets as many workers, an event loop each, to which its accepting thread hands the connections # round robin -- closer to -i than to one shared context. # -# With -2 the ASIO build over nghttp2 version 2, the rewrite, runs an HTTP/2 row as well: -# build-h2v2-release/, on OpenSSL again, so that the HTTP/2 library is all that differs from the -# asio row. Only HTTP/2 -- its HTTP/1.1 and HTTP/3 are the same backends as that row's. +# With -2 the builds over nghttp2 version 2, the rewrite, run an HTTP/2 row each as well: +# build-h2v2-release/ (ASIO) and build-corosio-h2v2-release/ (COROSIO), both on OpenSSL again, so +# that the HTTP/2 library is all that differs from the nghttp2 row of the same runtime. Only +# HTTP/2 -- their HTTP/1.1 and HTTP/3 are the same backends as the other rows'. +# +# On a terminal the runtimes and implementations are in colour, asio blue, corosio orange and +# nghttpd grey, and so are the ratios that differ from asio by more than 5%, green or red. +# NO_COLOR turns that off. # # Usage: scripts/bench.sh [-D seconds] [-c clients] [-m streams] [-M depth] [-t threads] [-i] # [-p port] [-u path] [-P] [-2] [-n] [-N] [-v] @@ -28,7 +35,7 @@ # -p port (default 18080) # -u request path (default /) # -P plaintext: HTTP/1.1 and HTTP/2 (prior knowledge) without TLS; no HTTP/3 -# -2 also run the nghttp2 version 2 build, for HTTP/2 +# -2 also run the nghttp2 version 2 builds, for HTTP/2 # -n don't build the servers first (otherwise a missing tree is configured with # scripts/configure.sh) # -N leave out nghttpd @@ -75,12 +82,56 @@ done root=$(cd "$(dirname "$0")/.." && pwd) cd "$root" +# A server here is one build, or nghttpd: what it is made of is in the columns of the table. declare -A trees=([asio]=build-openssl-release [corosio]=build-corosio-release - [h2v2]=build-h2v2-release) + [asio-h2v2]=build-h2v2-release [corosio-h2v2]=build-corosio-h2v2-release) +declare -A program=([asio]=anyhttp [corosio]=anyhttp [asio-h2v2]=anyhttp [corosio-h2v2]=anyhttp + [nghttpd]=nghttpd) +declare -A runtime=([asio]=asio [corosio]=corosio [asio-h2v2]=asio [corosio-h2v2]=corosio + [nghttpd]=libev) +declare -A impl=([h1]=beast [h2]=nghttp2 [h3]=ngtcp2+nghttp3 [asio-h2v2,h2]=nghttp2v2 + [corosio-h2v2,h2]=nghttp2v2) + +# Prints the implementation of protocol $2 in server $1 +impl_of() { + if [[ -v impl[$1,$2] ]]; then echo "${impl[$1,$2]}"; else echo "${impl[$2]}"; fi +} + +# SGR colours (256-colour palette) of the runtimes, of nghttpd, and of the implementations +declare -A color=([asio]='38;5;33' [corosio]='38;5;208' [libev]='38;5;245' [nghttpd]='38;5;245' + [beast]='38;5;141' [nghttp2]='38;5;37' [nghttp2v2]='38;5;44' [ngtcp2+nghttp3]='38;5;170' + [better]='38;5;70' [worse]='38;5;160') +color_out=0 color_err=0 +if [[ ! -v NO_COLOR ]]; then + [[ -t 1 ]] && color_out=1 + [[ -t 2 ]] && color_err=1 +fi + +# Prints $3 padded to $2 columns, in the colour of $4 (default: of $3) if $1 is 1. Padding first, +# so that the escape sequences do not count towards the width. +paint() { + local text key=${4:-$3} + printf -v text "%-${2}s" "$3" + if (($1)) && [[ -v color[$key] ]]; then + printf '\e[%sm%s\e[0m' "${color[$key]}" "$text" + else + printf '%s' "$text" + fi +} + +# Prints the columns that tell the servers apart: the program and runtime of server $2 and its +# implementation of protocol $3, in colour if $1 is 1 +columns() { + paint "$1" 8 "${program[$2]}" + printf ' ' + paint "$1" 8 "${runtime[$2]}" + printf ' ' + paint "$1" 15 "$(impl_of "$2" "$3")" +} protocols=(h1 h2 h3) [[ $scheme == http ]] && protocols=(h1 h2) servers=(asio corosio) -((h2v2)) && servers+=(h2v2) +((h2v2)) && servers+=(asio-h2v2 corosio-h2v2) if ((nghttpd)); then if command -v nghttpd >/dev/null; then servers+=(nghttpd) @@ -171,7 +222,7 @@ start_server() { kill -0 "$server_pid" 2>/dev/null || break sleep 0.1 done - echo "error: $server server did not come up on port $port, see $log" >&2 + echo "error: ${program[$server]} (${runtime[$server]}) did not come up on port $port, see $log" >&2 exit 1 } @@ -192,15 +243,15 @@ declare -A results for server in "${servers[@]}"; do start_server "$server" for proto in "${protocols[@]}"; do - # nghttpd serves HTTP/2 only, and the h2v2 build is the asio one below HTTP/2 - [[ $server == @(nghttpd|h2v2) && $proto != h2 ]] && continue + # nghttpd serves HTTP/2 only, and the h2v2 builds differ from the others only in HTTP/2 + [[ $server == @(nghttpd|*-h2v2) && $proto != h2 ]] && continue args=(-D "$duration" -c "$clients" -m "$streams" -t "$h2load_threads") case $proto in h1) args=(--h1 -D "$duration" -c "$clients" -m "$pipeline" -t "$h2load_threads") ;; h3) args+=(--h3) ;; esac log=$out/h2load-$server-$proto.log - printf '%-8s %-3s ... ' "$server" "$proto" >&2 + { printf '%-5s ' "$proto"; columns $color_err "$server" "$proto"; printf ' ... '; } >&2 if h2load "${args[@]}" "$scheme://127.0.0.1:$port$path" >"$log" 2>&1; then results[$server,$proto]=$(parse "$log") echo "$(cut -d' ' -f1 <<<"${results[$server,$proto]}") req/s" >&2 @@ -216,27 +267,33 @@ echo echo "h2load -D ${duration}s -c $clients -m $streams -t $h2load_threads (h1: -m $pipeline)," \ "$scheme://127.0.0.1:$port$path, server threads: $threads${independent:+ (independent)}" echo -row='%-8s %-5s %12s %11s %10s %7s %9s %9s\n' -printf "$row" server proto req/s MB/s succeeded failed median p99 -printf "$row" -------- ----- ------------ ----------- ---------- ------- --------- --------- +row='%-5s %-8s %-8s %-15s %12s %11s %10s %7s %9s %9s\n' +printf "$row" proto server runtime impl req/s MB/s succeeded failed median p99 +printf "$row" ----- -------- -------- --------------- ------------ ----------- ---------- ------- \ + --------- --------- for proto in "${protocols[@]}"; do for server in "${servers[@]}"; do [[ -v results[$server,$proto] ]] || continue read -r rps mbs ok failed median p99 <<<"${results[$server,$proto]}" - printf "$row" "$server" "$proto" "$rps" "$mbs" "$ok" "$failed" "$median" "$p99" + printf '%-5s %s %12s %11s %10s %7s %9s %9s\n' "$proto" \ + "$(columns $color_out "$server" "$proto")" "$rps" "$mbs" "$ok" "$failed" "$median" "$p99" done done -# corosio and nghttpd relative to asio, per protocol +# Every other row's req/s relative to the asio build's, per protocol echo +echo "req/s relative to anyhttp on asio with $(impl_of asio h1), $(impl_of asio h2)," \ + "$(impl_of asio h3):" for proto in "${protocols[@]}"; do a=$(cut -d' ' -f1 <<<"${results[asio,$proto]}") for server in "${servers[@]:1}"; do [[ -v results[$server,$proto] ]] || continue b=$(cut -d' ' -f1 <<<"${results[$server,$proto]}") if [[ $a != - && $b != - ]]; then - awk -v p="$proto" -v s="$server" -v a="$a" -v b="$b" \ - 'BEGIN { printf "%s: %s/asio = %.2f\n", p, s, (a > 0 ? b / a : 0) }' + ratio=$(awk -v a="$a" -v b="$b" 'BEGIN { printf "%.2f", (a > 0 ? b / a : 0) }') + key=$(awk -v r="$ratio" 'BEGIN { print (r >= 1.05 ? "better" : r <= 0.95 ? "worse" : "") }') + printf '%-5s %s %s\n' "$proto" "$(columns $color_out "$server" "$proto")" \ + "$(paint $color_out 5 "$ratio" "$key")" fi done done diff --git a/src/h2/stream.cpp b/src/h2/stream.cpp index 26937d8..8b01ad4 100644 --- a/src/h2/stream.cpp +++ b/src/h2/stream.cpp @@ -258,6 +258,19 @@ void NGHttp2Writer::async_submit(StatusHandler&& handler, unsigned int sta return; } + // + // A stream that has been closed has no say in nghttp2 any more, which would refuse the response + // anyway. Nor need there be an nghttp2 to ask: when the connection ends, nghttp2 closes the + // streams that are left and do_session() deletes the session, while a handler that is yet to + // submit can still hold one of them. Which of the two it meets is up to the scheduler. + // + if (stream->closed) + { + mlogw("async_submit: stream already closed"); + std::move(handler)(errors::connection_aborted); + return; + } + using namespace boost::beast::http; logd("[{}] {} {}", stream->log_prefix(), status_code, obsolete_reason(int_to_status(status_code))); @@ -493,7 +506,8 @@ void NGHttp2Stream::call_read_handler(boost::asio::const_buffer view) // // https://github.com/nghttp2/nghttp2/blob/a60e00c6285b5dd6a1aee80f70bce77d0c198288/lib/nghttp2_helper.c#L248 // - if (consumed) + // Not for a closed stream, whose rest of the body may be read after the connection is gone. + if (consumed && !closed) { nghttp2_session_consume_stream(parent.session, id, consumed); mlogd("consumed {} bytes, windows sizes: local {} / remote {}", consumed, diff --git a/src/h2v2/stream.cpp b/src/h2v2/stream.cpp index 71dc20e..ff2344c 100644 --- a/src/h2v2/stream.cpp +++ b/src/h2v2/stream.cpp @@ -272,6 +272,19 @@ void NGHttp2Writer::async_submit(StatusHandler&& handler, unsigned int sta return; } + // + // A stream that has been closed has no say in nghttp2 any more, which would refuse the response + // anyway. Nor need there be an nghttp2 to ask: when the connection ends, the send loop closes + // the streams that are left and do_session() deletes the connection, while a handler that is + // yet to submit can still hold one of them. Which of the two it meets is up to the scheduler. + // + if (stream->closed) + { + mlogw("async_submit: stream already closed"); + std::move(handler)(errors::connection_aborted); + return; + } + using namespace boost::beast::http; logd("[{}] {} {}", stream->log_prefix(), status_code, obsolete_reason(int_to_status(status_code))); @@ -497,7 +510,8 @@ void NGHttp2Stream::call_read_handler(boost::asio::const_buffer view) // The connection window was already reopened where the data arrived, see on_recv_data(): only // the stream-level credit is what the reader's pace is allowed to hold back. // - if (consumed) + // Not for a closed stream, whose rest of the body may be read after the connection is gone. + if (consumed && !closed) { nghttp2_conn_extend_max_stream_offset(parent.conn, id, consumed); mlogd("consumed {} bytes", consumed);