diff --git a/CHANGELOG.md b/CHANGELOG.md index 37f7d0bd..516d75b4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ### Fixed +- A POST request to the API no longer waits without a limit when the API does not reply. A POST or PATCH request now fails after 120 seconds without data. The CLI does not retry a POST or PATCH request after a read timeout or a dropped connection, because the API can have processed it. This also applies to the POST request that uploads a file. - `repos create`/`update`, `upstream create`/`update` and `policy deny|license|vulnerability create`/`update` now report a usage error when the config file isn't valid JSON or doesn't hold a JSON object, for example a file with a trailing comma. Previously the command crashed with a `JSONDecodeError` or `AttributeError` traceback. - Fixed `cloudsmith download` choosing a package that can't be downloaded. The command also found packages that failed to sync or haven't finished, such as a failed duplicate upload of the same version. With `--yes`, it then chose the failed package, because it was uploaded last, and the download failed with HTTP 404. The command now searches only packages that have finished syncing. diff --git a/cloudsmith_cli/core/rest.py b/cloudsmith_cli/core/rest.py index f436b8a6..87591532 100644 --- a/cloudsmith_cli/core/rest.py +++ b/cloudsmith_cli/core/rest.py @@ -12,13 +12,15 @@ import requests.exceptions from cloudsmith_api.rest import ApiException, RESTClientObject -from .session import create_requests_session +from .session import NON_IDEMPOTENT_METHODS, create_requests_session logger = logging.getLogger(__name__) # The API CDN returns a 504 after 60 seconds, so the client waits longer than that. READ_TIMEOUT_SECONDS = 90 REQUEST_TIMEOUT_SECONDS = 30 +# The API CDN returns a 504 after 60 seconds without a reply from the API. +NON_IDEMPOTENT_READ_TIMEOUT_SECONDS = 120 Timeout = float | tuple[float | None, float | None] @@ -30,10 +32,10 @@ def _is_seconds(value: object) -> bool: def _resolve_timeout(method: str, timeout: Timeout | None) -> Timeout: """Return the timeout for a request, or the default when there is none.""" if timeout is None: - # A retry after a read timeout sends the request again, so a POST gets - # only a connect timeout and cannot create a second entry. - if method == "POST": - return (REQUEST_TIMEOUT_SECONDS, None) + # The session does not retry a read timeout for these methods, so + # a slow but healthy request gets more time. + if method in NON_IDEMPOTENT_METHODS: + return (REQUEST_TIMEOUT_SECONDS, NON_IDEMPOTENT_READ_TIMEOUT_SECONDS) return (REQUEST_TIMEOUT_SECONDS, READ_TIMEOUT_SECONDS) if _is_seconds(timeout): return timeout @@ -110,7 +112,7 @@ def request( :param _request_timeout: timeout setting for this request. If one number provided, it will be total request timeout. It can also be a pair (tuple) of (connection, read) timeouts. Without it, the request uses REQUEST_TIMEOUT_SECONDS to connect and READ_TIMEOUT_SECONDS - to read, but a POST gets no read timeout. + to read, but a POST or PATCH waits NON_IDEMPOTENT_READ_TIMEOUT_SECONDS for a reply. """ # Based on the RESTClientObject class generated by Swagger method = method.upper() diff --git a/cloudsmith_cli/core/session.py b/cloudsmith_cli/core/session.py index 6b2f3d58..eef4ad16 100644 --- a/cloudsmith_cli/core/session.py +++ b/cloudsmith_cli/core/session.py @@ -39,6 +39,9 @@ def get_default_proxy(): return _default_proxy["url"] +NON_IDEMPOTENT_METHODS = frozenset({"POST", "PATCH"}) + + class ApiKeySafeSession(requests.Session): """A session that drops the API key on a redirect to another host.""" @@ -94,8 +97,20 @@ def increment( find_certificate_error(error) is not None or _is_unknown_host_error(error) ): raise MaxRetryError(_pool, url, error) from error + if self._can_repeat_processed_request(method, error): + raise error return super().increment(method, url, response, error, _pool, _stacktrace) + def _can_repeat_processed_request( + self, method: str | None, error: Exception | None + ) -> bool: + """Return True when a retry can send a request that the server processed.""" + return ( + isinstance(method, str) + and method.upper() in NON_IDEMPOTENT_METHODS + and self._is_read_error(error) + ) + def sleep_for_retry(self, response=None): retry_after = self.get_retry_after(response) if retry_after: diff --git a/cloudsmith_cli/core/tests/test_rest.py b/cloudsmith_cli/core/tests/test_rest.py index 1d57f78a..c5e497f6 100644 --- a/cloudsmith_cli/core/tests/test_rest.py +++ b/cloudsmith_cli/core/tests/test_rest.py @@ -1,14 +1,22 @@ import socket import ssl +import threading +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer import httpretty import pytest import requests +import urllib3 from cloudsmith_api.rest import ApiException from urllib3.exceptions import MaxRetryError, NameResolutionError, ProxyError, SSLError from ..api.init import initialise_api -from ..rest import READ_TIMEOUT_SECONDS, REQUEST_TIMEOUT_SECONDS, RestClient +from ..rest import ( + NON_IDEMPOTENT_READ_TIMEOUT_SECONDS, + READ_TIMEOUT_SECONDS, + REQUEST_TIMEOUT_SECONDS, + RestClient, +) from ..session import create_requests_session @@ -25,9 +33,35 @@ def patch_httpretty_socket(monkeypatch): ) +@pytest.fixture +def silent_api(): + """Start an API server that reads each request and never replies.""" + requests_seen = [] + release = threading.Event() + + class SilentHandler(BaseHTTPRequestHandler): + def _read_and_stall(self): + self.rfile.read(int(self.headers.get("Content-Length", 0))) + requests_seen.append(self.command) + if self.path == "/drop": + self.close_connection = True + return + release.wait() + + do_GET = do_POST = do_PATCH = _read_and_stall + + server = ThreadingHTTPServer(("127.0.0.1", 0), SilentHandler) + threading.Thread(target=server.serve_forever, daemon=True).start() + yield f"http://127.0.0.1:{server.server_port}/", requests_seen + release.set() + server.shutdown() + server.server_close() + + class TestRestClient: + @pytest.mark.parametrize("method", ["GET", "POST", "PATCH"]) @httpretty.activate(allow_net_connect=False, verbose=True) - def test_implicit_retry_for_status_codes(self): + def test_implicit_retry_for_status_codes(self, method): """Assert that the rest client retries certain status codes automatically.""" # initialise_api() needs to be called before RestClient can be instantiated, # because it adds default attributes to cloudsmith_api.Configuration which @@ -38,7 +72,6 @@ def test_implicit_retry_for_status_codes(self): client = RestClient(backoff_factor=0) - method = "GET" url = "https://test.site" httpretty.register_uri( @@ -75,15 +108,14 @@ def _record_timeout(*args, **kwargs): "method, expected_timeout", [ ("GET", (REQUEST_TIMEOUT_SECONDS, READ_TIMEOUT_SECONDS)), - ("POST", (REQUEST_TIMEOUT_SECONDS, None)), + ("POST", (REQUEST_TIMEOUT_SECONDS, NON_IDEMPOTENT_READ_TIMEOUT_SECONDS)), ], ) @httpretty.activate(allow_net_connect=False) def test_sets_a_default_timeout(self, monkeypatch, method, expected_timeout): """A stalled API does not block the CLI forever. - A POST gets no read timeout, because the retry after a read timeout - sends the POST again and can create a second entry. + A POST gets a longer read timeout, because the CLI does not retry it. """ initialise_api() client = RestClient() @@ -114,6 +146,64 @@ def test_rejects_an_invalid_timeout(self): assert raised.value.status == 0 + @pytest.mark.parametrize( + "method, expected_requests", [("POST", 1), ("PATCH", 1), ("GET", 3)] + ) + def test_retries_a_read_timeout_only_for_idempotent_methods( + self, silent_api, method, expected_requests + ): + """A retry after a read timeout can repeat a POST that the server processed.""" + url, requests_seen = silent_api + initialise_api() + client = RestClient(retries=2, backoff_factor=0) + + with pytest.raises(ApiException): + client.request(method, url, _request_timeout=(1, 0.2)) + + assert len(requests_seen) == expected_requests + + @pytest.mark.parametrize( + "method, expected_requests", [("POST", 1), ("PATCH", 1), ("GET", 3)] + ) + def test_retries_a_dropped_connection_only_for_idempotent_methods( + self, silent_api, method, expected_requests + ): + """The server can drop the connection after it processed the request.""" + url, requests_seen = silent_api + initialise_api() + client = RestClient(retries=2, backoff_factor=0) + + with pytest.raises(ApiException): + client.request(method, f"{url}drop", body={}, _request_timeout=(1, 5)) + + assert len(requests_seen) == expected_requests + + @pytest.mark.parametrize("method", ["POST", "PATCH"]) + def test_retries_a_connect_error_for_non_idempotent_methods( + self, monkeypatch, method + ): + """The server cannot process a request that did not connect.""" + with socket.socket() as unused_socket: + unused_socket.bind(("127.0.0.1", 0)) + closed_port = unused_socket.getsockname()[1] + connect_attempts = [] + original_connect = urllib3.connection.HTTPConnection.connect + + def _count_connect(connection): + connect_attempts.append(connection.port) + return original_connect(connection) + + monkeypatch.setattr( + urllib3.connection.HTTPConnection, "connect", _count_connect + ) + initialise_api() + client = RestClient(retries=2, backoff_factor=0) + + with pytest.raises(ApiException): + client.request(method, f"http://127.0.0.1:{closed_port}/", body={}) + + assert len(connect_attempts) == 3 + class TestCreateRequestsSession: @pytest.fixture(autouse=True)