From dd5aedf4b19bcd0acdb2fe1b63a058d1bb57e1b5 Mon Sep 17 00:00:00 2001 From: strtgbb <146047128+strtgbb@users.noreply.github.com> Date: Thu, 19 Nov 2026 12:17:09 -0500 Subject: [PATCH] Do not re-pull integration images the job already prefetched Port of https://github.com/ClickHouse/ClickHouse/pull/120107. `docker compose pull` asks the registry about every service image even when the daemon already has it, so a transient denial fails a module that needed no download. Skip that pull only when every image of the project is one this job actually pulled and is still present. The default server image on this branch is `altinityinfra/integration-test`. Co-authored-by: Cursor --- ci/jobs/integration_test_job.py | 43 +++++++++++++++---- .../scripts/prefetch-integration-test-images | 16 ++++++- tests/integration/helpers/cluster.py | 40 +++++++++++++---- 3 files changed, 82 insertions(+), 17 deletions(-) diff --git a/ci/jobs/integration_test_job.py b/ci/jobs/integration_test_job.py index 9285fa08402ae..f66b9bd702780 100644 --- a/ci/jobs/integration_test_job.py +++ b/ci/jobs/integration_test_job.py @@ -2,9 +2,10 @@ import os import re import subprocess +import tempfile import time from pathlib import Path -from typing import List, Optional, Tuple +from typing import List, Optional, Set, Tuple import yaml # NOTE (strtgbb): Used for loading broken tests rules @@ -340,13 +341,21 @@ def replace_var(m: re.Match) -> str: def prefetch_images( - images: List[str], retries: int = 3, pull_timeout: int = 300 + images: List[str], + retries: int = 3, + pull_timeout: int = 300, + fetched_out: Optional[Set[str]] = None, ) -> bool: """Pull every image in parallel using `ci/prefetch-integration-test-images`. Images with no manifest for the current architecture (e.g. amd64-only images on arm64 runners) are silently skipped. Returns True on success, False if any image fails to pull for a real reason. + + `fetched_out`, when given, receives the references the script reports as actually + pulled. A missing or short report can only leave references out, so a reporting + failure costs the skip in `tests/integration/helpers/cluster.py` instead of claiming + an image that was never fetched. """ if not images: print("No images to pre-fetch.") @@ -358,11 +367,19 @@ def prefetch_images( "PULL_RETRIES": str(retries), "PULL_TIMEOUT": str(pull_timeout), } - return Shell.check( - f"{script} {' '.join(images)}", - verbose=True, - env=env, - ) + report = "" + with tempfile.TemporaryDirectory(prefix="prefetch_", dir=temp_path) as report_dir: + if fetched_out is not None: + report = os.path.join(report_dir, "fetched.txt") + env["PREFETCH_FETCHED_FILE"] = report + ok = Shell.check( + f"{script} {' '.join(images)}", + verbose=True, + env=env, + ) + if fetched_out is not None and Path(report).is_file(): + fetched_out.update(Path(report).read_text(errors="replace").split()) + return ok def parse_args(): @@ -1012,12 +1029,19 @@ def main(): + ", ".join(str(f.name) for f in compose_files) ) images_to_prefetch = get_images_from_compose_files(compose_files) - if not prefetch_images(images_to_prefetch): + prefetched: Set[str] = set() + if not prefetch_images(images_to_prefetch, fetched_out=prefetched): Result.create_from( status=Result.Status.ERROR, info="Failed to pre-pull Docker images needed by the test batch", labels=[Result.Label.INFRA], ).complete_job() + # A batch's compose files need not yield the default server image, but a project's own + # enumeration can: it is the default instance image and Keeper's. So prefetch it separately, and + # ignore the result: a failed fetch only leaves it out of the export, which turns the skip off. + server_image = f"altinityinfra/integration-test:{os.environ['DOCKER_BASE_TAG']}" + if server_image not in prefetched: + prefetch_images([server_image], fetched_out=prefetched) test_env = { "CLICKHOUSE_TESTS_BASE_CONFIG_DIR": clickhouse_server_config_dir, @@ -1027,6 +1051,9 @@ def main(): "CLICKHOUSE_USE_OLD_ANALYZER": "1" if use_old_analyzer else "0", "CLICKHOUSE_USE_DISTRIBUTED_PLAN": "1" if use_distributed_plan else "0", "CLICKHOUSE_USE_DATABASE_DISK": "1" if use_database_disk else "0", + # Read by tests/integration/helpers/cluster.py: the references this job pulled. A reference + # outside this set was not fetched here and may be a stale floating tag, so it is pulled. + "CLICKHOUSE_TESTS_PREFETCHED_IMAGES": " ".join(sorted(prefetched)), "PYTEST_CLEANUP_CONTAINERS": "1", "JAVA_PATH": java_path, # PromQL compliance: deterministic JSON for post-hook (see promql_compliance_hook.py). diff --git a/ci/jobs/scripts/prefetch-integration-test-images b/ci/jobs/scripts/prefetch-integration-test-images index 0eca3f83817f2..e2e732ef1a28c 100755 --- a/ci/jobs/scripts/prefetch-integration-test-images +++ b/ci/jobs/scripts/prefetch-integration-test-images @@ -10,6 +10,7 @@ # Environment variables: # PULL_RETRIES Number of pull attempts per image (default: 3) # PULL_TIMEOUT Per-attempt timeout in seconds (default: 300) +# PREFETCH_FETCHED_FILE If set, the references actually pulled are reported to this path set -uo pipefail @@ -34,6 +35,7 @@ pull_one() { local image="$1" local fail_file="$2" + local fetched_file="$3" local attempt out start elapsed for ((attempt = 1; attempt <= PULL_RETRIES; attempt++)); do @@ -42,6 +44,7 @@ pull_one() if out=$(timeout "$PULL_TIMEOUT" docker pull "$image" 2>&1); then elapsed=$((SECONDS - start)) echo "Pulled $image in ${elapsed}s" + echo "$image" > "$fetched_file" return 0 fi elapsed=$((SECONDS - start)) @@ -67,7 +70,7 @@ pids=() for image in "${images[@]}"; do # Sanitize image name to a safe filename. safe="${image//[^a-zA-Z0-9._-]/_}" - pull_one "$image" "$work_dir/fail_${safe}" & + pull_one "$image" "$work_dir/fail_${safe}" "$work_dir/fetched_${safe}" & pids+=($!) done @@ -77,6 +80,17 @@ for pid in "${pids[@]}"; do wait "$pid" || any_fail=1 done +if [[ -n "${PREFETCH_FETCHED_FILE:-}" ]]; then + # Published by rename inside the report's own directory, so the publish is atomic and any + # write failure leaves no report at all, which the caller reads as "nothing was fetched". + shopt -s nullglob + records=("$work_dir"/fetched_*) + shopt -u nullglob + if cat /dev/null "${records[@]}" > "${PREFETCH_FETCHED_FILE}.partial"; then + mv "${PREFETCH_FETCHED_FILE}.partial" "$PREFETCH_FETCHED_FILE" + fi +fi + # Report failures. failed=() for f in "$work_dir"/fail_*; do diff --git a/tests/integration/helpers/cluster.py b/tests/integration/helpers/cluster.py index 2e81d9e52b21b..903afef97dc52 100644 --- a/tests/integration/helpers/cluster.py +++ b/tests/integration/helpers/cluster.py @@ -3739,6 +3739,30 @@ def login_to_ecr(self): input=password.encode(), ) + def _images_prefetched_and_present(self) -> bool: + """Whether this job pulled every image of this project and each is still in the daemon. + + A reference this job did not pull may be a floating tag whose local copy is stale, and a + reference it did pull can still be gone (the image store is shared with everything else on + this daemon), so both questions are asked. + + `docker image inspect` prints one id per reference it finds and nothing for one it does + not, so an equal count means none is missing. A non-zero exit or an empty enumeration + answers False. + """ + prefetched = set(os.environ.get("CLICKHOUSE_TESTS_PREFETCHED_IMAGES", "").split()) + if not prefetched: + return False + images = set(run_and_check(self.base_cmd + ["config", "--images"], nothrow=True).split()) + if not images: + return False + if not images <= prefetched: + return False + found = run_and_check( + ["docker", "image", "inspect", "--format", "{{.Id}}"] + sorted(images), nothrow=True + ).split() + return len(found) == len(images) + def start(self, connection_timeout=None): pytest_xdist_logging_to_separate_files.setup() logging.info("Running tests in {}".format(self.base_path)) @@ -3795,9 +3819,6 @@ def start(self, connection_timeout=None): common_opts = ["--verbose", "up", "-d"] - images_pull_cmd = self.base_cmd + ["pull"] - # sometimes dockerhub/proxy can be flaky - def logging_pulling_images(**kwargs): if "exception" in kwargs: logging.info( @@ -3805,7 +3826,14 @@ def logging_pulling_images(**kwargs): ) self.login_to_ecr() - retry(log_function=logging_pulling_images, retries=3, delay=8, jitter=8)(run_and_check, images_pull_cmd, timeout=600) + # `docker compose pull` re-verifies every service image against the registry even when + # it is cached, so a transient registry failure fails a module that needed no download. + if self._images_prefetched_and_present(): + logging.debug("Skipping `docker compose pull`: all images are present locally") + else: + # sometimes dockerhub/proxy can be flaky + images_pull_cmd = self.base_cmd + ["pull"] + retry(log_function=logging_pulling_images, retries=3, delay=8, jitter=8)(run_and_check, images_pull_cmd, timeout=600) def logging_compose_up(**kwargs): if "exception" in kwargs: @@ -4328,10 +4356,6 @@ def logging_azurite_initialization(exception, retry_number, sleep_time): self.wait_ytsaurus_to_start() if self.with_letsencrypt_pebble and self.base_letsencrypt_pebble_cmd: - letsencrypt_pebble_pull_cmd = self.base_letsencrypt_pebble_cmd + ["pull"] - retry(log_function=logging_pulling_images, retries=3, delay=8, jitter=8)( - run_and_check, letsencrypt_pebble_pull_cmd, timeout=600 - ) letsencrypt_pebble_start_cmd = self.base_letsencrypt_pebble_cmd + common_opts run_and_check(letsencrypt_pebble_start_cmd) self.wait_letsencrypt_pebble_to_start()