diff --git a/ci/jobs/integration_test_job.py b/ci/jobs/integration_test_job.py index 9285fa08402ae..f66b9bd702780 100644 --- a/ci/jobs/integration_test_job.py +++ b/ci/jobs/integration_test_job.py @@ -2,9 +2,10 @@ import os import re import subprocess +import tempfile import time from pathlib import Path -from typing import List, Optional, Tuple +from typing import List, Optional, Set, Tuple import yaml # NOTE (strtgbb): Used for loading broken tests rules @@ -340,13 +341,21 @@ def replace_var(m: re.Match) -> str: def prefetch_images( - images: List[str], retries: int = 3, pull_timeout: int = 300 + images: List[str], + retries: int = 3, + pull_timeout: int = 300, + fetched_out: Optional[Set[str]] = None, ) -> bool: """Pull every image in parallel using `ci/prefetch-integration-test-images`. Images with no manifest for the current architecture (e.g. amd64-only images on arm64 runners) are silently skipped. Returns True on success, False if any image fails to pull for a real reason. + + `fetched_out`, when given, receives the references the script reports as actually + pulled. A missing or short report can only leave references out, so a reporting + failure costs the skip in `tests/integration/helpers/cluster.py` instead of claiming + an image that was never fetched. """ if not images: print("No images to pre-fetch.") @@ -358,11 +367,19 @@ def prefetch_images( "PULL_RETRIES": str(retries), "PULL_TIMEOUT": str(pull_timeout), } - return Shell.check( - f"{script} {' '.join(images)}", - verbose=True, - env=env, - ) + report = "" + with tempfile.TemporaryDirectory(prefix="prefetch_", dir=temp_path) as report_dir: + if fetched_out is not None: + report = os.path.join(report_dir, "fetched.txt") + env["PREFETCH_FETCHED_FILE"] = report + ok = Shell.check( + f"{script} {' '.join(images)}", + verbose=True, + env=env, + ) + if fetched_out is not None and Path(report).is_file(): + fetched_out.update(Path(report).read_text(errors="replace").split()) + return ok def parse_args(): @@ -1012,12 +1029,19 @@ def main(): + ", ".join(str(f.name) for f in compose_files) ) images_to_prefetch = get_images_from_compose_files(compose_files) - if not prefetch_images(images_to_prefetch): + prefetched: Set[str] = set() + if not prefetch_images(images_to_prefetch, fetched_out=prefetched): Result.create_from( status=Result.Status.ERROR, info="Failed to pre-pull Docker images needed by the test batch", labels=[Result.Label.INFRA], ).complete_job() + # A batch's compose files need not yield the default server image, but a project's own + # enumeration can: it is the default instance image and Keeper's. So prefetch it separately, and + # ignore the result: a failed fetch only leaves it out of the export, which turns the skip off. + server_image = f"altinityinfra/integration-test:{os.environ['DOCKER_BASE_TAG']}" + if server_image not in prefetched: + prefetch_images([server_image], fetched_out=prefetched) test_env = { "CLICKHOUSE_TESTS_BASE_CONFIG_DIR": clickhouse_server_config_dir, @@ -1027,6 +1051,9 @@ def main(): "CLICKHOUSE_USE_OLD_ANALYZER": "1" if use_old_analyzer else "0", "CLICKHOUSE_USE_DISTRIBUTED_PLAN": "1" if use_distributed_plan else "0", "CLICKHOUSE_USE_DATABASE_DISK": "1" if use_database_disk else "0", + # Read by tests/integration/helpers/cluster.py: the references this job pulled. A reference + # outside this set was not fetched here and may be a stale floating tag, so it is pulled. + "CLICKHOUSE_TESTS_PREFETCHED_IMAGES": " ".join(sorted(prefetched)), "PYTEST_CLEANUP_CONTAINERS": "1", "JAVA_PATH": java_path, # PromQL compliance: deterministic JSON for post-hook (see promql_compliance_hook.py). diff --git a/ci/jobs/scripts/prefetch-integration-test-images b/ci/jobs/scripts/prefetch-integration-test-images index 0eca3f83817f2..e2e732ef1a28c 100755 --- a/ci/jobs/scripts/prefetch-integration-test-images +++ b/ci/jobs/scripts/prefetch-integration-test-images @@ -10,6 +10,7 @@ # Environment variables: # PULL_RETRIES Number of pull attempts per image (default: 3) # PULL_TIMEOUT Per-attempt timeout in seconds (default: 300) +# PREFETCH_FETCHED_FILE If set, the references actually pulled are reported to this path set -uo pipefail @@ -34,6 +35,7 @@ pull_one() { local image="$1" local fail_file="$2" + local fetched_file="$3" local attempt out start elapsed for ((attempt = 1; attempt <= PULL_RETRIES; attempt++)); do @@ -42,6 +44,7 @@ pull_one() if out=$(timeout "$PULL_TIMEOUT" docker pull "$image" 2>&1); then elapsed=$((SECONDS - start)) echo "Pulled $image in ${elapsed}s" + echo "$image" > "$fetched_file" return 0 fi elapsed=$((SECONDS - start)) @@ -67,7 +70,7 @@ pids=() for image in "${images[@]}"; do # Sanitize image name to a safe filename. safe="${image//[^a-zA-Z0-9._-]/_}" - pull_one "$image" "$work_dir/fail_${safe}" & + pull_one "$image" "$work_dir/fail_${safe}" "$work_dir/fetched_${safe}" & pids+=($!) done @@ -77,6 +80,17 @@ for pid in "${pids[@]}"; do wait "$pid" || any_fail=1 done +if [[ -n "${PREFETCH_FETCHED_FILE:-}" ]]; then + # Published by rename inside the report's own directory, so the publish is atomic and any + # write failure leaves no report at all, which the caller reads as "nothing was fetched". + shopt -s nullglob + records=("$work_dir"/fetched_*) + shopt -u nullglob + if cat /dev/null "${records[@]}" > "${PREFETCH_FETCHED_FILE}.partial"; then + mv "${PREFETCH_FETCHED_FILE}.partial" "$PREFETCH_FETCHED_FILE" + fi +fi + # Report failures. failed=() for f in "$work_dir"/fail_*; do diff --git a/tests/integration/helpers/cluster.py b/tests/integration/helpers/cluster.py index 2e81d9e52b21b..903afef97dc52 100644 --- a/tests/integration/helpers/cluster.py +++ b/tests/integration/helpers/cluster.py @@ -3739,6 +3739,30 @@ def login_to_ecr(self): input=password.encode(), ) + def _images_prefetched_and_present(self) -> bool: + """Whether this job pulled every image of this project and each is still in the daemon. + + A reference this job did not pull may be a floating tag whose local copy is stale, and a + reference it did pull can still be gone (the image store is shared with everything else on + this daemon), so both questions are asked. + + `docker image inspect` prints one id per reference it finds and nothing for one it does + not, so an equal count means none is missing. A non-zero exit or an empty enumeration + answers False. + """ + prefetched = set(os.environ.get("CLICKHOUSE_TESTS_PREFETCHED_IMAGES", "").split()) + if not prefetched: + return False + images = set(run_and_check(self.base_cmd + ["config", "--images"], nothrow=True).split()) + if not images: + return False + if not images <= prefetched: + return False + found = run_and_check( + ["docker", "image", "inspect", "--format", "{{.Id}}"] + sorted(images), nothrow=True + ).split() + return len(found) == len(images) + def start(self, connection_timeout=None): pytest_xdist_logging_to_separate_files.setup() logging.info("Running tests in {}".format(self.base_path)) @@ -3795,9 +3819,6 @@ def start(self, connection_timeout=None): common_opts = ["--verbose", "up", "-d"] - images_pull_cmd = self.base_cmd + ["pull"] - # sometimes dockerhub/proxy can be flaky - def logging_pulling_images(**kwargs): if "exception" in kwargs: logging.info( @@ -3805,7 +3826,14 @@ def logging_pulling_images(**kwargs): ) self.login_to_ecr() - retry(log_function=logging_pulling_images, retries=3, delay=8, jitter=8)(run_and_check, images_pull_cmd, timeout=600) + # `docker compose pull` re-verifies every service image against the registry even when + # it is cached, so a transient registry failure fails a module that needed no download. + if self._images_prefetched_and_present(): + logging.debug("Skipping `docker compose pull`: all images are present locally") + else: + # sometimes dockerhub/proxy can be flaky + images_pull_cmd = self.base_cmd + ["pull"] + retry(log_function=logging_pulling_images, retries=3, delay=8, jitter=8)(run_and_check, images_pull_cmd, timeout=600) def logging_compose_up(**kwargs): if "exception" in kwargs: @@ -4328,10 +4356,6 @@ def logging_azurite_initialization(exception, retry_number, sleep_time): self.wait_ytsaurus_to_start() if self.with_letsencrypt_pebble and self.base_letsencrypt_pebble_cmd: - letsencrypt_pebble_pull_cmd = self.base_letsencrypt_pebble_cmd + ["pull"] - retry(log_function=logging_pulling_images, retries=3, delay=8, jitter=8)( - run_and_check, letsencrypt_pebble_pull_cmd, timeout=600 - ) letsencrypt_pebble_start_cmd = self.base_letsencrypt_pebble_cmd + common_opts run_and_check(letsencrypt_pebble_start_cmd) self.wait_letsencrypt_pebble_to_start()