Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
43 changes: 35 additions & 8 deletions ci/jobs/integration_test_job.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,9 +2,10 @@
import os
import re
import subprocess
import tempfile
import time
from pathlib import Path
from typing import List, Optional, Tuple
from typing import List, Optional, Set, Tuple

import yaml # NOTE (strtgbb): Used for loading broken tests rules

Expand Down Expand Up @@ -340,13 +341,21 @@ def replace_var(m: re.Match) -> str:


def prefetch_images(
images: List[str], retries: int = 3, pull_timeout: int = 300
images: List[str],
retries: int = 3,
pull_timeout: int = 300,
fetched_out: Optional[Set[str]] = None,
) -> bool:
"""Pull every image in parallel using `ci/prefetch-integration-test-images`.

Images with no manifest for the current architecture (e.g. amd64-only images
on arm64 runners) are silently skipped. Returns True on success, False if any
image fails to pull for a real reason.

`fetched_out`, when given, receives the references the script reports as actually
pulled. A missing or short report can only leave references out, so a reporting
failure costs the skip in `tests/integration/helpers/cluster.py` instead of claiming
an image that was never fetched.
"""
if not images:
print("No images to pre-fetch.")
Expand All @@ -358,11 +367,19 @@ def prefetch_images(
"PULL_RETRIES": str(retries),
"PULL_TIMEOUT": str(pull_timeout),
}
return Shell.check(
f"{script} {' '.join(images)}",
verbose=True,
env=env,
)
report = ""
with tempfile.TemporaryDirectory(prefix="prefetch_", dir=temp_path) as report_dir:
if fetched_out is not None:
report = os.path.join(report_dir, "fetched.txt")
env["PREFETCH_FETCHED_FILE"] = report
ok = Shell.check(
f"{script} {' '.join(images)}",
verbose=True,
env=env,
)
if fetched_out is not None and Path(report).is_file():
fetched_out.update(Path(report).read_text(errors="replace").split())
return ok


def parse_args():
Expand Down Expand Up @@ -1012,12 +1029,19 @@ def main():
+ ", ".join(str(f.name) for f in compose_files)
)
images_to_prefetch = get_images_from_compose_files(compose_files)
if not prefetch_images(images_to_prefetch):
prefetched: Set[str] = set()
if not prefetch_images(images_to_prefetch, fetched_out=prefetched):
Result.create_from(
status=Result.Status.ERROR,
info="Failed to pre-pull Docker images needed by the test batch",
labels=[Result.Label.INFRA],
).complete_job()
# A batch's compose files need not yield the default server image, but a project's own
# enumeration can: it is the default instance image and Keeper's. So prefetch it separately, and
# ignore the result: a failed fetch only leaves it out of the export, which turns the skip off.
server_image = f"altinityinfra/integration-test:{os.environ['DOCKER_BASE_TAG']}"
if server_image not in prefetched:
prefetch_images([server_image], fetched_out=prefetched)

test_env = {
"CLICKHOUSE_TESTS_BASE_CONFIG_DIR": clickhouse_server_config_dir,
Expand All @@ -1027,6 +1051,9 @@ def main():
"CLICKHOUSE_USE_OLD_ANALYZER": "1" if use_old_analyzer else "0",
"CLICKHOUSE_USE_DISTRIBUTED_PLAN": "1" if use_distributed_plan else "0",
"CLICKHOUSE_USE_DATABASE_DISK": "1" if use_database_disk else "0",
# Read by tests/integration/helpers/cluster.py: the references this job pulled. A reference
# outside this set was not fetched here and may be a stale floating tag, so it is pulled.
"CLICKHOUSE_TESTS_PREFETCHED_IMAGES": " ".join(sorted(prefetched)),
"PYTEST_CLEANUP_CONTAINERS": "1",
"JAVA_PATH": java_path,
# PromQL compliance: deterministic JSON for post-hook (see promql_compliance_hook.py).
Expand Down
16 changes: 15 additions & 1 deletion ci/jobs/scripts/prefetch-integration-test-images
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@
# Environment variables:
# PULL_RETRIES Number of pull attempts per image (default: 3)
# PULL_TIMEOUT Per-attempt timeout in seconds (default: 300)
# PREFETCH_FETCHED_FILE If set, the references actually pulled are reported to this path

set -uo pipefail

Expand All @@ -34,6 +35,7 @@ pull_one()
{
local image="$1"
local fail_file="$2"
local fetched_file="$3"
local attempt out start elapsed

for ((attempt = 1; attempt <= PULL_RETRIES; attempt++)); do
Expand All @@ -42,6 +44,7 @@ pull_one()
if out=$(timeout "$PULL_TIMEOUT" docker pull "$image" 2>&1); then
elapsed=$((SECONDS - start))
echo "Pulled $image in ${elapsed}s"
echo "$image" > "$fetched_file"
return 0
fi
elapsed=$((SECONDS - start))
Expand All @@ -67,7 +70,7 @@ pids=()
for image in "${images[@]}"; do
# Sanitize image name to a safe filename.
safe="${image//[^a-zA-Z0-9._-]/_}"
pull_one "$image" "$work_dir/fail_${safe}" &
pull_one "$image" "$work_dir/fail_${safe}" "$work_dir/fetched_${safe}" &
pids+=($!)
done

Expand All @@ -77,6 +80,17 @@ for pid in "${pids[@]}"; do
wait "$pid" || any_fail=1
done

if [[ -n "${PREFETCH_FETCHED_FILE:-}" ]]; then
# Published by rename inside the report's own directory, so the publish is atomic and any
# write failure leaves no report at all, which the caller reads as "nothing was fetched".
shopt -s nullglob
records=("$work_dir"/fetched_*)
shopt -u nullglob
if cat /dev/null "${records[@]}" > "${PREFETCH_FETCHED_FILE}.partial"; then
mv "${PREFETCH_FETCHED_FILE}.partial" "$PREFETCH_FETCHED_FILE"
fi
fi

# Report failures.
failed=()
for f in "$work_dir"/fail_*; do
Expand Down
40 changes: 32 additions & 8 deletions tests/integration/helpers/cluster.py
Original file line number Diff line number Diff line change
Expand Up @@ -3739,6 +3739,30 @@ def login_to_ecr(self):
input=password.encode(),
)

def _images_prefetched_and_present(self) -> bool:
"""Whether this job pulled every image of this project and each is still in the daemon.

A reference this job did not pull may be a floating tag whose local copy is stale, and a
reference it did pull can still be gone (the image store is shared with everything else on
this daemon), so both questions are asked.

`docker image inspect` prints one id per reference it finds and nothing for one it does
not, so an equal count means none is missing. A non-zero exit or an empty enumeration
answers False.
"""
prefetched = set(os.environ.get("CLICKHOUSE_TESTS_PREFETCHED_IMAGES", "").split())
if not prefetched:
return False
images = set(run_and_check(self.base_cmd + ["config", "--images"], nothrow=True).split())
if not images:
return False
if not images <= prefetched:
return False
found = run_and_check(
["docker", "image", "inspect", "--format", "{{.Id}}"] + sorted(images), nothrow=True
).split()
return len(found) == len(images)

def start(self, connection_timeout=None):
pytest_xdist_logging_to_separate_files.setup()
logging.info("Running tests in {}".format(self.base_path))
Expand Down Expand Up @@ -3795,17 +3819,21 @@ def start(self, connection_timeout=None):

common_opts = ["--verbose", "up", "-d"]

images_pull_cmd = self.base_cmd + ["pull"]
# sometimes dockerhub/proxy can be flaky

def logging_pulling_images(**kwargs):
if "exception" in kwargs:
logging.info(
"Got exception pulling images: %s", kwargs["exception"]
)

self.login_to_ecr()
retry(log_function=logging_pulling_images, retries=3, delay=8, jitter=8)(run_and_check, images_pull_cmd, timeout=600)
# `docker compose pull` re-verifies every service image against the registry even when
# it is cached, so a transient registry failure fails a module that needed no download.
if self._images_prefetched_and_present():
logging.debug("Skipping `docker compose pull`: all images are present locally")
else:
# sometimes dockerhub/proxy can be flaky
images_pull_cmd = self.base_cmd + ["pull"]
retry(log_function=logging_pulling_images, retries=3, delay=8, jitter=8)(run_and_check, images_pull_cmd, timeout=600)

def logging_compose_up(**kwargs):
if "exception" in kwargs:
Expand Down Expand Up @@ -4328,10 +4356,6 @@ def logging_azurite_initialization(exception, retry_number, sleep_time):
self.wait_ytsaurus_to_start()

if self.with_letsencrypt_pebble and self.base_letsencrypt_pebble_cmd:
letsencrypt_pebble_pull_cmd = self.base_letsencrypt_pebble_cmd + ["pull"]
retry(log_function=logging_pulling_images, retries=3, delay=8, jitter=8)(
run_and_check, letsencrypt_pebble_pull_cmd, timeout=600
)
letsencrypt_pebble_start_cmd = self.base_letsencrypt_pebble_cmd + common_opts
run_and_check(letsencrypt_pebble_start_cmd)
self.wait_letsencrypt_pebble_to_start()
Expand Down
Loading