From b78424e20e1d6ffab473bc23ddedc81c0b1f7f98 Mon Sep 17 00:00:00 2001 From: MyroTk Date: Thu, 5 Nov 2026 15:43:14 -0500 Subject: [PATCH 1/2] Remove source upload and update docker server building. Backport of https://github.com/Altinity/ClickHouse/pull/2763 to antalya-26.6: drop the Source upload job and serve docker packages from local artifacts instead of S3 build reports. Co-authored-by: Cursor --- .github/workflows/fast_builds.yml | 47 +--------- .github/workflows/master.yml | 47 +--------- .github/workflows/pull_request.yml | 45 +-------- ci/defs/altinity_jobs.py | 10 -- ci/defs/job_configs.py | 16 +++- ci/jobs/docker_server.py | 136 +++++++++++++++++++--------- ci/jobs/source_upload.py | 67 -------------- ci/tests/test_clickhouse_version.py | 35 ------- ci/workflows/fast_builds.py | 1 - ci/workflows/master.py | 1 - ci/workflows/pull_request.py | 1 - 11 files changed, 109 insertions(+), 297 deletions(-) delete mode 100644 ci/jobs/source_upload.py diff --git a/.github/workflows/fast_builds.yml b/.github/workflows/fast_builds.yml index 660d5edeaec98..997c24644dd9b 100644 --- a/.github/workflows/fast_builds.yml +++ b/.github/workflows/fast_builds.yml @@ -1155,50 +1155,6 @@ jobs: . ./ci/tmp/praktika_setup_env.sh PYTHONUNBUFFERED=1 python3 -m praktika run 'Sign release (arm_release)' --workflow "Fast Builds" --ci --timestamp - source_upload: - runs-on: [self-hosted, altinity-on-demand, altinity-style-checker] - needs: [config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest] - if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'U291cmNlIHVwbG9hZA==') }} - name: "Source upload" - outputs: - data: ${{ steps.run.outputs.DATA }} - pipeline_status: ${{ steps.run.outputs.pipeline_status || 'undefined' }} - steps: - - name: Checkout code - uses: actions/checkout@v6 - with: - ref: ${{ env.CHECKOUT_REF }} - - - name: Setup - uses: ./.github/actions/runner_setup - - name: Docker setup - uses: ./.github/actions/docker_setup - with: - test_name: "Source upload" - - - name: Prepare env script - run: | - rm -rf ./ci/tmp - mkdir -p ./ci/tmp - cat > ./ci/tmp/praktika_setup_env.sh << 'ENV_SETUP_SCRIPT_EOF' - export PYTHONPATH=./ci:.: - cat > ./ci/tmp/workflow_inputs.json << 'EOF' - ${{ toJson(inputs) }} - EOF - cat > ./ci/tmp/workflow_job.json << 'EOF' - ${{ toJson(job) }} - EOF - cat > ./ci/tmp/workflow_status.json << 'EOF' - ${{ toJson(needs) }} - EOF - ENV_SETUP_SCRIPT_EOF - - - name: Run - id: run - run: | - . ./ci/tmp/praktika_setup_env.sh - PYTHONUNBUFFERED=1 python3 -m praktika run 'Source upload' --workflow "Fast Builds" --ci --timestamp - stateless_tests_arm_binary_parallel: runs-on: [self-hosted, altinity-on-demand, altinity-func-tester-aarch64, 16c] needs: [build_arm_binary, config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest] @@ -1456,7 +1412,7 @@ jobs: finish_workflow: runs-on: [self-hosted, altinity-on-demand, altinity-style-checker] - needs: [build_amd_asan_ubsan, build_amd_binary, build_amd_debug, build_amd_msan, build_amd_release, build_amd_tsan, build_arm_asan_ubsan, build_arm_binary, build_arm_debug, build_arm_msan, build_arm_release, build_arm_tsan, config_workflow, docker_keeper_image, docker_server_image, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest, install_packages_amd_release, install_packages_arm_release, sign_release_amd_release, sign_release_arm_release, source_upload, stateless_tests_amd_binary_cas_s3_storage_parallel, stateless_tests_amd_binary_cas_storage_parallel, stateless_tests_arm_binary_cas_s3_storage_parallel, stateless_tests_arm_binary_parallel, stateless_tests_arm_binary_sequential] + needs: [build_amd_asan_ubsan, build_amd_binary, build_amd_debug, build_amd_msan, build_amd_release, build_amd_tsan, build_arm_asan_ubsan, build_arm_binary, build_arm_debug, build_arm_msan, build_arm_release, build_arm_tsan, config_workflow, docker_keeper_image, docker_server_image, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest, install_packages_amd_release, install_packages_arm_release, sign_release_amd_release, sign_release_arm_release, stateless_tests_amd_binary_cas_s3_storage_parallel, stateless_tests_amd_binary_cas_storage_parallel, stateless_tests_arm_binary_cas_s3_storage_parallel, stateless_tests_arm_binary_parallel, stateless_tests_arm_binary_sequential] if: ${{ !cancelled() && needs.config_workflow.outputs.pipeline_status != '' }} name: "Finish Workflow" outputs: @@ -1549,7 +1505,6 @@ jobs: - install_packages_arm_release - sign_release_amd_release - sign_release_arm_release - - source_upload - stateless_tests_arm_binary_parallel - stateless_tests_arm_binary_sequential - stateless_tests_amd_binary_cas_s3_storage_parallel diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml index 73003c8dd7b64..05568c2a0afc7 100644 --- a/.github/workflows/master.yml +++ b/.github/workflows/master.yml @@ -1566,50 +1566,6 @@ jobs: . ./ci/tmp/praktika_setup_env.sh PYTHONUNBUFFERED=1 python3 -m praktika run 'Sign release (arm_release)' --workflow "MasterCI" --ci --timestamp - source_upload: - runs-on: [self-hosted, altinity-on-demand, altinity-style-checker] - needs: [config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest] - if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'U291cmNlIHVwbG9hZA==') }} - name: "Source upload" - outputs: - data: ${{ steps.run.outputs.DATA }} - pipeline_status: ${{ steps.run.outputs.pipeline_status || 'undefined' }} - steps: - - name: Checkout code - uses: actions/checkout@v6 - with: - ref: ${{ env.CHECKOUT_REF }} - - - name: Setup - uses: ./.github/actions/runner_setup - - name: Docker setup - uses: ./.github/actions/docker_setup - with: - test_name: "Source upload" - - - name: Prepare env script - run: | - rm -rf ./ci/tmp - mkdir -p ./ci/tmp - cat > ./ci/tmp/praktika_setup_env.sh << 'ENV_SETUP_SCRIPT_EOF' - export PYTHONPATH=./ci:.: - cat > ./ci/tmp/workflow_inputs.json << 'EOF' - ${{ toJson(inputs) }} - EOF - cat > ./ci/tmp/workflow_job.json << 'EOF' - ${{ toJson(job) }} - EOF - cat > ./ci/tmp/workflow_status.json << 'EOF' - ${{ toJson(needs) }} - EOF - ENV_SETUP_SCRIPT_EOF - - - name: Run - id: run - run: | - . ./ci/tmp/praktika_setup_env.sh - PYTHONUNBUFFERED=1 python3 -m praktika run 'Source upload' --workflow "MasterCI" --ci --timestamp - compatibility_check_amd_release: runs-on: [self-hosted, altinity-on-demand, altinity-style-checker] needs: [build_amd_release, config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest] @@ -7267,7 +7223,7 @@ jobs: finish_workflow: runs-on: [self-hosted, altinity-on-demand, altinity-style-checker] - needs: [ast_fuzzer_amd_debug, ast_fuzzer_amd_msan, ast_fuzzer_amd_tsan, ast_fuzzer_arm_asan_ubsan, build_amd_asan_ubsan, build_amd_binary, build_amd_debug, build_amd_llvm_coverage_per_test, build_amd_msan, build_amd_release, build_amd_release_pr_cache_warmup, build_amd_tsan, build_arm_asan_ubsan, build_arm_binary, build_arm_debug, build_arm_msan, build_arm_release, build_arm_release_pr_cache_warmup, build_arm_tsan, buzzhouse_amd_debug, buzzhouse_amd_msan, buzzhouse_amd_tsan, buzzhouse_arm_asan_ubsan, compatibility_check_amd_release, compatibility_check_arm_release, config_workflow, docker_keeper_image, docker_server_image, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest, install_packages_amd_release, install_packages_arm_release, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_1_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_2_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_3_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_4_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_5_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_6_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_7_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_8_8, integration_tests_amd_msan_10_10, integration_tests_amd_msan_1_10, integration_tests_amd_msan_2_10, integration_tests_amd_msan_3_10, integration_tests_amd_msan_4_10, integration_tests_amd_msan_5_10, integration_tests_amd_msan_6_10, integration_tests_amd_msan_7_10, integration_tests_amd_msan_8_10, integration_tests_amd_msan_9_10, integration_tests_amd_tsan_1_8, integration_tests_amd_tsan_2_8, integration_tests_amd_tsan_3_8, integration_tests_amd_tsan_4_8, integration_tests_amd_tsan_5_8, integration_tests_amd_tsan_6_8, integration_tests_amd_tsan_7_8, integration_tests_amd_tsan_8_8, integration_tests_arm_binary_distributed_plan_1_4, integration_tests_arm_binary_distributed_plan_2_4, integration_tests_arm_binary_distributed_plan_3_4, integration_tests_arm_binary_distributed_plan_4_4, sign_release_amd_release, sign_release_arm_release, source_upload, sqllogic_test, sqlstorm_test, sqltest, stateless_tests_amd_asan_ubsan_cas_s3_storage_parallel_1_2, stateless_tests_amd_asan_ubsan_cas_s3_storage_parallel_2_2, stateless_tests_amd_asan_ubsan_db_disk_distributed_plan_sequential_1_3, stateless_tests_amd_asan_ubsan_db_disk_distributed_plan_sequential_2_3, stateless_tests_amd_asan_ubsan_db_disk_distributed_plan_sequential_3_3, stateless_tests_amd_asan_ubsan_distributed_plan_parallel_1_2, stateless_tests_amd_asan_ubsan_distributed_plan_parallel_2_2, stateless_tests_amd_binary_cas_s3_storage_parallel, stateless_tests_amd_binary_cas_storage_parallel, stateless_tests_amd_debug_distributed_plan_s3_storage_parallel, stateless_tests_amd_debug_distributed_plan_s3_storage_sequential, stateless_tests_amd_debug_parallel, stateless_tests_amd_debug_sequential, stateless_tests_amd_llvm_coverage_per_test_per_test_coverage_1_8, stateless_tests_amd_llvm_coverage_per_test_per_test_coverage_2_8, stateless_tests_amd_llvm_coverage_per_test_per_test_coverage_3_8, stateless_tests_amd_llvm_coverage_per_test_per_test_coverage_4_8, stateless_tests_amd_llvm_coverage_per_test_per_test_coverage_5_8, stateless_tests_amd_llvm_coverage_per_test_per_test_coverage_6_8, stateless_tests_amd_llvm_coverage_per_test_per_test_coverage_7_8, stateless_tests_amd_llvm_coverage_per_test_per_test_coverage_8_8, stateless_tests_amd_msan_cas_s3_storage_parallel_1_3, stateless_tests_amd_msan_cas_s3_storage_parallel_2_3, stateless_tests_amd_msan_cas_s3_storage_parallel_3_3, stateless_tests_amd_msan_wasmedge_parallel_1_6, stateless_tests_amd_msan_wasmedge_parallel_2_6, stateless_tests_amd_msan_wasmedge_parallel_3_6, stateless_tests_amd_msan_wasmedge_parallel_4_6, stateless_tests_amd_msan_wasmedge_parallel_5_6, stateless_tests_amd_msan_wasmedge_parallel_6_6, stateless_tests_amd_msan_wasmedge_sequential_1_4, stateless_tests_amd_msan_wasmedge_sequential_2_4, stateless_tests_amd_msan_wasmedge_sequential_3_4, stateless_tests_amd_msan_wasmedge_sequential_4_4, stateless_tests_amd_tsan_cas_s3_storage_parallel_1_2, stateless_tests_amd_tsan_cas_s3_storage_parallel_2_2, stateless_tests_amd_tsan_parallel_1_4, stateless_tests_amd_tsan_parallel_2_4, stateless_tests_amd_tsan_parallel_3_4, stateless_tests_amd_tsan_parallel_4_4, stateless_tests_amd_tsan_s3_storage_parallel_1_3, stateless_tests_amd_tsan_s3_storage_parallel_2_3, stateless_tests_amd_tsan_s3_storage_parallel_3_3, stateless_tests_amd_tsan_s3_storage_sequential_1_2, stateless_tests_amd_tsan_s3_storage_sequential_2_2, stateless_tests_amd_tsan_sequential_1_2, stateless_tests_amd_tsan_sequential_2_2, stateless_tests_arm_asan_ubsan_azure_parallel_1_8, stateless_tests_arm_asan_ubsan_azure_parallel_2_8, stateless_tests_arm_asan_ubsan_azure_parallel_3_8, stateless_tests_arm_asan_ubsan_azure_parallel_4_8, stateless_tests_arm_asan_ubsan_azure_parallel_5_8, stateless_tests_arm_asan_ubsan_azure_parallel_6_8, stateless_tests_arm_asan_ubsan_azure_parallel_7_8, stateless_tests_arm_asan_ubsan_azure_parallel_8_8, stateless_tests_arm_asan_ubsan_azure_sequential_1_2, stateless_tests_arm_asan_ubsan_azure_sequential_2_2, stateless_tests_arm_binary_cas_s3_storage_parallel, stateless_tests_arm_binary_parallel, stateless_tests_arm_binary_sequential, stress_test_amd_asan_ubsan, stress_test_amd_debug, stress_test_amd_msan, stress_test_amd_tsan, stress_test_arm_asan_ubsan, stress_test_arm_asan_ubsan_s3, stress_test_arm_debug, stress_test_arm_msan, stress_test_arm_release, stress_test_arm_tsan, stress_test_azure_amd_msan, stress_test_azure_amd_tsan, unit_tests_asan_ubsan, unit_tests_asan_ubsan_function_prop_fuzzer, unit_tests_msan, unit_tests_msan_function_prop_fuzzer, unit_tests_tsan, unit_tests_tsan_function_prop_fuzzer] + needs: [ast_fuzzer_amd_debug, ast_fuzzer_amd_msan, ast_fuzzer_amd_tsan, ast_fuzzer_arm_asan_ubsan, build_amd_asan_ubsan, build_amd_binary, build_amd_debug, build_amd_llvm_coverage_per_test, build_amd_msan, build_amd_release, build_amd_release_pr_cache_warmup, build_amd_tsan, build_arm_asan_ubsan, build_arm_binary, build_arm_debug, build_arm_msan, build_arm_release, build_arm_release_pr_cache_warmup, build_arm_tsan, buzzhouse_amd_debug, buzzhouse_amd_msan, buzzhouse_amd_tsan, buzzhouse_arm_asan_ubsan, compatibility_check_amd_release, compatibility_check_arm_release, config_workflow, docker_keeper_image, docker_server_image, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest, install_packages_amd_release, install_packages_arm_release, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_1_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_2_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_3_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_4_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_5_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_6_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_7_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_8_8, integration_tests_amd_msan_10_10, integration_tests_amd_msan_1_10, integration_tests_amd_msan_2_10, integration_tests_amd_msan_3_10, integration_tests_amd_msan_4_10, integration_tests_amd_msan_5_10, integration_tests_amd_msan_6_10, integration_tests_amd_msan_7_10, integration_tests_amd_msan_8_10, integration_tests_amd_msan_9_10, integration_tests_amd_tsan_1_8, integration_tests_amd_tsan_2_8, integration_tests_amd_tsan_3_8, integration_tests_amd_tsan_4_8, integration_tests_amd_tsan_5_8, integration_tests_amd_tsan_6_8, integration_tests_amd_tsan_7_8, integration_tests_amd_tsan_8_8, integration_tests_arm_binary_distributed_plan_1_4, integration_tests_arm_binary_distributed_plan_2_4, integration_tests_arm_binary_distributed_plan_3_4, integration_tests_arm_binary_distributed_plan_4_4, sign_release_amd_release, sign_release_arm_release, sqllogic_test, sqlstorm_test, sqltest, stateless_tests_amd_asan_ubsan_cas_s3_storage_parallel_1_2, stateless_tests_amd_asan_ubsan_cas_s3_storage_parallel_2_2, stateless_tests_amd_asan_ubsan_db_disk_distributed_plan_sequential_1_3, stateless_tests_amd_asan_ubsan_db_disk_distributed_plan_sequential_2_3, stateless_tests_amd_asan_ubsan_db_disk_distributed_plan_sequential_3_3, stateless_tests_amd_asan_ubsan_distributed_plan_parallel_1_2, stateless_tests_amd_asan_ubsan_distributed_plan_parallel_2_2, stateless_tests_amd_binary_cas_s3_storage_parallel, stateless_tests_amd_binary_cas_storage_parallel, stateless_tests_amd_debug_distributed_plan_s3_storage_parallel, stateless_tests_amd_debug_distributed_plan_s3_storage_sequential, stateless_tests_amd_debug_parallel, stateless_tests_amd_debug_sequential, stateless_tests_amd_llvm_coverage_per_test_per_test_coverage_1_8, stateless_tests_amd_llvm_coverage_per_test_per_test_coverage_2_8, stateless_tests_amd_llvm_coverage_per_test_per_test_coverage_3_8, stateless_tests_amd_llvm_coverage_per_test_per_test_coverage_4_8, stateless_tests_amd_llvm_coverage_per_test_per_test_coverage_5_8, stateless_tests_amd_llvm_coverage_per_test_per_test_coverage_6_8, stateless_tests_amd_llvm_coverage_per_test_per_test_coverage_7_8, stateless_tests_amd_llvm_coverage_per_test_per_test_coverage_8_8, stateless_tests_amd_msan_cas_s3_storage_parallel_1_3, stateless_tests_amd_msan_cas_s3_storage_parallel_2_3, stateless_tests_amd_msan_cas_s3_storage_parallel_3_3, stateless_tests_amd_msan_wasmedge_parallel_1_6, stateless_tests_amd_msan_wasmedge_parallel_2_6, stateless_tests_amd_msan_wasmedge_parallel_3_6, stateless_tests_amd_msan_wasmedge_parallel_4_6, stateless_tests_amd_msan_wasmedge_parallel_5_6, stateless_tests_amd_msan_wasmedge_parallel_6_6, stateless_tests_amd_msan_wasmedge_sequential_1_4, stateless_tests_amd_msan_wasmedge_sequential_2_4, stateless_tests_amd_msan_wasmedge_sequential_3_4, stateless_tests_amd_msan_wasmedge_sequential_4_4, stateless_tests_amd_tsan_cas_s3_storage_parallel_1_2, stateless_tests_amd_tsan_cas_s3_storage_parallel_2_2, stateless_tests_amd_tsan_parallel_1_4, stateless_tests_amd_tsan_parallel_2_4, stateless_tests_amd_tsan_parallel_3_4, stateless_tests_amd_tsan_parallel_4_4, stateless_tests_amd_tsan_s3_storage_parallel_1_3, stateless_tests_amd_tsan_s3_storage_parallel_2_3, stateless_tests_amd_tsan_s3_storage_parallel_3_3, stateless_tests_amd_tsan_s3_storage_sequential_1_2, stateless_tests_amd_tsan_s3_storage_sequential_2_2, stateless_tests_amd_tsan_sequential_1_2, stateless_tests_amd_tsan_sequential_2_2, stateless_tests_arm_asan_ubsan_azure_parallel_1_8, stateless_tests_arm_asan_ubsan_azure_parallel_2_8, stateless_tests_arm_asan_ubsan_azure_parallel_3_8, stateless_tests_arm_asan_ubsan_azure_parallel_4_8, stateless_tests_arm_asan_ubsan_azure_parallel_5_8, stateless_tests_arm_asan_ubsan_azure_parallel_6_8, stateless_tests_arm_asan_ubsan_azure_parallel_7_8, stateless_tests_arm_asan_ubsan_azure_parallel_8_8, stateless_tests_arm_asan_ubsan_azure_sequential_1_2, stateless_tests_arm_asan_ubsan_azure_sequential_2_2, stateless_tests_arm_binary_cas_s3_storage_parallel, stateless_tests_arm_binary_parallel, stateless_tests_arm_binary_sequential, stress_test_amd_asan_ubsan, stress_test_amd_debug, stress_test_amd_msan, stress_test_amd_tsan, stress_test_arm_asan_ubsan, stress_test_arm_asan_ubsan_s3, stress_test_arm_debug, stress_test_arm_msan, stress_test_arm_release, stress_test_arm_tsan, stress_test_azure_amd_msan, stress_test_azure_amd_tsan, unit_tests_asan_ubsan, unit_tests_asan_ubsan_function_prop_fuzzer, unit_tests_msan, unit_tests_msan_function_prop_fuzzer, unit_tests_tsan, unit_tests_tsan_function_prop_fuzzer] if: ${{ !cancelled() && needs.config_workflow.outputs.pipeline_status != '' }} name: "Finish Workflow" outputs: @@ -7394,7 +7350,6 @@ jobs: - install_packages_arm_release - sign_release_amd_release - sign_release_arm_release - - source_upload - compatibility_check_amd_release - compatibility_check_arm_release - stateless_tests_amd_asan_ubsan_distributed_plan_parallel_1_2 diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index 1c4fae949b77c..13ebee4a14501 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -6326,51 +6326,9 @@ jobs: . ./ci/tmp/praktika_setup_env.sh PYTHONUNBUFFERED=1 python3 -m praktika run 'Build Toolchain (PGO, BOLT) (aarch64)' --workflow "PR" --ci --timestamp - source_upload: - runs-on: [self-hosted, altinity-on-demand, altinity-style-checker] - needs: [config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest] - if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'U291cmNlIHVwbG9hZA==') }} - name: "Source upload" - outputs: - data: ${{ steps.run.outputs.DATA }} - pipeline_status: ${{ steps.run.outputs.pipeline_status || 'undefined' }} - steps: - - name: Checkout code - uses: actions/checkout@v6 - with: - ref: ${{ env.CHECKOUT_REF }} - - - name: Setup - uses: ./.github/actions/runner_setup - - name: Docker setup - uses: ./.github/actions/docker_setup - with: - test_name: "Source upload" - - - name: Prepare env script - run: | - rm -rf ./ci/tmp - mkdir -p ./ci/tmp - cat > ./ci/tmp/praktika_setup_env.sh << 'ENV_SETUP_SCRIPT_EOF' - export PYTHONPATH=./ci:.: - - cat > ./ci/tmp/workflow_job.json << 'EOF' - ${{ toJson(job) }} - EOF - cat > ./ci/tmp/workflow_status.json << 'EOF' - ${{ toJson(needs) }} - EOF - ENV_SETUP_SCRIPT_EOF - - - name: Run - id: run - run: | - . ./ci/tmp/praktika_setup_env.sh - PYTHONUNBUFFERED=1 python3 -m praktika run 'Source upload' --workflow "PR" --ci --timestamp - finish_workflow: runs-on: [self-hosted, altinity-on-demand, altinity-style-checker] - needs: [ast_fuzzer_amd_debug, ast_fuzzer_amd_debug_targeted, ast_fuzzer_amd_debug_targeted_old_compatibility, ast_fuzzer_amd_msan, ast_fuzzer_amd_tsan, ast_fuzzer_arm_asan_ubsan, bugfix_validation_unit_tests, build_amd_asan_ubsan, build_amd_binary, build_amd_debug, build_amd_msan, build_amd_release, build_amd_tsan, build_arm_asan_ubsan, build_arm_binary, build_arm_debug, build_arm_msan, build_arm_release, build_arm_tsan, build_toolchain_pgo_bolt_aarch64, build_toolchain_pgo_bolt_amd64, build_wasm_parser, buzzhouse_amd_debug, buzzhouse_amd_msan, buzzhouse_amd_tsan, buzzhouse_arm_asan_ubsan, ci_tests, compatibility_check_amd_release, compatibility_check_arm_release, config_workflow, docker_keeper_image, docker_server_image, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest, fast_test, install_packages_amd_release, install_packages_arm_release, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_1_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_2_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_3_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_4_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_5_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_6_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_7_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_8_8, integration_tests_amd_asan_ubsan_targeted, integration_tests_amd_msan_10_10, integration_tests_amd_msan_1_10, integration_tests_amd_msan_2_10, integration_tests_amd_msan_3_10, integration_tests_amd_msan_4_10, integration_tests_amd_msan_5_10, integration_tests_amd_msan_6_10, integration_tests_amd_msan_7_10, integration_tests_amd_msan_8_10, integration_tests_amd_msan_9_10, integration_tests_amd_tsan_1_8, integration_tests_amd_tsan_2_8, integration_tests_amd_tsan_3_8, integration_tests_amd_tsan_4_8, integration_tests_amd_tsan_5_8, integration_tests_amd_tsan_6_8, integration_tests_amd_tsan_7_8, integration_tests_amd_tsan_8_8, integration_tests_arm_binary_distributed_plan_1_4, integration_tests_arm_binary_distributed_plan_2_4, integration_tests_arm_binary_distributed_plan_3_4, integration_tests_arm_binary_distributed_plan_4_4, keeper_stress_tests_pr, promql_compliance, quick_functional_tests, source_upload, sqllogic_test, sqlstorm_test, stateless_tests_amd_asan_ubsan_cas_s3_storage_parallel_1_2, stateless_tests_amd_asan_ubsan_cas_s3_storage_parallel_2_2, stateless_tests_amd_asan_ubsan_db_disk_distributed_plan_sequential_selected_tests, stateless_tests_amd_asan_ubsan_distributed_plan_parallel_selected_tests, stateless_tests_amd_binary_cas_s3_storage_parallel, stateless_tests_amd_binary_cas_storage_parallel, stateless_tests_amd_debug_distributed_plan_s3_storage_parallel, stateless_tests_amd_debug_distributed_plan_s3_storage_sequential, stateless_tests_amd_debug_parallel, stateless_tests_amd_debug_sequential, stateless_tests_amd_msan_cas_s3_storage_parallel_1_3, stateless_tests_amd_msan_cas_s3_storage_parallel_2_3, stateless_tests_amd_msan_cas_s3_storage_parallel_3_3, stateless_tests_amd_msan_wasmedge_parallel_1_6, stateless_tests_amd_msan_wasmedge_parallel_2_6, stateless_tests_amd_msan_wasmedge_parallel_3_6, stateless_tests_amd_msan_wasmedge_parallel_4_6, stateless_tests_amd_msan_wasmedge_parallel_5_6, stateless_tests_amd_msan_wasmedge_parallel_6_6, stateless_tests_amd_msan_wasmedge_sequential_1_4, stateless_tests_amd_msan_wasmedge_sequential_2_4, stateless_tests_amd_msan_wasmedge_sequential_3_4, stateless_tests_amd_msan_wasmedge_sequential_4_4, stateless_tests_amd_tsan_cas_s3_storage_parallel_1_2, stateless_tests_amd_tsan_cas_s3_storage_parallel_2_2, stateless_tests_amd_tsan_parallel_selected_tests, stateless_tests_amd_tsan_s3_storage_parallel_selected_tests, stateless_tests_amd_tsan_s3_storage_sequential_selected_tests, stateless_tests_amd_tsan_sequential_selected_tests, stateless_tests_arm_asan_ubsan_azure_parallel_1_8, stateless_tests_arm_asan_ubsan_azure_parallel_2_8, stateless_tests_arm_asan_ubsan_azure_parallel_3_8, stateless_tests_arm_asan_ubsan_azure_parallel_4_8, stateless_tests_arm_asan_ubsan_azure_parallel_5_8, stateless_tests_arm_asan_ubsan_azure_parallel_6_8, stateless_tests_arm_asan_ubsan_azure_parallel_7_8, stateless_tests_arm_asan_ubsan_azure_parallel_8_8, stateless_tests_arm_asan_ubsan_azure_sequential_1_2, stateless_tests_arm_asan_ubsan_azure_sequential_2_2, stateless_tests_arm_asan_ubsan_targeted, stateless_tests_arm_binary_cas_s3_storage_parallel, stateless_tests_arm_binary_parallel, stateless_tests_arm_binary_sequential, stress_test_amd_asan_ubsan, stress_test_amd_debug, stress_test_amd_msan, stress_test_amd_tsan, stress_test_arm_asan_ubsan, stress_test_arm_asan_ubsan_s3, stress_test_arm_debug, stress_test_arm_msan, stress_test_arm_release, stress_test_arm_tsan, unit_tests_asan_ubsan, unit_tests_asan_ubsan_function_prop_fuzzer, unit_tests_msan, unit_tests_msan_function_prop_fuzzer, unit_tests_tsan, unit_tests_tsan_function_prop_fuzzer] + needs: [ast_fuzzer_amd_debug, ast_fuzzer_amd_debug_targeted, ast_fuzzer_amd_debug_targeted_old_compatibility, ast_fuzzer_amd_msan, ast_fuzzer_amd_tsan, ast_fuzzer_arm_asan_ubsan, bugfix_validation_unit_tests, build_amd_asan_ubsan, build_amd_binary, build_amd_debug, build_amd_msan, build_amd_release, build_amd_tsan, build_arm_asan_ubsan, build_arm_binary, build_arm_debug, build_arm_msan, build_arm_release, build_arm_tsan, build_toolchain_pgo_bolt_aarch64, build_toolchain_pgo_bolt_amd64, build_wasm_parser, buzzhouse_amd_debug, buzzhouse_amd_msan, buzzhouse_amd_tsan, buzzhouse_arm_asan_ubsan, ci_tests, compatibility_check_amd_release, compatibility_check_arm_release, config_workflow, docker_keeper_image, docker_server_image, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest, fast_test, install_packages_amd_release, install_packages_arm_release, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_1_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_2_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_3_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_4_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_5_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_6_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_7_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_8_8, integration_tests_amd_asan_ubsan_targeted, integration_tests_amd_msan_10_10, integration_tests_amd_msan_1_10, integration_tests_amd_msan_2_10, integration_tests_amd_msan_3_10, integration_tests_amd_msan_4_10, integration_tests_amd_msan_5_10, integration_tests_amd_msan_6_10, integration_tests_amd_msan_7_10, integration_tests_amd_msan_8_10, integration_tests_amd_msan_9_10, integration_tests_amd_tsan_1_8, integration_tests_amd_tsan_2_8, integration_tests_amd_tsan_3_8, integration_tests_amd_tsan_4_8, integration_tests_amd_tsan_5_8, integration_tests_amd_tsan_6_8, integration_tests_amd_tsan_7_8, integration_tests_amd_tsan_8_8, integration_tests_arm_binary_distributed_plan_1_4, integration_tests_arm_binary_distributed_plan_2_4, integration_tests_arm_binary_distributed_plan_3_4, integration_tests_arm_binary_distributed_plan_4_4, keeper_stress_tests_pr, promql_compliance, quick_functional_tests, sqllogic_test, sqlstorm_test, stateless_tests_amd_asan_ubsan_cas_s3_storage_parallel_1_2, stateless_tests_amd_asan_ubsan_cas_s3_storage_parallel_2_2, stateless_tests_amd_asan_ubsan_db_disk_distributed_plan_sequential_selected_tests, stateless_tests_amd_asan_ubsan_distributed_plan_parallel_selected_tests, stateless_tests_amd_binary_cas_s3_storage_parallel, stateless_tests_amd_binary_cas_storage_parallel, stateless_tests_amd_debug_distributed_plan_s3_storage_parallel, stateless_tests_amd_debug_distributed_plan_s3_storage_sequential, stateless_tests_amd_debug_parallel, stateless_tests_amd_debug_sequential, stateless_tests_amd_msan_cas_s3_storage_parallel_1_3, stateless_tests_amd_msan_cas_s3_storage_parallel_2_3, stateless_tests_amd_msan_cas_s3_storage_parallel_3_3, stateless_tests_amd_msan_wasmedge_parallel_1_6, stateless_tests_amd_msan_wasmedge_parallel_2_6, stateless_tests_amd_msan_wasmedge_parallel_3_6, stateless_tests_amd_msan_wasmedge_parallel_4_6, stateless_tests_amd_msan_wasmedge_parallel_5_6, stateless_tests_amd_msan_wasmedge_parallel_6_6, stateless_tests_amd_msan_wasmedge_sequential_1_4, stateless_tests_amd_msan_wasmedge_sequential_2_4, stateless_tests_amd_msan_wasmedge_sequential_3_4, stateless_tests_amd_msan_wasmedge_sequential_4_4, stateless_tests_amd_tsan_cas_s3_storage_parallel_1_2, stateless_tests_amd_tsan_cas_s3_storage_parallel_2_2, stateless_tests_amd_tsan_parallel_selected_tests, stateless_tests_amd_tsan_s3_storage_parallel_selected_tests, stateless_tests_amd_tsan_s3_storage_sequential_selected_tests, stateless_tests_amd_tsan_sequential_selected_tests, stateless_tests_arm_asan_ubsan_azure_parallel_1_8, stateless_tests_arm_asan_ubsan_azure_parallel_2_8, stateless_tests_arm_asan_ubsan_azure_parallel_3_8, stateless_tests_arm_asan_ubsan_azure_parallel_4_8, stateless_tests_arm_asan_ubsan_azure_parallel_5_8, stateless_tests_arm_asan_ubsan_azure_parallel_6_8, stateless_tests_arm_asan_ubsan_azure_parallel_7_8, stateless_tests_arm_asan_ubsan_azure_parallel_8_8, stateless_tests_arm_asan_ubsan_azure_sequential_1_2, stateless_tests_arm_asan_ubsan_azure_sequential_2_2, stateless_tests_arm_asan_ubsan_targeted, stateless_tests_arm_binary_cas_s3_storage_parallel, stateless_tests_arm_binary_parallel, stateless_tests_arm_binary_sequential, stress_test_amd_asan_ubsan, stress_test_amd_debug, stress_test_amd_msan, stress_test_amd_tsan, stress_test_arm_asan_ubsan, stress_test_arm_asan_ubsan_s3, stress_test_arm_debug, stress_test_arm_msan, stress_test_arm_release, stress_test_arm_tsan, unit_tests_asan_ubsan, unit_tests_asan_ubsan_function_prop_fuzzer, unit_tests_msan, unit_tests_msan_function_prop_fuzzer, unit_tests_tsan, unit_tests_tsan_function_prop_fuzzer] if: ${{ !cancelled() && needs.config_workflow.outputs.pipeline_status != '' }} name: "Finish Workflow" outputs: @@ -6597,7 +6555,6 @@ jobs: - keeper_stress_tests_pr - build_toolchain_pgo_bolt_amd64 - build_toolchain_pgo_bolt_aarch64 - - source_upload - finish_workflow - GrypeScanServer - GrypeScanKeeper diff --git a/ci/defs/altinity_jobs.py b/ci/defs/altinity_jobs.py index 19a005a9f9bea..bc1cf9654373d 100644 --- a/ci/defs/altinity_jobs.py +++ b/ci/defs/altinity_jobs.py @@ -24,7 +24,6 @@ class AltinityArtifactConfigs: class AltinityJobNames: SIGN_RELEASE = "Sign release" - SOURCE_UPLOAD = "Source upload" class AltinityJobConfigs: @@ -62,15 +61,6 @@ class AltinityJobConfigs: provides=[AltinityArtifactNames.SIGNED_ARM_RELEASE], ), ) - # No digest_config: the job tars the whole source tree, which changes every - # commit, so it must never be skipped by the content-addressed CI cache. - source_upload_job = Job.Config( - name=AltinityJobNames.SOURCE_UPLOAD, - runs_on=RunnerLabels.STYLE_CHECK_AMD, - command="python3 ./ci/jobs/source_upload.py", - timeout=3600, - needs_submodules=True, - ) # Stateless tests with a content-addressed disk as the default MergeTree storage. cas_functional_tests_jobs = common_ft_job_config.parametrize( # CAS over S3: RustFS, not MinIO OSS, because the incarnation pool needs diff --git a/ci/defs/job_configs.py b/ci/defs/job_configs.py index 51c6c33ddab70..c80b34a0ef335 100644 --- a/ci/defs/job_configs.py +++ b/ci/defs/job_configs.py @@ -1698,7 +1698,7 @@ class JobConfigs: docker_server = Job.Config( name=JobNames.DOCKER_SERVER, runs_on=RunnerLabels.STYLE_CHECK_AMD, - command="python3 ./ci/jobs/docker_server.py --tag-type head --allow-build-reuse --push", + command="python3 ./ci/jobs/docker_server.py --tag-type head --allow-build-reuse --push --no-distroless", digest_config=Job.CacheDigestConfig( include_paths=[ "./ci/jobs/docker_server.py", @@ -1708,13 +1708,18 @@ class JobConfigs: ".github/workflows/grype_scan.yml", ], ), - requires=["Build (amd_release)", "Build (arm_release)"], + requires=[ + ArtifactNames.DEB_AMD_RELEASE, + ArtifactNames.DEB_ARM_RELEASE, + ArtifactNames.TGZ_AMD_RELEASE, + ArtifactNames.TGZ_ARM_RELEASE, + ], post_hooks=["python3 ./ci/jobs/scripts/job_hooks/docker_clean_up_hook.py"], ) docker_keeper = Job.Config( name=JobNames.DOCKER_KEEPER, runs_on=RunnerLabels.STYLE_CHECK_AMD, - command="python3 ./ci/jobs/docker_server.py --tag-type head --allow-build-reuse --push", + command="python3 ./ci/jobs/docker_server.py --tag-type head --allow-build-reuse --push --no-distroless", digest_config=Job.CacheDigestConfig( include_paths=[ "./ci/jobs/docker_server.py", @@ -1724,7 +1729,10 @@ class JobConfigs: ".github/workflows/grype_scan.yml", ], ), - requires=["Build (amd_release)", "Build (arm_release)"], + requires=[ + ArtifactNames.TGZ_AMD_RELEASE, + ArtifactNames.TGZ_ARM_RELEASE, + ], post_hooks=["python3 ./ci/jobs/scripts/job_hooks/docker_clean_up_hook.py"], ) sqlancer_master_jobs = Job.Config( diff --git a/ci/jobs/docker_server.py b/ci/jobs/docker_server.py index 02df5a62f90a1..d9a5bb58eb9fd 100644 --- a/ci/jobs/docker_server.py +++ b/ci/jobs/docker_server.py @@ -4,10 +4,14 @@ import logging import os import shlex +import shutil import tempfile +import threading import traceback +from http.server import SimpleHTTPRequestHandler, ThreadingHTTPServer from pathlib import Path from typing import Dict, List +from urllib.parse import unquote, urlparse from ci.defs.job_configs import JobConfigs from ci.jobs.scripts.clickhouse_version import CHVersion @@ -33,12 +37,74 @@ # atexit.register(os.remove, f.name) -def read_build_urls(build_name: str): - artifact_report = temp_path / f"artifact_report_build_{build_name}.json" - if artifact_report.is_file(): - with open(artifact_report, "r", encoding="utf-8") as f: - return json.load(f)["build_urls"] - return [] +def package_files_from_dir(pkg_dir: Path, image_repo: str) -> Dict[str, List[Path]]: + if "clickhouse-server" in image_repo: + packages = [ + "clickhouse-client", + "clickhouse-server", + "clickhouse-common-static", + ] + elif "clickhouse-keeper" in image_repo: + packages = ["clickhouse-common-static", "clickhouse-keeper"] + else: + assert False, "BUG" + + files = [f for f in pkg_dir.iterdir() if f.is_file() and "-dbg" not in f.name] + result: Dict[str, List[Path]] = {} + for arch in ARCH: + matched = [] + for f in files: + if not any(pkg in f.name for pkg in packages): + continue + if f"_{arch}." not in f.name and f"-{arch}." not in f.name: + continue + matched.append(f) + result[arch] = sorted(matched, key=lambda p: p.name) + return result + + +def start_package_http_server(files: List[Path]) -> ThreadingHTTPServer: + http_root = temp_path / "docker_pkg_http" + if http_root.exists(): + shutil.rmtree(http_root) + http_root.mkdir(parents=True) + allowed = set() + for f in files: + dest = http_root / f.name + if dest.exists(): + continue + try: + os.link(f, dest) + except OSError: + shutil.copy2(f, dest) + allowed.add(f.name) + + class Handler(SimpleHTTPRequestHandler): + def __init__(self, *args, **kwargs): + super().__init__(*args, directory=str(http_root), **kwargs) + + def list_directory(self, _path): + self.send_error(404, "Not found") + return None + + def do_GET(self): + name = Path(unquote(urlparse(self.path).path)).name + if name not in allowed: + self.send_error(404, "Not found") + return + self.path = "/" + name + super().do_GET() + + def log_message(self, fmt, *args): + logging.info("package-http " + fmt, *args) + + # 0.0.0.0: buildx uses the docker-container driver; wget runs in that + # container and reaches the runner via host.docker.internal/host-gateway. + httpd = ThreadingHTTPServer(("0.0.0.0", 0), Handler) + threading.Thread( + target=httpd.serve_forever, daemon=True, name="package-http" + ).start() + return httpd class DockerImageData: @@ -496,6 +562,8 @@ def build_and_push_image( else: urls = tgz_urls cmd_args = list(init_args) + if urls and any("host.docker.internal" in url for url in urls): + cmd_args.append("--add-host=host.docker.internal:host-gateway") cmd_args.extend( buildx_args( repo_urls, @@ -724,43 +792,27 @@ def main(): repo_urls = {} direct_urls: Dict[str, List[str]] = {} - for arch, build_name in zip(ARCH, ("amd_release", "arm_release")): - if args.allow_build_reuse: - # read s3 urls from pre-downloaded build reports - if "clickhouse-server" in image_repo: - PACKAGES = [ - "clickhouse-client", - "clickhouse-server", - "clickhouse-common-static", - ] - elif "clickhouse-keeper" in image_repo: - # Both packages are needed to cover all three keeper image variants: - # distroless: installs from .deb via dpkg; clickhouse-common-static - # provides the clickhouse multi-tool binary (clickhouse-keeper - # is a symlink to it). clickhouse-keeper .deb is not published - # separately, so the common-static .deb is the only source. - # alpine/ubuntu: installs from .tgz; clickhouse-keeper provides the - # standalone keeper binary and its symlinks. The common-static - # .tgz is implicitly excluded because the url filter below - # keeps only urls containing "clickhouse-keeper" in the name. - PACKAGES = ["clickhouse-common-static", "clickhouse-keeper"] - else: - assert False, "BUG" - urls = read_build_urls(build_name) - assert urls, "URLS has not been read from build report" - direct_urls[arch] = [ - url - for url in urls - if any(package in url for package in PACKAGES) and "-dbg" not in url - ] - elif args.bucket_prefix: - assert not args.allow_build_reuse + if args.bucket_prefix: + for arch, build_name in zip(ARCH, ("amd_release", "arm_release")): repo_urls[arch] = f"{args.bucket_prefix}/{build_name}" - print(f"Bucket prefix is set: Fetching packages from [{repo_urls}]") - else: - assert ( - False - ), "--sha, --bucket_prefix or --allow-build-reuse (to fetch packages from build report) must be provided" + print(f"Bucket prefix is set: Fetching packages from [{repo_urls}]") + else: + # Praktika downloads job.requires (DEB_*/TGZ_*) into ci/tmp. Serve only + # those package files so the Dockerfile wget path works without S3 creds. + files_by_arch = package_files_from_dir(temp_path, image_repo) + package_files = [f for files in files_by_arch.values() for f in files] + assert package_files, f"No packages found in {temp_path} for {image_repo}" + httpd = start_package_http_server(package_files) + atexit.register(httpd.shutdown) + port = httpd.server_address[1] + base_url = f"http://host.docker.internal:{port}" + logging.info("Serving %s package files at %s", len(package_files), base_url) + direct_urls = { + arch: [f"{base_url}/{f.name}" for f in files] + for arch, files in files_by_arch.items() + } + for arch, urls in direct_urls.items(): + logging.info("Package URLs [%s]: %s", arch, urls) if push: docker_login() diff --git a/ci/jobs/source_upload.py b/ci/jobs/source_upload.py deleted file mode 100644 index 3ef936473fb75..0000000000000 --- a/ci/jobs/source_upload.py +++ /dev/null @@ -1,67 +0,0 @@ -#!/usr/bin/env python3 -from pathlib import Path - -from ci.jobs.scripts.clickhouse_version import CHVersion -from ci.praktika._environment import _Environment -from ci.praktika.result import Result -from ci.praktika.s3 import S3 -from ci.praktika.settings import Settings -from ci.praktika.utils import Shell, Utils - -REPO_PATH = Path(Utils.cwd()) - -# The source tarball is published next to the release build's artifacts so that -# the legacy S3 layout (consumed outside this repo) keeps working. -S3_SUBDIR = "build_amd_release" - - -def checkout_submodules(): - return Shell.check( - "git submodule sync && git submodule init", verbose=True - ) and Shell.check( - "contrib/update-submodules.sh --max-procs 10", verbose=True, retries=3 - ) - - -def get_version() -> CHVersion: - version = CHVersion.get_current_version_from_ci_pipeline() - if not version: - version = CHVersion.get_current_version(no_strict=True) - return version - - -def main(): - stopwatch = Utils.Stopwatch() - - version = get_version() - tarball = REPO_PATH.parent / f"clickhouse-{version.string}.src.tar.gz" - - results = [ - Result.from_commands_run( - name="Checkout submodules", - command=checkout_submodules, - ) - ] - - if results[-1].is_ok(): - results.append( - Result.from_commands_run( - name="Create source tar", - command=f"tar czf {tarball} -C {REPO_PATH.parent} {REPO_PATH.name}", - ) - ) - - link = "" - if results[-1].is_ok(): - s3_path = f"{Settings.S3_ARTIFACT_PATH}/{_Environment.get().get_s3_prefix()}/{S3_SUBDIR}" - link = S3.copy_file_to_s3(s3_path=s3_path, local_path=str(tarball)) - - Result.create_from( - results=results, - stopwatch=stopwatch, - links=[link] if link else None, - ).complete_job() - - -if __name__ == "__main__": - main() diff --git a/ci/tests/test_clickhouse_version.py b/ci/tests/test_clickhouse_version.py index 4ccbd8fcbe80f..f64029a1cdae9 100644 --- a/ci/tests/test_clickhouse_version.py +++ b/ci/tests/test_clickhouse_version.py @@ -1,6 +1,5 @@ from pathlib import Path -from ci.jobs import source_upload from ci.jobs.scripts import clickhouse_version from ci.jobs.scripts.clickhouse_version import CHVersion from ci.jobs.scripts.workflow_hooks import version_log @@ -181,37 +180,3 @@ def store_version_data_in_ci_pipeline(self): assert stored["kv"] == config.custom_data["version"] assert "cidb" not in stored - - -def test_source_upload_prefers_pipeline_version(monkeypatch): - pipeline_version = CHVersion( - 26, - 8, - 1, - 54513, - tweak=1, - githash="pr-sha", - version_type="testing", - ) - monkeypatch.setattr( - CHVersion, - "get_current_version_from_ci_pipeline", - classmethod(lambda cls: pipeline_version), - ) - monkeypatch.setattr( - CHVersion, - "get_current_version", - classmethod( - lambda cls, **kwargs: (_ for _ in ()).throw( - AssertionError("repo fallback must not replace the pipeline version") - ) - ), - ) - - assert source_upload.get_version().string == "26.8.1.1" - - -def test_source_upload_restores_submodule_cache(): - from ci.defs.altinity_jobs import AltinityJobConfigs - - assert AltinityJobConfigs.source_upload_job.needs_submodules diff --git a/ci/workflows/fast_builds.py b/ci/workflows/fast_builds.py index d55b75d675485..654f513738b87 100644 --- a/ci/workflows/fast_builds.py +++ b/ci/workflows/fast_builds.py @@ -48,7 +48,6 @@ JobConfigs.docker_keeper, *JobConfigs.install_check_master_jobs, *AltinityJobConfigs.sign_release_jobs, - AltinityJobConfigs.source_upload_job, *[ job for job in FUNCTIONAL_TESTS_JOBS diff --git a/ci/workflows/master.py b/ci/workflows/master.py index bede045233dd6..e08ac8581d4c7 100644 --- a/ci/workflows/master.py +++ b/ci/workflows/master.py @@ -60,7 +60,6 @@ JobConfigs.docker_keeper, *JobConfigs.install_check_master_jobs, *AltinityJobConfigs.sign_release_jobs, - AltinityJobConfigs.source_upload_job, *JobConfigs.compatibility_test_jobs, *[ j diff --git a/ci/workflows/pull_request.py b/ci/workflows/pull_request.py index f7fdfdd22eb27..5e7cb2247fc34 100644 --- a/ci/workflows/pull_request.py +++ b/ci/workflows/pull_request.py @@ -244,7 +244,6 @@ .set_name("Keeper Stress Tests (PR)") .set_timeout(3 * 3600), *JobConfigs.toolchain_build_jobs, - AltinityJobConfigs.source_upload_job, ], additional_jobs=["GrypeScan", "RegressionPR", "CIReport"], artifacts=[ From 1ac83a2935df93ee622c3fdeeb4e595e7577b625 Mon Sep 17 00:00:00 2001 From: MyroTk Date: Thu, 19 Nov 2026 08:03:41 -0800 Subject: [PATCH 2/2] Drop the removed promql_compliance job from the PR workflow. finish_workflow still depended on a job antalya-26.8 no longer defines, which made the workflow file invalid. Co-authored-by: Cursor --- .github/workflows/pull_request.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index 58699219324e4..4f75c8c20fcd6 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -6286,7 +6286,7 @@ jobs: finish_workflow: runs-on: [self-hosted, altinity-on-demand, altinity-style-checker] - needs: [ast_fuzzer_amd_debug, ast_fuzzer_amd_debug_targeted, ast_fuzzer_amd_debug_targeted_old_compatibility, ast_fuzzer_amd_msan, ast_fuzzer_amd_tsan, ast_fuzzer_arm_asan_ubsan, bugfix_validation_unit_tests, build_amd_asan_ubsan, build_amd_binary, build_amd_debug, build_amd_msan, build_amd_release, build_amd_tsan, build_arm_asan_ubsan, build_arm_binary, build_arm_debug, build_arm_msan, build_arm_release, build_arm_tsan, build_toolchain_pgo_bolt_aarch64, build_toolchain_pgo_bolt_amd64, build_wasm_parser, buzzhouse_amd_debug, buzzhouse_amd_msan, buzzhouse_amd_tsan, buzzhouse_arm_asan_ubsan, ci_tests, compatibility_check_amd_release, compatibility_check_arm_release, config_workflow, docker_keeper_image, docker_server_image, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest, fast_test, install_packages_amd_release, install_packages_arm_release, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_1_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_2_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_3_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_4_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_5_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_6_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_7_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_8_8, integration_tests_amd_asan_ubsan_targeted, integration_tests_amd_msan_10_10, integration_tests_amd_msan_1_10, integration_tests_amd_msan_2_10, integration_tests_amd_msan_3_10, integration_tests_amd_msan_4_10, integration_tests_amd_msan_5_10, integration_tests_amd_msan_6_10, integration_tests_amd_msan_7_10, integration_tests_amd_msan_8_10, integration_tests_amd_msan_9_10, integration_tests_amd_tsan_1_8, integration_tests_amd_tsan_2_8, integration_tests_amd_tsan_3_8, integration_tests_amd_tsan_4_8, integration_tests_amd_tsan_5_8, integration_tests_amd_tsan_6_8, integration_tests_amd_tsan_7_8, integration_tests_amd_tsan_8_8, integration_tests_arm_binary_distributed_plan_1_4, integration_tests_arm_binary_distributed_plan_2_4, integration_tests_arm_binary_distributed_plan_3_4, integration_tests_arm_binary_distributed_plan_4_4, keeper_stress_tests_pr, promql_compliance, quick_functional_tests, sqllogic_test, sqlstorm_test, stateless_tests_amd_asan_ubsan_cas_s3_storage_parallel_1_2, stateless_tests_amd_asan_ubsan_cas_s3_storage_parallel_2_2, stateless_tests_amd_asan_ubsan_db_disk_distributed_plan_sequential_selected_tests, stateless_tests_amd_asan_ubsan_distributed_plan_parallel_selected_tests, stateless_tests_amd_binary_cas_s3_storage_parallel, stateless_tests_amd_binary_cas_storage_parallel, stateless_tests_amd_debug_distributed_plan_s3_storage_parallel, stateless_tests_amd_debug_distributed_plan_s3_storage_sequential, stateless_tests_amd_debug_parallel, stateless_tests_amd_debug_sequential, stateless_tests_amd_msan_cas_s3_storage_parallel_1_3, stateless_tests_amd_msan_cas_s3_storage_parallel_2_3, stateless_tests_amd_msan_cas_s3_storage_parallel_3_3, stateless_tests_amd_msan_wasmedge_parallel_1_6, stateless_tests_amd_msan_wasmedge_parallel_2_6, stateless_tests_amd_msan_wasmedge_parallel_3_6, stateless_tests_amd_msan_wasmedge_parallel_4_6, stateless_tests_amd_msan_wasmedge_parallel_5_6, stateless_tests_amd_msan_wasmedge_parallel_6_6, stateless_tests_amd_msan_wasmedge_sequential_1_4, stateless_tests_amd_msan_wasmedge_sequential_2_4, stateless_tests_amd_msan_wasmedge_sequential_3_4, stateless_tests_amd_msan_wasmedge_sequential_4_4, stateless_tests_amd_tsan_cas_s3_storage_parallel_1_2, stateless_tests_amd_tsan_cas_s3_storage_parallel_2_2, stateless_tests_amd_tsan_parallel_selected_tests, stateless_tests_amd_tsan_s3_storage_parallel_selected_tests, stateless_tests_amd_tsan_s3_storage_sequential_selected_tests, stateless_tests_amd_tsan_sequential_selected_tests, stateless_tests_arm_asan_ubsan_azure_parallel_1_8, stateless_tests_arm_asan_ubsan_azure_parallel_2_8, stateless_tests_arm_asan_ubsan_azure_parallel_3_8, stateless_tests_arm_asan_ubsan_azure_parallel_4_8, stateless_tests_arm_asan_ubsan_azure_parallel_5_8, stateless_tests_arm_asan_ubsan_azure_parallel_6_8, stateless_tests_arm_asan_ubsan_azure_parallel_7_8, stateless_tests_arm_asan_ubsan_azure_parallel_8_8, stateless_tests_arm_asan_ubsan_azure_sequential_1_2, stateless_tests_arm_asan_ubsan_azure_sequential_2_2, stateless_tests_arm_asan_ubsan_targeted, stateless_tests_arm_binary_cas_s3_storage_parallel, stateless_tests_arm_binary_parallel, stateless_tests_arm_binary_sequential, stress_test_amd_asan_ubsan, stress_test_amd_debug, stress_test_amd_msan, stress_test_amd_tsan, stress_test_arm_asan_ubsan, stress_test_arm_asan_ubsan_s3, stress_test_arm_debug, stress_test_arm_msan, stress_test_arm_release, stress_test_arm_tsan, unit_tests_asan_ubsan, unit_tests_asan_ubsan_function_prop_fuzzer, unit_tests_msan, unit_tests_msan_function_prop_fuzzer, unit_tests_tsan, unit_tests_tsan_function_prop_fuzzer] + needs: [ast_fuzzer_amd_debug, ast_fuzzer_amd_debug_targeted, ast_fuzzer_amd_debug_targeted_old_compatibility, ast_fuzzer_amd_msan, ast_fuzzer_amd_tsan, ast_fuzzer_arm_asan_ubsan, bugfix_validation_unit_tests, build_amd_asan_ubsan, build_amd_binary, build_amd_debug, build_amd_msan, build_amd_release, build_amd_tsan, build_arm_asan_ubsan, build_arm_binary, build_arm_debug, build_arm_msan, build_arm_release, build_arm_tsan, build_toolchain_pgo_bolt_aarch64, build_toolchain_pgo_bolt_amd64, build_wasm_parser, buzzhouse_amd_debug, buzzhouse_amd_msan, buzzhouse_amd_tsan, buzzhouse_arm_asan_ubsan, ci_tests, compatibility_check_amd_release, compatibility_check_arm_release, config_workflow, docker_keeper_image, docker_server_image, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest, fast_test, install_packages_amd_release, install_packages_arm_release, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_1_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_2_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_3_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_4_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_5_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_6_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_7_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_8_8, integration_tests_amd_asan_ubsan_targeted, integration_tests_amd_msan_10_10, integration_tests_amd_msan_1_10, integration_tests_amd_msan_2_10, integration_tests_amd_msan_3_10, integration_tests_amd_msan_4_10, integration_tests_amd_msan_5_10, integration_tests_amd_msan_6_10, integration_tests_amd_msan_7_10, integration_tests_amd_msan_8_10, integration_tests_amd_msan_9_10, integration_tests_amd_tsan_1_8, integration_tests_amd_tsan_2_8, integration_tests_amd_tsan_3_8, integration_tests_amd_tsan_4_8, integration_tests_amd_tsan_5_8, integration_tests_amd_tsan_6_8, integration_tests_amd_tsan_7_8, integration_tests_amd_tsan_8_8, integration_tests_arm_binary_distributed_plan_1_4, integration_tests_arm_binary_distributed_plan_2_4, integration_tests_arm_binary_distributed_plan_3_4, integration_tests_arm_binary_distributed_plan_4_4, keeper_stress_tests_pr, quick_functional_tests, sqllogic_test, sqlstorm_test, stateless_tests_amd_asan_ubsan_cas_s3_storage_parallel_1_2, stateless_tests_amd_asan_ubsan_cas_s3_storage_parallel_2_2, stateless_tests_amd_asan_ubsan_db_disk_distributed_plan_sequential_selected_tests, stateless_tests_amd_asan_ubsan_distributed_plan_parallel_selected_tests, stateless_tests_amd_binary_cas_s3_storage_parallel, stateless_tests_amd_binary_cas_storage_parallel, stateless_tests_amd_debug_distributed_plan_s3_storage_parallel, stateless_tests_amd_debug_distributed_plan_s3_storage_sequential, stateless_tests_amd_debug_parallel, stateless_tests_amd_debug_sequential, stateless_tests_amd_msan_cas_s3_storage_parallel_1_3, stateless_tests_amd_msan_cas_s3_storage_parallel_2_3, stateless_tests_amd_msan_cas_s3_storage_parallel_3_3, stateless_tests_amd_msan_wasmedge_parallel_1_6, stateless_tests_amd_msan_wasmedge_parallel_2_6, stateless_tests_amd_msan_wasmedge_parallel_3_6, stateless_tests_amd_msan_wasmedge_parallel_4_6, stateless_tests_amd_msan_wasmedge_parallel_5_6, stateless_tests_amd_msan_wasmedge_parallel_6_6, stateless_tests_amd_msan_wasmedge_sequential_1_4, stateless_tests_amd_msan_wasmedge_sequential_2_4, stateless_tests_amd_msan_wasmedge_sequential_3_4, stateless_tests_amd_msan_wasmedge_sequential_4_4, stateless_tests_amd_tsan_cas_s3_storage_parallel_1_2, stateless_tests_amd_tsan_cas_s3_storage_parallel_2_2, stateless_tests_amd_tsan_parallel_selected_tests, stateless_tests_amd_tsan_s3_storage_parallel_selected_tests, stateless_tests_amd_tsan_s3_storage_sequential_selected_tests, stateless_tests_amd_tsan_sequential_selected_tests, stateless_tests_arm_asan_ubsan_azure_parallel_1_8, stateless_tests_arm_asan_ubsan_azure_parallel_2_8, stateless_tests_arm_asan_ubsan_azure_parallel_3_8, stateless_tests_arm_asan_ubsan_azure_parallel_4_8, stateless_tests_arm_asan_ubsan_azure_parallel_5_8, stateless_tests_arm_asan_ubsan_azure_parallel_6_8, stateless_tests_arm_asan_ubsan_azure_parallel_7_8, stateless_tests_arm_asan_ubsan_azure_parallel_8_8, stateless_tests_arm_asan_ubsan_azure_sequential_1_2, stateless_tests_arm_asan_ubsan_azure_sequential_2_2, stateless_tests_arm_asan_ubsan_targeted, stateless_tests_arm_binary_cas_s3_storage_parallel, stateless_tests_arm_binary_parallel, stateless_tests_arm_binary_sequential, stress_test_amd_asan_ubsan, stress_test_amd_debug, stress_test_amd_msan, stress_test_amd_tsan, stress_test_arm_asan_ubsan, stress_test_arm_asan_ubsan_s3, stress_test_arm_debug, stress_test_arm_msan, stress_test_arm_release, stress_test_arm_tsan, unit_tests_asan_ubsan, unit_tests_asan_ubsan_function_prop_fuzzer, unit_tests_msan, unit_tests_msan_function_prop_fuzzer, unit_tests_tsan, unit_tests_tsan_function_prop_fuzzer] if: ${{ !cancelled() && needs.config_workflow.outputs.pipeline_status != '' }} name: "Finish Workflow" outputs: