From 4ffeb327eefd819092b0807e26480b4b4d07d22b Mon Sep 17 00:00:00 2001 From: John Cater Date: Fri, 13 Nov 2026 16:44:13 +0000 Subject: [PATCH 1/2] Test ios against each installed Xcode version Run the ios tests in CI against both Xcode 26.5 and 27.2, locally on macos and remotely from both the macos and linux jobs. Each ios matrix entry sets xcode_version and ios_sdk_version, which are passed to test-all.py as --xcode_version and --ios_sdk_version. test-all.py now forwards any arguments after `--` to `bazel test`. This replaces the APPLE_SDK_VERSION_OVERRIDE environment variable. The macos 27.2 jobs are disabled for now: local tests fail because Xcode 27.2 isn't installed on the macos CI fleet yet, and remote tests fail because remote-only Xcode versions aren't supported until #463 is merged. Part of BZL-94. AI-Assisted: Claude Code, Anthropic Opus 5.5 --- .github/workflows/main.yml | 109 ++++++++++++++++++++++++++++++++----- infra/test-all.py | 11 +++- 2 files changed, 105 insertions(+), 15 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index e940a2d..42954ad 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -33,13 +33,13 @@ jobs: strategy: fail-fast: false matrix: + # ios is tested separately in macos-ios-test-matrix. test_package: - cpp - csharp - docker - genrules - go - - ios - java - kotlin - perl @@ -67,9 +67,56 @@ jobs: ARCH: ${{ matrix.arch }} OS: ${{ matrix.os }} EXECUTION_TYPE: ${{ matrix.execution_type }} - APPLE_SDK_VERSION_OVERRIDE: 26.5 run: python3 infra/test-all.py --package=${{ matrix.test_package }} + macos-ios-test-matrix: + runs-on: + - self-hosted + - "os=macos" + - "arch=arm64" + - "os_distribution=tahoe" + - "os_version=26" + - "revision=384737b1a2f579dc6dea4ef4341e232d217a5161" + timeout-minutes: 10 + strategy: + fail-fast: false + matrix: + execution_type: [remote, local] + # Each installed Xcode version, with the iOS SDK versions to test it with. + xcode: + - { version: "26.5", ios_sdk: "26.5" } + - { version: "27.2", ios_sdk: "27.2" } + exclude: + # TODO: Re-enable when remote-only Xcode versions are supported. + - execution_type: remote + xcode: { version: "27.2", ios_sdk: "27.2" } + # TODO: Re-enable when Xcode 27.2 is installed on the macos CI fleet. + - execution_type: local + xcode: { version: "27.2", ios_sdk: "27.2" } + + steps: + - uses: "actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd" + + - name: Set up authentication + shell: bash + run: cp infra/bazelrc .bazelrc.user + + - name: Generate python requirements lock + shell: bash + run: bazel run //python:requirements.update + + - name: Build and test + env: + ARCH: arm64 + OS: macos + EXECUTION_TYPE: ${{ matrix.execution_type }} + run: >- + python3 infra/test-all.py + --package=ios + -- + --xcode_version=${{ matrix.xcode.version }} + --ios_sdk_version=${{ matrix.xcode.ios_sdk }} + linux-test-matrix: runs-on: - self-hosted @@ -86,13 +133,13 @@ jobs: strategy: fail-fast: false matrix: + # ios is tested separately in linux-ios-test-matrix. test_package: - cpp - csharp - docker - genrules - go - - ios - java - kotlin - perl @@ -104,16 +151,6 @@ jobs: - arch: x64 - os: linux - # Override remote ios tests to use macos/arm64 in the remote executor. - - test_package: ios - execution_type: remote - arch: arm64 - os: macos - exclude: - # Local ios tests are not supported on linux - - test_package: ios - execution_type: local - steps: - uses: "actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd" @@ -132,6 +169,52 @@ jobs: EXECUTION_TYPE: ${{ matrix.execution_type }} run: python3 infra/test-all.py --package=${{ matrix.test_package }} + # Local ios tests are not supported on linux, so this only runs remote tests. + linux-ios-test-matrix: + runs-on: + - self-hosted + - "os=linux" + - "arch=x64" + - "engflow-bento-name=example-gh-x64" + - "engflow-cluster=glass" + - "engflow-job-name=linux-test-matrix-remote" + - "engflow-job-type=${{github.event_name}}-${{github.ref_name}}" + - "engflow-run-id=${{ github.run_id }}" + - "engflow-bento-allow-save=false" + + timeout-minutes: 10 + strategy: + fail-fast: false + matrix: + # Each installed Xcode version, with the iOS SDK versions to test it with. + xcode: + - { version: "26.5", ios_sdk: "26.5" } + - { version: "27.2", ios_sdk: "27.2" } + + steps: + - uses: "actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd" + + - name: Set up authentication + shell: bash + run: cp infra/bazelrc .bazelrc.user + + - name: Generate python requirements lock + shell: bash + run: bazel run //python:requirements.update + + - name: Build and test + env: + # The ios tests use macos/arm64 in the remote executor. + ARCH: arm64 + OS: macos + EXECUTION_TYPE: remote + run: >- + python3 infra/test-all.py + --package=ios + -- + --xcode_version=${{ matrix.xcode.version }} + --ios_sdk_version=${{ matrix.xcode.ios_sdk }} + bento-snapshot: runs-on: - self-hosted diff --git a/infra/test-all.py b/infra/test-all.py index a3e50d7..5ec8b1c 100755 --- a/infra/test-all.py +++ b/infra/test-all.py @@ -43,7 +43,7 @@ def find_tests(package): return [t for t in output.split("\n") if t] -def run_tests(package): +def run_tests(package, extra_flags): targets = find_tests(package) if not targets: print(f"No targets found for {package}, skipping.") @@ -66,6 +66,8 @@ def run_tests(package): if package in EXTRA_FLAGS: flags += EXTRA_FLAGS[package] + flags += extra_flags + print(f"---------------\nTesting {package}...") sys.stdout.flush() @@ -90,6 +92,11 @@ def main(): action=argparse.BooleanOptionalAction, help="Whether to validate environment variables", ) + parser.add_argument( + "extra_flags", + nargs="*", + help="Extra flags to pass to bazel test, after --.", + ) opts = parser.parse_args() if opts.validate_env: @@ -97,7 +104,7 @@ def main(): package = opts.package - returncode = run_tests(opts.package) + returncode = run_tests(opts.package, opts.extra_flags) if returncode != 0: sys.exit(returncode) From 3e63087aa1cb39aea9496572519e045a0dc6253d Mon Sep 17 00:00:00 2001 From: John Cater Date: Fri, 13 Nov 2026 13:32:44 -0500 Subject: [PATCH 2/2] Add simulator device override --- .github/workflows/main.yml | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 42954ad..80f4829 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -83,16 +83,17 @@ jobs: matrix: execution_type: [remote, local] # Each installed Xcode version, with the iOS SDK versions to test it with. + # ios_simulator_device is optional, and overrides the default from .bazelrc. xcode: - { version: "26.5", ios_sdk: "26.5" } - - { version: "27.2", ios_sdk: "27.2" } + - { version: "27.2", ios_sdk: "27.2", ios_simulator_device: "iPhone Duo" } exclude: # TODO: Re-enable when remote-only Xcode versions are supported. - execution_type: remote - xcode: { version: "27.2", ios_sdk: "27.2" } + xcode: { version: "27.2", ios_sdk: "27.2", ios_simulator_device: "iPhone Duo" } # TODO: Re-enable when Xcode 27.2 is installed on the macos CI fleet. - execution_type: local - xcode: { version: "27.2", ios_sdk: "27.2" } + xcode: { version: "27.2", ios_sdk: "27.2", ios_simulator_device: "iPhone Duo" } steps: - uses: "actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd" @@ -116,6 +117,7 @@ jobs: -- --xcode_version=${{ matrix.xcode.version }} --ios_sdk_version=${{ matrix.xcode.ios_sdk }} + ${{ matrix.xcode.ios_simulator_device && format('--@rules_apple//apple/build_settings:ios_simulator_device="{0}"', matrix.xcode.ios_simulator_device) || '' }} linux-test-matrix: runs-on: @@ -187,9 +189,10 @@ jobs: fail-fast: false matrix: # Each installed Xcode version, with the iOS SDK versions to test it with. + # ios_simulator_device is optional, and overrides the default from .bazelrc. xcode: - { version: "26.5", ios_sdk: "26.5" } - - { version: "27.2", ios_sdk: "27.2" } + - { version: "27.2", ios_sdk: "27.2", ios_simulator_device: "iPhone Duo" } steps: - uses: "actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd" @@ -214,6 +217,7 @@ jobs: -- --xcode_version=${{ matrix.xcode.version }} --ios_sdk_version=${{ matrix.xcode.ios_sdk }} + ${{ matrix.xcode.ios_simulator_device && format('--@rules_apple//apple/build_settings:ios_simulator_device="{0}"', matrix.xcode.ios_simulator_device) || '' }} bento-snapshot: runs-on: