feat: add scheduler backend (incomplete)

.dockerignore

@@ -0,0 +1,11 @@
+**
+!dist/
+!dist/*.whl
+!deploy/
+!deploy/Dockerfile
+!deploy/project/
+!deploy/project/sqlbuild_project.toml
+!deploy/project/models/
+!deploy/project/models/*.sql
+!deploy/project/orchestration/
+!deploy/project/orchestration/*.py

.github/diagnostics-recipient.pem

@@ -0,0 +1,25 @@
+-----BEGIN CERTIFICATE-----
+MIIEJTCCAo2gAwIBAgIUUgxZ6C8ykh+91sFO8z78yEl0flIwDQYJKoZIhvcNAQEL
+BQAwIjEgMB4GA1UEAwwXc3FsYnVpbGQtY2ktZGlhZ25vc3RpY3MwHhcNMjYxMTAz
+MDQ0ODU4WhcNMzYxMDMxMDQ0ODU4WjAiMSAwHgYDVQQDDBdzcWxidWlsZC1jaS1k
+aWFnbm9zdGljczCCAaIwDQYJKoZIhvcNAQEBBQADggGPADCCAYoCggGBANhDATXs
+XtI0hyxJIba3/APxf8cVEZ7dTTHhyqyEpf2Pi5VKWvfLNkTJE+FIlXS37G5o4Hkb
+Tnm44fWdQg/UAnQCY5J9QjEGJQAIv+4jx9jcALtbh0VG0hXHuEXBMFi3pakazhVf
+rLHUtI4pDuU4Z0Zl7DsivZTgDiOAyCE+uIyCcKg5sn3vq3ITej7vIXaPHbrLV794
+M6hyPbQu8kiREW+0g6iktAiHiw4prYBlbGIVfWoCzkNUEtO6djMKQlg1es8bhCYo
+cVOwChPJrgsTX4tpN7I1y6kAYQ3EAd9AcoG+YmZ/Lf0wsawH4BrM0XZWV7Tr93CO
+nrLqegrCx8eYn9cweoCsWq4Ad5xDecYnbaxX3c9bFfuDmaU8tUKjm4KXTQgJFznW
+0cc1dRSbO7W8ogZyBj9KvRoJPboHiXaZQSHwcx/rwKmjZJ6yDcGTlTD7qY9WciuR
+KK5mWLJvsQ33fjMTGnaW5cd73UR3Y6phvHv1s4HpVeAWOjWJAVuEzDjU1QIDAQAB
+o1MwUTAdBgNVHQ4EFgQUg0RhY1QY4ID4KbGS710v48wx+jQwHwYDVR0jBBgwFoAU
+g0RhY1QY4ID4KbGS710v48wx+jQwDwYDVR0TAQH/BAUwAwEB/zANBgkqhkiG9w0B
+AQsFAAOCAYEAlMrRv+zYmWbdSTM5vZM1qJMlDifRbRqTqcbMzv8W0tOrm+Ch2b94
+VaPW0lCs+hOEUfAP2osegfLS6eAlv3MW774iVZzEcHjoQpvQQ2biaPgpJTe4jV+U
+iwWlYrBbOjFPOQsnvxHSKjsrkt5bHcEXxShSkCYLjS+HvUT2huZ5LGxBUR9yfia2
+gkOIa1zlPlbAMyUt6Fj66NL2+nbsCYLuGfQafqF4UUd3rG2sF1pkxSEuof1VpKTQ
+rKU9u0+csNFHF20CGZ7h9jUU1ATLLwBlPunFCCO70gIGcoNspqPRIOwsEXXGr81N
+aXJAeA5XMesS/v0k0x2GBLXujTT5+kC3iQA0+OfxH8t82D4xTXd2wYIKYwnzK+nv
+XkF3Of+Y/puUv7R+ADfG0K7aMdLkBDjllJJL7gUml+X281m+2S5UlJZIHxe2kYGB
+/i2XCahytLRBY9EGVSLEWjkhfnOydpjwuHfjkNaJDqNRDexMbzifxi0/liv8jbvV
+ZCqMfibxPUoT
+-----END CERTIFICATE-----

.github/workflows/build-distributions.yml

@@ -189,6 +189,11 @@ jobs:
           "${run_as[@]}" "${smoke_scripts}/sqb" --version
           "${run_as[@]}" "${smoke_scripts}/sqb" --help
           "${run_as[@]}" "${smoke_scripts}/sqlbuild" --help
+          if [[ "${{ inputs.prebuilt-run-id == '' }}" == "true" ]]; then
+            cd "${GITHUB_WORKSPACE}"
+            "${run_as[@]}" "${smoke_python}" -m tests.integration.src.sqlbuild.compiler.compiled_project.publication.wheel.helpers \
+              "${smoke_scripts}/sqb${{ runner.os == 'Windows' && '.exe' || '' }}"
+          fi
       - name: Check wheel contents and run a native command
         if: matrix.platform.name == 'linux-x86_64'
         run: |

.github/workflows/verify.yml

@@ -142,7 +142,7 @@ jobs:
               *) docs_only=false ;;
             esac
             case "${path}" in
-              crates/*|Cargo.toml|Cargo.lock|pyproject.toml|.github/workflows/verify.yml|.github/workflows/build-distributions.yml)
+              crates/*|Cargo.toml|Cargo.lock|pyproject.toml|.github/workflows/verify.yml|.github/workflows/build-distributions.yml|deploy/Dockerfile|deploy/project/*|deploy/compose*|compose.yml|.dockerignore|Makefile|tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_compose.py|tests/integration/src/sqlbuild/compiler/compiled_project/publication/docker/compose/*|benchmarks/duckshop/edge_cases/orchestration/container_deployment.json)
                 wheels=true
                 ;;
             esac
@@ -249,6 +249,8 @@ jobs:
         run: uv python install 3.12
       - name: Verify lockfile
         run: uv lock --check --python 3.12
+      - name: Verify native workspace lockfile
+        run: cargo metadata --locked --format-version 1 > /dev/null
 
   checks:
     name: Static checks and unit + integration tests
@@ -279,13 +281,59 @@ jobs:
         run: uv python install 3.12
       - name: Install dependencies
         run: uv sync --locked --all-groups --all-extras
+      - uses: actions/setup-node@v4
+        with:
+          node-version: "22"
+          cache: npm
+          cache-dependency-path: clients/typescript/package-lock.json
+      - name: Check generated control client against the built native API
+        run: |
+          npm --prefix clients/typescript ci
+          uv run --no-sync npm --prefix clients/typescript run check
+          uv run --no-sync npm --prefix clients/typescript test
       - name: Run static checks
         run: make check-ci
-      - name: Run native checks
-        run: make rust-check
+      - name: Prepare Docker worker tests
+        run: |
+          docker info > /dev/null
+          docker pull alpine:3.23
       - name: Run unit and integration tests
+        env:
+          SQLBUILD_REQUIRE_DOCKER_TESTS: "1"
         run: make test
 
+  native_checks:
+    name: Native workspace checks
+    needs: [changes, lockfile]
+    if: needs.changes.outputs.checks == 'true'
+    runs-on: ubuntu-24.04
+    timeout-minutes: 45
+    steps:
+      - uses: actions/checkout@v4
+      - uses: astral-sh/setup-uv@v5
+        with:
+          version: "0.12.3"
+          enable-cache: true
+          cache-dependency-glob: |
+            uv.lock
+            Cargo.lock
+      - uses: dtolnay/rust-toolchain@master
+        with:
+          toolchain: stable
+      - uses: Swatinem/rust-cache@v2
+        with:
+          shared-key: verify-checks
+          save-if: false
+      - name: Install Python
+        run: uv python install 3.12
+      - name: Install native test dependencies without rebuilding the packages
+        run: uv sync --locked --all-groups --all-extras --no-install-package sqlbuild --no-install-package sqlbuild-cli
+      - name: Run native checks
+        env:
+          UV_NO_SYNC: "1"
+          PYO3_PYTHON: ${{ github.workspace }}/.venv/bin/python
+        run: make rust-check
+
   e2e_shards:
     name: DuckDB E2E (${{ matrix.name }})
     needs: [changes, lockfile]
@@ -345,8 +393,23 @@ jobs:
               f"{os.environ.get('PYTEST_XDIST_AUTO_NUM_WORKERS', '<unset>')}"
           )
           PY
+      - name: Validate diagnostic encryption recipient
+        if: matrix.target == 'test-e2e-duckdb-cli'
+        run: openssl x509 -in .github/diagnostics-recipient.pem -noout
       - name: Run DuckDB E2E shard
+        env:
+          SQLBUILD_COMPARISON_ENCRYPTED_DIR: ${{ matrix.target == 'test-e2e-duckdb-cli' && format('{0}/comparison-encrypted', runner.temp) || '' }}
+          SQLBUILD_COMPARISON_RECIPIENT: ${{ github.workspace }}/.github/diagnostics-recipient.pem
         run: make ${{ matrix.target }}
+      - name: Retain encrypted comparison failures
+        if: failure() && matrix.target == 'test-e2e-duckdb-cli'
+        uses: actions/upload-artifact@v4
+        with:
+          name: encrypted-comparison-${{ github.run_id }}-${{ github.run_attempt }}
+          path: ${{ runner.temp }}/comparison-encrypted/failure-*/failure.cms
+          if-no-files-found: warn
+          compression-level: 0
+          retention-days: 7
 
   compiler_differential:
     name: Compiler golden outputs
@@ -966,6 +1029,33 @@ jobs:
     uses: ./.github/workflows/build-distributions.yml
     with:
       sdist: false
+      upload: true
+      retention-days: 3
+
+  compose:
+    name: Packaged Compose lifecycle
+    needs: [changes, lockfile, wheels]
+    if: needs.changes.outputs.wheels == 'true'
+    runs-on: ubuntu-24.04
+    timeout-minutes: 20
+    steps:
+      - uses: actions/checkout@v4
+      - uses: astral-sh/setup-uv@v5
+        with:
+          version: "0.12.3"
+          enable-cache: true
+      - name: Install Python test dependencies without rebuilding the packages
+        run: |
+          uv python install 3.12
+          uv sync --locked --all-groups --all-extras --no-install-package sqlbuild --no-install-package sqlbuild-cli
+      - uses: actions/download-artifact@v4
+        with:
+          name: dist-wheel-linux-x86_64
+          path: dist
+      - name: Install the same wheels used by the images
+        run: uv pip install --no-deps dist/*.whl
+      - name: Verify packaged Compose deployment and operational recovery
+        run: make test-compose-packaged
 
   verify:
     # The required Verify of a Release Please pull request is the commit status that the trusted
@@ -985,6 +1075,7 @@ jobs:
         changes,
         lockfile,
         checks,
+        native_checks,
         e2e_shards,
         compiler_differential,
         compiler_differential_python_versions,
@@ -999,6 +1090,7 @@ jobs:
         postgres,
         clickhouse,
         wheels,
+        compose,
       ]
     runs-on: ubuntu-24.04
     steps:
@@ -1008,6 +1100,7 @@ jobs:
           CHECKS_SELECTED: ${{ needs.changes.outputs.checks }}
           LOCKFILE_RESULT: ${{ needs.lockfile.result }}
           CHECKS_RESULT: ${{ needs.checks.result }}
+          NATIVE_CHECKS_RESULT: ${{ needs.native_checks.result }}
           E2E_SHARDS_RESULT: ${{ needs.e2e_shards.result }}
           COMPILER_DIFFERENTIAL_RESULT: ${{ needs.compiler_differential.result }}
           COMPILER_DIFFERENTIAL_PYTHON_VERSIONS_RESULT: ${{ needs.compiler_differential_python_versions.result }}
@@ -1022,14 +1115,21 @@ jobs:
           POSTGRES_RESULT: ${{ needs.postgres.result }}
           CLICKHOUSE_RESULT: ${{ needs.clickhouse.result }}
           WHEELS_RESULT: ${{ needs.wheels.result }}
+          COMPOSE_RESULT: ${{ needs.compose.result }}
         run: |
           if [[ "${CHANGES_RESULT}" != "success" || "${LOCKFILE_RESULT}" != "success" ]]; then
             echo "Required classification or lockfile checks did not pass" >&2
             exit 1
           fi
-          if [[ "${CHECKS_RESULT}" != "success" ]] && \
-             ! [[ "${CHECKS_SELECTED}" == "false" && "${CHECKS_RESULT}" == "skipped" ]]; then
-            echo "Required static checks did not pass: ${CHECKS_RESULT}" >&2
+          for result in "${CHECKS_RESULT}" "${NATIVE_CHECKS_RESULT}"; do
+            if [[ "${result}" != "success" ]] && \
+               ! [[ "${CHECKS_SELECTED}" == "false" && "${result}" == "skipped" ]]; then
+              echo "Required static or native checks did not pass: ${result}" >&2
+              exit 1
+            fi
+          done
+          if [[ "${WHEELS_RESULT}" == "success" && "${COMPOSE_RESULT}" != "success" ]]; then
+            echo "Required packaged Compose verification did not pass: ${COMPOSE_RESULT}" >&2
             exit 1
           fi
           for result in \
@@ -1046,7 +1146,8 @@ jobs:
             "${RULES_PERFORMANCE_RESULT}" \
             "${POSTGRES_RESULT}" \
             "${CLICKHOUSE_RESULT}" \
-            "${WHEELS_RESULT}"; do
+            "${WHEELS_RESULT}" \
+            "${COMPOSE_RESULT}"; do
             if [[ "${result}" != "success" && "${result}" != "skipped" ]]; then
               echo "A selected verification suite did not pass: ${result}" >&2
               exit 1

.gitignore

@@ -58,6 +58,7 @@ target/
 *.dylib
 .dlt/
 .amp/portals/*
+/.env
 
 # Local SQLBuild overrides
 sqlbuild_local.yml

CONTRIBUTING.md

@@ -42,6 +42,19 @@ For a change confined to one Rust crate, run its focused tests before rebuilding
 cargo test -p sqlbuild-sqltext given_model_files_when_matching_headers_then_offsets_follow_the_python_pattern
 ```
 
+Control-plane integration tests require Docker and run both SQLite and PostgreSQL:
+
+```bash
+make test-native NATIVE_TEST_ARGS='-p sqlbuild-orchestration --lib'
+```
+
+The same Testcontainers harness runs under `make rust-check` in CI. Each invocation starts one
+digest-pinned PostgreSQL container, waits for database readiness, shares it across the suite with
+an isolated schema per fixture, and removes it after success or failure. No host PostgreSQL setup
+is needed. Missing Docker, startup failures or missing database configuration fail the gate;
+PostgreSQL cases never silently skip. `NATIVE_TEST_ARGS` accepts Cargo test arguments; its default
+runs the workspace with all features.
+
 Keep the toolchain, `CARGO_TARGET_DIR`, profile and `RUSTFLAGS` consistent between iterations.
 Changing them can invalidate compiled artifacts. Coordinate builds that share a target directory;
 do not delete another task's artifacts or change global compiler-cache settings. The first build