Devops/integration testing

.github/workflows/integration-tests.yml

@@ -0,0 +1,90 @@
+name: Integration Tests
+
+# Docker-based CDA integration tests (integration-tests/). The `cda_stack`
+# fixture in integration-tests/conftest.py brings up db + data-api with
+# docker compose and seeds them with cda-etl, so this job only has to run the
+# Gradle task. It is slow (Oracle image pull, ~12 min ETL reseed, calculations),
+# so it runs on demand and on pull requests that touch the code or seed data
+# it exercises, not on every push.
+
+on:
+  workflow_dispatch:
+  pull_request:
+    paths:
+      - "integration-tests/**"
+      - "regi-headless/**"
+      - "gradle/**"
+      - "build.gradle"
+      - "settings.gradle"
+      - "buildSrc/**"
+      - ".github/workflows/integration-tests.yml"
+
+permissions:
+  contents: read
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  integration-tests:
+    name: CDA integration tests
+    runs-on: ubuntu-latest
+    # Image pull + db start (compose healthcheck allows up to 40 min on a cold
+    # start) + ETL reseed + the calculation tests.
+    timeout-minutes: 120
+
+    env:
+      JUNIT_XML: ${{ github.workspace }}/integration-tests/build/integration-junit.xml
+      # conftest.py writes the db/data-api/etl container logs here just before
+      # it tears the stack down (afterwards there is nothing left to collect).
+      CDA_COMPOSE_LOG: ${{ github.workspace }}/integration-tests/build/docker-compose.log
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v7
+        with:
+          # The git-version plugin derives the wheel version from tags/history.
+          fetch-depth: 0
+
+      # The Oracle "database-ready" image is large; free space on the runner
+      # before docker pulls it.
+      - name: Free disk space
+        run: |
+          sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc /opt/hostedtoolcache/CodeQL
+          docker image prune --all --force
+          df -h /
+
+      - name: Set up Java
+        uses: actions/setup-java@v6
+        with:
+          distribution: temurin
+          java-version: 21
+
+      - name: Set up Gradle
+        uses: gradle/actions/setup-gradle@v6
+
+      - name: Make Gradle wrapper executable
+        run: chmod +x ./gradlew
+
+      # integrationTest builds and installs the regi_python wheel and the
+      # test dependencies into its own venv, then runs `pytest -m integration`.
+      # A missing prerequisite or a failed docker bring-up is a test error (not a
+      # skip), so pytest, and therefore this step, fails. PYTEST_ADDOPTS is picked
+      # up by that pytest and writes the JUnit report that is uploaded below.
+      - name: Run integration tests
+        env:
+          PYTEST_ADDOPTS: "-rs --junitxml=${{ env.JUNIT_XML }}"
+        run: ./gradlew :integration-tests:integrationTest --stacktrace
+
+      - name: Upload test reports and logs
+        if: always()
+        uses: actions/upload-artifact@v7
+        with:
+          name: integration-test-reports
+          path: |
+            integration-tests/build/integration-junit.xml
+            integration-tests/build/docker-compose.log
+            integration-tests/regi-projects/regi-python/regi-python.log
+          if-no-files-found: warn
+          retention-days: 14

.gitignore

@@ -4,4 +4,9 @@
 *.iml
 .DS_STORE
 */build/
-.coverage*
+.coverage*
+__pycache__/
+.pytest_cache/
+*.env
+regi-projects/
+/gradle.properties

build.gradle

@@ -1,6 +1,7 @@
 plugins {
     id "com.palantir.git-version" version "5.0.0"
     id "org.sonarqube" version "7.3.1.8318"
+    alias(libs.plugins.python.plugin) apply false
 }
 
 def gitOutput(String... args) {

buildSrc/src/main/groovy/regi-headless.java-conventions.gradle

@@ -7,9 +7,9 @@ tasks.withType(JavaCompile).configureEach {
 }
 
 dependencies {
-    annotationProcessor(libs.service.annotations)
     configurations.compileClasspath.extendsFrom(configurations.annotationProcessor)
-    testAnnotationProcessor(libs.service.annotations)
+    annotationProcessor(libs.lookup)
+    testAnnotationProcessor(libs.lookup)
 }
 
 test {

cda-etl-expander/src/python/cda-etl-expander/__init__.py

@@ -0,0 +1,38 @@
+#  MIT License
+#  Copyright (c) 2026 Hydrologic Engineering Center
+#  Permission is hereby granted, free of charge, to any person obtaining a copy
+#  of this software and associated documentation files (the "Software"), to deal
+#  in the Software without restriction, including without limitation the rights
+#  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+#  copies of the Software, and to permit persons to whom the Software is
+#  furnished to do so, subject to the following conditions:
+#  The above copyright notice and this permission notice shall be included in all
+#  copies or substantial portions of the Software.
+#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+#  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+#  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+#  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+#  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+#  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+#  SOFTWARE.
+"""
+cda-expander - resolves association-based ids into a literal-id cda-etl config.
+
+This package is a standalone preprocessor. It reads a "template" config
+describing how to *find* timeseries/rating/location-level ids (by reading
+CWMS properties, and later by reading the A2W/PublishedTimeSeries API),
+resolves them, and writes a fully expanded YAML file containing only literal
+ids in cda-etl's own, unmodified config schema.
+
+cda-etl itself knows nothing about any of this: it consumes the generated
+file exactly as it would a hand-written one.
+
+Dependency rule: this package must not import anything from cda_etl, and
+cda_etl must not import anything from here. The two only ever meet through
+the generated YAML file. Enforced by tests/expander/test_import_isolation.py.
+"""
+
+# Bumped whenever the generated output format changes in a way that would
+# alter the bytes written for an unchanged input. Recorded in the generated
+# file's header so a reviewer can tell what produced it.
+__version__ = "1.0.0"

cda-etl-expander/src/python/cda-etl-expander/__main__.py

@@ -0,0 +1,23 @@
+#  MIT License
+#  Copyright (c) 2026 Hydrologic Engineering Center
+#  Permission is hereby granted, free of charge, to any person obtaining a copy
+#  of this software and associated documentation files (the "Software"), to deal
+#  in the Software without restriction, including without limitation the rights
+#  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+#  copies of the Software, and to permit persons to whom the Software is
+#  furnished to do so, subject to the following conditions:
+#  The above copyright notice and this permission notice shall be included in all
+#  copies or substantial portions of the Software.
+#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+#  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+#  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+#  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+#  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+#  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+#  SOFTWARE.
+import sys
+
+from cda_expander.cli import main
+
+if __name__ == "__main__":
+    sys.exit(main())