Build Go SDK e2e example bundle natively in CI

.github/workflows/airflow-e2e-tests.yml

@@ -121,16 +121,17 @@ jobs:
           use-uv: ${{ inputs.use-uv }}
           make-mnt-writeable-and-cleanup: true
         id: breeze
-      # Only the java_sdk mode runs the Gradle builds; every other e2e mode skips the
-      # steps below. LANG_SDK_NATIVE_TOOLCHAIN=true (the same switch the lang-SDK k8s
-      # job uses) makes the e2e conftest build with the host toolchain provisioned
-      # here instead of ephemeral eclipse-temurin containers, skipping the
-      # toolchain-image pull; local runs keep the containerized build. Without the
-      # cache each run re-downloads the example bundles' full dependency tree (Spark
-      # alone is ~1GB) into a fresh ~/.gradle. The key carries a ``-vN-`` salt (bump
-      # to force-invalidate a poisoned cache) and runner.arch keys the cache per
-      # architecture: ~/.gradle/jdks holds arch-specific auto-provisioned JDKs (the
-      # example pins a Java 11 toolchain) and this job runs on amd64 and arm64.
+      # The java_sdk mode runs the Gradle builds and the go_sdk mode runs the Go
+      # build; every other e2e mode skips the toolchain steps below.
+      # LANG_SDK_NATIVE_TOOLCHAIN=true (the same switch the lang-SDK k8s job uses)
+      # makes the e2e conftest build with the host toolchain provisioned here
+      # instead of an ephemeral toolchain container, skipping the image pull; local
+      # runs keep the containerized builds. The cache keys carry a ``-vN-`` salt
+      # (bump to force-invalidate a poisoned cache) and runner.arch keys each cache
+      # per architecture, as this job runs on amd64 and arm64. Without its cache
+      # each java_sdk run re-downloads the example bundles' full dependency tree
+      # (Spark alone is ~1GB) into a fresh ~/.gradle, whose ~/.gradle/jdks holds
+      # arch-specific auto-provisioned JDKs (the example pins a Java 11 toolchain).
       - name: "Setup Java for the Java SDK build"
         if: inputs.e2e_test_mode == 'java_sdk'
         uses: actions/setup-java@1bcf9fb12cf4aa7d266a90ae39939e61372fe520  # v5.4.0
@@ -151,12 +152,32 @@ jobs:
           key: e2e-java-sdk-gradle-v1-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('java-sdk/**/*.gradle*', 'java-sdk/**/gradle.properties', 'java-sdk/gradle/wrapper/gradle-wrapper.properties', 'java-sdk/gradle/libs.versions.toml') }}  # yamllint disable-line rule:line-length
           restore-keys: |
             e2e-java-sdk-gradle-v1-${{ runner.os }}-${{ runner.arch }}-
+      # go-version-file sources the Go toolchain from go-sdk/go.mod, so there is no
+      # separate version pin to keep in sync; cache: false because the explicit
+      # restore/save pair below persists the module + build caches even on red runs.
+      - name: "Setup Go for the Go SDK build"
+        if: inputs.e2e_test_mode == 'go_sdk'
+        uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16  # v6.5.0
+        with:
+          go-version-file: go-sdk/go.mod
+          cache: false
+      - name: "Restore Go SDK module + build cache"
+        if: inputs.e2e_test_mode == 'go_sdk'
+        id: go-cache
+        uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9  # v6.1.0
+        with:
+          path: |
+            ~/.cache/go-build
+            ~/go/pkg/mod
+          key: e2e-go-sdk-go-v1-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('go-sdk/go.sum') }}
+          restore-keys: |
+            e2e-go-sdk-go-v1-${{ runner.os }}-${{ runner.arch }}-
       - name: "Test e2e integration tests"
         run: breeze testing airflow-e2e-tests
         env:
           DOCKER_IMAGE: "${{ inputs.docker-image-tag }}"
           E2E_TEST_MODE: "${{ inputs.e2e_test_mode }}"
-          LANG_SDK_NATIVE_TOOLCHAIN: "${{ inputs.e2e_test_mode == 'java_sdk' && 'true' || 'false' }}"
+          LANG_SDK_NATIVE_TOOLCHAIN: "${{ (inputs.e2e_test_mode == 'java_sdk' || inputs.e2e_test_mode == 'go_sdk') && 'true' || 'false' }}"  # yamllint disable-line rule:line-length
       - name: "Save Java SDK Gradle dependency cache"
         # Saved even when the e2e tests fail: the Gradle warm-up is independent of
         # test outcome, and actions/cache's post step would drop it on every red run.
@@ -170,6 +191,15 @@ jobs:
             !~/.gradle/caches/*.lock
             !~/.gradle/caches/journal-1
           key: ${{ steps.gradle-cache.outputs.cache-primary-key }}
+      - name: "Save Go SDK module + build cache"
+        # Saved even when the e2e tests fail, for the same reason as the Gradle cache above.
+        if: always() && inputs.e2e_test_mode == 'go_sdk' && steps.go-cache.outputs.cache-hit != 'true'
+        uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9  # v6.1.0
+        with:
+          path: |
+            ~/.cache/go-build
+            ~/go/pkg/mod
+          key: ${{ steps.go-cache.outputs.cache-primary-key }}
       - name: Zip logs
         run: |
           cd ./airflow-e2e-tests && zip -r logs.zip logs

airflow-e2e-tests/tests/airflow_e2e_tests/conftest.py

@@ -44,6 +44,7 @@
     GO_SDK_BUNDLE_NAME,
     GO_SDK_DAGS_PATH,
     GO_SDK_EXAMPLE_BUNDLE_PKG,
+    GO_SDK_ROOT_PATH,
     JAVA_COMPOSE_PATH,
     JAVA_DOCKERFILE_PATH,
     JAVA_SDK_EXAMPLE_DAGS_PATH,
@@ -499,53 +500,73 @@ def _setup_go_sdk_integration(dot_env_file, tmp_dir):
     ``CGO_ENABLED=0``), so the stock Airflow worker image can exec it directly
     without a Go toolchain or any extra runtime installed -- see ``go.yml``.
     """
-    # Build + pack the example bundle inside an ephemeral Go container so the
-    # host does not need Go installed.
+    # `go tool airflow-go-pack` builds the bundle package, reads its
+    # --airflow-metadata, and appends the source + airflow-metadata.yaml + the
+    # AFBNDL01 trailer, writing a single self-contained executable bundle.
+    # CGO_ENABLED=0 yields a fully static binary that runs on the stock worker.
+    #
+    # In native mode (used in CI, where the host already has a Go toolchain plus
+    # restored module/build caches via actions/setup-go) the host `go` runs the
+    # build directly, skipping the toolchain-image pull and the container
+    # workarounds below. The containerized path stays the default for local runs
+    # so a dev host needs no Go installed:
     #
     # --user keeps build outputs owned by the current user (not root).
     # HOME points at a writable, gitignored dir under go-sdk/bin so the Go build
     # and module caches persist between runs (first run downloads modules once;
     # subsequent runs skip straight to compilation).
-    # CGO_ENABLED=0 yields a fully static binary that runs on the stock worker.
     # USER/HOME must be set because the SDK calls user.Current() at init; with
     # cgo disabled Go's pure-Go resolver reads those env vars instead of libc,
     # and panics if either is empty (the same vars are set on the worker in
     # go.yml so the packed binary runs the same way at execution time).
-    # `go tool airflow-go-pack` builds the bundle package, reads its
-    # --airflow-metadata, and appends the source + airflow-metadata.yaml + the
-    # AFBNDL01 trailer, writing a single self-contained executable bundle.
-    go_cache_home = "/repo/go-sdk/bin/.home"
-    bundle_out = f"/repo/go-sdk/bin/{GO_SDK_BUNDLE_NAME}"
-    console.print(f"[yellow]Building Go SDK example bundle ({GO_BUILDER_IMAGE})...")
-    subprocess.run(
-        [
-            "docker",
-            "run",
-            "--rm",
-            "--user",
-            f"{os.getuid()}:{os.getgid()}",
-            "-e",
-            f"HOME={go_cache_home}",
-            "-e",
-            "USER=airflow",
-            "-e",
-            "CGO_ENABLED=0",
-            # Mount the repo so the whole go-sdk module (go.mod, tool directive,
-            # example sources) is visible to `go tool`.
-            "-v",
-            f"{AIRFLOW_ROOT_PATH}:/repo",
-            "-w",
-            "/repo/go-sdk",
-            GO_BUILDER_IMAGE,
-            "go",
-            "tool",
-            "airflow-go-pack",
-            "--output",
-            bundle_out,
-            GO_SDK_EXAMPLE_BUNDLE_PKG,
-        ],
-        check=True,
-    )
+    if LANG_SDK_NATIVE_TOOLCHAIN:
+        console.print("[yellow]Building Go SDK example bundle (host toolchain)...")
+        subprocess.run(
+            [
+                "go",
+                "tool",
+                "airflow-go-pack",
+                "--output",
+                str(GO_SDK_BIN_PATH / GO_SDK_BUNDLE_NAME),
+                GO_SDK_EXAMPLE_BUNDLE_PKG,
+            ],
+            cwd=GO_SDK_ROOT_PATH,
+            env={**os.environ, "CGO_ENABLED": "0"},
+            check=True,
+        )
+    else:
+        go_cache_home = "/repo/go-sdk/bin/.home"
+        bundle_out = f"/repo/go-sdk/bin/{GO_SDK_BUNDLE_NAME}"
+        console.print(f"[yellow]Building Go SDK example bundle ({GO_BUILDER_IMAGE})...")
+        subprocess.run(
+            [
+                "docker",
+                "run",
+                "--rm",
+                "--user",
+                f"{os.getuid()}:{os.getgid()}",
+                "-e",
+                f"HOME={go_cache_home}",
+                "-e",
+                "USER=airflow",
+                "-e",
+                "CGO_ENABLED=0",
+                # Mount the repo so the whole go-sdk module (go.mod, tool directive,
+                # example sources) is visible to `go tool`.
+                "-v",
+                f"{AIRFLOW_ROOT_PATH}:/repo",
+                "-w",
+                "/repo/go-sdk",
+                GO_BUILDER_IMAGE,
+                "go",
+                "tool",
+                "airflow-go-pack",
+                "--output",
+                bundle_out,
+                GO_SDK_EXAMPLE_BUNDLE_PKG,
+            ],
+            check=True,
+        )
 
     # Copy the compose override into the temp directory.
     copyfile(GO_COMPOSE_PATH, tmp_dir / "go.yml")

airflow-e2e-tests/tests/airflow_e2e_tests/constants.py

@@ -83,7 +83,8 @@
 # Where airflow-go-pack writes the packed bundle inside the repo (go-sdk/bin is gitignored).
 GO_SDK_BIN_PATH = GO_SDK_ROOT_PATH / "bin"
 GO_COMPOSE_PATH = AIRFLOW_ROOT_PATH / "airflow-e2e-tests" / "docker" / "go.yml"
-# Go toolchain image used to build the bundle; must satisfy go-sdk/go.mod's toolchain.
+# Go toolchain image used to build the bundle in the containerized path (i.e. unless
+# LANG_SDK_NATIVE_TOOLCHAIN is set); must satisfy go-sdk/go.mod's toolchain.
 # The Alpine variant is ~7x smaller than the Debian one and is safe here because the
 # bundle is built with CGO_ENABLED=0 (a fully static binary, independent of musl/glibc)
 # and module fetches go through the HTTPS proxy (no git/gcc needed).