feat(run-task): fetch secrets listed in RUN_TASK_SECRETS

.taskcluster.yml

@@ -224,6 +224,11 @@ tasks:
                                 TASKGRAPH_HEAD_REF: '${head_ref}'
                                 TASKGRAPH_HEAD_REV: '${head_sha}'
                                 TASKGRAPH_REPOSITORY_TYPE: git
+                                RUN_TASK_SECRETS:
+                                    $json:
+                                        - type: github-repo-token
+                                          app: read
+                                          repo: '${baseRepoUrl}'
                                 REPOSITORIES: {$json: {taskgraph: Taskgraph}}
                               - $if: 'isPullRequest'
                                 then:

docs/reference/index.rst

@@ -6,6 +6,7 @@ Reference
 
    cli
    parameters
+   run-task
    transforms/index
    optimization-strategies
    source/modules

docs/reference/run-task.rst

@@ -0,0 +1,50 @@
+run-task
+========
+
+`run-task`_ is a script in the Docker images provided by Taskgraph. It wraps a
+task's command and handles setup such as cloning repositories, fetching
+artifacts and installing Python requirements.
+
+.. _run-task: https://github.com/taskcluster/taskgraph/blob/main/src/taskgraph/run-task/run-task
+
+.. _run-task-secrets:
+
+Secrets
+-------
+
+``run-task`` can fetch secrets before it clones anything, so the checkout and
+the task can both use them. List them as JSON in ``RUN_TASK_SECRETS``:
+
+.. code-block:: yaml
+
+   payload:
+       env:
+           RUN_TASK_SECRETS:
+               $json:
+                   - type: github-repo-token
+                     app: read
+                     repo: https://github.com/myorg/myrepo
+
+The only type so far is ``github-repo-token``. It asks Taskcluster's auth service
+for an installation token of the GitHub App ``app``, scoped to ``repo``. Its
+keys are:
+
+``app`` (required)
+   The name of the GitHub App registered with Taskcluster.
+``repo`` (required)
+   The GitHub repository the token is for.
+``permissions``
+   The token's permissions. Defaults to ``{"contents": "read"}``.
+``env``
+   The environment variable to export the token in. Defaults to
+   ``GITHUB_TOKEN``. A variable that is already set is never replaced.
+
+``run-task`` also keeps the token for its own GitHub requests. The task needs
+the ``taskclusterProxy`` feature and an
+``auth:github-repo-token:<app>/<owner>/<repo>:<permission>:<level>`` scope for
+each permission. If a request fails, for example because a pull request from a
+fork lacks the scope, ``run-task`` logs a warning and carries on without the
+token. The same happens when the auth service rejects the request, such as for
+an unknown ``app`` or invalid ``permissions``. Invalid JSON, an unknown ``type``
+or key, or a ``repo`` that isn't on GitHub fails the task. Installation tokens
+expire after an hour.

src/taskgraph/run-task/run-task

@@ -37,9 +37,10 @@ import urllib.request
 from pathlib import Path
 from typing import Dict, List, Literal, NamedTuple, Optional
 
-SECRET_BASEURL_TPL = "{}/secrets/v1/secret/{{}}".format(
-    os.environ.get("TASKCLUSTER_PROXY_URL", "http://taskcluster").rstrip("/")
-)
+TASKCLUSTER_PROXY_URL = os.environ.get(
+    "TASKCLUSTER_PROXY_URL", "http://taskcluster"
+).rstrip("/")
+SECRET_BASEURL_TPL = TASKCLUSTER_PROXY_URL + "/secrets/v1/secret/{}"
 
 GITHUB_SSH_FINGERPRINT = (
     b"github.com ssh-ed25519 "
@@ -1156,6 +1157,89 @@ def git_checkout(
     return commit_hash
 
 
+def _github_owner_repo(url):
+    """Return the owner and repository for a github.com HTTPS or SSH URL."""
+    match = re.fullmatch(
+        r"(?:https?://github\.com/|ssh://git@github\.com/|git@github\.com:)"
+        r"([^/]+)/([^/]+?)(?:\.git)?/?",
+        url or "",
+    )
+    return match.groups() if match else None
+
+
+# GitHub tokens minted from RUN_TASK_SECRETS, keyed by lowercase (owner, repo),
+# so run-task can authenticate its own GitHub requests wherever the task asked
+# for the token to go.
+GITHUB_TOKENS: Dict[tuple, str] = {}
+
+
+def fetch_github_token(app, owner, repo, permissions):
+    """Request a GitHub installation token through the task proxy."""
+    request = urllib.request.Request(
+        f"{TASKCLUSTER_PROXY_URL}/api/auth/v1/github/{app}/{owner}/repo-token",
+        data=json.dumps({"repositories": [repo], "permissions": permissions}).encode(
+            "utf-8"
+        ),
+        headers={"Content-Type": "application/json"},
+        method="POST",
+    )
+    try:
+        with urllib.request.urlopen(request, timeout=10) as response:
+            return json.load(response)["token"]
+    except (urllib.error.URLError, OSError, ValueError, KeyError, TypeError):
+        print_line(
+            b"vcs",
+            b"warning: unable to retrieve GitHub token; continuing unauthenticated\n",
+        )
+        return None
+
+
+def fetch_secrets():
+    """Fetch the secrets listed in RUN_TASK_SECRETS and export them."""
+    if not (raw := os.environ.get("RUN_TASK_SECRETS")):
+        return
+    try:
+        entries = json.loads(raw)
+    except ValueError as e:
+        raise RuntimeError(f"RUN_TASK_SECRETS is not valid JSON: {e}") from None
+    if not isinstance(entries, list) or not all(isinstance(e, dict) for e in entries):
+        raise RuntimeError("RUN_TASK_SECRETS must be a JSON list of objects")
+
+    for entry in entries:
+        if entry.get("type") == "github-repo-token":
+            unknown = set(entry) - {"type", "app", "repo", "permissions", "env"}
+            if unknown:
+                raise RuntimeError(f"RUN_TASK_SECRETS: unknown keys {sorted(unknown)}")
+
+            repo = entry.get("repo")
+            github_repo = isinstance(repo, str) and _github_owner_repo(repo)
+            if not github_repo:
+                raise RuntimeError("RUN_TASK_SECRETS: 'repo' must be a GitHub repository")
+
+            owner, name = github_repo
+            envvar = entry.get("env", "GITHUB_TOKEN")
+            token = fetch_github_token(
+                entry.get("app"),
+                owner,
+                name,
+                entry.get("permissions", {"contents": "read"}),
+            )
+            if not token:
+                continue
+            GITHUB_TOKENS.setdefault((owner.lower(), name.lower()), token)
+            if envvar in os.environ:
+                print_line(
+                    b"setup",
+                    f"{envvar} is already set; not replacing it\n".encode(),
+                )
+            else:
+                os.environ[envvar] = token
+        else:
+            raise RuntimeError(
+                f"RUN_TASK_SECRETS: unsupported type {entry.get('type')!r}"
+            )
+
+
 def fetch_ssh_secret(secret_name):
     """Retrieves the private ssh key, and returns it as a StringIO object"""
     secret_url = SECRET_BASEURL_TPL.format(secret_name)
@@ -1720,6 +1804,9 @@ def main(args):
         os.setresgid(gid, gid, gid)
         os.setresuid(uid, uid, uid)
 
+    # Fetch secrets before checkout, so the checkout can use them.
+    fetch_secrets()
+
     if hook_path := os.environ.get("RUN_TASK_STARTUP_HOOK"):
         run_python_script(hook_path)
 

test/test_scripts_run_task.py

@@ -6,6 +6,7 @@
 import subprocess
 import sys
 import tempfile
+import urllib.error
 from argparse import Namespace
 from importlib.machinery import SourceFileLoader
 from importlib.util import module_from_spec, spec_from_loader
@@ -1153,3 +1154,142 @@ def test_git_checkout_sparse_cone_converts_for_globs(
     after = materialized_files(destination)
     assert set(before) <= set(after)
     assert "c/four.md" in after
+
+
+@pytest.mark.parametrize(
+    "url,expected",
+    [
+        ("https://github.com/owner/repo.git", ("owner", "repo")),
+        ("https://github.com/owner/repo/", ("owner", "repo")),
+        ("git@github.com:owner/repo.git", ("owner", "repo")),
+        ("ssh://git@github.com/owner/repo", ("owner", "repo")),
+        ("https://github.com.evil/owner/repo", None),
+        ("https://example.com/owner/repo", None),
+        (None, None),
+    ],
+)
+def test_github_owner_repo(run_task_mod, url, expected):
+    assert run_task_mod._github_owner_repo(url) == expected
+
+
+def test_fetch_github_token(run_task_mod, mocker):
+    response = mocker.MagicMock()
+    response.__enter__.return_value.read.return_value = b'{"token": "secret"}'
+    urlopen = mocker.patch("urllib.request.urlopen", return_value=response)
+    permissions = {"contents": "read"}
+    assert (
+        run_task_mod.fetch_github_token("read", "owner", "repo", permissions)
+        == "secret"
+    )
+    request = urlopen.call_args.args[0]
+    assert (
+        request.full_url
+        == f"{run_task_mod.TASKCLUSTER_PROXY_URL}/api/auth/v1/github/read/owner/repo-token"
+    )
+    assert request.method == "POST"
+    assert json.loads(request.data) == {
+        "repositories": ["repo"],
+        "permissions": {"contents": "read"},
+    }
+    assert urlopen.call_args.kwargs == {"timeout": 10}
+    urlopen.side_effect = urllib.error.URLError("denied")
+    assert run_task_mod.fetch_github_token("read", "owner", "repo", permissions) is None
+
+
+GITHUB_SECRET = {
+    "type": "github-repo-token",
+    "app": "read",
+    "repo": "https://github.com/Owner/Repo",
+}
+
+
+@pytest.fixture
+def github_tokens(run_task_mod, monkeypatch):
+    tokens = {}
+    monkeypatch.setattr(run_task_mod, "GITHUB_TOKENS", tokens)
+    return tokens
+
+
+def test_fetch_secrets_unset(run_task_mod, monkeypatch, mocker):
+    monkeypatch.setattr(os, "environ", {})
+    fetch = mocker.patch.object(run_task_mod, "fetch_github_token")
+    run_task_mod.fetch_secrets()
+    fetch.assert_not_called()
+    assert os.environ == {}
+
+
+@pytest.mark.parametrize(
+    "value,error",
+    [
+        pytest.param("not json", "not valid JSON", id="invalid_json"),
+        pytest.param({"type": "github-repo-token"}, "list of objects", id="not_list"),
+        pytest.param(["string"], "list of objects", id="not_objects"),
+        pytest.param([{**GITHUB_SECRET, "type": "secret"}], "unsupported type"),
+        pytest.param([{**GITHUB_SECRET, "extra": 1}], "unknown keys"),
+        pytest.param(
+            [{**GITHUB_SECRET, "repo": "https://example.com/o/r"}], "GitHub repository"
+        ),
+        pytest.param(
+            [{**GITHUB_SECRET, "repo": 1}], "GitHub repository", id="repo_type"
+        ),
+        pytest.param(
+            [{k: v for k, v in GITHUB_SECRET.items() if k != "repo"}],
+            "GitHub repository",
+            id="no_repo",
+        ),
+        pytest.param([{**GITHUB_SECRET, "file": "token"}], "unknown keys", id="file"),
+    ],
+)
+def test_fetch_secrets_invalid(run_task_mod, monkeypatch, mocker, value, error):
+    if not isinstance(value, str):
+        value = json.dumps(value)
+    monkeypatch.setattr(os, "environ", {"RUN_TASK_SECRETS": value})
+    fetch = mocker.patch.object(run_task_mod, "fetch_github_token")
+    with pytest.raises(RuntimeError, match=error):
+        run_task_mod.fetch_secrets()
+    fetch.assert_not_called()
+
+
+def test_fetch_secrets(run_task_mod, monkeypatch, mocker, github_tokens):
+    entries = [
+        # Defaults to read access to contents, exported as GITHUB_TOKEN.
+        GITHUB_SECRET,
+        {
+            **GITHUB_SECRET,
+            "repo": "https://github.com/owner/other",
+            "permissions": {"pull_requests": "write"},
+            "env": "EXISTING",
+        },
+        {**GITHUB_SECRET, "env": "FAILED"},
+    ]
+    # fetch_secrets sets variables, so keep them out of the real environment.
+    monkeypatch.setattr(
+        os,
+        "environ",
+        {"RUN_TASK_SECRETS": json.dumps(entries), "EXISTING": "explicit"},
+    )
+    fetch = mocker.patch.object(
+        run_task_mod, "fetch_github_token", side_effect=["t1", "t2", None]
+    )
+
+    run_task_mod.fetch_secrets()
+
+    assert fetch.call_args_list == [
+        mocker.call("read", "Owner", "Repo", {"contents": "read"}),
+        mocker.call("read", "owner", "other", {"pull_requests": "write"}),
+        mocker.call("read", "Owner", "Repo", {"contents": "read"}),
+    ]
+    assert {k: v for k, v in os.environ.items() if k != "RUN_TASK_SECRETS"} == {
+        "GITHUB_TOKEN": "t1",
+        "EXISTING": "explicit",
+    }
+    assert github_tokens == {("owner", "repo"): "t1", ("owner", "other"): "t2"}
+
+
+def test_main_exports_secrets(run_main, mocker, run_task_mod, github_tokens):
+    mocker.patch.object(run_task_mod, "fetch_github_token", return_value="t1")
+
+    result, env = run_main(env={"RUN_TASK_SECRETS": json.dumps([GITHUB_SECRET])})
+
+    assert result == 0
+    assert env["GITHUB_TOKEN"] == "t1"