From 01cbd8bc034ab75681066e5b898ee0d91eedabca Mon Sep 17 00:00:00 2001 From: Andrew Halberstadt Date: Tue, 10 Nov 2026 15:20:25 -0500 Subject: [PATCH] feat(run-task): fetch secrets listed in RUN_TASK_SECRETS RUN_TASK_SECRETS is a JSON list of secrets for run-task to fetch before checkout. The first type, github-repo-token, requests a GitHub App installation token from Taskcluster's auth service and exports it to an environment variable, GITHUB_TOKEN by default. Tokens are also kept in memory so run-task can authenticate its own GitHub requests. --- .taskcluster.yml | 5 ++ docs/reference/index.rst | 1 + docs/reference/run-task.rst | 50 ++++++++++++ src/taskgraph/run-task/run-task | 93 ++++++++++++++++++++- test/test_scripts_run_task.py | 140 ++++++++++++++++++++++++++++++++ 5 files changed, 286 insertions(+), 3 deletions(-) create mode 100644 docs/reference/run-task.rst diff --git a/.taskcluster.yml b/.taskcluster.yml index 67063664a..f7380fd31 100644 --- a/.taskcluster.yml +++ b/.taskcluster.yml @@ -224,6 +224,11 @@ tasks: TASKGRAPH_HEAD_REF: '${head_ref}' TASKGRAPH_HEAD_REV: '${head_sha}' TASKGRAPH_REPOSITORY_TYPE: git + RUN_TASK_SECRETS: + $json: + - type: github-repo-token + app: read + repo: '${baseRepoUrl}' REPOSITORIES: {$json: {taskgraph: Taskgraph}} - $if: 'isPullRequest' then: diff --git a/docs/reference/index.rst b/docs/reference/index.rst index ab6e87e5d..52eed157e 100644 --- a/docs/reference/index.rst +++ b/docs/reference/index.rst @@ -6,6 +6,7 @@ Reference cli parameters + run-task transforms/index optimization-strategies source/modules diff --git a/docs/reference/run-task.rst b/docs/reference/run-task.rst new file mode 100644 index 000000000..5a24d49c0 --- /dev/null +++ b/docs/reference/run-task.rst @@ -0,0 +1,50 @@ +run-task +======== + +`run-task`_ is a script in the Docker images provided by Taskgraph. It wraps a +task's command and handles setup such as cloning repositories, fetching +artifacts and installing Python requirements. + +.. _run-task: https://github.com/taskcluster/taskgraph/blob/main/src/taskgraph/run-task/run-task + +.. _run-task-secrets: + +Secrets +------- + +``run-task`` can fetch secrets before it clones anything, so the checkout and +the task can both use them. List them as JSON in ``RUN_TASK_SECRETS``: + +.. code-block:: yaml + + payload: + env: + RUN_TASK_SECRETS: + $json: + - type: github-repo-token + app: read + repo: https://github.com/myorg/myrepo + +The only type so far is ``github-repo-token``. It asks Taskcluster's auth service +for an installation token of the GitHub App ``app``, scoped to ``repo``. Its +keys are: + +``app`` (required) + The name of the GitHub App registered with Taskcluster. +``repo`` (required) + The GitHub repository the token is for. +``permissions`` + The token's permissions. Defaults to ``{"contents": "read"}``. +``env`` + The environment variable to export the token in. Defaults to + ``GITHUB_TOKEN``. A variable that is already set is never replaced. + +``run-task`` also keeps the token for its own GitHub requests. The task needs +the ``taskclusterProxy`` feature and an +``auth:github-repo-token://::`` scope for +each permission. If a request fails, for example because a pull request from a +fork lacks the scope, ``run-task`` logs a warning and carries on without the +token. The same happens when the auth service rejects the request, such as for +an unknown ``app`` or invalid ``permissions``. Invalid JSON, an unknown ``type`` +or key, or a ``repo`` that isn't on GitHub fails the task. Installation tokens +expire after an hour. diff --git a/src/taskgraph/run-task/run-task b/src/taskgraph/run-task/run-task index 8508318af..54b1ddfa0 100755 --- a/src/taskgraph/run-task/run-task +++ b/src/taskgraph/run-task/run-task @@ -37,9 +37,10 @@ import urllib.request from pathlib import Path from typing import Dict, List, Literal, NamedTuple, Optional -SECRET_BASEURL_TPL = "{}/secrets/v1/secret/{{}}".format( - os.environ.get("TASKCLUSTER_PROXY_URL", "http://taskcluster").rstrip("/") -) +TASKCLUSTER_PROXY_URL = os.environ.get( + "TASKCLUSTER_PROXY_URL", "http://taskcluster" +).rstrip("/") +SECRET_BASEURL_TPL = TASKCLUSTER_PROXY_URL + "/secrets/v1/secret/{}" GITHUB_SSH_FINGERPRINT = ( b"github.com ssh-ed25519 " @@ -1156,6 +1157,89 @@ def git_checkout( return commit_hash +def _github_owner_repo(url): + """Return the owner and repository for a github.com HTTPS or SSH URL.""" + match = re.fullmatch( + r"(?:https?://github\.com/|ssh://git@github\.com/|git@github\.com:)" + r"([^/]+)/([^/]+?)(?:\.git)?/?", + url or "", + ) + return match.groups() if match else None + + +# GitHub tokens minted from RUN_TASK_SECRETS, keyed by lowercase (owner, repo), +# so run-task can authenticate its own GitHub requests wherever the task asked +# for the token to go. +GITHUB_TOKENS: Dict[tuple, str] = {} + + +def fetch_github_token(app, owner, repo, permissions): + """Request a GitHub installation token through the task proxy.""" + request = urllib.request.Request( + f"{TASKCLUSTER_PROXY_URL}/api/auth/v1/github/{app}/{owner}/repo-token", + data=json.dumps({"repositories": [repo], "permissions": permissions}).encode( + "utf-8" + ), + headers={"Content-Type": "application/json"}, + method="POST", + ) + try: + with urllib.request.urlopen(request, timeout=10) as response: + return json.load(response)["token"] + except (urllib.error.URLError, OSError, ValueError, KeyError, TypeError): + print_line( + b"vcs", + b"warning: unable to retrieve GitHub token; continuing unauthenticated\n", + ) + return None + + +def fetch_secrets(): + """Fetch the secrets listed in RUN_TASK_SECRETS and export them.""" + if not (raw := os.environ.get("RUN_TASK_SECRETS")): + return + try: + entries = json.loads(raw) + except ValueError as e: + raise RuntimeError(f"RUN_TASK_SECRETS is not valid JSON: {e}") from None + if not isinstance(entries, list) or not all(isinstance(e, dict) for e in entries): + raise RuntimeError("RUN_TASK_SECRETS must be a JSON list of objects") + + for entry in entries: + if entry.get("type") == "github-repo-token": + unknown = set(entry) - {"type", "app", "repo", "permissions", "env"} + if unknown: + raise RuntimeError(f"RUN_TASK_SECRETS: unknown keys {sorted(unknown)}") + + repo = entry.get("repo") + github_repo = isinstance(repo, str) and _github_owner_repo(repo) + if not github_repo: + raise RuntimeError("RUN_TASK_SECRETS: 'repo' must be a GitHub repository") + + owner, name = github_repo + envvar = entry.get("env", "GITHUB_TOKEN") + token = fetch_github_token( + entry.get("app"), + owner, + name, + entry.get("permissions", {"contents": "read"}), + ) + if not token: + continue + GITHUB_TOKENS.setdefault((owner.lower(), name.lower()), token) + if envvar in os.environ: + print_line( + b"setup", + f"{envvar} is already set; not replacing it\n".encode(), + ) + else: + os.environ[envvar] = token + else: + raise RuntimeError( + f"RUN_TASK_SECRETS: unsupported type {entry.get('type')!r}" + ) + + def fetch_ssh_secret(secret_name): """Retrieves the private ssh key, and returns it as a StringIO object""" secret_url = SECRET_BASEURL_TPL.format(secret_name) @@ -1720,6 +1804,9 @@ def main(args): os.setresgid(gid, gid, gid) os.setresuid(uid, uid, uid) + # Fetch secrets before checkout, so the checkout can use them. + fetch_secrets() + if hook_path := os.environ.get("RUN_TASK_STARTUP_HOOK"): run_python_script(hook_path) diff --git a/test/test_scripts_run_task.py b/test/test_scripts_run_task.py index 86a6a8c44..a375d700e 100644 --- a/test/test_scripts_run_task.py +++ b/test/test_scripts_run_task.py @@ -6,6 +6,7 @@ import subprocess import sys import tempfile +import urllib.error from argparse import Namespace from importlib.machinery import SourceFileLoader from importlib.util import module_from_spec, spec_from_loader @@ -1153,3 +1154,142 @@ def test_git_checkout_sparse_cone_converts_for_globs( after = materialized_files(destination) assert set(before) <= set(after) assert "c/four.md" in after + + +@pytest.mark.parametrize( + "url,expected", + [ + ("https://github.com/owner/repo.git", ("owner", "repo")), + ("https://github.com/owner/repo/", ("owner", "repo")), + ("git@github.com:owner/repo.git", ("owner", "repo")), + ("ssh://git@github.com/owner/repo", ("owner", "repo")), + ("https://github.com.evil/owner/repo", None), + ("https://example.com/owner/repo", None), + (None, None), + ], +) +def test_github_owner_repo(run_task_mod, url, expected): + assert run_task_mod._github_owner_repo(url) == expected + + +def test_fetch_github_token(run_task_mod, mocker): + response = mocker.MagicMock() + response.__enter__.return_value.read.return_value = b'{"token": "secret"}' + urlopen = mocker.patch("urllib.request.urlopen", return_value=response) + permissions = {"contents": "read"} + assert ( + run_task_mod.fetch_github_token("read", "owner", "repo", permissions) + == "secret" + ) + request = urlopen.call_args.args[0] + assert ( + request.full_url + == f"{run_task_mod.TASKCLUSTER_PROXY_URL}/api/auth/v1/github/read/owner/repo-token" + ) + assert request.method == "POST" + assert json.loads(request.data) == { + "repositories": ["repo"], + "permissions": {"contents": "read"}, + } + assert urlopen.call_args.kwargs == {"timeout": 10} + urlopen.side_effect = urllib.error.URLError("denied") + assert run_task_mod.fetch_github_token("read", "owner", "repo", permissions) is None + + +GITHUB_SECRET = { + "type": "github-repo-token", + "app": "read", + "repo": "https://github.com/Owner/Repo", +} + + +@pytest.fixture +def github_tokens(run_task_mod, monkeypatch): + tokens = {} + monkeypatch.setattr(run_task_mod, "GITHUB_TOKENS", tokens) + return tokens + + +def test_fetch_secrets_unset(run_task_mod, monkeypatch, mocker): + monkeypatch.setattr(os, "environ", {}) + fetch = mocker.patch.object(run_task_mod, "fetch_github_token") + run_task_mod.fetch_secrets() + fetch.assert_not_called() + assert os.environ == {} + + +@pytest.mark.parametrize( + "value,error", + [ + pytest.param("not json", "not valid JSON", id="invalid_json"), + pytest.param({"type": "github-repo-token"}, "list of objects", id="not_list"), + pytest.param(["string"], "list of objects", id="not_objects"), + pytest.param([{**GITHUB_SECRET, "type": "secret"}], "unsupported type"), + pytest.param([{**GITHUB_SECRET, "extra": 1}], "unknown keys"), + pytest.param( + [{**GITHUB_SECRET, "repo": "https://example.com/o/r"}], "GitHub repository" + ), + pytest.param( + [{**GITHUB_SECRET, "repo": 1}], "GitHub repository", id="repo_type" + ), + pytest.param( + [{k: v for k, v in GITHUB_SECRET.items() if k != "repo"}], + "GitHub repository", + id="no_repo", + ), + pytest.param([{**GITHUB_SECRET, "file": "token"}], "unknown keys", id="file"), + ], +) +def test_fetch_secrets_invalid(run_task_mod, monkeypatch, mocker, value, error): + if not isinstance(value, str): + value = json.dumps(value) + monkeypatch.setattr(os, "environ", {"RUN_TASK_SECRETS": value}) + fetch = mocker.patch.object(run_task_mod, "fetch_github_token") + with pytest.raises(RuntimeError, match=error): + run_task_mod.fetch_secrets() + fetch.assert_not_called() + + +def test_fetch_secrets(run_task_mod, monkeypatch, mocker, github_tokens): + entries = [ + # Defaults to read access to contents, exported as GITHUB_TOKEN. + GITHUB_SECRET, + { + **GITHUB_SECRET, + "repo": "https://github.com/owner/other", + "permissions": {"pull_requests": "write"}, + "env": "EXISTING", + }, + {**GITHUB_SECRET, "env": "FAILED"}, + ] + # fetch_secrets sets variables, so keep them out of the real environment. + monkeypatch.setattr( + os, + "environ", + {"RUN_TASK_SECRETS": json.dumps(entries), "EXISTING": "explicit"}, + ) + fetch = mocker.patch.object( + run_task_mod, "fetch_github_token", side_effect=["t1", "t2", None] + ) + + run_task_mod.fetch_secrets() + + assert fetch.call_args_list == [ + mocker.call("read", "Owner", "Repo", {"contents": "read"}), + mocker.call("read", "owner", "other", {"pull_requests": "write"}), + mocker.call("read", "Owner", "Repo", {"contents": "read"}), + ] + assert {k: v for k, v in os.environ.items() if k != "RUN_TASK_SECRETS"} == { + "GITHUB_TOKEN": "t1", + "EXISTING": "explicit", + } + assert github_tokens == {("owner", "repo"): "t1", ("owner", "other"): "t2"} + + +def test_main_exports_secrets(run_main, mocker, run_task_mod, github_tokens): + mocker.patch.object(run_task_mod, "fetch_github_token", return_value="t1") + + result, env = run_main(env={"RUN_TASK_SECRETS": json.dumps([GITHUB_SECRET])}) + + assert result == 0 + assert env["GITHUB_TOKEN"] == "t1"