diff --git a/http_service/bugbug_http/app.py b/http_service/bugbug_http/app.py index 3c8d99abcf5..d184c2929fb 100644 --- a/http_service/bugbug_http/app.py +++ b/http_service/bugbug_http/app.py @@ -8,6 +8,7 @@ import itertools import logging import os +import re import uuid from dataclasses import dataclass, field from datetime import datetime, timedelta, timezone @@ -34,11 +35,13 @@ from bugbug.model import CommentModel from bugbug.models import get_model_class from bugbug_http.models import ( + COMMIT_MODELS, MODELS_NAMES, PUSH_CLASSIFIERS, classify_broken_site_report, classify_bug, classify_comment, + classify_commits, classify_issue, get_config_specific_groups, schedule_tests, @@ -83,10 +86,17 @@ ssl_cert_reqs=None, ) +# A full hg commit hash. +COMMIT_HASH_RE = re.compile(r"[0-9a-f]{40}") + # Kill jobs which don't finish within 12 minutes. JOB_TIMEOUT = 12 * 60 # Kill Bugzilla jobs which don't finish within 5 minutes. BUGZILLA_JOB_TIMEOUT = 5 * 60 +# The maximum number of commits which can be classified in a single request. +MAX_COMMITS = 100 +# The number of commits classified by a single job. +COMMITS_PER_JOB = 25 # Remove jobs from the queue if they haven't started within 7 minutes. QUEUE_TIMEOUT = 7 * 60 # Store the information that a job failed for 3 minutes. @@ -128,6 +138,17 @@ class PerformanceRegressionPrediction(Schema): extra_data = fields.Dict() +class CommitPrediction(Schema): + node = fields.String() + bug_id = fields.Integer(allow_none=True) + prob = fields.List(fields.Float(), allow_none=True) + index = fields.Integer(allow_none=True) + predicted_class = fields.Integer(data_key="class", allow_none=True) + risk_band = fields.String(allow_none=True) + ignored_reason = fields.String(allow_none=True) + extra_data = fields.Dict() + + class NotAvailableYet(Schema): ready = fields.Boolean(metadata={"enum": [False]}) @@ -145,6 +166,10 @@ class PushModelName(Schema): ) +class CommitModelName(Schema): + model_name = fields.String(metadata={"enum": COMMIT_MODELS, "example": "regressor"}) + + class UnauthorizedError(Schema): message = fields.String(dump_default="Error, missing X-API-KEY") @@ -167,9 +192,11 @@ class Schedules(Schema): PerformanceRegressionPrediction.__name__, schema=PerformanceRegressionPrediction, ) +spec.components.schema(CommitPrediction.__name__, schema=CommitPrediction) spec.components.schema(NotAvailableYet.__name__, schema=NotAvailableYet) spec.components.schema(ModelName.__name__, schema=ModelName) spec.components.schema(PushModelName.__name__, schema=PushModelName) +spec.components.schema(CommitModelName.__name__, schema=CommitModelName) spec.components.schema(UnauthorizedError.__name__, schema=UnauthorizedError) spec.components.schema(BranchName.__name__, schema=BranchName) spec.components.schema(Schedules.__name__, schema=Schedules) @@ -658,6 +685,233 @@ def model_prediction_push(model_name: str, branch: str, rev: str): return compress_response(data, status_code) +def get_commit_predictions( + model_name: str, branch: str, revs: Sequence[str] +) -> tuple[dict[str, Any], int]: + """Get the results of the classification of commits, queuing the missing ones. + + Returns the result for each revision ({"ready": False} for the ones being + classified), and 200 if all of them are available or 202 otherwise. + """ + # Support the string 'autoland' for convenience. + if branch == "autoland": + branch = "integration/autoland" + + status_code = 200 + data: dict[str, Any] = {} + missing_revs = [] + for rev in dict.fromkeys(revs): + job = JobInfo(classify_commits, model_name, branch, rev) + data[rev] = get_result(job) + if not data[rev]: + if not is_pending(job): + missing_revs.append(rev) + status_code = 202 + data[rev] = {"ready": False} + + queue_jobs: Queue = [] + for batch in itertools.batched(missing_revs, COMMITS_PER_JOB, strict=False): + job_id = get_job_id() + # Set the mapping before queuing to avoid some race conditions. + redis_conn.mset( + { + JobInfo(classify_commits, model_name, branch, rev).mapping_key: job_id + for rev in batch + } + ) + queue_jobs.append( + prepare_queue_job( + JobInfo(classify_commits, model_name, branch, list(batch)), + job_id=job_id, + ) + ) + if queue_jobs: + q.enqueue_many(queue_jobs) + + return data, status_code + + +@application.route("//predict/commit//") +@cross_origin() +def model_prediction_commit(model_name: str, branch: str, rev: str): + """ + --- + get: + description: > + Classify a commit using the given model, answer either 200 if the commit + is processed or 202 if the commit is being processed. +

+ The answer is {"available": False} when the commit can't be found on + the branch (or is a merge commit). Backouts and commits ignored by the + model (e.g. reformatting commits) are not scored, and have their + ignored_reason set. + summary: Classify a single commit + parameters: + - name: model_name + in: path + schema: CommitModelName + - name: branch + in: path + required: true + schema: + BranchName + - name: rev + in: path + required: true + schema: + type: str + example: 8ba995b74e18334ab3707f27e9eb8f4e37ba3d29 + responses: + 200: + description: A single commit prediction + content: + application/json: + schema: CommitPrediction + 202: + description: A temporary answer for the commit being processed + content: + application/json: + schema: NotAvailableYet + 400: + description: The revision is not a full commit hash + 401: + description: API key is missing + content: + application/json: + schema: UnauthorizedError + """ + if not request.headers.get(API_TOKEN): + return jsonify(UnauthorizedError().dump({})), 401 + + if model_name not in COMMIT_MODELS: + return ( + jsonify( + {"error": f"Model {model_name} doesn't support commit predictions"} + ), + 404, + ) + + if not COMMIT_HASH_RE.fullmatch(rev): + return jsonify({"error": f"{rev} is not a full commit hash"}), 400 + + LOGGER.info( + "Received %s commit prediction request for %s @ %s", model_name, branch, rev + ) + + data, status_code = get_commit_predictions(model_name, branch, [rev]) + return compress_response(data[rev], status_code) + + +@application.route("//predict/commits/", methods=["POST"]) +@cross_origin() +def model_prediction_commits(model_name: str, branch: str): + """ + --- + post: + description: > + Post a batch of commit hashes to classify, answer either 200 if all + commits are processed or 202 if at least one commit is not processed. +

+ Send the full hashes of commits on the given branch like this:
+ ``` + {"commits": ["8ba995b74e18334ab3707f27e9eb8f4e37ba3d29", "1f2e3d4c5b6a79880f1e2d3c4b5a69788f9e0d1c"]} + ```

+ + and call back the same endpoint with the same commits until all of the + results are available. The answer for each commit is the same as the + single commit prediction endpoint, or {"ready": False} while it is + being processed:
+ ``` + {"commits": {"8ba995b74e18334ab3707f27e9eb8f4e37ba3d29": {"ready": False}, "1f2e3d4c5b6a79880f1e2d3c4b5a69788f9e0d1c": {"node": "1f2e3d4c5b6a79880f1e2d3c4b5a69788f9e0d1c", "prob": [0.9, 0.1], ...}}} + ``` + summary: Classify a batch of commits + parameters: + - name: model_name + in: path + schema: CommitModelName + - name: branch + in: path + required: true + schema: + BranchName + requestBody: + description: The list of commits to classify + content: + application/json: + schema: + type: object + properties: + commits: + type: array + items: + type: string + examples: + commits: + summary: An example of payload + value: + commits: + ["8ba995b74e18334ab3707f27e9eb8f4e37ba3d29", "1f2e3d4c5b6a79880f1e2d3c4b5a69788f9e0d1c"] + responses: + 200: + description: The result for each commit + content: + application/json: + schema: + type: object + properties: + commits: + type: object + additionalProperties: CommitPrediction + 202: + description: A temporary answer for commits being processed + 400: + description: The request is not valid + 401: + description: API key is missing + content: + application/json: + schema: UnauthorizedError + """ + if not request.headers.get(API_TOKEN): + return jsonify(UnauthorizedError().dump({})), 401 + + if model_name not in COMMIT_MODELS: + return ( + jsonify( + {"error": f"Model {model_name} doesn't support commit predictions"} + ), + 404, + ) + + try: + body = orjson.loads(request.data) + except orjson.JSONDecodeError: + return jsonify({"error": "The body is not valid JSON"}), 400 + + schema = { + "commits": { + "type": "list", + "required": True, + "minlength": 1, + "maxlength": MAX_COMMITS, + "schema": {"type": "string", "regex": COMMIT_HASH_RE.pattern}, + } + } + validator = Validator() + if not isinstance(body, dict) or not validator.validate(body, schema): + return jsonify({"errors": validator.errors}), 400 + + LOGGER.info( + "Received %s prediction request for %d commits @ %s", + model_name, + len(body["commits"]), + branch, + ) + + data, status_code = get_commit_predictions(model_name, branch, body["commits"]) + return compress_response({"commits": data}, status_code) + + @application.route( "//predict/github///" ) diff --git a/http_service/bugbug_http/models.py b/http_service/bugbug_http/models.py index 95690dbf838..10ca854072c 100644 --- a/http_service/bugbug_http/models.py +++ b/http_service/bugbug_http/models.py @@ -50,6 +50,7 @@ "worksforme", "fenixcomponent", "perfregressionpredictor", + "regressor", ] DEFAULT_EXPIRATION_TTL = 7 * 24 * 3600 # A week @@ -475,6 +476,119 @@ def classify_perf_regression(branch: str, rev: str) -> str: PERF_REGRESSION_PREDICTOR: classify_perf_regression, } +# Models that classify single commits (see classify_commits). +COMMIT_MODELS = ["regressor"] + + +def _get_ignored_reason(commit: repository.CommitDict) -> str | None: + if commit["backsout"]: + return "backout" + if commit["ignored"]: + return "ignored" + return None + + +def classify_commits(model_name: str, branch: str, revs: Sequence[str]) -> str: + """Classify commits with a commit model, and store the result of each in Redis. + + The commits, identified by their full hash, are pulled from the + hg.mozilla.org repository `branch` (e.g. "integration/autoland" or "try") + and mined from the service's local clone, so their features are computed in + the same way as during training. Backouts and commits ignored during + training (e.g. reformatting commits) are listed with their `ignored_reason` + but not scored, unknown revisions and merges are marked as not available. + """ + from bugbug_http import REPO_DIR + from bugbug_http.app import JobInfo + + def store(node: str, data: dict) -> None: + job = JobInfo(classify_commits, model_name, branch, node) + setkey(job.result_key, orjson.dumps(data), compress=True) + + LOGGER.info("Classifying %s @ %s with %s...", revs, branch, model_name) + + # Pull the commits to the local repository, like for test selection. + nodes = [] + for node in dict.fromkeys(revs): + try: + repository.pull(REPO_DIR, branch, node, update=False) + nodes.append(node) + except RuntimeError: + LOGGER.warning("Could not pull %s @ %s", branch, node) + store(node, {"available": False}) + + if not nodes: + return "OK" + + # The commits are mined in the same way as for the commits DB, but without + # storing them (or their contribution to the experience features). + commits = repository.download_commits( + REPO_DIR, + revs=[node.encode("ascii") for node in nodes], + branch=None, + save=False, + include_no_bug=True, + include_backouts=True, + include_ignored=True, + ) + commits_by_node = {commit["node"]: commit for commit in commits} + + to_score = [commit for commit in commits if _get_ignored_reason(commit) is None] + + model = MODEL_CACHE.get(model_name) + model_extra_data = model.get_extra_data() + + results: dict[str, dict] = {} + if to_score: + probs = model.classify(to_score, probabilities=True) + indexes = probs.argmax(axis=-1) + classes = model.le.inverse_transform(indexes) + for commit, prob, index, class_ in zip( + to_score, probs.tolist(), indexes.tolist(), classes.tolist() + ): + try: + risk_band = model.find_risk_band(prob[1]) + except Exception: + # The risk bands are not configured. + risk_band = None + + results[commit["node"]] = { + "prob": prob, + "index": index, + "class": class_, + "risk_band": risk_band, + "ignored_reason": None, + } + + for node in nodes: + # Merge commits are not mined. + mined_commit = commits_by_node.get(node) + if mined_commit is None: + store(node, {"available": False}) + continue + + data = results.get( + node, + { + "prob": None, + "index": None, + "class": None, + "risk_band": None, + "ignored_reason": _get_ignored_reason(mined_commit), + }, + ) + store( + node, + { + "node": node, + "bug_id": mined_commit["bug_id"], + **data, + "extra_data": model_extra_data, + }, + ) + + return "OK" + @lru_cache(maxsize=None) def get_known_tasks() -> tuple[str, ...]: diff --git a/http_service/tests/test_commit_classification.py b/http_service/tests/test_commit_classification.py new file mode 100644 index 00000000000..110347589a5 --- /dev/null +++ b/http_service/tests/test_commit_classification.py @@ -0,0 +1,234 @@ +# -*- coding: utf-8 -*- +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. + +import gzip + +import hglib +import numpy as np +import orjson +import zstandard + +from bugbug_http import models +from bugbug_http.app import API_TOKEN, JobInfo + +REV1 = "a" * 40 +REV2 = "b" * 40 + + +def _response_json(response): + if response.headers.get("Content-Encoding") == "gzip": + return orjson.loads(gzip.decompress(response.data)) + return response.json + + +def _job_key(rev: str, branch: str = "integration/autoland") -> str: + return str(JobInfo(models.classify_commits, "regressor", branch, rev)) + + +def _get_result(rev: str, branch: str = "integration/autoland"): + job = JobInfo(models.classify_commits, "regressor", branch, rev) + return orjson.loads( + zstandard.ZstdDecompressor().decompress(models.redis.get(job.result_key)) + ) + + +PREDICTION = { + "node": REV2, + "bug_id": 123456, + "prob": [0.9, 0.1], + "index": 0, + "class": 0, + "risk_band": "l", + "ignored_reason": None, + "extra_data": {}, +} + + +def test_commit_endpoint(client, jobs, add_result) -> None: + endpoint = f"/regressor/predict/commit/autoland/{REV2}" + + assert client.get(endpoint).status_code == 401 + + # Only commit models can be used with the commit prediction endpoint. + response = client.get( + f"/component/predict/commit/autoland/{REV2}", headers={API_TOKEN: "test"} + ) + assert response.status_code == 404 + + for rev in ("tip", REV2[:12]): + response = client.get( + f"/regressor/predict/commit/autoland/{rev}", headers={API_TOKEN: "test"} + ) + assert response.status_code == 400 + + response = client.get(endpoint, headers={API_TOKEN: "test"}) + assert response.status_code == 202 + assert _response_json(response) == {"ready": False} + assert [keys[0] for keys in jobs.values()] == [_job_key(REV2)] + + # The commit is not queued again while it is being processed. + response = client.get(endpoint, headers={API_TOKEN: "test"}) + assert response.status_code == 202 + assert len(jobs) == 1 + + add_result(_job_key(REV2), PREDICTION) + + response = client.get(endpoint, headers={API_TOKEN: "test"}) + assert response.status_code == 200 + assert _response_json(response) == PREDICTION + + +def test_commits_endpoint(client, jobs, add_result) -> None: + endpoint = "/regressor/predict/commits/autoland" + + assert client.post(endpoint, json={"commits": [REV1]}).status_code == 401 + + bodies: tuple[object, ...] = ( + {}, + {"commits": []}, + {"commits": ["not a hash"]}, + {"commits": [REV1[:12]]}, + {"commits": [REV1] * 101}, + [REV1], + ) + for body in bodies: + response = client.post(endpoint, json=body, headers={API_TOKEN: "test"}) + assert response.status_code == 400, body + + add_result(_job_key(REV2), PREDICTION) + + response = client.post( + endpoint, json={"commits": [REV1, REV2, REV1]}, headers={API_TOKEN: "test"} + ) + assert response.status_code == 202 + assert _response_json(response) == { + "commits": {REV1: {"ready": False}, REV2: PREDICTION} + } + # Only the missing commit is queued. + assert [keys[0] for keys in jobs.values()] == [_job_key(REV1)] + + add_result(_job_key(REV1), {"available": False}) + + response = client.post( + endpoint, json={"commits": [REV1, REV2]}, headers={API_TOKEN: "test"} + ) + assert response.status_code == 200 + assert _response_json(response) == { + "commits": {REV1: {"available": False}, REV2: PREDICTION} + } + + +class MockRegressorModel: + def __init__(self): + self.classified = [] + + class le: + @staticmethod + def inverse_transform(indexes): + return indexes + + def classify(self, commits, probabilities): + self.classified.extend(commits) + return np.array([[0.25, 0.75]] * len(commits)) + + def find_risk_band(self, risk): + return "h" if risk > 0.5 else "l" + + def get_extra_data(self): + return {"model_version": "1"} + + +def test_classify_commits( + monkeypatch, + mock_repo, + mock_hgmo, + mock_component_taskcluster_artifact, + mock_coverage_mapping_artifact, +) -> None: + local_dir, remote_dir = mock_repo + with hglib.open(str(remote_dir)) as hg: + remote_node = hg.log(limit=1)[0].node.decode("ascii") + with hglib.open(str(local_dir)) as hg: + local_node = hg.log(limit=1)[0].node.decode("ascii") + + model = MockRegressorModel() + monkeypatch.setattr(models.MODEL_CACHE, "get", lambda model_name: model) + + unknown = "c" * 40 + pulled = [] + orig_pull = models.repository.pull + + def pull(repo_dir, branch, rev, update): + pulled.append(rev) + if rev == unknown: + raise RuntimeError("unknown revision") + orig_pull(repo_dir, branch, rev, update) + + monkeypatch.setattr(models.repository, "pull", pull) + + assert ( + models.classify_commits( + "regressor", "integration/autoland", [remote_node, local_node, unknown] + ) + == "OK" + ) + + assert pulled == [remote_node, local_node, unknown] + with hglib.open(str(local_dir)) as hg: + assert hg.log(limit=1)[0].node.decode("ascii") == remote_node + + assert sorted(commit["node"] for commit in model.classified) == sorted( + [remote_node, local_node] + ) + for node in (remote_node, local_node): + assert _get_result(node) == { + "node": node, + "bug_id": None, + "prob": [0.25, 0.75], + "index": 1, + "class": 1, + "risk_band": "h", + "ignored_reason": None, + "extra_data": {"model_version": "1"}, + } + assert _get_result(unknown) == {"available": False} + + +def test_classify_commits_not_scored(monkeypatch) -> None: + nodes = { + "backout": "1" * 40, + "ignored": "2" * 40, + "scored": "3" * 40, + "merge": "4" * 40, + } + monkeypatch.setattr(models.repository, "pull", lambda *args, **kwargs: None) + + def download_commits(repo_dir, revs, **kwargs): + # Merge commits are not mined. + return tuple( + { + "node": node, + "bug_id": 1, + "backsout": ["0" * 40] if name == "backout" else [], + "ignored": name == "ignored", + } + for name, node in nodes.items() + if name != "merge" + ) + + monkeypatch.setattr(models.repository, "download_commits", download_commits) + model = MockRegressorModel() + monkeypatch.setattr(models.MODEL_CACHE, "get", lambda model_name: model) + + assert models.classify_commits("regressor", "try", list(nodes.values())) == "OK" + + assert [commit["node"] for commit in model.classified] == [nodes["scored"]] + for reason in ("backout", "ignored"): + result = _get_result(nodes[reason], "try") + assert result["ignored_reason"] == reason + assert result["prob"] is None + assert result["risk_band"] is None + assert _get_result(nodes["scored"], "try")["prob"] == [0.25, 0.75] + assert _get_result(nodes["merge"], "try") == {"available": False}