From 4b842fc43ea9a92d78fa00ee36dd0bce43f762b2 Mon Sep 17 00:00:00 2001 From: Marco Castelluccio Date: Mon, 16 Nov 2026 15:57:24 +0100 Subject: [PATCH 1/5] Train the regressor model on two years of data for both classes Regressors were taken from all history while clean commits only from the last two years, which let the model learn "old-looking patch" (e.g. lower experience counts) instead of "risky patch". Use the same two-year window for both classes. On held-out windows this improves bug-level AUC by about +0.015/+0.027. --- bugbug/models/regressor.py | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/bugbug/models/regressor.py b/bugbug/models/regressor.py index a2a40b2786b..0d1d5cdd100 100644 --- a/bugbug/models/regressor.py +++ b/bugbug/models/regressor.py @@ -191,21 +191,20 @@ def get_labels(self): if push_date > datetime.utcnow() - relativedelta(months=EVALUATION_MONTHS): continue + # Only use the last two years of data, for both classes. The clean set can't go + # further back anyway, as "Regressed by" data is only reliable since then (see + # https://groups.google.com/g/mozilla.dev.platform/c/SjjW6_O-FqM/m/G-CrIVT2BAAJ). + # Taking regressors from a longer period than clean commits would let the model + # learn "old-looking patch" (e.g. lower experience counts) instead of "risky patch". + if push_date < datetime.utcnow() - relativedelta(years=2): + continue + node = commit_data["node"] if commit_data["bug_id"] in regressor_bugs or ( self.use_finder and node in regressors ): classes[node] = 1 elif not self.exclude_finder or node not in regressors: - # The labels we have are only from two years ago (see https://groups.google.com/g/mozilla.dev.platform/c/SjjW6_O-FqM/m/G-CrIVT2BAAJ). - # While we can go further back with the regressor finder script, it isn't remotely - # as precise as the "Regressed By" data. - # In the future, we might want to re-evaluate this limit (e.g. extend ), but we - # have to be careful (using too old patches might cause worse results as patch - # characteristics evolve over time). - if push_date < datetime.utcnow() - relativedelta(years=2): - continue - # We remove the last 3 months, as there could be regressions which haven't been # filed yet. While it is true that some regressions might not be found for a long # time, more than 3 months seems overly conservative. From 1abcf0bac3a5d75d5e5ea53901b091436b314af5 Mon Sep 17 00:00:00 2001 From: Marco Castelluccio Date: Mon, 16 Nov 2026 15:57:24 +0100 Subject: [PATCH 2/5] Tune the regressor model's XGBoost hyperparameters Use 600 trees with a 0.03 learning rate, depth 6, row/column subsampling and a minimum child weight of 5 instead of the defaults. On held-out windows this improves bug-level AUC by about +0.015/+0.02. --- bugbug/models/regressor.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/bugbug/models/regressor.py b/bugbug/models/regressor.py index 0d1d5cdd100..cc2c95b64e9 100644 --- a/bugbug/models/regressor.py +++ b/bugbug/models/regressor.py @@ -140,7 +140,15 @@ def __init__( ), ] ) - estimator = xgboost.XGBClassifier(n_jobs=utils.get_physical_cpu_count()) + estimator = xgboost.XGBClassifier( + n_estimators=600, + learning_rate=0.03, + max_depth=6, + subsample=0.8, + colsample_bytree=0.5, + min_child_weight=5, + n_jobs=utils.get_physical_cpu_count(), + ) if calibration: estimator = IsotonicRegressionCalibrator(estimator) # This is a temporary workaround for the error : "Model type not yet supported by TreeExplainer" From b3cf53f4680ec1d9f9d763e6c8b14a3b0f267b1b Mon Sep 17 00:00:00 2001 From: Marco Castelluccio Date: Mon, 16 Nov 2026 15:57:24 +0100 Subject: [PATCH 3/5] Use commit messages in the regressor model by default Enable the commit message TF-IDF features (interpretable=False) by default: the message is also available when classifying patches under review. On held-out windows this improves bug-level AUC by about +0.010/+0.016. --- bugbug/models/regressor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bugbug/models/regressor.py b/bugbug/models/regressor.py index cc2c95b64e9..a69036e5722 100644 --- a/bugbug/models/regressor.py +++ b/bugbug/models/regressor.py @@ -56,7 +56,7 @@ def __init__( self, calibration: bool = True, lemmatization: bool = False, - interpretable: bool = True, + interpretable: bool = False, use_finder: bool = False, exclude_finder: bool = True, finder_regressions_only: bool = False, From 3a695de23d49f032672d0f8c5d29c3e5c1f694de Mon Sep 17 00:00:00 2001 From: Marco Castelluccio Date: Mon, 16 Nov 2026 15:57:24 +0100 Subject: [PATCH 4/5] Compute regressor risk bands at training time, with a new very high band The risk band thresholds were set manually through the REGRESSOR_RISK_BANDS secret, and a gap between bands made find_risk_band raise. Compute four contiguous bands (lower 50%, average 30%, higher 15%, very high 5% of bugs) from the risk distribution of the evaluation period at training time and store them in the model; the secret is only used for older models. Thresholds are chosen among distinct risk values, as calibrated risks have many ties. Show the new band in the changes UI and count it in the landings risk report (which also counted average-risk changes under a band name that doesn't exist). --- bugbug/models/regressor.py | 206 ++++++++++------------- scripts/commit_classifier.py | 3 +- scripts/generate_landings_risk_report.py | 26 ++- tests/test_regressor_risk_bands.py | 47 ++++++ ui/changes/src/common.js | 17 +- ui/changes/src/feature.html | 1 + ui/changes/src/index.html | 1 + 7 files changed, 178 insertions(+), 123 deletions(-) create mode 100644 tests/test_regressor_risk_bands.py diff --git a/bugbug/models/regressor.py b/bugbug/models/regressor.py index a69036e5722..a9ceff70981 100644 --- a/bugbug/models/regressor.py +++ b/bugbug/models/regressor.py @@ -49,8 +49,67 @@ EVALUATION_MONTHS = 3 +# Risk bands, as (name, share of bugs in the band). The model separates patches much better +# at the high end of the risk scale than at the low end, so the bands get narrower as the +# risk increases. +RISK_BAND_SHARES = ( + ("l", 0.5), # Lower than average risk. + ("a", 0.3), # Average risk. + ("h", 0.15), # Higher than average risk. + ("v", 0.05), # Very high risk. +) + + +def compute_risk_bands(risks: np.ndarray) -> list[tuple[str, float, float]]: + """Split the risk scale into contiguous bands holding the RISK_BAND_SHARES of risks. + + Calibrated risks take few distinct values, so a plain quantile could fall in a large + group of tied risks and leave a band almost empty. Instead, each threshold is the + distinct risk value whose share of risks at or below it is closest to the target. + A risk equal to a threshold belongs to the lower band. + """ + values, counts = np.unique(risks, return_counts=True) + # Share of risks lower than or equal to each distinct value. + cumulative = np.cumsum(counts) / counts.sum() + targets = np.cumsum([share for _, share in RISK_BAND_SHARES[:-1]]) + + thresholds = [ + 0.0, + *[float(values[np.argmin(np.abs(cumulative - target))]) for target in targets], + 1.0, + ] + + return [ + (name, thresholds[i], thresholds[i + 1]) + for i, (name, _) in enumerate(RISK_BAND_SHARES) + ] + + +def log_risk_bands( + risk_bands: list[tuple[str, float, float]], + risks: np.ndarray, + is_regressor: np.ndarray, +) -> None: + logger.info("Average regression rate is %0.3f", is_regressor.mean()) + + lower_bound = -1.0 + for name, start, end in risk_bands: + mask = (risks > lower_bound) & (risks <= end) + lower_bound = end + logger.info( + "Risk band %s (%0.4f-%0.4f): %d bugs, regression rate %0.3f", + name, + start, + end, + mask.sum(), + is_regressor[mask].mean() if mask.any() else 0.0, + ) + + class RegressorModel(CommitModel): - RISK_BANDS = None + # Risk bands from the secret, for models trained before bands were stored in the model. + RISK_BANDS: list[tuple[str, float, float]] | None = None + risk_bands: list[tuple[str, float, float]] | None = None def __init__( self, @@ -235,23 +294,29 @@ def get_labels(self): return classes, [0, 1] - @staticmethod - def find_risk_band(risk: float) -> str: - if RegressorModel.RISK_BANDS is None: - - def _parse_risk_band(risk_band: str) -> tuple[str, float, float]: - name, start, end = risk_band.split("-") - return (name, float(start), float(end)) + def find_risk_band(self, risk: float) -> str: + risk_bands = self.risk_bands + if risk_bands is None: + # Models trained before risk bands were computed at training time. + if RegressorModel.RISK_BANDS is None: + + def _parse_risk_band(risk_band: str) -> tuple[str, float, float]: + name, start, end = risk_band.split("-") + return (name, float(start), float(end)) + + RegressorModel.RISK_BANDS = sorted( + ( + _parse_risk_band(risk_band) + for risk_band in utils.get_secret("REGRESSOR_RISK_BANDS").split( + ";" + ) + ), + key=lambda x: x[1], + ) - RegressorModel.RISK_BANDS = sorted( - ( - _parse_risk_band(risk_band) - for risk_band in utils.get_secret("REGRESSOR_RISK_BANDS").split(";") - ), - key=lambda x: x[1], - ) + risk_bands = RegressorModel.RISK_BANDS - for name, start, end in RegressorModel.RISK_BANDS: + for name, start, end in risk_bands: if start <= risk <= end: return name @@ -273,7 +338,7 @@ def evaluation(self) -> None: push_date = dateutil.parser.parse(commit_data["pushdate"]) - # Use the past two months of data (make sure it is not also used for training!). + # Use the past months of data (make sure it is not also used for training!). if push_date < datetime.utcnow() - relativedelta(months=EVALUATION_MONTHS): continue @@ -286,107 +351,22 @@ def evaluation(self) -> None: # Sort commits by bug ID, so we can use itertools.groupby to group them by bug ID. commits.sort(key=lambda x: x["bug_id"]) - results = [] + risks = [] + is_regressor = [] for bug_id, commit_iter in itertools.groupby(commits, lambda x: x["bug_id"]): probs = self.classify(list(commit_iter), probabilities=True) - results.append((max(probs[:, 1]), bug_id in bug_regressors)) - - # Let's define the risk bands relatively to average risk. - # On average, around 1 out of 8 (13%) patches cause regressions. - # Risk band 1 - around 1 out of 15 (7%) patches within this risk band cause regressions. - # Risk band 2 - around 1 out of 7 (15%) patches within this risk band cause regressions. - # Risk bank 3 - around 1 out of 3 (35%) patches within this risk band cause regressions. - - # Step 1. Calculate % of patches which cause regressions. - total_landings = len(results) - total_regressions = sum(is_reg for _, is_reg in results) - average_regression_rate = total_regressions / total_landings - - logger.info("Average risk is %0.2f", average_regression_rate) - - MIN_SAMPLE = 200 - - # Step 2. Define risk band 1 (half than average risk). - max_band1_prob = 1.0 - total_landings = 0 - total_regressions = 0 - results.sort(key=lambda x: x[0]) - for prob, is_reg in results: - total_landings += 1 - if is_reg: - total_regressions += 1 - - if total_landings < MIN_SAMPLE: - continue - - print( - f"{total_regressions} out of {total_landings} patches with risk lower than {prob} caused regressions ({total_regressions / total_landings}" - ) - - # No need to go further, since we are interested in half than average risk. - if ( - total_regressions / total_landings - >= (average_regression_rate / 2) + 0.01 - ): - max_band1_prob = prob - break - - print("\n\n") - - # Step 3. Define risk band 3 (double than average risk). - min_band3_prob = 0.0 - total_landings = 0 - total_regressions = 0 - results.sort(key=lambda x: x[0], reverse=True) - for prob, is_reg in results: - total_landings += 1 - if is_reg: - total_regressions += 1 - - if total_landings < MIN_SAMPLE: - continue + risks.append(max(probs[:, 1])) + is_regressor.append(bug_id in bug_regressors) - print( - f"{total_regressions} out of {total_landings} patches with risk higher than {prob} caused regressions ({total_regressions / total_landings}" - ) - - # No need to go further, since we are interested in double than average risk. - if ( - total_regressions / total_landings - <= (average_regression_rate * 2) - 0.01 - ): - min_band3_prob = prob - break - - print("\n\n") - - # Step 4. Define risk band 2 (average risk). - results.sort(key=lambda x: x[0]) - for prob_start in np.arange(max_band1_prob / 2, max_band1_prob + 0.02, 0.01): - for prob_end in np.arange(min_band3_prob - 0.02, 0.99, 0.01): - total_landings = 0 - total_regressions = 0 - for prob, is_reg in results: - if prob < prob_start or prob > prob_end: - continue - - total_landings += 1 - if is_reg: - total_regressions += 1 - - if total_landings < MIN_SAMPLE: - continue + if len(risks) == 0: + logger.warning("No bugs in the evaluation set, risk bands not computed") + return - if ( - (average_regression_rate / 2) + 0.01 - > total_regressions / total_landings - > (average_regression_rate * 2) - 0.01 - ): - continue + self.risk_bands = compute_risk_bands(np.array(risks)) - print( - f"{total_regressions} out of {total_landings} patches with risk between {prob_start} and {prob_end} caused regressions ({total_regressions / total_landings}" - ) + # Regressions for recent patches might not have been filed yet, so the rates are + # lower than the actual ones, but their ratios between bands are still meaningful. + log_risk_bands(self.risk_bands, np.array(risks), np.array(is_regressor)) def get_feature_names(self): return self.clf.named_steps["union"].get_feature_names_out() diff --git a/scripts/commit_classifier.py b/scripts/commit_classifier.py index 50d7fdf2fa7..3c7963f961b 100644 --- a/scripts/commit_classifier.py +++ b/scripts/commit_classifier.py @@ -633,7 +633,8 @@ def classify_regressor(self, commits: tuple[repository.CommitDict, ...]) -> None "probs": probs[0].tolist(), } if self.model_name == "regressor": - results["risk_band"] = RegressorModel.find_risk_band(probs[0][1]) + assert isinstance(self.model, RegressorModel) + results["risk_band"] = self.model.find_risk_band(probs[0][1]) with open("results.json", "w") as f: json.dump(results, f) diff --git a/scripts/generate_landings_risk_report.py b/scripts/generate_landings_risk_report.py index 268b87a76b7..0a1adfc0777 100644 --- a/scripts/generate_landings_risk_report.py +++ b/scripts/generate_landings_risk_report.py @@ -686,7 +686,7 @@ def get_commit_data(commit_list: list[repository.CommitDict]) -> list[dict]: "commits": commit_data, "meta_ids": list(blocker_to_meta[bug_id]), "risk": max_risk, - "risk_band": RegressorModel.find_risk_band(max_risk) + "risk_band": self.regressor_model.find_risk_band(max_risk) if max_risk is not None else None, "fuzz": "b" @@ -995,6 +995,7 @@ def get_tracking_info(bug): for team in all_teams: cur_team_data = team_data[team] = {} cur_team_data["month_changes"] = 0 + cur_team_data["very_high_risk_changes"] = 0 cur_team_data["high_risk_changes"] = 0 cur_team_data["medium_risk_changes"] = 0 cur_team_data["low_risk_changes"] = 0 @@ -1055,9 +1056,14 @@ def get_tracking_info(bug): (fix_date - creation_date).total_seconds() / 86400 ) - if bug["risk_band"] == "h": + # "high_risk_changes" counts all higher than average risk changes, including + # the very high risk ones. + if bug["risk_band"] == "v": + cur_team_data["very_high_risk_changes"] += 1 cur_team_data["high_risk_changes"] += 1 - elif bug["risk_band"] == "m": + elif bug["risk_band"] == "h": + cur_team_data["high_risk_changes"] += 1 + elif bug["risk_band"] == "a": cur_team_data["medium_risk_changes"] += 1 elif bug["risk_band"] == "l": cur_team_data["low_risk_changes"] += 1 @@ -1106,7 +1112,7 @@ def get_tracking_info(bug): (fix_date - creation_date).total_seconds() / 86400 ) - if bug["risk_band"] == "h": + if bug["risk_band"] in ("h", "v"): cur_team_data["prev_high_risk_changes"] += 1 cur_team_data["prev_review_times"] += [ @@ -1460,10 +1466,16 @@ def get_top_crashes(team: str, channel: str) -> str | None: try: month_changes = cur_team_data["month_changes"] high_risk_changes = cur_team_data["high_risk_changes"] + # Regression rates observed in each risk band. predicted_regressions = round( - 0.33 * cur_team_data["high_risk_changes"] - + 0.16 * cur_team_data["medium_risk_changes"] - + 0.06 * cur_team_data["low_risk_changes"], + 0.40 * cur_team_data["very_high_risk_changes"] + + 0.24 + * ( + cur_team_data["high_risk_changes"] + - cur_team_data["very_high_risk_changes"] + ) + + 0.12 * cur_team_data["medium_risk_changes"] + + 0.045 * cur_team_data["low_risk_changes"], 1, ) prev_high_risk_changes = cur_team_data["prev_high_risk_changes"] diff --git a/tests/test_regressor_risk_bands.py b/tests/test_regressor_risk_bands.py new file mode 100644 index 00000000000..5755174f1f7 --- /dev/null +++ b/tests/test_regressor_risk_bands.py @@ -0,0 +1,47 @@ +# -*- coding: utf-8 -*- +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. + +from collections import Counter + +import numpy as np + +from bugbug.models.regressor import RegressorModel, compute_risk_bands + + +def test_compute_risk_bands(): + risks = np.random.RandomState(0).uniform(size=10000) + + risk_bands = compute_risk_bands(risks) + + assert [name for name, _, _ in risk_bands] == ["l", "a", "h", "v"] + # The bands are contiguous and cover the whole risk scale. + assert risk_bands[0][1] == 0.0 + assert risk_bands[-1][2] == 1.0 + for (_, _, end), (_, start, _) in zip(risk_bands, risk_bands[1:]): + assert end == start + + model = RegressorModel() + model.risk_bands = risk_bands + counts = Counter(model.find_risk_band(risk) for risk in risks) + assert counts == {"l": 5000, "a": 3000, "h": 1500, "v": 500} + + # Boundaries belong to the lower band. + assert model.find_risk_band(risk_bands[0][2]) == "l" + assert model.find_risk_band(0.0) == "l" + assert model.find_risk_band(1.0) == "v" + + +def test_compute_risk_bands_ties(): + # Isotonic calibration outputs few distinct values, so many risks are tied. + risks = np.repeat([0.1, 0.2, 0.5, 0.9], [600, 250, 120, 30]) + + risk_bands = compute_risk_bands(risks) + + model = RegressorModel() + model.risk_bands = risk_bands + counts = Counter(model.find_risk_band(risk) for risk in risks) + # Each tied group goes to the band whose target share it fits best, and the top + # band is not left empty even if the 95th percentile falls in a tied group. + assert counts == {"l": 600, "a": 250, "h": 120, "v": 30} diff --git a/ui/changes/src/common.js b/ui/changes/src/common.js index f4959ba42fb..a1bf46d6d54 100644 --- a/ui/changes/src/common.js +++ b/ui/changes/src/common.js @@ -148,6 +148,7 @@ export const TESTING_TAGS = { }, }; +const VERY_HIGH_RISK_COLOR = "darkred"; const HIGH_RISK_COLOR = "rgb(255, 13, 87)"; const MEDIUM_RISK_COLOR = "darkkhaki"; const LOW_RISK_COLOR = "green"; @@ -564,13 +565,16 @@ export async function renderRiskChart(chartEl, bugSummaries) { counterObj.low += 1; } else if (summary.risk_band == "a") { counterObj.medium += 1; - } else { + } else if (summary.risk_band == "h") { counterObj.high += 1; + } else { + counterObj.veryHigh += 1; } } ); let categories = []; + let veryHigh = []; let high = []; let medium = []; let low = []; @@ -579,11 +583,16 @@ export async function renderRiskChart(chartEl, bugSummaries) { low.push(summaryData[date].low); medium.push(summaryData[date].medium); high.push(summaryData[date].high); + veryHigh.push(summaryData[date].veryHigh); } renderChart( chartEl, [ + { + name: "Very high", + data: veryHigh, + }, { name: "Higher", data: high, @@ -598,7 +607,7 @@ export async function renderRiskChart(chartEl, bugSummaries) { }, ], categories, - "Evolution of lower/average/higher risk changes", + "Evolution of lower/average/higher/very high risk changes", "# of patches" ); } @@ -2919,6 +2928,10 @@ function addRow( // Higher than average risk. risk_text.style.color = HIGH_RISK_COLOR; risk_text.textContent = "Higher"; + } else if (bugSummary.risk_band == "v") { + // Very high risk. + risk_text.style.color = VERY_HIGH_RISK_COLOR; + risk_text.textContent = "Very high"; } else if (bugSummary.risk_band == null) { // No risk available (there are no commits associated to the bug). risk_text.textContent = "N/A"; diff --git a/ui/changes/src/feature.html b/ui/changes/src/feature.html index 192d870e9f7..d039317df45 100644 --- a/ui/changes/src/feature.html +++ b/ui/changes/src/feature.html @@ -109,6 +109,7 @@

Filters