diff --git a/bugbug/model.py b/bugbug/model.py index 5ff6e5c07ee..cae4c70239c 100644 --- a/bugbug/model.py +++ b/bugbug/model.py @@ -370,12 +370,12 @@ def get_test_subsets(self) -> dict[str, np.ndarray]: """Subclasses can return boolean masks over the test set, to also report metrics on those subsets.""" return {} - def get_sample_weights(self, y) -> np.ndarray | None: + def get_sample_weights(self, X, y) -> np.ndarray | None: """Subclasses can return a weight for each training sample, passed to the estimator.""" return None def fit_classifier(self, X, y): - sample_weight = self.get_sample_weights(y) + sample_weight = self.get_sample_weights(X, y) if sample_weight is None: self.clf.fit(X, y) else: @@ -419,8 +419,21 @@ def train(self, importance_cutoff=0.15, limit=None): if len(self.class_names) == 2: scorings += ["precision", "recall"] + # The folds are weighted like the final model (see fit_classifier); the + # weights are computed once on the whole training set and split with it. + cv_y = self.le.transform(y_train) + sample_weight = self.get_sample_weights(X_train, cv_y) scores = cross_validate( - self.clf, X_train, self.le.transform(y_train), scoring=scorings, cv=5 + self.clf, + X_train, + cv_y, + scoring=scorings, + cv=5, + params=( + {"estimator__sample_weight": sample_weight} + if sample_weight is not None + else None + ), ) logger.info("Cross Validation scores:") diff --git a/bugbug/model_calibration.py b/bugbug/model_calibration.py index 64392efb405..b3e4d825d2a 100644 --- a/bugbug/model_calibration.py +++ b/bugbug/model_calibration.py @@ -7,6 +7,7 @@ from typing import Sequence +import numpy as np from sklearn.base import BaseEstimator, ClassifierMixin from sklearn.calibration import CalibratedClassifierCV from sklearn.model_selection import train_test_split @@ -39,12 +40,15 @@ def __init__(self, base_clf): base_clf, cv="prefit", method="isotonic" ) - def fit(self, X_train, y_train): - X_train, X_val, y_train, y_val = train_test_split( - X_train, y_train, test_size=0.2, random_state=42 + def fit(self, X_train, y_train, sample_weight=None): + if sample_weight is None: + sample_weight = np.ones(len(y_train)) + + X_train, X_val, y_train, y_val, w_train, w_val = train_test_split( + X_train, y_train, sample_weight, test_size=0.2, random_state=42 ) - self.base_clf.fit(X_train, y_train) - self.calibrated_clf.fit(X_val, y_val) + self.base_clf.fit(X_train, y_train, sample_weight=w_train) + self.calibrated_clf.fit(X_val, y_val, sample_weight=w_val) def predict(self, X): return self.calibrated_clf.predict(X) diff --git a/bugbug/models/regressor.py b/bugbug/models/regressor.py index a2a40b2786b..ef31c480c77 100644 --- a/bugbug/models/regressor.py +++ b/bugbug/models/regressor.py @@ -11,8 +11,6 @@ import numpy as np import xgboost from dateutil.relativedelta import relativedelta -from imblearn.pipeline import Pipeline as ImblearnPipeline -from imblearn.under_sampling import RandomUnderSampler from sklearn.compose import ColumnTransformer from sklearn.feature_extraction import DictVectorizer from sklearn.feature_extraction.text import CountVectorizer @@ -49,14 +47,73 @@ EVALUATION_MONTHS = 3 +# Risk bands, as (name, share of bugs in the band). The model separates patches much better +# at the high end of the risk scale than at the low end, so the bands get narrower as the +# risk increases. +RISK_BAND_SHARES = ( + ("l", 0.5), # Lower than average risk. + ("a", 0.3), # Average risk. + ("h", 0.15), # Higher than average risk. + ("v", 0.05), # Very high risk. +) + + +def compute_risk_bands(risks: np.ndarray) -> list[tuple[str, float, float]]: + """Split the risk scale into contiguous bands holding the RISK_BAND_SHARES of risks. + + Calibrated risks take few distinct values, so a plain quantile could fall in a large + group of tied risks and leave a band almost empty. Instead, each threshold is the + distinct risk value whose share of risks at or below it is closest to the target. + A risk equal to a threshold belongs to the lower band. + """ + values, counts = np.unique(risks, return_counts=True) + # Share of risks lower than or equal to each distinct value. + cumulative = np.cumsum(counts) / counts.sum() + targets = np.cumsum([share for _, share in RISK_BAND_SHARES[:-1]]) + + thresholds = [ + 0.0, + *[float(values[np.argmin(np.abs(cumulative - target))]) for target in targets], + 1.0, + ] + + return [ + (name, thresholds[i], thresholds[i + 1]) + for i, (name, _) in enumerate(RISK_BAND_SHARES) + ] + + +def log_risk_bands( + risk_bands: list[tuple[str, float, float]], + risks: np.ndarray, + is_regressor: np.ndarray, +) -> None: + logger.info("Average regression rate is %0.3f", is_regressor.mean()) + + lower_bound = -1.0 + for name, start, end in risk_bands: + mask = (risks > lower_bound) & (risks <= end) + lower_bound = end + logger.info( + "Risk band %s (%0.4f-%0.4f): %d bugs, regression rate %0.3f", + name, + start, + end, + mask.sum(), + is_regressor[mask].mean() if mask.any() else 0.0, + ) + + class RegressorModel(CommitModel): - RISK_BANDS = None + # Risk bands from the secret, for models trained before bands were stored in the model. + RISK_BANDS: list[tuple[str, float, float]] | None = None + risk_bands: list[tuple[str, float, float]] | None = None def __init__( self, calibration: bool = True, lemmatization: bool = False, - interpretable: bool = True, + interpretable: bool = False, use_finder: bool = False, exclude_finder: bool = True, finder_regressions_only: bool = False, @@ -140,18 +197,47 @@ def __init__( ), ] ) - estimator = xgboost.XGBClassifier(n_jobs=utils.get_physical_cpu_count()) + estimator = xgboost.XGBClassifier( + n_estimators=600, + learning_rate=0.03, + max_depth=6, + subsample=0.8, + colsample_bytree=0.5, + min_child_weight=5, + n_jobs=utils.get_physical_cpu_count(), + ) if calibration: estimator = IsotonicRegressionCalibrator(estimator) # This is a temporary workaround for the error : "Model type not yet supported by TreeExplainer" self.calculate_importance = False - self.clf = ImblearnPipeline( + self.clf = Pipeline( [ ("union", ColumnTransformer(column_transformers)), - ("sampler", RandomUnderSampler(random_state=0)), ("estimator", estimator), ] ) + self.train_bug_ids: list[int] = [] + + def items_gen(self, classes): + # Remember the bug of each item, to weight training samples by bug (see get_sample_weights). + self.train_bug_ids = [] + for commit, label in super().items_gen(classes): + self.train_bug_ids.append(commit["bug_id"]) + yield commit, label + + def get_sample_weights(self, X, y): + # Labels are per bug (a commit is a regressor if its bug caused a regression), and + # predictions are evaluated per bug, so bugs with many commits would otherwise dominate + # training: each bug gets the same total weight, split among its commits. The classes + # are balanced with weights too (instead of undersampling, which would not keep the + # weights aligned with the samples). + bug_ids = np.array(self.train_bug_ids)[np.asarray(X.index)] + _, inverse, counts = np.unique(bug_ids, return_inverse=True, return_counts=True) + sample_weight = 1.0 / counts[inverse] + for label in np.unique(y): + mask = y == label + sample_weight[mask] *= len(y) / (2 * sample_weight[mask].sum()) + return sample_weight def get_labels(self): classes = {} @@ -191,21 +277,20 @@ def get_labels(self): if push_date > datetime.utcnow() - relativedelta(months=EVALUATION_MONTHS): continue + # Only use the last two years of data, for both classes. The clean set can't go + # further back anyway, as "Regressed by" data is only reliable since then (see + # https://groups.google.com/g/mozilla.dev.platform/c/SjjW6_O-FqM/m/G-CrIVT2BAAJ). + # Taking regressors from a longer period than clean commits would let the model + # learn "old-looking patch" (e.g. lower experience counts) instead of "risky patch". + if push_date < datetime.utcnow() - relativedelta(years=2): + continue + node = commit_data["node"] if commit_data["bug_id"] in regressor_bugs or ( self.use_finder and node in regressors ): classes[node] = 1 elif not self.exclude_finder or node not in regressors: - # The labels we have are only from two years ago (see https://groups.google.com/g/mozilla.dev.platform/c/SjjW6_O-FqM/m/G-CrIVT2BAAJ). - # While we can go further back with the regressor finder script, it isn't remotely - # as precise as the "Regressed By" data. - # In the future, we might want to re-evaluate this limit (e.g. extend ), but we - # have to be careful (using too old patches might cause worse results as patch - # characteristics evolve over time). - if push_date < datetime.utcnow() - relativedelta(years=2): - continue - # We remove the last 3 months, as there could be regressions which haven't been # filed yet. While it is true that some regressions might not be found for a long # time, more than 3 months seems overly conservative. @@ -228,23 +313,29 @@ def get_labels(self): return classes, [0, 1] - @staticmethod - def find_risk_band(risk: float) -> str: - if RegressorModel.RISK_BANDS is None: - - def _parse_risk_band(risk_band: str) -> tuple[str, float, float]: - name, start, end = risk_band.split("-") - return (name, float(start), float(end)) + def find_risk_band(self, risk: float) -> str: + risk_bands = self.risk_bands + if risk_bands is None: + # Models trained before risk bands were computed at training time. + if RegressorModel.RISK_BANDS is None: + + def _parse_risk_band(risk_band: str) -> tuple[str, float, float]: + name, start, end = risk_band.split("-") + return (name, float(start), float(end)) + + RegressorModel.RISK_BANDS = sorted( + ( + _parse_risk_band(risk_band) + for risk_band in utils.get_secret("REGRESSOR_RISK_BANDS").split( + ";" + ) + ), + key=lambda x: x[1], + ) - RegressorModel.RISK_BANDS = sorted( - ( - _parse_risk_band(risk_band) - for risk_band in utils.get_secret("REGRESSOR_RISK_BANDS").split(";") - ), - key=lambda x: x[1], - ) + risk_bands = RegressorModel.RISK_BANDS - for name, start, end in RegressorModel.RISK_BANDS: + for name, start, end in risk_bands: if start <= risk <= end: return name @@ -266,7 +357,7 @@ def evaluation(self) -> None: push_date = dateutil.parser.parse(commit_data["pushdate"]) - # Use the past two months of data (make sure it is not also used for training!). + # Use the past months of data (make sure it is not also used for training!). if push_date < datetime.utcnow() - relativedelta(months=EVALUATION_MONTHS): continue @@ -279,107 +370,22 @@ def evaluation(self) -> None: # Sort commits by bug ID, so we can use itertools.groupby to group them by bug ID. commits.sort(key=lambda x: x["bug_id"]) - results = [] + risks = [] + is_regressor = [] for bug_id, commit_iter in itertools.groupby(commits, lambda x: x["bug_id"]): probs = self.classify(list(commit_iter), probabilities=True) - results.append((max(probs[:, 1]), bug_id in bug_regressors)) - - # Let's define the risk bands relatively to average risk. - # On average, around 1 out of 8 (13%) patches cause regressions. - # Risk band 1 - around 1 out of 15 (7%) patches within this risk band cause regressions. - # Risk band 2 - around 1 out of 7 (15%) patches within this risk band cause regressions. - # Risk bank 3 - around 1 out of 3 (35%) patches within this risk band cause regressions. - - # Step 1. Calculate % of patches which cause regressions. - total_landings = len(results) - total_regressions = sum(is_reg for _, is_reg in results) - average_regression_rate = total_regressions / total_landings - - logger.info("Average risk is %0.2f", average_regression_rate) - - MIN_SAMPLE = 200 - - # Step 2. Define risk band 1 (half than average risk). - max_band1_prob = 1.0 - total_landings = 0 - total_regressions = 0 - results.sort(key=lambda x: x[0]) - for prob, is_reg in results: - total_landings += 1 - if is_reg: - total_regressions += 1 - - if total_landings < MIN_SAMPLE: - continue + risks.append(max(probs[:, 1])) + is_regressor.append(bug_id in bug_regressors) - print( - f"{total_regressions} out of {total_landings} patches with risk lower than {prob} caused regressions ({total_regressions / total_landings}" - ) + if len(risks) == 0: + logger.warning("No bugs in the evaluation set, risk bands not computed") + return - # No need to go further, since we are interested in half than average risk. - if ( - total_regressions / total_landings - >= (average_regression_rate / 2) + 0.01 - ): - max_band1_prob = prob - break - - print("\n\n") - - # Step 3. Define risk band 3 (double than average risk). - min_band3_prob = 0.0 - total_landings = 0 - total_regressions = 0 - results.sort(key=lambda x: x[0], reverse=True) - for prob, is_reg in results: - total_landings += 1 - if is_reg: - total_regressions += 1 - - if total_landings < MIN_SAMPLE: - continue + self.risk_bands = compute_risk_bands(np.array(risks)) - print( - f"{total_regressions} out of {total_landings} patches with risk higher than {prob} caused regressions ({total_regressions / total_landings}" - ) - - # No need to go further, since we are interested in double than average risk. - if ( - total_regressions / total_landings - <= (average_regression_rate * 2) - 0.01 - ): - min_band3_prob = prob - break - - print("\n\n") - - # Step 4. Define risk band 2 (average risk). - results.sort(key=lambda x: x[0]) - for prob_start in np.arange(max_band1_prob / 2, max_band1_prob + 0.02, 0.01): - for prob_end in np.arange(min_band3_prob - 0.02, 0.99, 0.01): - total_landings = 0 - total_regressions = 0 - for prob, is_reg in results: - if prob < prob_start or prob > prob_end: - continue - - total_landings += 1 - if is_reg: - total_regressions += 1 - - if total_landings < MIN_SAMPLE: - continue - - if ( - (average_regression_rate / 2) + 0.01 - > total_regressions / total_landings - > (average_regression_rate * 2) - 0.01 - ): - continue - - print( - f"{total_regressions} out of {total_landings} patches with risk between {prob_start} and {prob_end} caused regressions ({total_regressions / total_landings}" - ) + # Regressions for recent patches might not have been filed yet, so the rates are + # lower than the actual ones, but their ratios between bands are still meaningful. + log_risk_bands(self.risk_bands, np.array(risks), np.array(is_regressor)) def get_feature_names(self): return self.clf.named_steps["union"].get_feature_names_out() diff --git a/bugbug/models/testconfigdimensions.py b/bugbug/models/testconfigdimensions.py index b66d182ca9a..cfd78037767 100644 --- a/bugbug/models/testconfigdimensions.py +++ b/bugbug/models/testconfigdimensions.py @@ -489,7 +489,7 @@ def train_test_split(self, X, y): train_len = sum(1 for index in self.row_push_index if index < split) return X[:train_len], X[train_len:], y[:train_len], y[train_len:] - def get_sample_weights(self, y): + def get_sample_weights(self, X, y): return np.array(self.row_weights[: len(y)]) def _predict_proba(self, features: Sequence[dict[str, float]]) -> np.ndarray: @@ -596,7 +596,7 @@ def fit_classifier(self, X, y): y = np.concatenate([y, missing]) super().fit_classifier(X, y) - def get_sample_weights(self, y): + def get_sample_weights(self, X, y): placeholders = getattr(self, "_placeholders", 0) weights = np.array(self.row_weights[: len(y) - placeholders]) return np.concatenate([weights, np.full(placeholders, 1e-6)]) diff --git a/bugbug/models/testconfigselect.py b/bugbug/models/testconfigselect.py index 7b36bc1663d..df99e7f5597 100644 --- a/bugbug/models/testconfigselect.py +++ b/bugbug/models/testconfigselect.py @@ -790,7 +790,7 @@ def train_test_split(self, X, y): ) return X[:train_len], X[train_len:], y[:train_len], y[train_len:] - def get_sample_weights(self, y): + def get_sample_weights(self, X, y): # The training samples are a prefix of the rows generated by items_gen. return np.array(self.row_weights[: len(y)]) diff --git a/bugbug/models/testselect.py b/bugbug/models/testselect.py index ee5e6c8dfe9..c5ac7a2f13b 100644 --- a/bugbug/models/testselect.py +++ b/bugbug/models/testselect.py @@ -643,7 +643,7 @@ def __getstate__(self): state.pop("row_push_index", None) return state - def get_sample_weights(self, y): + def get_sample_weights(self, X, y): if not self.balance_with_weights: return None diff --git a/scripts/commit_classifier.py b/scripts/commit_classifier.py index 50d7fdf2fa7..3c7963f961b 100644 --- a/scripts/commit_classifier.py +++ b/scripts/commit_classifier.py @@ -633,7 +633,8 @@ def classify_regressor(self, commits: tuple[repository.CommitDict, ...]) -> None "probs": probs[0].tolist(), } if self.model_name == "regressor": - results["risk_band"] = RegressorModel.find_risk_band(probs[0][1]) + assert isinstance(self.model, RegressorModel) + results["risk_band"] = self.model.find_risk_band(probs[0][1]) with open("results.json", "w") as f: json.dump(results, f) diff --git a/scripts/generate_landings_risk_report.py b/scripts/generate_landings_risk_report.py index 268b87a76b7..0a1adfc0777 100644 --- a/scripts/generate_landings_risk_report.py +++ b/scripts/generate_landings_risk_report.py @@ -686,7 +686,7 @@ def get_commit_data(commit_list: list[repository.CommitDict]) -> list[dict]: "commits": commit_data, "meta_ids": list(blocker_to_meta[bug_id]), "risk": max_risk, - "risk_band": RegressorModel.find_risk_band(max_risk) + "risk_band": self.regressor_model.find_risk_band(max_risk) if max_risk is not None else None, "fuzz": "b" @@ -995,6 +995,7 @@ def get_tracking_info(bug): for team in all_teams: cur_team_data = team_data[team] = {} cur_team_data["month_changes"] = 0 + cur_team_data["very_high_risk_changes"] = 0 cur_team_data["high_risk_changes"] = 0 cur_team_data["medium_risk_changes"] = 0 cur_team_data["low_risk_changes"] = 0 @@ -1055,9 +1056,14 @@ def get_tracking_info(bug): (fix_date - creation_date).total_seconds() / 86400 ) - if bug["risk_band"] == "h": + # "high_risk_changes" counts all higher than average risk changes, including + # the very high risk ones. + if bug["risk_band"] == "v": + cur_team_data["very_high_risk_changes"] += 1 cur_team_data["high_risk_changes"] += 1 - elif bug["risk_band"] == "m": + elif bug["risk_band"] == "h": + cur_team_data["high_risk_changes"] += 1 + elif bug["risk_band"] == "a": cur_team_data["medium_risk_changes"] += 1 elif bug["risk_band"] == "l": cur_team_data["low_risk_changes"] += 1 @@ -1106,7 +1112,7 @@ def get_tracking_info(bug): (fix_date - creation_date).total_seconds() / 86400 ) - if bug["risk_band"] == "h": + if bug["risk_band"] in ("h", "v"): cur_team_data["prev_high_risk_changes"] += 1 cur_team_data["prev_review_times"] += [ @@ -1460,10 +1466,16 @@ def get_top_crashes(team: str, channel: str) -> str | None: try: month_changes = cur_team_data["month_changes"] high_risk_changes = cur_team_data["high_risk_changes"] + # Regression rates observed in each risk band. predicted_regressions = round( - 0.33 * cur_team_data["high_risk_changes"] - + 0.16 * cur_team_data["medium_risk_changes"] - + 0.06 * cur_team_data["low_risk_changes"], + 0.40 * cur_team_data["very_high_risk_changes"] + + 0.24 + * ( + cur_team_data["high_risk_changes"] + - cur_team_data["very_high_risk_changes"] + ) + + 0.12 * cur_team_data["medium_risk_changes"] + + 0.045 * cur_team_data["low_risk_changes"], 1, ) prev_high_risk_changes = cur_team_data["prev_high_risk_changes"] diff --git a/tests/test_regressor_risk_bands.py b/tests/test_regressor_risk_bands.py new file mode 100644 index 00000000000..dc4439ffa2b --- /dev/null +++ b/tests/test_regressor_risk_bands.py @@ -0,0 +1,119 @@ +# -*- coding: utf-8 -*- +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. + +from collections import Counter + +import numpy as np + +from bugbug.models.regressor import RegressorModel, compute_risk_bands + + +def test_compute_risk_bands(): + risks = np.random.RandomState(0).uniform(size=10000) + + risk_bands = compute_risk_bands(risks) + + assert [name for name, _, _ in risk_bands] == ["l", "a", "h", "v"] + # The bands are contiguous and cover the whole risk scale. + assert risk_bands[0][1] == 0.0 + assert risk_bands[-1][2] == 1.0 + for (_, _, end), (_, start, _) in zip(risk_bands, risk_bands[1:]): + assert end == start + + model = RegressorModel() + model.risk_bands = risk_bands + counts = Counter(model.find_risk_band(risk) for risk in risks) + assert counts == {"l": 5000, "a": 3000, "h": 1500, "v": 500} + + # Boundaries belong to the lower band. + assert model.find_risk_band(risk_bands[0][2]) == "l" + assert model.find_risk_band(0.0) == "l" + assert model.find_risk_band(1.0) == "v" + + +def test_compute_risk_bands_ties(): + # Isotonic calibration outputs few distinct values, so many risks are tied. + risks = np.repeat([0.1, 0.2, 0.5, 0.9], [600, 250, 120, 30]) + + risk_bands = compute_risk_bands(risks) + + model = RegressorModel() + model.risk_bands = risk_bands + counts = Counter(model.find_risk_band(risk) for risk in risks) + # Each tied group goes to the band whose target share it fits best, and the top + # band is not left empty even if the 95th percentile falls in a tied group. + assert counts == {"l": 600, "a": 250, "h": 120, "v": 30} + + +def test_fit_classifier_bug_weights(monkeypatch): + import pandas as pd + + model = RegressorModel() + # Bug 1: 3 commits (regressor), bug 2: 1 commit (regressor), bugs 3-6: clean. + model.train_bug_ids = [1, 1, 1, 2, 3, 4, 5, 6] + # As after train_test_split: a subset of rows, shuffled, keeping the original index. + X = pd.DataFrame({"a": range(8)}, index=[7, 0, 3, 1, 4, 2, 5, 6]) + y = np.array([0, 1, 1, 1, 0, 1, 0, 0]) + + captured = {} + monkeypatch.setattr( + model.clf, + "fit", + lambda X, y, estimator__sample_weight: captured.update( + w=estimator__sample_weight + ), + ) + model.fit_classifier(X, y) + + w = dict(zip(X.index, captured["w"])) + # Each regressor bug gets the same total weight, split among its commits. + assert np.isclose(w[0] + w[1] + w[2], w[3]) + assert np.isclose(w[0], w[1]) and np.isclose(w[1], w[2]) + # Both classes have the same total weight. + assert np.isclose(captured["w"][y == 1].sum(), captured["w"][y == 0].sum()) + + +def test_cross_validation_bug_weights(monkeypatch): + import pandas as pd + import pytest + + import bugbug.model + + model = RegressorModel() + # 40 commits from 20 bugs (2 commits each), the first 10 bugs are regressors. + bug_ids = [i // 2 for i in range(40)] + labels = [1 if bug_id < 10 else 0 for bug_id in bug_ids] + + def items_gen(classes): + model.train_bug_ids = bug_ids + return iter(zip(range(40), labels)) + + monkeypatch.setattr(model, "get_labels", lambda: ({}, [0, 1])) + monkeypatch.setattr(model, "items_gen", items_gen) + monkeypatch.setattr( + model.extraction_pipeline, + "transform", + lambda X_gen: pd.DataFrame({"a": list(X_gen())}), + ) + + class Stop(Exception): + pass + + captured = {} + + def cross_validate(clf, X, y, params, **kwargs): + captured.update(X=X, y=y, params=params) + raise Stop + + monkeypatch.setattr(bugbug.model, "cross_validate", cross_validate) + with pytest.raises(Stop): + model.train() + + # The folds get the same weights as the final model. + weights = captured["params"]["estimator__sample_weight"] + assert np.allclose(weights, model.get_sample_weights(captured["X"], captured["y"])) + assert np.isclose( + weights[captured["y"] == 1].sum(), weights[captured["y"] == 0].sum() + ) diff --git a/tests/test_testconfigselect.py b/tests/test_testconfigselect.py index 9a955b1d100..516a008bf8c 100644 --- a/tests/test_testconfigselect.py +++ b/tests/test_testconfigselect.py @@ -214,7 +214,7 @@ def trained_model( assert {r.push_index for r, _ in model.evaluation_regressions} == set( range(540, 600, 10) ) - assert len(model.get_sample_weights(y_train)) == len(y_train) + assert len(model.get_sample_weights(None, y_train)) == len(y_train) model.le.fit(y) model.fit_classifier(X, model.le.transform(y)) diff --git a/tests/test_testselect.py b/tests/test_testselect.py index 7dde6843b81..c26dfae5f66 100644 --- a/tests/test_testselect.py +++ b/tests/test_testselect.py @@ -841,7 +841,7 @@ def test_group_model_balances_with_weights() -> None: assert "sampler" not in model.clf.named_steps config_group_model = testselect.TestConfigGroupSelectModel() assert "sampler" in config_group_model.clf.named_steps - assert config_group_model.get_sample_weights(np.array([1, 0])) is None + assert config_group_model.get_sample_weights(None, np.array([1, 0])) is None rng = np.random.default_rng(0) y = (rng.random(200) < 0.2).astype(int) @@ -928,7 +928,7 @@ def test_group_model_sample_weights() -> None: model.row_push_failures = [10, 10, 1, 1, 1, 1] model.row_non_run = [False] * 6 model.row_push_index = [0] * 6 - assert list(model.get_sample_weights(y)) == [0.5, 0.5, 1.0, 0.5, 0.5, 0.5] + assert list(model.get_sample_weights(None, y)) == [0.5, 0.5, 1.0, 0.5, 0.5, 0.5] # Non-run negatives (when enabled) get a lower weight, and don't count when balancing the # classes. @@ -938,7 +938,7 @@ def test_group_model_sample_weights() -> None: model.row_push_failures = [1, 1, 1, 1] model.row_non_run = [False, False, True, True] model.row_push_index = [0] * 4 - assert list(model.get_sample_weights(y)) == [1.0, 1.0, 0.3, 0.3] + assert list(model.get_sample_weights(None, y)) == [1.0, 1.0, 0.3, 0.3] def test_group_model_uses_manifest_suite() -> None: @@ -1152,8 +1152,8 @@ def train_test_split(self, X, y): weights = [] orig_weights = model_class.get_sample_weights - def get_sample_weights(self, y): - w = orig_weights(self, y) + def get_sample_weights(self, X, y): + w = orig_weights(self, X, y) weights.append((len(y), w)) return w diff --git a/ui/changes/src/common.js b/ui/changes/src/common.js index f4959ba42fb..a1bf46d6d54 100644 --- a/ui/changes/src/common.js +++ b/ui/changes/src/common.js @@ -148,6 +148,7 @@ export const TESTING_TAGS = { }, }; +const VERY_HIGH_RISK_COLOR = "darkred"; const HIGH_RISK_COLOR = "rgb(255, 13, 87)"; const MEDIUM_RISK_COLOR = "darkkhaki"; const LOW_RISK_COLOR = "green"; @@ -564,13 +565,16 @@ export async function renderRiskChart(chartEl, bugSummaries) { counterObj.low += 1; } else if (summary.risk_band == "a") { counterObj.medium += 1; - } else { + } else if (summary.risk_band == "h") { counterObj.high += 1; + } else { + counterObj.veryHigh += 1; } } ); let categories = []; + let veryHigh = []; let high = []; let medium = []; let low = []; @@ -579,11 +583,16 @@ export async function renderRiskChart(chartEl, bugSummaries) { low.push(summaryData[date].low); medium.push(summaryData[date].medium); high.push(summaryData[date].high); + veryHigh.push(summaryData[date].veryHigh); } renderChart( chartEl, [ + { + name: "Very high", + data: veryHigh, + }, { name: "Higher", data: high, @@ -598,7 +607,7 @@ export async function renderRiskChart(chartEl, bugSummaries) { }, ], categories, - "Evolution of lower/average/higher risk changes", + "Evolution of lower/average/higher/very high risk changes", "# of patches" ); } @@ -2919,6 +2928,10 @@ function addRow( // Higher than average risk. risk_text.style.color = HIGH_RISK_COLOR; risk_text.textContent = "Higher"; + } else if (bugSummary.risk_band == "v") { + // Very high risk. + risk_text.style.color = VERY_HIGH_RISK_COLOR; + risk_text.textContent = "Very high"; } else if (bugSummary.risk_band == null) { // No risk available (there are no commits associated to the bug). risk_text.textContent = "N/A"; diff --git a/ui/changes/src/feature.html b/ui/changes/src/feature.html index 192d870e9f7..d039317df45 100644 --- a/ui/changes/src/feature.html +++ b/ui/changes/src/feature.html @@ -109,6 +109,7 @@