Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 16 additions & 3 deletions bugbug/model.py
Original file line number Diff line number Diff line change
Expand Up @@ -370,12 +370,12 @@ def get_test_subsets(self) -> dict[str, np.ndarray]:
"""Subclasses can return boolean masks over the test set, to also report metrics on those subsets."""
return {}

def get_sample_weights(self, y) -> np.ndarray | None:
def get_sample_weights(self, X, y) -> np.ndarray | None:
"""Subclasses can return a weight for each training sample, passed to the estimator."""
return None

def fit_classifier(self, X, y):
sample_weight = self.get_sample_weights(y)
sample_weight = self.get_sample_weights(X, y)
if sample_weight is None:
self.clf.fit(X, y)
else:
Expand Down Expand Up @@ -419,8 +419,21 @@ def train(self, importance_cutoff=0.15, limit=None):
if len(self.class_names) == 2:
scorings += ["precision", "recall"]

# The folds are weighted like the final model (see fit_classifier); the
# weights are computed once on the whole training set and split with it.
cv_y = self.le.transform(y_train)
sample_weight = self.get_sample_weights(X_train, cv_y)
scores = cross_validate(
self.clf, X_train, self.le.transform(y_train), scoring=scorings, cv=5
self.clf,
X_train,
cv_y,
scoring=scorings,
cv=5,
params=(
{"estimator__sample_weight": sample_weight}
if sample_weight is not None
else None
),
)

logger.info("Cross Validation scores:")
Expand Down
14 changes: 9 additions & 5 deletions bugbug/model_calibration.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@

from typing import Sequence

import numpy as np
from sklearn.base import BaseEstimator, ClassifierMixin
from sklearn.calibration import CalibratedClassifierCV
from sklearn.model_selection import train_test_split
Expand Down Expand Up @@ -39,12 +40,15 @@ def __init__(self, base_clf):
base_clf, cv="prefit", method="isotonic"
)

def fit(self, X_train, y_train):
X_train, X_val, y_train, y_val = train_test_split(
X_train, y_train, test_size=0.2, random_state=42
def fit(self, X_train, y_train, sample_weight=None):
if sample_weight is None:
sample_weight = np.ones(len(y_train))

X_train, X_val, y_train, y_val, w_train, w_val = train_test_split(
X_train, y_train, sample_weight, test_size=0.2, random_state=42
)
self.base_clf.fit(X_train, y_train)
self.calibrated_clf.fit(X_val, y_val)
self.base_clf.fit(X_train, y_train, sample_weight=w_train)
self.calibrated_clf.fit(X_val, y_val, sample_weight=w_val)

def predict(self, X):
return self.calibrated_clf.predict(X)
Expand Down
Loading